cctally 1.82.0 → 1.83.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +52 -74
  3. package/bin/_cctally_alerts.py +8 -1
  4. package/bin/_cctally_cache.py +963 -149
  5. package/bin/_cctally_config.py +43 -4
  6. package/bin/_cctally_core.py +933 -759
  7. package/bin/_cctally_dashboard.py +157 -47
  8. package/bin/_cctally_dashboard_cache_report.py +13 -6
  9. package/bin/_cctally_dashboard_conversation.py +1 -0
  10. package/bin/_cctally_dashboard_envelope.py +186 -8
  11. package/bin/_cctally_dashboard_share.py +60 -20
  12. package/bin/_cctally_dashboard_sources.py +427 -128
  13. package/bin/_cctally_db.py +605 -128
  14. package/bin/_cctally_doctor.py +413 -28
  15. package/bin/_cctally_five_hour.py +12 -5
  16. package/bin/_cctally_journal.py +2050 -156
  17. package/bin/_cctally_journal_repair.py +519 -0
  18. package/bin/_cctally_milestone_history.py +142 -56
  19. package/bin/_cctally_milestones.py +179 -111
  20. package/bin/_cctally_parser.py +42 -0
  21. package/bin/_cctally_project.py +24 -18
  22. package/bin/_cctally_quota.py +139 -25
  23. package/bin/_cctally_record.py +279 -108
  24. package/bin/_cctally_rederive.py +1052 -0
  25. package/bin/_cctally_reporting.py +58 -53
  26. package/bin/_cctally_setup.py +1 -0
  27. package/bin/_cctally_source_analytics.py +4 -1
  28. package/bin/_cctally_statusline.py +11 -11
  29. package/bin/_cctally_store.py +1039 -31
  30. package/bin/_cctally_sync_week.py +17 -8
  31. package/bin/_cctally_tui.py +421 -54
  32. package/bin/_cctally_update.py +133 -8
  33. package/bin/_cctally_weekrefs.py +14 -0
  34. package/bin/_lib_aggregators.py +10 -6
  35. package/bin/_lib_cache_report.py +101 -9
  36. package/bin/_lib_codex_pools.py +82 -0
  37. package/bin/_lib_conversation_query.py +126 -33
  38. package/bin/_lib_dashboard_sources.py +126 -1
  39. package/bin/_lib_diff_kernel.py +28 -15
  40. package/bin/_lib_doctor.py +342 -4
  41. package/bin/_lib_journal.py +924 -2
  42. package/bin/_lib_jsonl.py +43 -14
  43. package/bin/_lib_pricing.py +140 -21
  44. package/bin/_lib_readme_refresh.py +401 -0
  45. package/bin/_lib_rederive.py +395 -0
  46. package/bin/_lib_share.py +58 -2
  47. package/bin/cctally +56 -8
  48. package/dashboard/static/assets/{index-DJP4gEB7.js → index-3bgCMVHb.js} +52 -52
  49. package/dashboard/static/assets/index-D27EIHEI.css +1 -0
  50. package/dashboard/static/dashboard.html +2 -2
  51. package/package.json +6 -1
  52. package/dashboard/static/assets/index-Dk1nplOz.css +0 -1
@@ -117,7 +117,7 @@ _CACHE_DISPATCH_WRITER_LOCK_HELD = contextvars.ContextVar(
117
117
  # (no shim drift); _lib_pricing is a stdlib-only leaf module so no cycle
118
118
  # risk. Other siblings (_cctally_record, _cctally_dashboard) follow the
119
119
  # same direct-import pattern.
120
- from _lib_pricing import _calculate_entry_cost
120
+ from _lib_pricing import _calculate_entry_cost, claude_usage_dict
121
121
 
122
122
 
123
123
  # Module-level back-ref shim for the one Z-high callable that STAYS in
@@ -276,11 +276,20 @@ class StatsDbCorruptError(sqlite3.DatabaseError):
276
276
 
277
277
 
278
278
  class StatsDbMaintenanceError(sqlite3.OperationalError):
279
- """A guided repair owns stats.db; new cctally opens must stay out."""
279
+ """A guided repair owns stats.db; new cctally opens must stay out.
280
280
 
281
- def __init__(self) -> None:
281
+ ``reason`` is optional and defaults to the historical repair wording, which
282
+ several regressions assert verbatim. #386 added a second cause — the
283
+ opener's BOUNDED wait for ``stats.db.maintenance.lock`` expiring while some
284
+ other maintenance command (rebuild / vacuum / rederive / skip / unskip)
285
+ holds it exclusive — which is not a `db repair` and must not claim to be.
286
+ """
287
+
288
+ def __init__(self, reason: "str | None" = None) -> None:
282
289
  super().__init__(
283
- "stats.db repair is in progress; retry after the repair command exits"
290
+ reason
291
+ or "stats.db repair is in progress; retry after the repair "
292
+ "command exits"
284
293
  )
285
294
 
286
295
 
@@ -325,6 +334,15 @@ def _stats_corruption_guidance() -> str:
325
334
  )
326
335
 
327
336
 
337
+ # #195: the cache_meta flag cache migration 030 sets to arm the one-time
338
+ # cost-side re-walk that lands the cache-write TTL split on existing rows.
339
+ # `sync_cache` reads it to select SESSION_ENTRY_UPSERT_SQL_REWALK (the chained
340
+ # ON CONFLICT variant) and clears it at the end of a clean, non-targeted full
341
+ # walk. Deliberately NOT `claude_ingest_walk_complete` — see the migration's
342
+ # docstring for why reusing that marker is both unnecessary and over-broad.
343
+ CACHE_CREATION_SPLIT_REWALK_KEY = "cache_creation_split_rewalk_pending"
344
+
345
+
328
346
  class MigrationGateNotMet(Exception):
329
347
  """Migration cannot run yet because a cross-DB prerequisite is unsatisfied.
330
348
 
@@ -667,17 +685,96 @@ def _forensics_iso(epoch_or_now) -> str:
667
685
  return d.isoformat(timespec="seconds").replace("+00:00", "Z")
668
686
 
669
687
 
688
+ class CorruptionProbeDisposition(str, enum.Enum):
689
+ """Whether the locked forensics probe confirmed physical corruption."""
690
+
691
+ CONFIRMED = "confirmed"
692
+ UNCONFIRMED = "unconfirmed"
693
+
694
+
695
+ @dataclass(frozen=True)
696
+ class CorruptionForensicsResult:
697
+ """Typed result used by destructive recovery decision points."""
698
+
699
+ path: "pathlib.Path | None"
700
+ disposition: CorruptionProbeDisposition
701
+ reason: str
702
+ integrity_check: "tuple[str, ...] | str | None"
703
+
704
+
705
+ _FORENSICS_ORIGIN_MAX = 160
706
+ _FORENSICS_EXCEPTION_TYPE_MAX = 160
707
+ _FORENSICS_EXCEPTION_MESSAGE_MAX = 512
708
+ _FORENSICS_CALLSITE_TEXT_MAX = 160
709
+
710
+
711
+ def _bounded_forensics_text(value: object, limit: int) -> str:
712
+ text = str(value)
713
+ if len(text) <= limit:
714
+ return text
715
+ return text[: max(0, limit - 1)] + "…"
716
+
717
+
718
+ def _corruption_trigger_record(
719
+ origin: str, exc: BaseException,
720
+ ) -> dict[str, object]:
721
+ """Build bounded, caller-owned trigger metadata without parsing display text."""
722
+ frames = traceback.extract_tb(exc.__traceback__, limit=32)
723
+ call_site: dict[str, object] | None = None
724
+ if frames:
725
+ frame = frames[-1]
726
+ call_site = {
727
+ "file": _bounded_forensics_text(
728
+ pathlib.Path(frame.filename).name,
729
+ _FORENSICS_CALLSITE_TEXT_MAX,
730
+ ),
731
+ "function": _bounded_forensics_text(
732
+ frame.name, _FORENSICS_CALLSITE_TEXT_MAX,
733
+ ),
734
+ "line": int(frame.lineno),
735
+ }
736
+ code = getattr(exc, "sqlite_errorcode", None)
737
+ name = getattr(exc, "sqlite_errorname", None)
738
+ exc_type = type(exc)
739
+ return {
740
+ "origin": _bounded_forensics_text(origin, _FORENSICS_ORIGIN_MAX),
741
+ "exceptionType": _bounded_forensics_text(
742
+ f"{exc_type.__module__}.{exc_type.__qualname__}",
743
+ _FORENSICS_EXCEPTION_TYPE_MAX,
744
+ ),
745
+ "message": _bounded_forensics_text(
746
+ exc, _FORENSICS_EXCEPTION_MESSAGE_MAX,
747
+ ),
748
+ "sqliteErrorCode": code if isinstance(code, int) else None,
749
+ "sqliteErrorName": name if isinstance(name, str) else None,
750
+ "tracebackCallSite": call_site,
751
+ }
752
+
753
+
670
754
  def write_corruption_forensics(
671
- db_path, *, db_label: str = "stats",
672
- ) -> "pathlib.Path | None":
755
+ db_path,
756
+ *,
757
+ db_label: str = "stats",
758
+ trigger_origin: "str | None" = None,
759
+ trigger_exception: "BaseException | None" = None,
760
+ return_result: bool = False,
761
+ ) -> "pathlib.Path | None | CorruptionForensicsResult":
673
762
  """Write the #336-defense-2 corruption forensics bundle FIRST — before any
674
763
  quarantine/rebuild disturbs the evidence (spec §6.3). Captures the file family
675
764
  sizes+mtimes, a best-effort ``PRAGMA integrity_check``, an ``lsof`` of the
676
765
  family, and a scan of running processes for known cloud-sync/backup holders.
677
- Every leg degrades to a captured error string rather than raising, so a heal
678
- never fails because forensics could not fully run. Returns the bundle path (or
679
- None if it could not be written). Shared by the auto-heal HEAL_HOOK and
680
- ``db rebuild`` (and, later, doctor's incident leg)."""
766
+ Every leg degrades to a captured error string rather than raising.
767
+
768
+ ``return_result=False`` preserves the historical path-or-None API for stats
769
+ heal/rebuild callers. Cache recovery opts into the typed result: only a
770
+ persisted bundle plus either a non-``ok`` integrity row or a positively
771
+ classified corruption failure during the probe confirms destructive
772
+ quarantine. Probe/write unavailability is always unconfirmed.
773
+ """
774
+ if (trigger_origin is None) != (trigger_exception is None):
775
+ raise ValueError(
776
+ "trigger_origin and trigger_exception must be supplied together"
777
+ )
681
778
  db_path = pathlib.Path(db_path)
682
779
  ts = _db_backup_timestamp()
683
780
  try:
@@ -692,9 +789,15 @@ def write_corruption_forensics(
692
789
  "path": str(db_path),
693
790
  "family": {},
694
791
  "integrityCheck": None,
792
+ "probeDisposition": CorruptionProbeDisposition.UNCONFIRMED.value,
793
+ "probeReason": "integrity_check_not_run",
695
794
  "lsof": None,
696
795
  "holders": [],
697
796
  }
797
+ if trigger_origin is not None and trigger_exception is not None:
798
+ bundle["trigger"] = _corruption_trigger_record(
799
+ trigger_origin, trigger_exception,
800
+ )
698
801
  for suffix in ("", "-wal", "-shm"):
699
802
  p = pathlib.Path(str(db_path) + suffix)
700
803
  try:
@@ -709,12 +812,34 @@ def write_corruption_forensics(
709
812
  try:
710
813
  c.execute("PRAGMA busy_timeout=2000")
711
814
  bundle["integrityCheck"] = [
712
- r[0] for r in c.execute("PRAGMA integrity_check").fetchall()
815
+ str(r[0]) for r in c.execute("PRAGMA integrity_check").fetchall()
713
816
  ]
714
817
  finally:
715
818
  c.close()
819
+ rows = bundle["integrityCheck"]
820
+ assert isinstance(rows, list)
821
+ if any(row.strip().casefold() != "ok" for row in rows):
822
+ disposition = CorruptionProbeDisposition.CONFIRMED
823
+ reason = "integrity_check_non_ok"
824
+ elif len(rows) == 1 and rows[0].strip().casefold() == "ok":
825
+ disposition = CorruptionProbeDisposition.UNCONFIRMED
826
+ reason = "integrity_check_ok"
827
+ else:
828
+ disposition = CorruptionProbeDisposition.UNCONFIRMED
829
+ reason = "integrity_check_inconclusive"
716
830
  except Exception as exc:
717
- bundle["integrityCheck"] = f"error: {exc}"
831
+ bundle["integrityCheck"] = (
832
+ "error: "
833
+ + _bounded_forensics_text(exc, _FORENSICS_EXCEPTION_MESSAGE_MAX)
834
+ )
835
+ if _is_sqlite_corruption_error(exc):
836
+ disposition = CorruptionProbeDisposition.CONFIRMED
837
+ reason = "integrity_check_corruption_error"
838
+ else:
839
+ disposition = CorruptionProbeDisposition.UNCONFIRMED
840
+ reason = "integrity_check_unavailable"
841
+ bundle["probeDisposition"] = disposition.value
842
+ bundle["probeReason"] = reason
718
843
  try:
719
844
  cp = subprocess.run(
720
845
  ["lsof", "--", str(db_path)],
@@ -741,8 +866,28 @@ def write_corruption_forensics(
741
866
  except OSError:
742
867
  pass
743
868
  except OSError:
744
- return None
745
- return out
869
+ result = CorruptionForensicsResult(
870
+ path=None,
871
+ disposition=CorruptionProbeDisposition.UNCONFIRMED,
872
+ reason="bundle_write_failed",
873
+ integrity_check=(
874
+ tuple(bundle["integrityCheck"])
875
+ if isinstance(bundle["integrityCheck"], list)
876
+ else bundle["integrityCheck"]
877
+ ),
878
+ )
879
+ return result if return_result else None
880
+ result = CorruptionForensicsResult(
881
+ path=out,
882
+ disposition=disposition,
883
+ reason=reason,
884
+ integrity_check=(
885
+ tuple(bundle["integrityCheck"])
886
+ if isinstance(bundle["integrityCheck"], list)
887
+ else bundle["integrityCheck"]
888
+ ),
889
+ )
890
+ return result if return_result else out
746
891
 
747
892
 
748
893
  def _quarantine_pending_path(db_path: pathlib.Path) -> pathlib.Path:
@@ -949,15 +1094,20 @@ def quarantine_db_family(
949
1094
  return incident
950
1095
 
951
1096
 
1097
+ #: How many quarantined same-revision groups `db rebuild` names in TEXT output
1098
+ #: before summarising the rest (#374). `--json` always carries every group.
1099
+ _REBUILD_CONFLICT_SAMPLE = 10
1100
+
1101
+
952
1102
  def cmd_db_rebuild(args: argparse.Namespace) -> int:
953
1103
  """``db rebuild --db stats`` — explicit journal replay into a fresh index
954
- (spec §9). Forensics-quarantines the current stats.db the SAME forensics-first
955
- way auto-heal does (even when the DB is healthy this is an operator's
956
- deliberate reset), then rebuilds from the journal and reports counts +
957
- duration. Held under the stats maintenance lock so it serializes with a
958
- concurrent auto-heal. #146 prod guard: a dev/worktree binary refuses to
959
- rebuild the real prod stats.db unless ``CCTALLY_ALLOW_PROD_MIGRATION=1``.
960
- Exit 0 on success, 2 for the prod guard, 3 for a rebuild failure."""
1104
+ (spec §9). Captures forensics first, builds and validates a fresh index while
1105
+ the old one remains available, then lets the common rebuild cutover preserve
1106
+ the old family and atomically publish the replacement. Held under the stats
1107
+ maintenance lock so it serializes with a concurrent auto-heal. #146 prod
1108
+ guard: a dev/worktree binary refuses to rebuild the real prod stats.db unless
1109
+ ``CCTALLY_ALLOW_PROD_MIGRATION=1``. Exit 0 on success, 2 for the prod guard,
1110
+ 3 for a rebuild failure."""
961
1111
  from _lib_json_envelope import stamp_schema_version
962
1112
  import _cctally_journal
963
1113
 
@@ -980,26 +1130,51 @@ def cmd_db_rebuild(args: argparse.Namespace) -> int:
980
1130
  incident = None
981
1131
  try:
982
1132
  fcntl.flock(maint_fd, fcntl.LOCK_EX)
1133
+ # #386: record the hold so a nested live `open_db()` does not request
1134
+ # SHARED on a second fd of this same file and self-deadlock.
1135
+ _cctally_core.note_stats_maintenance_acquired()
983
1136
  import _cctally_store
984
1137
  # Symmetry with the auto-heal path (Task-8 P3-3): take the bounded ingest
985
1138
  # lock so a concurrent ingest cycle can't write into the stats.db we are
986
1139
  # about to quarantine + rebuild. Journal-is-truth makes this
987
1140
  # correctness-neutral, but keeping the two rebuild paths identical avoids
988
1141
  # a latent write-into-quarantined-inode surprise.
989
- ingest_fd = _cctally_store._heal_flock_bounded(
990
- _cctally_core.JOURNAL_INGEST_LOCK_PATH, 5.0)
1142
+ # #386: the bounded acquire now returns None on timeout instead of an
1143
+ # UNLOCKED fd, so "we are serialized" is no longer a claim the code makes
1144
+ # about a lock it does not hold. A context that already holds the ingest
1145
+ # lock IS the serialized writer and must not wait on itself.
1146
+ if _cctally_store.holds_ingest_lock():
1147
+ ingest_fd = None
1148
+ else:
1149
+ ingest_fd = _cctally_store._heal_flock_bounded(
1150
+ _cctally_core.JOURNAL_INGEST_LOCK_PATH, 5.0)
1151
+ if ingest_fd is None:
1152
+ eprint(
1153
+ "cctally: stats.db rebuild declined: another ingest holds "
1154
+ "journal.ingest.lock. Retry shortly."
1155
+ )
1156
+ return 3
991
1157
  try:
992
1158
  if path.exists():
1159
+ # Forensics FIRST. The common cutover performs the final drain
1160
+ # check only after the scratch index is complete.
993
1161
  forensics = write_corruption_forensics(path, db_label="stats")
994
- incident = quarantine_db_family(path)
995
- result = _cctally_journal.rebuild_stats_index()
1162
+ # #386: declare the sanctioned maintenance regime around the
1163
+ # replacement — the rebuild writes its scratch index through an
1164
+ # authorizer-armed `open_db(_target_path=...)` connection, and we
1165
+ # hold maintenance exclusive, which is what spec §3.1 sanctions.
1166
+ with _cctally_store.stats_write_scope("maintenance-rebuild"):
1167
+ result = _cctally_journal.rebuild_stats_index()
1168
+ incident = result.quarantine_dir
996
1169
  except Exception as exc:
997
1170
  eprint(f"cctally: stats.db rebuild failed: {exc}")
998
1171
  return 3
999
1172
  finally:
1000
- _cctally_store._heal_release_flock(ingest_fd)
1173
+ if ingest_fd is not None:
1174
+ _cctally_store._heal_release_flock(ingest_fd)
1001
1175
  finally:
1002
1176
  try:
1177
+ _cctally_core.note_stats_maintenance_released()
1003
1178
  fcntl.flock(maint_fd, fcntl.LOCK_UN)
1004
1179
  finally:
1005
1180
  os.close(maint_fd)
@@ -1016,6 +1191,19 @@ def cmd_db_rebuild(args: argparse.Namespace) -> int:
1016
1191
  "totalRows": total_rows,
1017
1192
  "quarantineDir": str(incident) if incident else None,
1018
1193
  "forensicsPath": str(forensics) if forensics else None,
1194
+ # #374 — additive; DISTINCT from `db rederive --json`'s pre-existing
1195
+ # `conflicts` key (command-validation failures). Exit stays 0.
1196
+ "journalConflicts": [c.to_dict() for c in result.conflicts],
1197
+ # #402 Task A — the usable index omitted every action from these
1198
+ # whole tainted batches. Complete and deterministic in JSON.
1199
+ "journalProtocolViolations": [
1200
+ violation.to_dict()
1201
+ for violation in result.protocol_violations
1202
+ ],
1203
+ "journalAcknowledgedProtocolViolations": [
1204
+ violation.to_dict()
1205
+ for violation in result.acknowledged_protocol_violations
1206
+ ],
1019
1207
  }
1020
1208
  print(json.dumps(stamp_schema_version(payload, version=1)))
1021
1209
  else:
@@ -1028,6 +1216,40 @@ def cmd_db_rebuild(args: argparse.Namespace) -> int:
1028
1216
  for tbl, n in result.rows_by_table.items():
1029
1217
  if n:
1030
1218
  print(f" {tbl}: {n}")
1219
+ if result.conflicts:
1220
+ print(
1221
+ f" {len(result.conflicts)} quarantined same-revision "
1222
+ "group(s) — the first-written variant was used:"
1223
+ )
1224
+ # A real journal can carry hundreds of groups (292 on the maintainer's
1225
+ # own), so the text view lists a sample and points at `--json` for the
1226
+ # complete set, which is never truncated.
1227
+ for conflict in result.conflicts[:_REBUILD_CONFLICT_SAMPLE]:
1228
+ print(f" {conflict.event_id} rev {conflict.rev} "
1229
+ f"({len(conflict.content_hashes)} variants)")
1230
+ hidden = len(result.conflicts) - _REBUILD_CONFLICT_SAMPLE
1231
+ if hidden > 0:
1232
+ print(f" ... and {hidden} more (see --json for all)")
1233
+ print(" Resolve with: cctally db rederive --family claude-usage")
1234
+ if result.protocol_violations:
1235
+ print(
1236
+ f" {len(result.protocol_violations)} structural journal "
1237
+ "protocol violation(s) — affected correction batches were "
1238
+ "tainted and omitted:"
1239
+ )
1240
+ for violation in result.protocol_violations:
1241
+ print(f" {violation.batch_id}: {violation.kind}")
1242
+ if result.acknowledged_protocol_violations:
1243
+ print(
1244
+ f" {len(result.acknowledged_protocol_violations)} acknowledged "
1245
+ "structural journal protocol violation(s) — affected correction "
1246
+ "batches remain tainted and omitted:"
1247
+ )
1248
+ for violation in result.acknowledged_protocol_violations:
1249
+ print(
1250
+ f" {violation.batch_id}: {violation.kind} "
1251
+ f"(audit {violation.audit_id})"
1252
+ )
1031
1253
  if incident is not None:
1032
1254
  print(f" previous stats.db quarantined -> {incident}")
1033
1255
  if forensics is not None:
@@ -1145,6 +1367,30 @@ def _stamp_applied(conn, name, applied_at_utc=None):
1145
1367
  conn.commit()
1146
1368
 
1147
1369
 
1370
+ def _reconcile_durable_applied_migration_errors(
1371
+ conn: sqlite3.Connection,
1372
+ registry: list[Migration],
1373
+ db_label: str,
1374
+ ) -> None:
1375
+ """Retry stale-sentinel cleanup for every durably applied migration."""
1376
+ if not _cctally_core.MIGRATION_ERROR_LOG_PATH.exists():
1377
+ return
1378
+ try:
1379
+ applied = {
1380
+ row[0]
1381
+ for row in conn.execute(
1382
+ "SELECT name FROM schema_migrations"
1383
+ ).fetchall()
1384
+ }
1385
+ except sqlite3.OperationalError:
1386
+ return
1387
+ for migration in registry:
1388
+ if migration.name in applied:
1389
+ _clear_migration_error_log_entries(
1390
+ f"{db_label}:{migration.name}"
1391
+ )
1392
+
1393
+
1148
1394
  def _run_pending_migrations(
1149
1395
  conn: sqlite3.Connection,
1150
1396
  *,
@@ -1167,7 +1413,12 @@ def _run_pending_migrations(
1167
1413
  - Fresh install (schema_migrations just CREATE'd, zero rows
1168
1414
  post-bootstrap, AND the DB's primary data table is empty or
1169
1415
  absent) → stamp every migration applied without invoking
1170
- handlers. The data-emptiness probe (D1) defends against the
1416
+ handlers, commit the markers, then clear matching stale
1417
+ migration-error sentinel entries. The marker commit precedes
1418
+ sentinel cleanup so a failed commit preserves failure evidence;
1419
+ later slow and fast opens retry cleanup for every durable applied
1420
+ marker so an interruption after commit converges.
1421
+ The data-emptiness probe (D1) defends against the
1171
1422
  pre-framework upgrade case where cache.db was populated by
1172
1423
  a pre-v1.12.0 build that wrote ``session_entries`` without
1173
1424
  ever creating ``schema_migrations`` — pre-fix that landscape
@@ -1240,6 +1491,9 @@ def _run_pending_migrations(
1240
1491
  _clear_migration_error_log_entries(
1241
1492
  f"{db_label}:_bootstrap_rename_legacy_markers"
1242
1493
  )
1494
+ _reconcile_durable_applied_migration_errors(
1495
+ conn, registry, db_label,
1496
+ )
1243
1497
  return # fast path
1244
1498
 
1245
1499
  # Prod-migration guard (issue #142): a git-checkout binary must not
@@ -1321,6 +1575,9 @@ def _run_pending_migrations(
1321
1575
  skipped = {
1322
1576
  row[0] for row in conn.execute("SELECT name FROM schema_migrations_skipped").fetchall()
1323
1577
  }
1578
+ _reconcile_durable_applied_migration_errors(
1579
+ conn, registry, db_label,
1580
+ )
1324
1581
 
1325
1582
  # D1 — fresh install requires BOTH "schema_migrations table did not
1326
1583
  # exist" AND "the DB's primary data table is empty (or absent)".
@@ -1406,17 +1663,19 @@ def _run_pending_migrations(
1406
1663
  break
1407
1664
 
1408
1665
  now_iso = now_utc_iso()
1666
+ stamp_only_applied: list[str] = []
1409
1667
  for m in registry:
1410
1668
  if m.name in applied or m.name in skipped:
1411
1669
  continue
1670
+ qualified_name = f"{db_label}:{m.name}"
1412
1671
  if fresh_install:
1413
1672
  conn.execute(
1414
1673
  "INSERT OR IGNORE INTO schema_migrations (name, applied_at_utc) VALUES (?, ?)",
1415
1674
  (m.name, now_iso),
1416
1675
  )
1417
1676
  applied.add(m.name)
1677
+ stamp_only_applied.append(qualified_name)
1418
1678
  continue
1419
- qualified_name = f"{db_label}:{m.name}"
1420
1679
  try:
1421
1680
  m.handler(conn)
1422
1681
  _stamp_applied(conn, m.name, now_iso) # central stamp (#140)
@@ -1467,6 +1726,8 @@ def _run_pending_migrations(
1467
1726
 
1468
1727
  if fresh_install:
1469
1728
  conn.commit() # commit fresh-install stamps so they're durable
1729
+ for qualified_name in stamp_only_applied:
1730
+ _clear_migration_error_log_entries(qualified_name)
1470
1731
 
1471
1732
  # Advance user_version only when every migration is applied OR skipped.
1472
1733
  if all((m.name in applied or m.name in skipped) for m in registry):
@@ -3271,6 +3532,14 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
3271
3532
  add_column_if_missing(conn, "codex_session_entries", "account_key", "TEXT")
3272
3533
  add_column_if_missing(conn, "codex_session_files", "account_key", "TEXT")
3273
3534
  add_column_if_missing(conn, "quota_window_snapshots", "account_key", "TEXT")
3535
+ # #195: the cache-write TTL split. NULLable with NO DEFAULT — NULL is the
3536
+ # "split unknown" sentinel a pre-#195 row produces for free, and a real
3537
+ # zero (an all-5m turn) must stay distinguishable from it. Added via
3538
+ # add_column_if_missing ONLY, never in the CREATE TABLE DDL (the
3539
+ # account_key precedent): ALTER TABLE ADD COLUMN appends, so a column in
3540
+ # both places would sit at a different ordinal on fresh vs migrated DBs.
3541
+ add_column_if_missing(conn, "session_entries", "cache_create_1h_tokens", "INTEGER")
3542
+ add_column_if_missing(conn, "session_entries", "cache_create_5m_tokens", "INTEGER")
3274
3543
  conn.execute(
3275
3544
  "CREATE INDEX IF NOT EXISTS idx_codex_files_source_root "
3276
3545
  "ON codex_session_files(source_root_key)"
@@ -5498,6 +5767,92 @@ def _029_backfill_claude_account(conn: sqlite3.Connection) -> None:
5498
5767
  raise
5499
5768
 
5500
5769
 
5770
+ @cache_migration("030_session_entries_cache_creation_split")
5771
+ def _030_session_entries_cache_creation_split(conn: sqlite3.Connection) -> None:
5772
+ """#195: arm a one-time cost-side re-walk so existing session_entries rows
5773
+ gain their cache-write TTL split.
5774
+
5775
+ The columns themselves arrive via add_column_if_missing (the column-addition
5776
+ pattern); this migration exists only to arm the WALK. Note the lever: the
5777
+ six _REINGEST_FLAG_KEYS drive _resumable_reingest_conversation_messages,
5778
+ which re-walks conversation_messages and never touches session_entries. The
5779
+ COST store's re-walk lever is the per-file cursor reset below: invalidating
5780
+ ``size_bytes``/``last_byte_offset`` makes every file fail the
5781
+ ``size == prev_size`` early-exit and take the ``size > prev_size`` branch
5782
+ with ``start_offset = prev_offset == 0``, i.e. a full re-ingest from offset 0
5783
+ WITHOUT tripping the truncation escalation (which would wipe).
5784
+
5785
+ Why ``-1`` and not ``0`` for the invalidated cursor
5786
+ --------------------------------------------------
5787
+ ``session_files.size_bytes`` is overloaded: it is the delta-resume cursor
5788
+ AND the "this path had ingested bytes" bit that BOTH orphan gates read —
5789
+ ``_prune_orphaned_cache_entries`` (``if sz and p not in on_disk``) and
5790
+ ``sync_cache``'s detect-only leg (``if size_bytes and p not in
5791
+ on_disk_paths``). A path no longer on disk is never revisited by the
5792
+ re-walk, so a blanket ``size_bytes = 0`` would pin that bit to false
5793
+ FOREVER and silently turn ``cache-sync --prune-orphans`` and the dashboard
5794
+ self-heal into no-ops for every orphan that predated the upgrade. (The
5795
+ truncation escalation in ``sync_cache`` may zero it because it DELETEs
5796
+ ``session_entries`` first, so there is no orphaned accounting left to
5797
+ reclaim; 030 deliberately preserves every row, so it may not.)
5798
+
5799
+ ``-1`` satisfies both roles at once: it is never equal to a real
5800
+ ``st_size`` (so the early-exit misses), it is always LESS than one (so the
5801
+ truncation branch cannot fire), and it is truthy in Python (so both orphan
5802
+ gates still see the evidence). The ``CASE`` guard keeps genuine
5803
+ ``size_bytes = 0`` rows at 0 — a never-ingested row holds no
5804
+ ``session_entries``, so its absence from disk leaves no orphan, and
5805
+ fixtures seed exactly such rows to stay out of the orphan set. The next
5806
+ successful ingest of a surviving file overwrites the sentinel with its real
5807
+ size, so it is transient for everything still on disk. Only these two
5808
+ readers exist; grep ``session_files`` before adding a third.
5809
+
5810
+ Unlike the truncation path this does NOT wipe session_entries: rows are
5811
+ preserved and UPSERTed in place, which is why sync_cache must use the
5812
+ chained-conflict SQL variant while the walk is armed (see
5813
+ SESSION_ENTRY_UPSERT_SQL_REWALK).
5814
+
5815
+ Why a DEDICATED flag rather than deleting ``claude_ingest_walk_complete``
5816
+ (the spec's explicitly-designated open design point, resolved here on
5817
+ measured evidence):
5818
+
5819
+ * That marker is not the re-walk mechanism — the cursor reset above is. The
5820
+ truncation path deletes it because it WIPES the cache; this migration does
5821
+ not wipe, so there is nothing to invalidate.
5822
+ * The marker is the input to ``_gate_001_post_ingest_completed``, whose
5823
+ protective concern is a HALF-POPULATED ``session_entries`` causing the
5824
+ stats recomputes to zero out real dollars (spec D5). 030 preserves every
5825
+ row, so that hazard does not exist; clearing the marker would defer stats
5826
+ migrations 008/009/010 for a reason that is not real. Measured: deleting
5827
+ it failed 60 tests across 12 modules with ``MigrationGateNotMet``.
5828
+ * A dedicated flag is strictly TIGHTER as the ``rewalk_armed`` predicate.
5829
+ Marker-absence is also true on a fresh cache, after truncation, after
5830
+ orphan invalidation and after any unclean walk — all cases where the
5831
+ chained clause is unnecessary (rows are wiped or absent, so no physical-key
5832
+ conflict is possible). Scoping the flag to this migration keeps migration
5833
+ 020's deliberately LOUD duplicate-physical-key backstop intact on every
5834
+ other ingest path, which is the resolution the spec states it prefers.
5835
+
5836
+ ``sync_cache`` clears the flag at the end of a clean, non-targeted full walk
5837
+ — the same condition under which it (re)writes ``claude_ingest_walk_complete``.
5838
+
5839
+ Idempotent: re-running re-arms a walk that is already correct. NO self-stamp
5840
+ — the dispatcher central-stamps on a clean return (#140); the handler commits
5841
+ its own work first, per the cache-migration contract.
5842
+ """
5843
+ conn.execute(
5844
+ "INSERT INTO cache_meta(key, value) VALUES(?, ?) "
5845
+ "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
5846
+ (CACHE_CREATION_SPLIT_REWALK_KEY, "1"),
5847
+ )
5848
+ conn.execute(
5849
+ "UPDATE session_files "
5850
+ "SET size_bytes = CASE WHEN size_bytes > 0 THEN -1 ELSE size_bytes END, "
5851
+ " last_byte_offset = 0"
5852
+ )
5853
+ conn.commit()
5854
+
5855
+
5501
5856
  # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
5502
5857
 
5503
5858
  @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
@@ -5634,7 +5989,7 @@ def _008_recompute_weekly_cost_snapshots_dedup_fix(
5634
5989
  entries = cache_ro.execute(
5635
5990
  "SELECT model, input_tokens, output_tokens, "
5636
5991
  "cache_create_tokens, cache_read_tokens, "
5637
- "usage_extra_json, cost_usd_raw "
5992
+ "usage_extra_json, cost_usd_raw, cache_create_1h_tokens, speed "
5638
5993
  "FROM session_entries "
5639
5994
  "WHERE timestamp_utc >= ? AND timestamp_utc <= ?",
5640
5995
  (
@@ -5643,19 +5998,13 @@ def _008_recompute_weekly_cost_snapshots_dedup_fix(
5643
5998
  ),
5644
5999
  ).fetchall()
5645
6000
  total = 0.0
5646
- for model, i, o, cc, cr, extras_json, raw in entries:
5647
- usage = {
5648
- "input_tokens": i,
5649
- "output_tokens": o,
5650
- "cache_creation_input_tokens": cc,
5651
- "cache_read_input_tokens": cr,
5652
- }
5653
- # #181: usage_extra_json is cost-irrelevant (cost is
5654
- # token-only); parsed here only for pre-008 rows that may
5655
- # still carry the legacy blob — NOT a speed reader, so the
5656
- # write-side NULL going forward is safe.
5657
- if extras_json:
5658
- usage.update(json.loads(extras_json))
6001
+ for model, i, o, cc, cr, extras_json, raw, cc1h, speed in entries:
6002
+ if speed is None and extras_json:
6003
+ speed = json.loads(extras_json).get("speed")
6004
+ usage = claude_usage_dict( # #195 chokepoint
6005
+ input_tokens=i, output_tokens=o,
6006
+ cache_creation_tokens=cc, cache_read_tokens=cr,
6007
+ cache_1h_tokens=cc1h, speed=speed)
5659
6008
  total += _calculate_entry_cost(
5660
6009
  model, usage, mode="auto", cost_usd=raw,
5661
6010
  )
@@ -5937,7 +6286,7 @@ def _009_recompute_five_hour_blocks_dedup_fix(
5937
6286
  "SELECT se.model, se.input_tokens, se.output_tokens, "
5938
6287
  " se.cache_create_tokens, se.cache_read_tokens, "
5939
6288
  " se.usage_extra_json, se.cost_usd_raw, "
5940
- " sf.project_path "
6289
+ " sf.project_path, se.cache_create_1h_tokens, se.speed "
5941
6290
  "FROM session_entries se "
5942
6291
  "LEFT JOIN session_files sf "
5943
6292
  " ON sf.path = se.source_path "
@@ -5960,20 +6309,14 @@ def _009_recompute_five_hour_blocks_dedup_fix(
5960
6309
  by_project: dict[str, dict[str, Any]] = {}
5961
6310
  for (
5962
6311
  model, in_t, out_t, cc_t, cr_t,
5963
- extras_json, raw_cost, project_path,
6312
+ extras_json, raw_cost, project_path, cc1h, speed,
5964
6313
  ) in entries:
5965
- usage = {
5966
- "input_tokens": in_t,
5967
- "output_tokens": out_t,
5968
- "cache_creation_input_tokens": cc_t,
5969
- "cache_read_input_tokens": cr_t,
5970
- }
5971
- # #181: usage_extra_json is cost-irrelevant (cost is
5972
- # token-only); parsed here only for pre-008 rows that may
5973
- # still carry the legacy blob — NOT a speed reader, so the
5974
- # write-side NULL going forward is safe.
5975
- if extras_json:
5976
- usage.update(json.loads(extras_json))
6314
+ if speed is None and extras_json:
6315
+ speed = json.loads(extras_json).get("speed")
6316
+ usage = claude_usage_dict( # #195 chokepoint
6317
+ input_tokens=in_t, output_tokens=out_t,
6318
+ cache_creation_tokens=cc_t, cache_read_tokens=cr_t,
6319
+ cache_1h_tokens=cc1h, speed=speed)
5977
6320
  cost = _calculate_entry_cost(
5978
6321
  model, usage, mode="auto", cost_usd=raw_cost,
5979
6322
  )
@@ -6106,9 +6449,10 @@ def _010_recompute_percent_milestones_dedup_fix(
6106
6449
 
6107
6450
  Scope (B2)
6108
6451
  ----------
6109
- ``percent_milestones`` is normally write-once forward-only (per
6110
- "Write-once milestones" gotcha): the cost-at-moment-of-crossing is
6111
- captured at insert time and never recomputed. After the upstream
6452
+ A ``percent_milestones`` natural key denotes one durable logical first
6453
+ crossing: its first captured cost, logical refs, and alert latch survive
6454
+ disposable-index replacement and replay. An explicit audited correction
6455
+ may supersede that fact at a higher journal revision. After the upstream
6112
6456
  dedup fix, every historical milestone's ``cumulative_cost_usd`` is
6113
6457
  inflated by the same factor that inflated
6114
6458
  ``weekly_cost_snapshots`` — keeping them as-recorded would leave
@@ -6129,9 +6473,9 @@ def _010_recompute_percent_milestones_dedup_fix(
6129
6473
  reset_event_id)``. First milestone of a week has
6130
6474
  ``marginal == cumulative``.
6131
6475
 
6132
- Forward-going behavior is unchanged — new crossings keep their
6133
- "write-once at moment of crossing" semantics. This migration only
6134
- rewrites the historical rows once.
6476
+ Forward-going behavior is unchanged — new crossings keep their logical
6477
+ first-capture semantics. This pre-journal migration only rewrites the
6478
+ historical physical rows once.
6135
6479
 
6136
6480
  Timestamp comparison
6137
6481
  --------------------
@@ -6225,7 +6569,8 @@ def _010_recompute_percent_milestones_dedup_fix(
6225
6569
  entries = cache_ro.execute(
6226
6570
  "SELECT model, input_tokens, output_tokens, "
6227
6571
  " cache_create_tokens, cache_read_tokens, "
6228
- " usage_extra_json, cost_usd_raw "
6572
+ " usage_extra_json, cost_usd_raw, cache_create_1h_tokens, "
6573
+ " speed "
6229
6574
  "FROM session_entries "
6230
6575
  "WHERE timestamp_utc >= ? AND timestamp_utc <= ?",
6231
6576
  (
@@ -6236,20 +6581,14 @@ def _010_recompute_percent_milestones_dedup_fix(
6236
6581
 
6237
6582
  cumulative = 0.0
6238
6583
  for (
6239
- model, i, o, cc, cr, extras_json, raw,
6584
+ model, i, o, cc, cr, extras_json, raw, cc1h, speed,
6240
6585
  ) in entries:
6241
- usage = {
6242
- "input_tokens": i,
6243
- "output_tokens": o,
6244
- "cache_creation_input_tokens": cc,
6245
- "cache_read_input_tokens": cr,
6246
- }
6247
- # #181: usage_extra_json is cost-irrelevant (cost is
6248
- # token-only); parsed here only for pre-008 rows that may
6249
- # still carry the legacy blob — NOT a speed reader, so the
6250
- # write-side NULL going forward is safe.
6251
- if extras_json:
6252
- usage.update(json.loads(extras_json))
6586
+ if speed is None and extras_json:
6587
+ speed = json.loads(extras_json).get("speed")
6588
+ usage = claude_usage_dict( # #195 chokepoint
6589
+ input_tokens=i, output_tokens=o,
6590
+ cache_creation_tokens=cc, cache_read_tokens=cr,
6591
+ cache_1h_tokens=cc1h, speed=speed)
6253
6592
  cumulative += _calculate_entry_cost(
6254
6593
  model, usage, mode="auto", cost_usd=raw,
6255
6594
  )
@@ -6560,10 +6899,25 @@ def cmd_db_status(args: argparse.Namespace) -> int:
6560
6899
  Spec: docs/superpowers/specs/2026-05-06-migration-framework-design.md §4.2.
6561
6900
  Glyphs: ✓ applied, ✗ failed, · pending, ~ skipped.
6562
6901
  """
6902
+ # #386: the stats leg now opens under the maintenance-shared replacement
6903
+ # protocol, so it can legitimately DECLINE while a repair / quarantine owns
6904
+ # the family. Report that the way doctor already reports the cache
6905
+ # equivalent (`_open_error`) instead of surfacing a traceback.
6906
+ try:
6907
+ stats_status = _db_status_for(
6908
+ _cctally_core.DB_PATH, _STATS_MIGRATIONS, "stats.db")
6909
+ except sqlite3.DatabaseError as exc:
6910
+ stats_status = {
6911
+ "path": str(_cctally_core.DB_PATH),
6912
+ "user_version": 0,
6913
+ "registry_size": len(_STATS_MIGRATIONS),
6914
+ "migrations": [],
6915
+ "_open_error": str(exc),
6916
+ }
6563
6917
  payload = {
6564
6918
  "schema_version": 1,
6565
6919
  "databases": {
6566
- "stats.db": _db_status_for(_cctally_core.DB_PATH, _STATS_MIGRATIONS, "stats.db"),
6920
+ "stats.db": stats_status,
6567
6921
  "cache.db": _db_status_for(_cctally_core.CACHE_DB_PATH, _CACHE_MIGRATIONS, "cache.db"),
6568
6922
  "conversations.db": _db_status_for(
6569
6923
  _cctally_core.CONVERSATIONS_DB_PATH, _CONVERSATIONS_MIGRATIONS,
@@ -6582,6 +6936,8 @@ def cmd_db_status(args: argparse.Namespace) -> int:
6582
6936
  f"{db_label} ({info['path']}) "
6583
6937
  f"version {info['user_version']} / {info['registry_size']} known{suffix}"
6584
6938
  )
6939
+ if info.get("_open_error"):
6940
+ print(f" (not readable: {info['_open_error']})")
6585
6941
  for m in info["migrations"]:
6586
6942
  line = _db_status_format_row(m)
6587
6943
  print(line)
@@ -6590,7 +6946,11 @@ def cmd_db_status(args: argparse.Namespace) -> int:
6590
6946
 
6591
6947
 
6592
6948
  def _db_status_for(
6593
- db_path: pathlib.Path, registry: list[Migration], db_label: str,
6949
+ db_path: pathlib.Path,
6950
+ registry: list[Migration],
6951
+ db_label: str,
6952
+ *,
6953
+ recover_interrupted_stats: bool = True,
6594
6954
  ) -> dict:
6595
6955
  """Build per-DB status dict.
6596
6956
 
@@ -6609,7 +6969,21 @@ def _db_status_for(
6609
6969
  for m in registry
6610
6970
  ],
6611
6971
  }
6612
- conn = sqlite3.connect(db_path)
6972
+ if db_path == _cctally_core.DB_PATH:
6973
+ # #386: participate in the replacement protocol, and connect `mode=rw`
6974
+ # so this diagnostic can never MATERIALISE the very stats.db a
6975
+ # quarantine is trying to keep absent. The `exists()` check above is a
6976
+ # TOCTOU, not a guarantee: a plain read-write `sqlite3.connect` recreates
6977
+ # a file removed in the gap. Reached from `db status` and from
6978
+ # `doctor_gather_state`.
6979
+ import _cctally_store
6980
+ conn = _cctally_store.stats_open_guarded(
6981
+ db_path,
6982
+ connect=lambda p: sqlite3.connect(f"file:{p}?mode=rw", uri=True),
6983
+ recover_interruptions=recover_interrupted_stats,
6984
+ )
6985
+ else:
6986
+ conn = sqlite3.connect(db_path)
6613
6987
  try:
6614
6988
  user_version = conn.execute("PRAGMA user_version").fetchone()[0]
6615
6989
  # Tolerate missing tables (e.g., cache.db never opened by framework).
@@ -6775,30 +7149,47 @@ def _db_path_for_label(db_label: str) -> pathlib.Path:
6775
7149
  raise ValueError(f"unknown db_label: {db_label}")
6776
7150
 
6777
7151
 
6778
- def _acquire_cache_admin_writer_flocks(
7152
+ def _acquire_db_admin_writer_flocks(
6779
7153
  db_label: str,
6780
7154
  *,
6781
7155
  timeout: float = 15.0,
6782
7156
  ) -> "list[int] | None":
6783
- """Serialize raw cache admin mutations without affecting other stores."""
6784
- if db_label != "cache.db":
6785
- return []
7157
+ """Serialize raw admin mutations for ``db_label`` without affecting others.
7158
+
7159
+ #386: this returned ``[]`` — no lock at all — for every non-cache DB, and
7160
+ `db skip` / `db unskip` then ran raw DDL, a raw INSERT/DELETE and
7161
+ ``PRAGMA user_version = 0`` against the live stats index. stats.db now takes
7162
+ its maintenance lock EXCLUSIVE, which is the regime spec section 3.1 puts
7163
+ administrative mutation in.
7164
+ """
6786
7165
  from _lib_cache_writer_lock import acquire_ordered_flocks
6787
7166
 
6788
- return acquire_ordered_flocks(
6789
- [
6790
- (_cctally_core.CACHE_LOCK_MAINTENANCE_PATH, fcntl.LOCK_SH),
6791
- (_cctally_core.CACHE_LOCK_PATH, fcntl.LOCK_EX),
6792
- ],
6793
- timeout=timeout,
6794
- )
7167
+ if db_label == "cache.db":
7168
+ return acquire_ordered_flocks(
7169
+ [
7170
+ (_cctally_core.CACHE_LOCK_MAINTENANCE_PATH, fcntl.LOCK_SH),
7171
+ (_cctally_core.CACHE_LOCK_PATH, fcntl.LOCK_EX),
7172
+ ],
7173
+ timeout=timeout,
7174
+ )
7175
+ if db_label == "stats.db":
7176
+ held = acquire_ordered_flocks(
7177
+ [(_cctally_core.STATS_LOCK_MAINTENANCE_PATH, fcntl.LOCK_EX)],
7178
+ timeout=timeout,
7179
+ )
7180
+ if held is not None:
7181
+ _cctally_core.note_stats_maintenance_acquired()
7182
+ return held
7183
+ return []
6795
7184
 
6796
7185
 
6797
- def _release_cache_admin_writer_flocks(held: list[int]) -> None:
7186
+ def _release_db_admin_writer_flocks(db_label: str, held: list[int]) -> None:
6798
7187
  if not held:
6799
7188
  return
6800
7189
  from _lib_cache_writer_lock import release_cache_writer_flocks
6801
7190
 
7191
+ if db_label == "stats.db":
7192
+ _cctally_core.note_stats_maintenance_released()
6802
7193
  release_cache_writer_flocks(held)
6803
7194
 
6804
7195
 
@@ -6827,14 +7218,14 @@ def cmd_db_skip(args: argparse.Namespace) -> int:
6827
7218
  # ~/.local/share/cctally/ yet, and sqlite3.connect() does NOT
6828
7219
  # create parent directories (only the DB file itself).
6829
7220
  path.parent.mkdir(parents=True, exist_ok=True)
6830
- held = _acquire_cache_admin_writer_flocks(db_label)
7221
+ held = _acquire_db_admin_writer_flocks(db_label)
6831
7222
  if held is None:
6832
- eprint("cctally: cache.db writer busy; could not record migration skip.")
7223
+ eprint(f"cctally: {db_label} is busy; could not record migration skip.")
6833
7224
  return 3
6834
7225
  try:
6835
7226
  conn = sqlite3.connect(path)
6836
7227
  except BaseException:
6837
- _release_cache_admin_writer_flocks(held)
7228
+ _release_db_admin_writer_flocks(db_label, held)
6838
7229
  raise
6839
7230
  try:
6840
7231
  conn.execute(
@@ -6882,7 +7273,7 @@ def cmd_db_skip(args: argparse.Namespace) -> int:
6882
7273
  conn.commit()
6883
7274
  finally:
6884
7275
  conn.close()
6885
- _release_cache_admin_writer_flocks(held)
7276
+ _release_db_admin_writer_flocks(db_label, held)
6886
7277
  print(f"Skipped: {name}")
6887
7278
  return 0
6888
7279
 
@@ -6921,14 +7312,14 @@ def cmd_db_unskip(args: argparse.Namespace) -> int:
6921
7312
  if not path.exists():
6922
7313
  print(f"cctally: {name} is not skipped; nothing to do.")
6923
7314
  return 0
6924
- held = _acquire_cache_admin_writer_flocks(db_label)
7315
+ held = _acquire_db_admin_writer_flocks(db_label)
6925
7316
  if held is None:
6926
- eprint("cctally: cache.db writer busy; could not remove migration skip.")
7317
+ eprint(f"cctally: {db_label} is busy; could not remove migration skip.")
6927
7318
  return 3
6928
7319
  try:
6929
7320
  conn = sqlite3.connect(path)
6930
7321
  except BaseException:
6931
- _release_cache_admin_writer_flocks(held)
7322
+ _release_db_admin_writer_flocks(db_label, held)
6932
7323
  raise
6933
7324
  try:
6934
7325
  try:
@@ -6947,7 +7338,7 @@ def cmd_db_unskip(args: argparse.Namespace) -> int:
6947
7338
  conn.commit()
6948
7339
  finally:
6949
7340
  conn.close()
6950
- _release_cache_admin_writer_flocks(held)
7341
+ _release_db_admin_writer_flocks(db_label, held)
6951
7342
  print(f"Unskipped: {name} (will run on next open).")
6952
7343
  return 0
6953
7344
 
@@ -6986,14 +7377,14 @@ def cmd_db_recover(args: argparse.Namespace) -> int:
6986
7377
  print(f"cctally: {label} not present; nothing to recover.")
6987
7378
  return 0
6988
7379
 
6989
- held = _acquire_cache_admin_writer_flocks(label)
7380
+ held = _acquire_db_admin_writer_flocks(label)
6990
7381
  if held is None:
6991
7382
  eprint("cctally: cache.db writer busy; could not recover schema state.")
6992
7383
  return 3
6993
7384
  try:
6994
7385
  conn = sqlite3.connect(path)
6995
7386
  except BaseException:
6996
- _release_cache_admin_writer_flocks(held)
7387
+ _release_db_admin_writer_flocks(label, held)
6997
7388
  raise
6998
7389
  try:
6999
7390
  cur_version = conn.execute("PRAGMA user_version").fetchone()[0]
@@ -7017,7 +7408,7 @@ def cmd_db_recover(args: argparse.Namespace) -> int:
7017
7408
  return 0
7018
7409
  finally:
7019
7410
  conn.close()
7020
- _release_cache_admin_writer_flocks(held)
7411
+ _release_db_admin_writer_flocks(label, held)
7021
7412
 
7022
7413
 
7023
7414
  def _db_backup_timestamp() -> str:
@@ -7235,6 +7626,14 @@ def _db_family_open_pids(path: pathlib.Path) -> "set[int] | None":
7235
7626
  for suffix in ("", "-wal", "-shm")
7236
7627
  if pathlib.Path(str(path) + suffix).exists()
7237
7628
  ]
7629
+ if not family:
7630
+ # #386: an ABSENT family cannot be open — and this guard is load-bearing,
7631
+ # not defensive. `lsof -F p --` with no path operands lists EVERY open
7632
+ # file on the machine, so without it this returns essentially every PID
7633
+ # on the host and any caller gated on "is the family drained?" refuses
7634
+ # forever. Surfaced by the stats drain gate; the cache callers only ever
7635
+ # reached here with an existing family, which is why it lay dormant.
7636
+ return set()
7238
7637
  lsof = shutil.which("lsof")
7239
7638
  if lsof:
7240
7639
  try:
@@ -7535,6 +7934,44 @@ def cmd_db_repair(args: argparse.Namespace) -> int:
7535
7934
 
7536
7935
 
7537
7936
  def _cmd_db_repair_exclusive(args: argparse.Namespace, path: pathlib.Path) -> int:
7937
+ """Take maintenance exclusive, verify no pre-marker handle remains, then
7938
+ enter the repair body.
7939
+
7940
+ #386: repair used to rely on its marker plus the lsof scan alone, so it
7941
+ could race a command that honours `stats.db.maintenance.lock` and ignores
7942
+ the marker (`db rebuild`, auto-heal, the epoch resolver, `db rederive`) —
7943
+ two processes replacing the same file family at once. The ordering is
7944
+ deliberate and unchanged: the MARKER is claimed first (in `cmd_db_repair`,
7945
+ which fences new openers), THEN the lock, THEN the lsof drain check. Marker
7946
+ before lsof is what lets repair succeed under a live storm; adding the lock
7947
+ between them adds mutual exclusion against the lock-honouring paths without
7948
+ disturbing that.
7949
+ """
7950
+ from _lib_cache_writer_lock import (
7951
+ acquire_ordered_flocks,
7952
+ release_cache_writer_flocks,
7953
+ )
7954
+
7955
+ _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
7956
+ held = acquire_ordered_flocks(
7957
+ [(_cctally_core.STATS_LOCK_MAINTENANCE_PATH, fcntl.LOCK_EX)],
7958
+ timeout=15.0,
7959
+ )
7960
+ if held is None:
7961
+ eprint(
7962
+ "cctally: stats.db repair declined: another maintenance operation "
7963
+ "holds stats.db.maintenance.lock. Retry shortly."
7964
+ )
7965
+ return 3
7966
+ _cctally_core.note_stats_maintenance_acquired()
7967
+ try:
7968
+ return _cmd_db_repair_locked(args, path)
7969
+ finally:
7970
+ _cctally_core.note_stats_maintenance_released()
7971
+ release_cache_writer_flocks(held)
7972
+
7973
+
7974
+ def _cmd_db_repair_locked(args: argparse.Namespace, path: pathlib.Path) -> int:
7538
7975
  """Verify no pre-marker handle remains, then enter the repair body."""
7539
7976
  open_pids = _db_family_open_pids(path)
7540
7977
  if open_pids is None:
@@ -7547,7 +7984,8 @@ def _cmd_db_repair_exclusive(args: argparse.Namespace, path: pathlib.Path) -> in
7547
7984
  eprint(
7548
7985
  "cctally: stats.db is still open in process(es) "
7549
7986
  + ", ".join(str(pid) for pid in sorted(open_pids))
7550
- + ". Stop the dashboard and other cctally processes, then retry."
7987
+ + ". The usual holder is a running `cctally dashboard` stop it "
7988
+ "first (and any `cctally tui`), then retry."
7551
7989
  )
7552
7990
  return 3
7553
7991
  return _cmd_db_repair_claimed(args, path)
@@ -7615,7 +8053,8 @@ def _cmd_db_repair_claimed(args: argparse.Namespace, path: pathlib.Path) -> int:
7615
8053
  "cctally: could not establish a quiescent stats.db writer "
7616
8054
  f"guard ({preflight_reason}). The DB is still in use or too "
7617
8055
  "damaged to lock safely. "
7618
- "Stop the dashboard and other cctally processes, then retry; "
8056
+ "The usual holder is a running `cctally dashboard` stop it "
8057
+ "first (and any `cctally tui`), then retry; "
7619
8058
  "nothing was changed."
7620
8059
  )
7621
8060
  return 3
@@ -7807,9 +8246,23 @@ def cmd_db_backup(args: argparse.Namespace) -> int:
7807
8246
  prefix=f".{output.name}.tmp-", dir=output.parent
7808
8247
  ) as scratch_raw:
7809
8248
  temp_path = pathlib.Path(scratch_raw) / output.name
7810
- source = sqlite3.connect(
7811
- f"file:{path}?mode=ro", uri=True, timeout=max(timeout_ms, 0) / 1000
7812
- )
8249
+ # #386: `db backup --db stats` holds this handle across the entire
8250
+ # `source.backup(destination)` loop — a long-lived READ TRANSACTION,
8251
+ # which Stage 1 measured as the only thing that pins the stats WAL —
8252
+ # so it participates in the replacement protocol. The `mode=ro` open
8253
+ # is preserved verbatim through the `connect` seam.
8254
+ def _backup_source_connect(_p, _timeout_ms=timeout_ms):
8255
+ return sqlite3.connect(
8256
+ f"file:{_p}?mode=ro", uri=True,
8257
+ timeout=max(_timeout_ms, 0) / 1000,
8258
+ )
8259
+
8260
+ if which == "stats":
8261
+ import _cctally_store
8262
+ source = _cctally_store.stats_open_guarded(
8263
+ path, connect=_backup_source_connect)
8264
+ else:
8265
+ source = _backup_source_connect(path)
7813
8266
  destination = sqlite3.connect(temp_path)
7814
8267
  try:
7815
8268
  source.execute(f"PRAGMA busy_timeout={max(timeout_ms, 0)}")
@@ -7890,26 +8343,35 @@ def cmd_db_checkpoint(args: argparse.Namespace) -> int:
7890
8343
  print(f"cctally: no {label} database file present; nothing to drain.")
7891
8344
  return 0
7892
8345
 
8346
+ from _lib_cache_writer_lock import acquire_ordered_flocks
8347
+
8348
+ # #386: the stats leg previously took NO advisory lock — the flock branch was
8349
+ # gated `if which == "cache"` — while running a real
8350
+ # `wal_checkpoint(TRUNCATE)` against the live family. Maintenance SHARED is
8351
+ # the right strength: a checkpoint is not a physical replacement, it just
8352
+ # must not overlap one.
8353
+ lock_plan = {
8354
+ "cache": [
8355
+ (_cctally_core.CACHE_LOCK_MAINTENANCE_PATH, fcntl.LOCK_SH),
8356
+ (_cctally_core.CACHE_LOCK_PATH, fcntl.LOCK_EX),
8357
+ ],
8358
+ "stats": [
8359
+ (_cctally_core.STATS_LOCK_MAINTENANCE_PATH, fcntl.LOCK_SH),
8360
+ ],
8361
+ }[which]
8362
+
7893
8363
  held: list[int] = []
7894
8364
  try:
7895
- if which == "cache":
7896
- from _lib_cache_writer_lock import acquire_ordered_flocks
7897
-
7898
- acquired = acquire_ordered_flocks(
7899
- [
7900
- (_cctally_core.CACHE_LOCK_MAINTENANCE_PATH, fcntl.LOCK_SH),
7901
- (_cctally_core.CACHE_LOCK_PATH, fcntl.LOCK_EX),
7902
- ],
7903
- timeout=max(timeout, 0) / 1000,
8365
+ acquired = acquire_ordered_flocks(
8366
+ lock_plan, timeout=max(timeout, 0) / 1000
8367
+ )
8368
+ if acquired is None:
8369
+ wal_bytes = _cctally_cache._wal_file_size(path)
8370
+ result = _cctally_cache.CheckpointResult(
8371
+ label, wal_bytes, wal_bytes, 0, True, False
7904
8372
  )
7905
- if acquired is None:
7906
- wal_bytes = _cctally_cache._wal_file_size(path)
7907
- result = _cctally_cache.CheckpointResult(
7908
- label, wal_bytes, wal_bytes, 0, True, False
7909
- )
7910
- else:
7911
- held = acquired
7912
- if which != "cache" or held:
8373
+ else:
8374
+ held = acquired
7913
8375
  conn = sqlite3.connect(f"file:{path}?mode=rw", uri=True)
7914
8376
  try:
7915
8377
  conn.execute(f"PRAGMA busy_timeout={timeout}")
@@ -8026,11 +8488,21 @@ def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
8026
8488
  # dedicated maintenance flock (F13/F7), then the provider flocks for
8027
8489
  # cache.db. All non-blocking: fail promptly rather than hang.
8028
8490
  conversation_store = path == core.CONVERSATIONS_DB_PATH
8029
- maintenance_path = (
8030
- core.CONVERSATIONS_LOCK_MAINTENANCE_PATH
8031
- if conversation_store else core.CACHE_LOCK_MAINTENANCE_PATH
8032
- )
8491
+ stats_store = path == core.DB_PATH
8492
+ if conversation_store:
8493
+ maintenance_path = core.CONVERSATIONS_LOCK_MAINTENANCE_PATH
8494
+ elif stats_store:
8495
+ # #386: a stats VACUUM rewrites the ENTIRE database file under
8496
+ # `locking_mode=EXCLUSIVE`. The two-way selection below sent every
8497
+ # non-conversations DB — stats included — to the CACHE maintenance lock,
8498
+ # so it serialized against the wrong database and not at all against
8499
+ # stats ingest / auto-heal / rebuild / rederive. Standalone corruption
8500
+ # vector; spec section 1.1 Gap D calls it the headline defect.
8501
+ maintenance_path = core.STATS_LOCK_MAINTENANCE_PATH
8502
+ else:
8503
+ maintenance_path = core.CACHE_LOCK_MAINTENANCE_PATH
8033
8504
  maint_fh = open(maintenance_path, "w")
8505
+ noted = False
8034
8506
  try:
8035
8507
  try:
8036
8508
  fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -8040,6 +8512,9 @@ def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
8040
8512
  f"is running. Retry shortly."
8041
8513
  )
8042
8514
  return 3
8515
+ if stats_store:
8516
+ _cctally_core.note_stats_maintenance_acquired()
8517
+ noted = True
8043
8518
  held = []
8044
8519
  try:
8045
8520
  if provider_locked:
@@ -8077,6 +8552,8 @@ def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
8077
8552
  fh.close()
8078
8553
  finally:
8079
8554
  try:
8555
+ if noted:
8556
+ _cctally_core.note_stats_maintenance_released()
8080
8557
  fcntl.flock(maint_fh, fcntl.LOCK_UN)
8081
8558
  except OSError:
8082
8559
  pass