cctally 1.113.0 → 1.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_account.py +32 -8
  4. package/bin/_cctally_cache.py +6035 -722
  5. package/bin/_cctally_completion.py +1287 -0
  6. package/bin/_cctally_config.py +37 -0
  7. package/bin/_cctally_core.py +400 -2
  8. package/bin/_cctally_dashboard.py +98 -20
  9. package/bin/_cctally_dashboard_conversation.py +292 -23
  10. package/bin/_cctally_dashboard_envelope.py +16 -3
  11. package/bin/_cctally_dashboard_perf.py +128 -9
  12. package/bin/_cctally_dashboard_sources.py +285 -213
  13. package/bin/_cctally_db.py +1177 -23
  14. package/bin/_cctally_diagnosis_sources.py +22 -4
  15. package/bin/_cctally_doctor.py +572 -45
  16. package/bin/_cctally_five_hour.py +84 -33
  17. package/bin/_cctally_forecast.py +108 -15
  18. package/bin/_cctally_journal.py +3950 -116
  19. package/bin/_cctally_journal_repair.py +101 -12
  20. package/bin/_cctally_milestone_history.py +82 -9
  21. package/bin/_cctally_parser.py +42 -0
  22. package/bin/_cctally_percent_breakdown.py +3 -2
  23. package/bin/_cctally_project.py +8 -2
  24. package/bin/_cctally_quarantine.py +652 -0
  25. package/bin/_cctally_quota.py +227 -37
  26. package/bin/_cctally_quota_model.py +360 -83
  27. package/bin/_cctally_record.py +691 -149
  28. package/bin/_cctally_rederive.py +320 -41
  29. package/bin/_cctally_refresh.py +84 -13
  30. package/bin/_cctally_source_analytics.py +83 -17
  31. package/bin/_cctally_statusline.py +6154 -517
  32. package/bin/_cctally_store.py +367 -53
  33. package/bin/_cctally_transcript.py +8 -7
  34. package/bin/_cctally_tui.py +61 -56
  35. package/bin/_cctally_weekrefs.py +128 -21
  36. package/bin/_lib_account_binding.py +152 -0
  37. package/bin/_lib_account_protocol.py +386 -0
  38. package/bin/_lib_aggregators.py +4 -0
  39. package/bin/_lib_cache_coverage.py +13 -1
  40. package/bin/_lib_cache_report.py +35 -6
  41. package/bin/_lib_cache_writer_lock.py +19 -3
  42. package/bin/_lib_clone_copy.py +142 -0
  43. package/bin/_lib_codex_conversation.py +32 -0
  44. package/bin/_lib_codex_conversation_query.py +297 -49
  45. package/bin/_lib_codex_conversation_watch.py +59 -14
  46. package/bin/_lib_codex_drain_admission.py +214 -0
  47. package/bin/_lib_codex_metadata.py +18 -1
  48. package/bin/_lib_codex_publication_barrier.py +277 -0
  49. package/bin/_lib_codex_rollout_container.py +502 -0
  50. package/bin/_lib_conversation_dispatch.py +3 -0
  51. package/bin/_lib_conversation_query.py +106 -59
  52. package/bin/_lib_conversation_retention.py +1605 -640
  53. package/bin/_lib_cost_provenance.py +53 -2
  54. package/bin/_lib_detector_access_registry.py +3817 -0
  55. package/bin/_lib_doctor.py +714 -69
  56. package/bin/_lib_five_hour.py +126 -0
  57. package/bin/_lib_fmt.py +11 -0
  58. package/bin/_lib_ingest_frontier.py +414 -20
  59. package/bin/_lib_journal.py +141 -10
  60. package/bin/_lib_jsonl.py +5 -4
  61. package/bin/_lib_legacy_spool.py +126 -0
  62. package/bin/_lib_legacy_writer_evidence.py +136 -0
  63. package/bin/_lib_merge_gate.py +231 -86
  64. package/bin/_lib_meter_rate_change.py +10 -0
  65. package/bin/_lib_outcome_quarantine.py +690 -0
  66. package/bin/_lib_perf.py +27 -1
  67. package/bin/_lib_pricing.py +199 -9
  68. package/bin/_lib_pricing_check.py +20 -4
  69. package/bin/_lib_quarantine.py +154 -0
  70. package/bin/_lib_quota_calibration.py +53 -8
  71. package/bin/_lib_quota_copy.py +2 -1
  72. package/bin/_lib_quota_model.py +288 -46
  73. package/bin/_lib_reclaim_planner.py +792 -0
  74. package/bin/_lib_rederive.py +319 -82
  75. package/bin/_lib_segment_summary.py +35 -2
  76. package/bin/_lib_snapshot_cache.py +21 -0
  77. package/bin/_lib_sqlite_close.py +81 -0
  78. package/bin/_lib_statusline_candidates.py +292 -23
  79. package/bin/_lib_subscription_weeks.py +100 -17
  80. package/bin/_lib_tick_stats.py +217 -17
  81. package/bin/_lib_wal_checkpoint.py +1077 -0
  82. package/bin/_lib_write_budget.py +483 -0
  83. package/bin/_lib_write_io.py +703 -0
  84. package/bin/cctally +31 -1
  85. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  86. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  87. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  88. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  89. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  90. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  91. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  92. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  93. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  94. package/dashboard/static/dashboard.html +2 -2
  95. package/package.json +19 -1
  96. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  97. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -232,6 +232,10 @@ class DoctorState:
232
232
  # evidence. None is the normal state; each record contains counts/reasons
233
233
  # but never configured paths or provider identifiers.
234
234
  codex_prune_refusals: Optional[list[dict]] = None
235
+ codex_compressed_census: Optional[dict] = None
236
+ codex_decoder_capability: Optional[str] = None
237
+ codex_container_status: Optional[dict] = None
238
+ codex_maintenance_debt: Optional[dict] = None
235
239
  # #705: the conversations store's refused-pricing-write record, read
236
240
  # verbatim from cache_meta. None means no refusal is recorded. A record
237
241
  # that exists but cannot be parsed arrives as {"__malformed__": True}
@@ -343,6 +347,18 @@ class DoctorState:
343
347
  # retained corpus from unreadable metadata.
344
348
  codex_project_metadata_health: Optional[dict] = None
345
349
  codex_project_metadata_error: Optional[str] = None
350
+ # #825/#832 (spec section 8, F9): the account-bound legacy-writer evidence.
351
+ # `{"available": bool, "cutoverId", "excludedCount", "coordinateDigest",
352
+ # "evidence": <sidecar document or None>}` from the validated admission
353
+ # summary (a deep run substitutes its own whole-journal recomputation).
354
+ # None / available=False is reported as unavailable, never as zero.
355
+ legacy_writer: Optional[dict] = None
356
+ # #825/#832 (spec section 8, F16): the outcome quarantine, read from the
357
+ # validated index. `{"available": bool, "triggered": bool, "accounts":
358
+ # [{"account", "accountLabel", "accountingPaused", "controlPaused",
359
+ # "accountingSince", "controlSince"}]}`. None / available=False is
360
+ # reported as unavailable, never as OK.
361
+ outcome_quarantine: Optional[dict] = None
346
362
  # Beta-channel (spec 2026-07-21 §3): the configured RELEASE channel
347
363
  # (stable|beta) that `cctally update` tracks — DISTINCT from the preview
348
364
  # `channel` (prod|preview) above. Populated by doctor_gather_state from a
@@ -391,6 +407,11 @@ class DoctorState:
391
407
  journal_cursor_lag_bytes: Optional[int] = None
392
408
  journal_hw_segment: Optional[str] = None
393
409
  journal_cursor_segment: Optional[str] = None
410
+ # * journal_cursor_invalid (#946) — the applied cursor names no safe
411
+ # endpoint: past its segment's size, or inside a line. The lag beside
412
+ # it is then not a measure of anything consumed, so it is never
413
+ # reported as caught up.
414
+ journal_cursor_invalid: bool = False
394
415
  journal_heal_incidents: Optional[list] = None
395
416
  journal_heal_detections: Optional[list] = None
396
417
  # * retained_artifacts (#496 S6 §7.3) — the read-only retention scan
@@ -423,6 +444,15 @@ class DoctorState:
423
444
  journal_protocol_violations: Optional[list] = None
424
445
  journal_protocol_acknowledged: Optional[list] = None
425
446
  journal_protocol_error: Optional[str] = None
447
+ # * journal_listing_error (#825/#832 fix round K11, K9K10-R6) — why the
448
+ # segment listing failed. An unlistable journal is never an empty one.
449
+ # * journal_scan_error (K9K10-R6/R11) — why the deep scan could not
450
+ # complete: an unreadable segment, a raising admission-boundary lookup
451
+ # or an unreadable excluded population. The scan's checks are then
452
+ # unavailable — never zero excluded records, never an unfiltered
453
+ # conflict scan.
454
+ journal_listing_error: Optional[str] = None
455
+ journal_scan_error: Optional[str] = None
426
456
  # Multi-account attribution (#341). A dict (None when unavailable) with:
427
457
  # * claude_identity_status — "identified" | "stably_absent" | "torn"
428
458
  # * claude_email — the active Claude email (or None)
@@ -441,6 +471,14 @@ class DoctorState:
441
471
  # known file-level backup/sync root. Only provider/status are retained so
442
472
  # reports never expose the machine-specific data path.
443
473
  backup_sync_state: Optional[dict] = None
474
+ # #901: transcript-store page size, so `db.conversations_reclaimable`
475
+ # judges the paced-reclaim thresholds on observed bytes.
476
+ conversations_db_page_size: Optional[int] = None
477
+ # #901 §5.6: running dashboards' write telemetry, gathered read-only —
478
+ # the local instance's own snapshot in the dashboard process, or
479
+ # descriptor discovery plus one bounded loopback probe per live
480
+ # instance elsewhere. {"mode", "instances": [...]}; None = not gathered.
481
+ dashboard_disk_writes: Optional[dict] = None
444
482
 
445
483
 
446
484
  @dataclasses.dataclass(frozen=True)
@@ -1011,7 +1049,7 @@ def _check_db_version_ahead(s: DoctorState) -> CheckResult:
1011
1049
  # current index as a mismatch; keep this in lockstep with core
1012
1050
  # (#496 S5b §6.1). It stays a literal because this kernel is pure and
1013
1051
  # must not import `_cctally_core`.
1014
- epoch = 1016
1052
+ epoch = 1018
1015
1053
  mismatch = uv > legacy_head and uv != epoch
1016
1054
  return {"user_version": uv, "legacy_head": legacy_head, "epoch": epoch,
1017
1055
  "mismatch": mismatch}
@@ -1346,6 +1384,95 @@ def _check_data_codex_replay(s: DoctorState) -> CheckResult:
1346
1384
  )
1347
1385
 
1348
1386
 
1387
+ def _check_data_codex_compressed_rollouts(s: DoctorState) -> CheckResult:
1388
+ """#968 5.6 with operator clarifications D2, D3 and D6.
1389
+
1390
+ `warn` when any of three conditions holds: compressed rollouts exist and
1391
+ no decoder is available; a verification ended `unverified` (D3, at once,
1392
+ never debt); verification, reading or deletion work has been pending for
1393
+ more than 24 hours (D2, D6). The summary states each true condition's
1394
+ sentence once, joined by a single space, in that order. Otherwise pending
1395
+ work is `ok` "continuing", and the all-clear sentence is used only when
1396
+ no compressed rollout is unread, pending or unverified. Never FAIL: a
1397
+ missing optional decoder is an environment limitation.
1398
+ """
1399
+ if s.codex_compressed_census is None:
1400
+ return CheckResult(
1401
+ id="data.codex_compressed_rollouts", title="Compressed Codex rollouts",
1402
+ severity="warn", summary="unavailable (compressed Codex rollout census)",
1403
+ remediation=None, details={"available": False})
1404
+ census = s.codex_compressed_census
1405
+ compressed = int(census.get("compressedOnly", 0) or 0)
1406
+ never = int(census.get("neverIngested", 0) or 0)
1407
+ incomplete = int(census.get("retainedIncomplete", 0) or 0)
1408
+ unverified = int(census.get("unverified", 0) or 0)
1409
+ debt = s.codex_maintenance_debt or {}
1410
+ maintenance_partial = (s.codex_maintenance_debt is None
1411
+ or debt.get("coverage") == "partial")
1412
+ verifications = int(debt.get("verifications", 0) or 0)
1413
+ deletions = int(debt.get("deletions", 0) or 0)
1414
+ no_decoder = s.codex_decoder_capability == "unavailable"
1415
+ # D6: with a decoder, an unread or partly read compressed rollout is
1416
+ # pending reading work; without one it is the no-decoder condition.
1417
+ reads = 0 if no_decoder else never + incomplete
1418
+ details = {
1419
+ "compressedOnly": compressed,
1420
+ "retainedComplete": int(census.get("retainedComplete", 0) or 0),
1421
+ "retainedIncomplete": incomplete,
1422
+ "neverIngested": never,
1423
+ "unverified": unverified,
1424
+ "verificationDebt": verifications,
1425
+ "readDebt": reads,
1426
+ "pruneDebt": deletions,
1427
+ }
1428
+ if maintenance_partial:
1429
+ details["maintenanceCoverage"] = "partial"
1430
+ if s.codex_maintenance_debt is None:
1431
+ details["verificationDebt"] = None
1432
+ details["pruneDebt"] = None
1433
+ status = s.codex_container_status or {}
1434
+ if status.get("census") == "partial":
1435
+ details["census"] = "partial"
1436
+ details["observedAt"] = status.get("observedAt")
1437
+ if compressed:
1438
+ details["decoderAvailable"] = not no_decoder
1439
+ pending = bool(verifications or reads or deletions)
1440
+ sentences = []
1441
+ if compressed and no_decoder:
1442
+ sentences.append(
1443
+ f"{compressed} compressed Codex rollouts can't be read on this "
1444
+ f"Python ({never} never ingested). Cached history was kept. "
1445
+ "Install `backports.zstd` or use Python 3.14+.")
1446
+ if unverified:
1447
+ sentences.append(
1448
+ f"{unverified} compressed Codex rollouts kept their cached history "
1449
+ "but couldn't be verified against it.")
1450
+ if pending and debt.get("olderThan24h"):
1451
+ sentences.append(
1452
+ "Codex history maintenance has been pending for more than 24 hours "
1453
+ f"({verifications} verifications, {reads} reads, {deletions} "
1454
+ "deletions).")
1455
+ if sentences:
1456
+ severity, summary = "warn", " ".join(sentences)
1457
+ elif pending:
1458
+ severity = "ok"
1459
+ summary = "Codex history maintenance is continuing in the background."
1460
+ elif maintenance_partial:
1461
+ severity = "warn"
1462
+ summary = "unavailable (compressed Codex rollout census)"
1463
+ else:
1464
+ severity = "ok"
1465
+ summary = "All compressed Codex rollouts are readable and verified."
1466
+ return CheckResult(
1467
+ id="data.codex_compressed_rollouts",
1468
+ title="Compressed Codex rollouts",
1469
+ severity=severity,
1470
+ summary=summary,
1471
+ remediation=None,
1472
+ details=details,
1473
+ )
1474
+
1475
+
1349
1476
  def _check_data_codex_prune_safety(s: DoctorState) -> CheckResult:
1350
1477
  """Surface a fail-closed orphan-prune decision instead of silent loss."""
1351
1478
  records = [
@@ -2916,92 +3043,119 @@ def _check_db_reclaimable(s: DoctorState) -> CheckResult:
2916
3043
 
2917
3044
 
2918
3045
  def _check_db_conversations_reclaimable(s: DoctorState) -> CheckResult:
2919
- """Surface transcript-store free pages without mutating the DB (#320)."""
3046
+ """Transcript free space against the paced-reclaim policy (#320, #780,
3047
+ #901). Read-only; judged on the OBSERVED freelist, never on a record.
3048
+
3049
+ OK for retained slack below the reclaim start threshold — reusable pages
3050
+ kept on purpose (operator decision Q2). WARN for an eligible backlog being
3051
+ paced, including one whose attempts are failing (`reclaim_failure`); a
3052
+ scheduled wait is not "stuck". FAIL at the hard ceiling, where transcript
3053
+ rebuilds are refused."""
3054
+ import _lib_conversation_retention as _conv_retention
3055
+
2920
3056
  page_count = s.conversations_db_page_count
2921
3057
  freelist_count = s.conversations_db_freelist_count
2922
- ratio = None
2923
- if (
2924
- isinstance(page_count, int)
2925
- and not isinstance(page_count, bool)
2926
- and page_count > 0
2927
- and isinstance(freelist_count, int)
2928
- and not isinstance(freelist_count, bool)
3058
+ page_size = s.conversations_db_page_size
3059
+ measured = (
3060
+ all(isinstance(v, int) and not isinstance(v, bool)
3061
+ for v in (page_count, freelist_count, page_size))
3062
+ and page_count > 0 and page_size > 0
2929
3063
  and 0 <= freelist_count <= page_count
2930
- ):
2931
- ratio = freelist_count / page_count
3064
+ )
3065
+ geometry = (
3066
+ _conv_retention.StoreGeometry(page_count, freelist_count, page_size)
3067
+ if measured else None)
3068
+ record = (s.conversations_reclaim_pending
3069
+ if isinstance(s.conversations_reclaim_pending, dict) else None)
3070
+ episode = bool(record and record.get("eligible") is True)
3071
+ failure = record.get("last_failure") if record else None
3072
+ if not isinstance(failure, dict):
3073
+ failure = None
3074
+ free_bytes = geometry.free_bytes if geometry is not None else None
2932
3075
  details = {
2933
3076
  "conversations_db_page_count": page_count,
2934
3077
  "conversations_db_freelist_count": freelist_count,
2935
- "conversations_db_free_ratio": ratio,
2936
- "warn_ratio": DOCTOR_RECLAIMABLE_WARN_RATIO,
3078
+ "conversations_db_page_size": page_size,
3079
+ "conversations_db_free_ratio": (
3080
+ freelist_count / page_count if geometry is not None else None),
3081
+ "reclaimable_bytes": free_bytes,
3082
+ "reclaim_start_bytes": _conv_retention.RECLAIM_START_BYTES,
3083
+ "reclaim_start_ratio": _conv_retention.RECLAIM_START_RATIO,
3084
+ "reclaim_stop_bytes": _conv_retention.RECLAIM_STOP_BYTES,
3085
+ "reclaim_stop_ratio": _conv_retention.RECLAIM_STOP_RATIO,
3086
+ "reclaim_ceiling_bytes": _conv_retention.RECLAIM_CEILING_BYTES,
3087
+ "reclaim_episode_active": episode if record is not None else None,
3088
+ "reclaim_failure": failure,
2937
3089
  }
2938
- # #780: the reclaim backlog rides on this check rather than adding a new
2939
- # check id, and it contributes NOTHING — no detail key, no severity change
2940
- # — on a store with no pending record. That is every store that has never
2941
- # fallen behind, so an install in the ordinary state renders exactly what
2942
- # it rendered before.
2943
- pending = s.conversations_reclaim_pending
2944
- if pending:
2945
- import _lib_conversation_retention as _conv_retention
2946
-
2947
- backlog = _conv_retention.reclaim_backlog_bytes(pending)
2948
- details["reclaim_pending"] = pending
2949
- details["reclaim_backlog_bytes"] = backlog
2950
- details["reclaim_ceiling_bytes"] = _conv_retention.RECLAIM_CEILING_BYTES
2951
- if _conv_retention.reclaim_backlog_over_ceiling(pending):
2952
- return CheckResult(
2953
- id="db.conversations_reclaimable",
2954
- title="Reclaimable transcript space",
2955
- severity="fail",
2956
- summary=(
2957
- f"reclaim backlog {_gib_text(backlog)} is at or over the "
2958
- f"{_gib_text(_conv_retention.RECLAIM_CEILING_BYTES)} "
2959
- "ceiling; transcript rebuilds are refused"
2960
- ),
2961
- remediation=(
2962
- "Run `cctally db vacuum --db conversations` to drain the "
2963
- "backlog, then `cctally db checkpoint --db conversations`."
2964
- ),
2965
- details=details,
2966
- )
2967
- if _conv_retention.reclaim_backlog_escalated(pending):
2968
- return CheckResult(
2969
- id="db.conversations_reclaimable",
2970
- title="Reclaimable transcript space",
2971
- severity="warn",
2972
- summary=(
2973
- f"reclaim backlog {_gib_text(backlog)} is draining across "
2974
- "continuation passes"
2975
- ),
2976
- remediation=(
2977
- "No action is required unless it keeps growing; "
2978
- "`cctally db vacuum --db conversations` drains it now."
2979
- ),
2980
- details=details,
2981
- )
2982
- if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
3090
+ title = "Reclaimable transcript space"
3091
+ if _conv_retention.backlog_over_ceiling(free_bytes):
2983
3092
  return CheckResult(
2984
- id="db.conversations_reclaimable",
2985
- title="Reclaimable transcript space",
2986
- severity="warn",
3093
+ id="db.conversations_reclaimable", title=title, severity="fail",
2987
3094
  summary=(
2988
- f"high — {ratio * 100:.1f}% of conversations.db pages are free"
3095
+ f"reclaim backlog {_gib_text(free_bytes)} is at or over the "
3096
+ f"{_gib_text(_conv_retention.RECLAIM_CEILING_BYTES)} "
3097
+ "ceiling; transcript rebuilds are refused"
2989
3098
  ),
2990
3099
  remediation=(
2991
- "Run `cctally db vacuum --db conversations` to reclaim disk space."
3100
+ "Run `cctally db vacuum --db conversations` to drain the "
3101
+ "backlog, then `cctally db checkpoint --db conversations`."
2992
3102
  ),
2993
3103
  details=details,
2994
3104
  )
3105
+ if _conv_retention.reclaim_episode_active(geometry, episode):
3106
+ # Q9: a paced wait is not a refused attempt. A planner refusal is an
3107
+ # in-memory pass record of the running dashboard (it writes nothing
3108
+ # durable), read from `writeIo.reclaimRefusal`; a connection-settings
3109
+ # or SQLite failure stays the durable `reclaim_failure`, as before.
3110
+ refusal = _latest_reclaim_refusal(s.dashboard_disk_writes)
3111
+ wait = ("refused" if refusal is not None
3112
+ else "failing" if failure is not None else "paced")
3113
+ remediation = (
3114
+ "No action is required unless it keeps growing; "
3115
+ "`cctally db vacuum --db conversations` drains it now.")
3116
+ if refusal is not None:
3117
+ remediation = (
3118
+ "Reclaim attempts are being refused "
3119
+ f"({refusal['reason'].replace('_', ' ')}); "
3120
+ "`cctally db vacuum --db conversations` reclaims the space "
3121
+ "at once.")
3122
+ return CheckResult(
3123
+ id="db.conversations_reclaimable", title=title, severity="warn",
3124
+ summary=(
3125
+ f"Reclaiming {_gib_text(free_bytes)} of free transcript "
3126
+ "space at a paced rate."
3127
+ ),
3128
+ remediation=remediation,
3129
+ details={**details, "reclaim_wait": wait,
3130
+ "reclaim_refusal": refusal},
3131
+ )
2995
3132
  return CheckResult(
2996
- id="db.conversations_reclaimable",
2997
- title="Reclaimable transcript space",
2998
- severity="ok",
2999
- summary="below threshold",
3000
- remediation=None,
3001
- details=details,
3133
+ id="db.conversations_reclaimable", title=title, severity="ok",
3134
+ summary="below threshold", remediation=None, details=details,
3002
3135
  )
3003
3136
 
3004
3137
 
3138
+ def _latest_reclaim_refusal(gathered) -> "dict | None":
3139
+ """The newest planner refusal any running dashboard reports in its
3140
+ `writeIo.reclaimRefusal`, with that dashboard's port; None when none
3141
+ does (or no dashboard runs)."""
3142
+ latest = None
3143
+ entries = gathered.get("instances") if isinstance(gathered, dict) else None
3144
+ for entry in entries or ():
3145
+ if not isinstance(entry, dict) or entry.get("probe") != "ok":
3146
+ continue
3147
+ refusal = (entry.get("write_io") or {}).get("reclaimRefusal")
3148
+ if (not isinstance(refusal, dict)
3149
+ or not isinstance(refusal.get("reason"), str)
3150
+ or not isinstance(refusal.get("at"), str)):
3151
+ continue
3152
+ candidate = {"reason": refusal["reason"], "at": refusal["at"],
3153
+ "port": entry.get("port")}
3154
+ if latest is None or candidate["at"] > latest["at"]:
3155
+ latest = candidate
3156
+ return latest
3157
+
3158
+
3005
3159
  #: A stuck reclaim entry has persisted past this, so no pass will clear it.
3006
3160
  _RECLAIM_STUCK_AFTER_SECONDS = 24 * 3600
3007
3161
 
@@ -3238,6 +3392,28 @@ _JOURNAL_CURSOR_LAG_WARN_BYTES = 4 * 1024 * 1024
3238
3392
  _JOURNAL_HEAL_RECENT_SECONDS = 7 * 24 * 3600
3239
3393
 
3240
3394
 
3395
+ def _journal_unreadable_remediation(s: DoctorState) -> str:
3396
+ """K9K10-R6: the remediation for a journal doctor could not read. Fix
3397
+ round K12 (K11R-12): it names the resolved data directory (a dev
3398
+ checkout or `CCTALLY_DATA_DIR` is not the default path)."""
3399
+ journal = f"{str(s.app_dir or '').rstrip('/')}/journal/"
3400
+ return (f"Check the permissions of {journal} and the disk, then run "
3401
+ "`cctally doctor --deep` again")
3402
+
3403
+
3404
+ def _journal_scan_unavailable(check_id: str, title: str, s: DoctorState,
3405
+ details: dict) -> CheckResult:
3406
+ """A deep-scan check the scan could not complete (K9K10-R6): WARN,
3407
+ unavailable, with the cause."""
3408
+ return CheckResult(
3409
+ id=check_id, title=title, severity="warn",
3410
+ summary=f"unavailable ({s.journal_scan_error})",
3411
+ remediation=_journal_unreadable_remediation(s),
3412
+ details={**details, "scanned": True, "available": False,
3413
+ "error": s.journal_scan_error},
3414
+ )
3415
+
3416
+
3241
3417
  def _check_journal_presence(s: DoctorState) -> CheckResult:
3242
3418
  """The journal directory exists and is appendable. A pre-cutover (legacy)
3243
3419
  install has NO journal yet — that is INFO (OK), never a FAIL."""
@@ -3249,6 +3425,15 @@ def _check_journal_presence(s: DoctorState) -> CheckResult:
3249
3425
  )
3250
3426
  details = {"present": True, "appendable": s.journal_appendable,
3251
3427
  "segments": s.journal_segment_count}
3428
+ if s.journal_listing_error:
3429
+ return CheckResult(
3430
+ id="journal.presence", title="Journal", severity="warn",
3431
+ summary=(f"segment listing unavailable "
3432
+ f"({s.journal_listing_error})"),
3433
+ remediation=_journal_unreadable_remediation(s),
3434
+ details={**details, "segments": None,
3435
+ "listingError": s.journal_listing_error},
3436
+ )
3252
3437
  if s.journal_appendable is False:
3253
3438
  return CheckResult(
3254
3439
  id="journal.presence", title="Journal", severity="warn",
@@ -3272,6 +3457,9 @@ def _check_journal_integrity(s: DoctorState) -> CheckResult:
3272
3457
  id="journal.integrity", title="Journal integrity", severity="ok",
3273
3458
  summary="no journal", remediation=None, details={"present": False},
3274
3459
  )
3460
+ if s.journal_scan_error:
3461
+ return _journal_scan_unavailable(
3462
+ "journal.integrity", "Journal integrity", s, {})
3275
3463
  if s.journal_malformed_count is None:
3276
3464
  return CheckResult(
3277
3465
  id="journal.integrity", title="Journal integrity", severity="ok",
@@ -3313,6 +3501,16 @@ def _check_journal_index_freshness(s: DoctorState) -> CheckResult:
3313
3501
  details = {"lag_bytes": lag, "hw_segment": s.journal_hw_segment,
3314
3502
  "cursor_segment": s.journal_cursor_segment,
3315
3503
  "warn_bytes": _JOURNAL_CURSOR_LAG_WARN_BYTES}
3504
+ if s.journal_cursor_invalid:
3505
+ # #946: lag is computed from the safe applied cursor, and this one is
3506
+ # not safe, so no amount of lag here means "caught up".
3507
+ details["cursor_invalid"] = True
3508
+ return CheckResult(
3509
+ id="journal.index_freshness", title="Journal index", severity="warn",
3510
+ summary="index cursor is not at a journal line boundary",
3511
+ remediation="Run `cctally db rebuild --db stats`",
3512
+ details=details,
3513
+ )
3316
3514
  if lag == 0:
3317
3515
  return CheckResult(
3318
3516
  id="journal.index_freshness", title="Journal index", severity="ok",
@@ -3547,6 +3745,9 @@ def _check_journal_conflicts(s: DoctorState) -> CheckResult:
3547
3745
  remediation=None,
3548
3746
  details={"scanned": True, "available": False, "conflicts": None},
3549
3747
  )
3748
+ if s.journal_scan_error:
3749
+ return _journal_scan_unavailable(
3750
+ "journal.conflicts", "Journal conflicts", s, {"conflicts": None})
3550
3751
  if s.journal_conflicts is None:
3551
3752
  return CheckResult(
3552
3753
  id="journal.conflicts", title="Journal conflicts", severity="ok",
@@ -3585,6 +3786,283 @@ def _check_journal_conflicts(s: DoctorState) -> CheckResult:
3585
3786
  )
3586
3787
 
3587
3788
 
3789
+ def _check_journal_legacy_writer(s: DoctorState) -> CheckResult:
3790
+ """#825/#832 (spec section 8, F9, operator Q4): records an older cctally
3791
+ process wrote after the account-bound cutover are excluded from current
3792
+ accounting. WARN while the latest excluded population was first observed
3793
+ under seven days ago; OK with a historical note after seven quiet days.
3794
+ Never asserts that an old writer is alive, and never reports a missing
3795
+ summary as zero. An incomplete legacy spool inventory is also a WARN,
3796
+ independently of the excluded-record count (spec revision 5, section
3797
+ 8; `_with_legacy_spool_inventory`)."""
3798
+ result = _legacy_writer_count_result(s)
3799
+ inventory = (s.legacy_writer or {}).get("legacySpoolInventory")
3800
+ if not inventory or inventory.get("available", False):
3801
+ return result
3802
+ return _with_legacy_spool_inventory(result, inventory)
3803
+
3804
+
3805
+ def _with_legacy_spool_inventory(result: CheckResult,
3806
+ inventory: dict) -> CheckResult:
3807
+ """Spec revision 5 (section 8): an incomplete legacy spool inventory is a
3808
+ WARN independently of the excluded-record count. It names the path, the
3809
+ failed operation and the error, never claims an old writer exists, and
3810
+ says detection is paused only when the validated index holds the
3811
+ durable `inventory-incomplete` marker. An inventory that failed for any
3812
+ other reason (fix round K12, K11R-13) names no file: it is reported
3813
+ unavailable with its cause."""
3814
+ if inventory.get("unavailableCause") and not inventory.get("path"):
3815
+ return _with_unavailable_spool_inventory(
3816
+ result, inventory, inventory["unavailableCause"])
3817
+ text = LEGACY_SPOOL_INVENTORY_TEXT.format(
3818
+ path=inventory.get("path"), operation=inventory.get("operation"),
3819
+ error=inventory.get("error"))
3820
+ if inventory.get("paused"):
3821
+ text += " " + LEGACY_SPOOL_INVENTORY_PAUSED_TEXT
3822
+ note = "legacy status-line inventory unavailable"
3823
+ if result.severity == "warn":
3824
+ summary = f"{result.summary}; {note}"
3825
+ remediation = (f"{result.remediation} {text}" if result.remediation
3826
+ else text)
3827
+ else:
3828
+ summary = (f"{note} ({inventory.get('operation')} "
3829
+ f"{inventory.get('path')}: {inventory.get('error')})")
3830
+ remediation = text
3831
+ return CheckResult(
3832
+ id=result.id, title=result.title, severity="warn", summary=summary,
3833
+ remediation=remediation,
3834
+ details={**(result.details or {}),
3835
+ "legacySpoolInventory": dict(inventory)},
3836
+ )
3837
+
3838
+
3839
+ def _with_unavailable_spool_inventory(result: CheckResult, inventory: dict,
3840
+ cause: str) -> CheckResult:
3841
+ """`_with_legacy_spool_inventory` for an inventory that names no file:
3842
+ a WARN, "legacy status-line inventory unavailable ({cause})", which keeps
3843
+ the excluded-record half's own summary and remediation."""
3844
+ note = f"legacy status-line inventory unavailable ({cause})"
3845
+ if result.severity == "warn":
3846
+ summary, remediation = f"{result.summary}; {note}", result.remediation
3847
+ else:
3848
+ summary, remediation = note, None
3849
+ return CheckResult(
3850
+ id=result.id, title=result.title, severity="warn", summary=summary,
3851
+ remediation=remediation,
3852
+ details={**(result.details or {}),
3853
+ "legacySpoolInventory": dict(inventory)},
3854
+ )
3855
+
3856
+
3857
+ def _legacy_writer_count_result(s: DoctorState) -> CheckResult:
3858
+ """The excluded-record half of `journal.legacy_writer`."""
3859
+ import _lib_legacy_writer_evidence as _lw
3860
+
3861
+ data = s.legacy_writer
3862
+ if data and data.get("unavailableCause"):
3863
+ # K9K10-R6/R11: the deep recomputation could not complete. The count
3864
+ # is unavailable — never zero, never the stale summary.
3865
+ return CheckResult(
3866
+ id="journal.legacy_writer", title="Legacy writer",
3867
+ severity="warn",
3868
+ summary=f"unavailable ({data['unavailableCause']})",
3869
+ remediation=_journal_unreadable_remediation(s),
3870
+ details={"available": False,
3871
+ "unavailableCause": data["unavailableCause"]},
3872
+ )
3873
+ if not data or not data.get("available"):
3874
+ # No current summary (for example an older process republished its
3875
+ # own epoch). The durable sidecar alone still proves an excluded
3876
+ # population and when it was first observed, so it decides the leg;
3877
+ # without one the leg is unavailable, never zero.
3878
+ sidecar = _lw.from_document((data or {}).get("evidence"))
3879
+ if sidecar is None or sidecar.excluded_count <= 0:
3880
+ return CheckResult(
3881
+ id="journal.legacy_writer", title="Legacy writer",
3882
+ severity="ok",
3883
+ summary="unavailable (no current admission summary)",
3884
+ remediation=None,
3885
+ details={"available": False},
3886
+ )
3887
+ verdict = _lw.classify(sidecar, excluded_count=sidecar.excluded_count,
3888
+ now=s.now_utc)
3889
+ return _legacy_writer_verdict_result(
3890
+ verdict, count=sidecar.excluded_count,
3891
+ details={"available": False, "source": "evidence-sidecar",
3892
+ "cutoverId": sidecar.cutover_id,
3893
+ "excludedCount": sidecar.excluded_count,
3894
+ "state": verdict.state,
3895
+ "lastSeenAt": verdict.last_seen_at})
3896
+ cutover_id = data.get("cutoverId")
3897
+ count = int(data.get("excludedCount") or 0)
3898
+ if cutover_id is None:
3899
+ return CheckResult(
3900
+ id="journal.legacy_writer", title="Legacy writer", severity="ok",
3901
+ summary="no account-bound cutover yet", remediation=None,
3902
+ details={"available": True, "cutoverId": None,
3903
+ "excludedCount": count, "state": "none",
3904
+ "lastSeenAt": None},
3905
+ )
3906
+ # A missing or stale sidecar reads as "first observed now", which can only
3907
+ # lengthen a warning (spec section 8).
3908
+ evidence = _lw.observe(
3909
+ _lw.from_document(data.get("evidence")), cutover_id=cutover_id,
3910
+ excluded_count=count, coordinate_digest=data.get("coordinateDigest"),
3911
+ now=s.now_utc)
3912
+ verdict = _lw.classify(evidence, excluded_count=count, now=s.now_utc)
3913
+ details = {"available": True, "cutoverId": cutover_id,
3914
+ "excludedCount": count, "state": verdict.state,
3915
+ "lastSeenAt": verdict.last_seen_at}
3916
+ if verdict.state == "none":
3917
+ return CheckResult(
3918
+ id="journal.legacy_writer", title="Legacy writer", severity="ok",
3919
+ summary="no legacy writes since the account-bound cutover",
3920
+ remediation=None, details=details,
3921
+ )
3922
+ return _legacy_writer_verdict_result(verdict, count=count, details=details)
3923
+
3924
+
3925
+ def _legacy_writer_verdict_result(verdict, *, count: int,
3926
+ details: dict) -> CheckResult:
3927
+ """The WARN (recent) or historical OK (quiet) result in the approved
3928
+ spec-section-8 content: the count, the last-seen time and the remedy."""
3929
+ import _lib_legacy_writer_evidence as _lw
3930
+
3931
+ if verdict.state == "recent":
3932
+ return CheckResult(
3933
+ id="journal.legacy_writer", title="Legacy writer", severity="warn",
3934
+ summary=(f"{count} record(s) an older cctally process wrote after "
3935
+ "the upgrade were excluded from accounting (last seen "
3936
+ f"{verdict.last_seen_at})"),
3937
+ remediation=_lw.REMEDY, details=details,
3938
+ )
3939
+ return CheckResult(
3940
+ id="journal.legacy_writer", title="Legacy writer", severity="ok",
3941
+ summary=(f"no legacy writes in the last {_lw.QUIET_DAYS} days "
3942
+ f"({count} excluded in total, last seen "
3943
+ f"{verdict.last_seen_at})"),
3944
+ remediation=None, details=details,
3945
+ )
3946
+
3947
+
3948
+ #: Spec revision 5 (section 8), operator-owned wording, verbatim: an
3949
+ #: incomplete legacy spool inventory, and the sentence added when the
3950
+ #: validated index holds its durable `inventory-incomplete` marker.
3951
+ LEGACY_SPOOL_INVENTORY_TEXT = (
3952
+ "cctally cannot read {path} ({operation}: {error}), one of the "
3953
+ "status-line files an older cctally release used. Until it can, cctally "
3954
+ "cannot tell whether an older process changed it. Check the file's "
3955
+ "permissions and the disk.")
3956
+ LEGACY_SPOOL_INVENTORY_PAUSED_TEXT = (
3957
+ "Automatic credit and reset detection is paused because of this, and "
3958
+ "making the file readable again does not end the pause (see "
3959
+ "`accounting.outcome_quarantine`); usage recording continues.")
3960
+
3961
+
3962
+ #: Operator-approved wording (#825/#832 spec section 8), verbatim.
3963
+ OUTCOME_QUARANTINE_ACCOUNTING_TEXT = (
3964
+ "Automatic credit and reset detection is paused for {account} since "
3965
+ "{time} because an older cctally process wrote data after the upgrade. It "
3966
+ "resumes only after cctally verifies a clean reading for this account, "
3967
+ "typically the first one after an idle period spanning a weekly reset; "
3968
+ "there is no guaranteed time. If your usage was reset in place meanwhile, "
3969
+ "record it with `cctally record-credit`.")
3970
+ OUTCOME_QUARANTINE_CONTROL_TEXT = (
3971
+ "Status-line drop detection also waits for a successful usage refresh for "
3972
+ "this account.")
3973
+ #: Spec revision 3 (section 8; operator decision dec-2b5bcf910816), verbatim:
3974
+ #: an account paused by an unresolved spool marker. It replaces the "resumes
3975
+ #: only after..." sentence and already names drop detection, so no control
3976
+ #: sentence is appended.
3977
+ OUTCOME_QUARANTINE_SPOOL_MARKER_TEXT = (
3978
+ "Automatic credit, reset and status-line drop detection is paused for "
3979
+ "{account} since {time} because an older cctally process wrote "
3980
+ "status-line data after the upgrade. This version of cctally cannot yet "
3981
+ "confirm a safe point to resume, so the pause does not end on its own; "
3982
+ "usage recording continues. If your usage was reset in place meanwhile, "
3983
+ "record it with `cctally record-credit`.")
3984
+ #: Spec revision 5 (section 8), operator-owned wording, verbatim: an account
3985
+ #: paused by a marker an incomplete legacy spool inventory left.
3986
+ OUTCOME_QUARANTINE_INVENTORY_MARKER_TEXT = (
3987
+ "Automatic credit, reset and status-line drop detection is paused for "
3988
+ "{account} since {time} because cctally could not read the status-line "
3989
+ "files an older cctally release used, so it cannot rule out that an "
3990
+ "older process changed them. This version of cctally cannot yet confirm "
3991
+ "a safe point to resume, so the pause does not end on its own; usage "
3992
+ "recording continues. If your usage was reset in place meanwhile, record "
3993
+ "it with `cctally record-credit`.")
3994
+ #: Operator decision dec-5d2a9c2b004a (escalation esc-8a386c207966): the
3995
+ #: standalone sentence for an account missing only its control certificate.
3996
+ #: The approved control sentence above opens with "also", so it reads only as
3997
+ #: an addendum to the accounting one.
3998
+ OUTCOME_QUARANTINE_CONTROL_ONLY_TEXT = (
3999
+ "Status-line drop detection is paused for {account} since {time}. It "
4000
+ "resumes after a successful usage refresh for this account.")
4001
+
4002
+
4003
+ def _outcome_quarantine_message(row: dict) -> str:
4004
+ """One account's paused outcome classes, in the approved wording. A
4005
+ missing control certificate appends its sentence to the accounting one;
4006
+ alone, it gets its own operator-approved standalone sentence."""
4007
+ label = row.get("accountLabel") or row.get("account")
4008
+ if row.get("unresolvedSpoolMarker"):
4009
+ text = (OUTCOME_QUARANTINE_INVENTORY_MARKER_TEXT
4010
+ if row.get("spoolMarkerCause") == "inventory-incomplete"
4011
+ else OUTCOME_QUARANTINE_SPOOL_MARKER_TEXT)
4012
+ return text.format(
4013
+ account=label, time=row.get("accountingSince") or "an unknown time")
4014
+ if row.get("accountingPaused"):
4015
+ text = OUTCOME_QUARANTINE_ACCOUNTING_TEXT.format(
4016
+ account=label, time=row.get("accountingSince") or "an unknown time")
4017
+ if row.get("controlPaused"):
4018
+ text += " " + OUTCOME_QUARANTINE_CONTROL_TEXT
4019
+ return text
4020
+ return OUTCOME_QUARANTINE_CONTROL_ONLY_TEXT.format(
4021
+ account=label, time=row.get("controlSince") or "an unknown time")
4022
+
4023
+
4024
+ def _check_accounting_outcome_quarantine(s: DoctorState) -> CheckResult:
4025
+ """#825/#832 (spec section 8, F16, operator Q9): after a post-cutover
4026
+ legacy write or legacy spool change, automatic outcomes are withheld per
4027
+ account until certified recovery. OK when no account is quarantined; WARN
4028
+ naming per account what is paused and since when; unavailable — WARN,
4029
+ never OK — on an unavailable or unvalidated index. Separate from
4030
+ `journal.legacy_writer`: seven quiet days do not mean outcomes resumed.
4031
+ Never claims when recovery will happen. While a spool marker is
4032
+ unresolved (spec revision 3) every row uses the marker wording and the
4033
+ details carry the markers."""
4034
+ data = s.outcome_quarantine
4035
+ title = "Outcome quarantine"
4036
+ if not data or not data.get("available"):
4037
+ return CheckResult(
4038
+ id="accounting.outcome_quarantine", title=title, severity="warn",
4039
+ summary="unavailable (no validated stats index)",
4040
+ remediation=("Run any cctally command to bring the stats index "
4041
+ "up to date, then run `cctally doctor` again."),
4042
+ details={"available": False, "accounts": []},
4043
+ )
4044
+ rows = []
4045
+ for row in data.get("accounts") or []:
4046
+ rows.append({**row, "message": _outcome_quarantine_message(row)})
4047
+ details = {"available": True, "triggered": bool(data.get("triggered")),
4048
+ "accounts": rows}
4049
+ if data.get("unresolvedSpoolMarkers"):
4050
+ details["unresolvedSpoolMarkers"] = list(data["unresolvedSpoolMarkers"])
4051
+ if not rows:
4052
+ return CheckResult(
4053
+ id="accounting.outcome_quarantine", title=title, severity="ok",
4054
+ summary=("no account is quarantined" if data.get("triggered")
4055
+ else "no legacy write or spool change since the upgrade"),
4056
+ remediation=None, details=details,
4057
+ )
4058
+ return CheckResult(
4059
+ id="accounting.outcome_quarantine", title=title, severity="warn",
4060
+ summary=(f"automatic outcomes paused for {len(rows)} account(s)"),
4061
+ remediation=" ".join(row["message"] for row in rows),
4062
+ details=details,
4063
+ )
4064
+
4065
+
3588
4066
  def _check_journal_protocol(s: DoctorState) -> CheckResult:
3589
4067
  """Report selector failure or whole-batch tainting without false health."""
3590
4068
  if (
@@ -4086,6 +4564,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
4086
4564
  ("journal.writer_guard", "_check_journal_writer_guard"),
4087
4565
  ("journal.conflicts", "_check_journal_conflicts"),
4088
4566
  ("journal.protocol", "_check_journal_protocol"),
4567
+ ("journal.legacy_writer", "_check_journal_legacy_writer"),
4089
4568
  ("journal.quota_projection", "_check_journal_quota_projection"),
4090
4569
  )),
4091
4570
  ("data", "Data", (
@@ -4099,6 +4578,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
4099
4578
  # after the pair rather than between them.
4100
4579
  ("data.codex_project_metadata", "_check_data_codex_project_metadata"),
4101
4580
  ("data.codex_prune_safety", "_check_data_codex_prune_safety"),
4581
+ ("data.codex_compressed_rollouts", "_check_data_codex_compressed_rollouts"),
4102
4582
  ("data.codex_replay", "_check_data_codex_replay"),
4103
4583
  ("data.codex_ingest_backlog", "_check_data_codex_ingest_backlog"),
4104
4584
  ("data.codex_quota", "_check_data_codex_quota"),
@@ -4120,6 +4600,12 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
4120
4600
  ("accounts.codex_window_attribution",
4121
4601
  "_check_accounts_codex_window_attribution"),
4122
4602
  )),
4603
+ # #825/#832 spec section 8: reported separately from the seven-day
4604
+ # `journal.legacy_writer` recency, because quiet time never decides
4605
+ # recovery.
4606
+ ("accounting", "Accounting", (
4607
+ ("accounting.outcome_quarantine", "_check_accounting_outcome_quarantine"),
4608
+ )),
4123
4609
  ("pricing", "Pricing", (
4124
4610
  ("pricing.coverage", "_check_pricing_coverage"),
4125
4611
  # #705. Cannot FAIL, so `doctor`'s exit code is unaffected — the same
@@ -4149,6 +4635,12 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
4149
4635
  ("telemetry", "Telemetry", (
4150
4636
  ("telemetry.state", "_check_telemetry"),
4151
4637
  )),
4638
+ # #901 §5.6. Never FAILs, so `doctor`'s exit code is unaffected. Last, so
4639
+ # every existing check keeps its order.
4640
+ ("performance", "Performance", (
4641
+ ("performance.dashboard_disk_writes",
4642
+ "_check_performance_dashboard_disk_writes"),
4643
+ )),
4152
4644
  )
4153
4645
 
4154
4646
 
@@ -4322,6 +4814,159 @@ def _check_quota_meter_drift(s: DoctorState) -> CheckResult:
4322
4814
  )
4323
4815
 
4324
4816
 
4817
+ _DISK_WRITES_ID = "performance.dashboard_disk_writes"
4818
+ _DISK_WRITES_TITLE = "Dashboard disk writes"
4819
+ _DISK_WRITES_REMEDIATION = (
4820
+ "Run `cctally dashboard-perf` for the breakdown, and report it if it "
4821
+ "persists."
4822
+ )
4823
+
4824
+
4825
+ def _words(reason) -> str:
4826
+ return str(reason or "unavailable").replace("_", " ")
4827
+
4828
+
4829
+ #: PR-9: probe failures doctor cannot overcome from here (it sends no token
4830
+ #: and probes over loopback only), each with its sentence fragment.
4831
+ _NOT_MEASURABLE_FROM_HERE = {
4832
+ "authentication_required": "needs its access token",
4833
+ "not_loopback_reachable": "is not reachable over loopback",
4834
+ }
4835
+
4836
+
4837
+ def _check_performance_dashboard_disk_writes(s: DoctorState) -> CheckResult:
4838
+ """Running dashboards' disk writes against the write budget, and
4839
+ transcript maintenance against its allowance (#901 §5.6, §5.7).
4840
+
4841
+ WARN when a measured running dashboard is over budget, when a live
4842
+ instance cannot supply valid telemetry, or when the durable maintenance
4843
+ ledger is over its allowance (so that leg needs no running dashboard).
4844
+ OK otherwise: none running, within budget, a statistic that has no
4845
+ verdict yet (named by its reason), a dashboard doctor cannot measure from
4846
+ here (a token-mode or non-loopback bind), or a platform with no counter.
4847
+ Never FAIL."""
4848
+ import _lib_write_budget as wb
4849
+
4850
+ gathered = (s.dashboard_disk_writes
4851
+ if isinstance(s.dashboard_disk_writes, dict)
4852
+ else {"mode": "error", "instances": []})
4853
+ record = (s.conversations_reclaim_pending
4854
+ if isinstance(s.conversations_reclaim_pending, dict) else {})
4855
+ maintenance = wb.maintenance_statistic(record.get("ledger"), s.now_utc)
4856
+ rows, over, cannot, within, warming, platform = [], [], [], [], [], []
4857
+ unmeasured = []
4858
+ for entry in gathered.get("instances") or ():
4859
+ if not isinstance(entry, dict):
4860
+ continue
4861
+ port = entry.get("port")
4862
+ write_io = entry.get("write_io") if entry.get("probe") == "ok" else None
4863
+ budget = (write_io or {}).get("budget") or {}
4864
+ if write_io is None:
4865
+ verdict, reason = "unavailable", entry.get("reason") or "unreachable"
4866
+ if reason in _NOT_MEASURABLE_FROM_HERE:
4867
+ # PR-9: doctor sends no token and probes over loopback only,
4868
+ # so these never answer it: a note, not a warning.
4869
+ unmeasured.append((port, reason))
4870
+ else:
4871
+ cannot.append((port, reason))
4872
+ elif write_io.get("status") != "ok":
4873
+ verdict = "unavailable"
4874
+ reason = write_io.get("reason") or "counter_error"
4875
+ if reason == "unsupported_platform":
4876
+ platform.append((port, reason))
4877
+ else:
4878
+ cannot.append((port, reason))
4879
+ else:
4880
+ verdict, reason = budget.get("verdict"), None
4881
+ if verdict == "over":
4882
+ over.append((port, write_io, budget))
4883
+ elif verdict == "ok":
4884
+ within.append((port, write_io))
4885
+ else:
4886
+ verdict = "insufficient"
4887
+ reason = (budget.get("reasons") or ["warming_up"])[0]
4888
+ warming.append((port, reason))
4889
+ rows.append({
4890
+ "instance_id": entry.get("instance_id"),
4891
+ "port": port,
4892
+ "pid": entry.get("pid"),
4893
+ "verdict": verdict,
4894
+ "reason": reason,
4895
+ "bytes_per_minute": (write_io or {}).get("bytesPerMinute"),
4896
+ "mean_bytes_per_publication": (write_io or {}).get(
4897
+ "meanBytesPerTick"),
4898
+ "bytes_per_minute_limit": budget.get("bytesPerMinuteLimit"),
4899
+ "mean_bytes_per_publication_limit": budget.get(
4900
+ "meanBytesPerTickLimit"),
4901
+ })
4902
+ rows.sort(key=lambda row: (row["port"] is None, row["port"] or 0))
4903
+ details = {
4904
+ "policy_version": wb.POLICY_VERSION,
4905
+ "mode": gathered.get("mode"),
4906
+ "instances": rows,
4907
+ "maintenance": maintenance.as_details(),
4908
+ }
4909
+ warn = []
4910
+ for port, write_io, budget in sorted(over, key=lambda item: item[0] or 0):
4911
+ reasons = budget.get("reasons") or []
4912
+ if ("publication_over_limit" in reasons
4913
+ and "rate_over_limit" not in reasons):
4914
+ warn.append(
4915
+ f"The running dashboard (port {port}) is writing "
4916
+ f"{wb.format_mib(write_io.get('meanBytesPerTick') or 0)} per "
4917
+ f"publication, over its "
4918
+ f"{wb.format_mib(budget.get('meanBytesPerTickLimit') or 0)} "
4919
+ f"per publication budget.")
4920
+ else:
4921
+ warn.append(
4922
+ f"The running dashboard (port {port}) is writing "
4923
+ f"{wb.format_mib(write_io.get('bytesPerMinute') or 0)}/min, "
4924
+ f"over its "
4925
+ f"{wb.format_mib(budget.get('bytesPerMinuteLimit') or 0)}/min "
4926
+ f"budget.")
4927
+ for port, reason in sorted(cannot, key=lambda item: item[0] or 0):
4928
+ warn.append(f"The running dashboard (port {port}) cannot report its "
4929
+ f"disk writes ({_words(reason)}).")
4930
+ if maintenance.verdict == "over":
4931
+ warn.append(
4932
+ f"Transcript maintenance was charged "
4933
+ f"{wb.format_size(maintenance.charged_bytes)} over the last "
4934
+ f"{wb.format_window_hm(maintenance.window_minutes)}, over its "
4935
+ f"{wb.format_size(maintenance.allowance_bytes)} allowance.")
4936
+ if warn:
4937
+ return CheckResult(
4938
+ id=_DISK_WRITES_ID, title=_DISK_WRITES_TITLE, severity="warn",
4939
+ summary=" ".join(warn), remediation=_DISK_WRITES_REMEDIATION,
4940
+ details=details)
4941
+ if within:
4942
+ port, write_io = sorted(within, key=lambda item: item[0] or 0)[0]
4943
+ summary = (f"Within budget: "
4944
+ f"{wb.format_mib(write_io.get('bytesPerMinute') or 0)}/min "
4945
+ f"over the last 5 minutes.")
4946
+ elif warming:
4947
+ # PR-8: the first instance's own reason; only `warming_up` is
4948
+ # "no samples yet".
4949
+ _port, reason = sorted(warming, key=lambda item: item[0] or 0)[0]
4950
+ summary = f"Disk writes: {wb.statistic_reason_phrase(reason)}."
4951
+ elif unmeasured:
4952
+ port, reason = sorted(unmeasured, key=lambda item: item[0] or 0)[0]
4953
+ summary = (f"Disk writes: the dashboard on port {port} "
4954
+ f"{_NOT_MEASURABLE_FROM_HERE[reason]}, so it is not "
4955
+ "measured from here.")
4956
+ elif platform:
4957
+ summary = f"Disk writes: unavailable ({_words(platform[0][1])})."
4958
+ elif gathered.get("mode") == "error":
4959
+ summary = "Disk writes: unavailable (gather failed)."
4960
+ elif gathered.get("mode") == "not_probed":
4961
+ summary = ("Disk writes: not measured here; run `cctally doctor` to "
4962
+ "check running dashboards.")
4963
+ else:
4964
+ summary = "No dashboard is running."
4965
+ return CheckResult(
4966
+ id=_DISK_WRITES_ID, title=_DISK_WRITES_TITLE, severity="ok",
4967
+ summary=summary, remediation=None, details=details)
4968
+
4969
+
4325
4970
  def _evaluate_one(check_id: str, check_fn_name: str,
4326
4971
  state: DoctorState) -> CheckResult:
4327
4972
  """Invoke a single check evaluator by name, catching any exception so