cctally 1.113.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +6035 -722
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +400 -2
- package/bin/_cctally_dashboard.py +98 -20
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +285 -213
- package/bin/_cctally_db.py +1177 -23
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +572 -45
- package/bin/_cctally_five_hour.py +84 -33
- package/bin/_cctally_forecast.py +108 -15
- package/bin/_cctally_journal.py +3950 -116
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +227 -37
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +691 -149
- package/bin/_cctally_rederive.py +320 -41
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +83 -17
- package/bin/_cctally_statusline.py +6154 -517
- package/bin/_cctally_store.py +367 -53
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +61 -56
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +297 -49
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +106 -59
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +714 -69
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +231 -86
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +199 -9
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +319 -82
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +31 -1
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +19 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
package/bin/_lib_doctor.py
CHANGED
|
@@ -232,6 +232,10 @@ class DoctorState:
|
|
|
232
232
|
# evidence. None is the normal state; each record contains counts/reasons
|
|
233
233
|
# but never configured paths or provider identifiers.
|
|
234
234
|
codex_prune_refusals: Optional[list[dict]] = None
|
|
235
|
+
codex_compressed_census: Optional[dict] = None
|
|
236
|
+
codex_decoder_capability: Optional[str] = None
|
|
237
|
+
codex_container_status: Optional[dict] = None
|
|
238
|
+
codex_maintenance_debt: Optional[dict] = None
|
|
235
239
|
# #705: the conversations store's refused-pricing-write record, read
|
|
236
240
|
# verbatim from cache_meta. None means no refusal is recorded. A record
|
|
237
241
|
# that exists but cannot be parsed arrives as {"__malformed__": True}
|
|
@@ -343,6 +347,18 @@ class DoctorState:
|
|
|
343
347
|
# retained corpus from unreadable metadata.
|
|
344
348
|
codex_project_metadata_health: Optional[dict] = None
|
|
345
349
|
codex_project_metadata_error: Optional[str] = None
|
|
350
|
+
# #825/#832 (spec section 8, F9): the account-bound legacy-writer evidence.
|
|
351
|
+
# `{"available": bool, "cutoverId", "excludedCount", "coordinateDigest",
|
|
352
|
+
# "evidence": <sidecar document or None>}` from the validated admission
|
|
353
|
+
# summary (a deep run substitutes its own whole-journal recomputation).
|
|
354
|
+
# None / available=False is reported as unavailable, never as zero.
|
|
355
|
+
legacy_writer: Optional[dict] = None
|
|
356
|
+
# #825/#832 (spec section 8, F16): the outcome quarantine, read from the
|
|
357
|
+
# validated index. `{"available": bool, "triggered": bool, "accounts":
|
|
358
|
+
# [{"account", "accountLabel", "accountingPaused", "controlPaused",
|
|
359
|
+
# "accountingSince", "controlSince"}]}`. None / available=False is
|
|
360
|
+
# reported as unavailable, never as OK.
|
|
361
|
+
outcome_quarantine: Optional[dict] = None
|
|
346
362
|
# Beta-channel (spec 2026-07-21 §3): the configured RELEASE channel
|
|
347
363
|
# (stable|beta) that `cctally update` tracks — DISTINCT from the preview
|
|
348
364
|
# `channel` (prod|preview) above. Populated by doctor_gather_state from a
|
|
@@ -391,6 +407,11 @@ class DoctorState:
|
|
|
391
407
|
journal_cursor_lag_bytes: Optional[int] = None
|
|
392
408
|
journal_hw_segment: Optional[str] = None
|
|
393
409
|
journal_cursor_segment: Optional[str] = None
|
|
410
|
+
# * journal_cursor_invalid (#946) — the applied cursor names no safe
|
|
411
|
+
# endpoint: past its segment's size, or inside a line. The lag beside
|
|
412
|
+
# it is then not a measure of anything consumed, so it is never
|
|
413
|
+
# reported as caught up.
|
|
414
|
+
journal_cursor_invalid: bool = False
|
|
394
415
|
journal_heal_incidents: Optional[list] = None
|
|
395
416
|
journal_heal_detections: Optional[list] = None
|
|
396
417
|
# * retained_artifacts (#496 S6 §7.3) — the read-only retention scan
|
|
@@ -423,6 +444,15 @@ class DoctorState:
|
|
|
423
444
|
journal_protocol_violations: Optional[list] = None
|
|
424
445
|
journal_protocol_acknowledged: Optional[list] = None
|
|
425
446
|
journal_protocol_error: Optional[str] = None
|
|
447
|
+
# * journal_listing_error (#825/#832 fix round K11, K9K10-R6) — why the
|
|
448
|
+
# segment listing failed. An unlistable journal is never an empty one.
|
|
449
|
+
# * journal_scan_error (K9K10-R6/R11) — why the deep scan could not
|
|
450
|
+
# complete: an unreadable segment, a raising admission-boundary lookup
|
|
451
|
+
# or an unreadable excluded population. The scan's checks are then
|
|
452
|
+
# unavailable — never zero excluded records, never an unfiltered
|
|
453
|
+
# conflict scan.
|
|
454
|
+
journal_listing_error: Optional[str] = None
|
|
455
|
+
journal_scan_error: Optional[str] = None
|
|
426
456
|
# Multi-account attribution (#341). A dict (None when unavailable) with:
|
|
427
457
|
# * claude_identity_status — "identified" | "stably_absent" | "torn"
|
|
428
458
|
# * claude_email — the active Claude email (or None)
|
|
@@ -441,6 +471,14 @@ class DoctorState:
|
|
|
441
471
|
# known file-level backup/sync root. Only provider/status are retained so
|
|
442
472
|
# reports never expose the machine-specific data path.
|
|
443
473
|
backup_sync_state: Optional[dict] = None
|
|
474
|
+
# #901: transcript-store page size, so `db.conversations_reclaimable`
|
|
475
|
+
# judges the paced-reclaim thresholds on observed bytes.
|
|
476
|
+
conversations_db_page_size: Optional[int] = None
|
|
477
|
+
# #901 §5.6: running dashboards' write telemetry, gathered read-only —
|
|
478
|
+
# the local instance's own snapshot in the dashboard process, or
|
|
479
|
+
# descriptor discovery plus one bounded loopback probe per live
|
|
480
|
+
# instance elsewhere. {"mode", "instances": [...]}; None = not gathered.
|
|
481
|
+
dashboard_disk_writes: Optional[dict] = None
|
|
444
482
|
|
|
445
483
|
|
|
446
484
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -1011,7 +1049,7 @@ def _check_db_version_ahead(s: DoctorState) -> CheckResult:
|
|
|
1011
1049
|
# current index as a mismatch; keep this in lockstep with core
|
|
1012
1050
|
# (#496 S5b §6.1). It stays a literal because this kernel is pure and
|
|
1013
1051
|
# must not import `_cctally_core`.
|
|
1014
|
-
epoch =
|
|
1052
|
+
epoch = 1018
|
|
1015
1053
|
mismatch = uv > legacy_head and uv != epoch
|
|
1016
1054
|
return {"user_version": uv, "legacy_head": legacy_head, "epoch": epoch,
|
|
1017
1055
|
"mismatch": mismatch}
|
|
@@ -1346,6 +1384,95 @@ def _check_data_codex_replay(s: DoctorState) -> CheckResult:
|
|
|
1346
1384
|
)
|
|
1347
1385
|
|
|
1348
1386
|
|
|
1387
|
+
def _check_data_codex_compressed_rollouts(s: DoctorState) -> CheckResult:
|
|
1388
|
+
"""#968 5.6 with operator clarifications D2, D3 and D6.
|
|
1389
|
+
|
|
1390
|
+
`warn` when any of three conditions holds: compressed rollouts exist and
|
|
1391
|
+
no decoder is available; a verification ended `unverified` (D3, at once,
|
|
1392
|
+
never debt); verification, reading or deletion work has been pending for
|
|
1393
|
+
more than 24 hours (D2, D6). The summary states each true condition's
|
|
1394
|
+
sentence once, joined by a single space, in that order. Otherwise pending
|
|
1395
|
+
work is `ok` "continuing", and the all-clear sentence is used only when
|
|
1396
|
+
no compressed rollout is unread, pending or unverified. Never FAIL: a
|
|
1397
|
+
missing optional decoder is an environment limitation.
|
|
1398
|
+
"""
|
|
1399
|
+
if s.codex_compressed_census is None:
|
|
1400
|
+
return CheckResult(
|
|
1401
|
+
id="data.codex_compressed_rollouts", title="Compressed Codex rollouts",
|
|
1402
|
+
severity="warn", summary="unavailable (compressed Codex rollout census)",
|
|
1403
|
+
remediation=None, details={"available": False})
|
|
1404
|
+
census = s.codex_compressed_census
|
|
1405
|
+
compressed = int(census.get("compressedOnly", 0) or 0)
|
|
1406
|
+
never = int(census.get("neverIngested", 0) or 0)
|
|
1407
|
+
incomplete = int(census.get("retainedIncomplete", 0) or 0)
|
|
1408
|
+
unverified = int(census.get("unverified", 0) or 0)
|
|
1409
|
+
debt = s.codex_maintenance_debt or {}
|
|
1410
|
+
maintenance_partial = (s.codex_maintenance_debt is None
|
|
1411
|
+
or debt.get("coverage") == "partial")
|
|
1412
|
+
verifications = int(debt.get("verifications", 0) or 0)
|
|
1413
|
+
deletions = int(debt.get("deletions", 0) or 0)
|
|
1414
|
+
no_decoder = s.codex_decoder_capability == "unavailable"
|
|
1415
|
+
# D6: with a decoder, an unread or partly read compressed rollout is
|
|
1416
|
+
# pending reading work; without one it is the no-decoder condition.
|
|
1417
|
+
reads = 0 if no_decoder else never + incomplete
|
|
1418
|
+
details = {
|
|
1419
|
+
"compressedOnly": compressed,
|
|
1420
|
+
"retainedComplete": int(census.get("retainedComplete", 0) or 0),
|
|
1421
|
+
"retainedIncomplete": incomplete,
|
|
1422
|
+
"neverIngested": never,
|
|
1423
|
+
"unverified": unverified,
|
|
1424
|
+
"verificationDebt": verifications,
|
|
1425
|
+
"readDebt": reads,
|
|
1426
|
+
"pruneDebt": deletions,
|
|
1427
|
+
}
|
|
1428
|
+
if maintenance_partial:
|
|
1429
|
+
details["maintenanceCoverage"] = "partial"
|
|
1430
|
+
if s.codex_maintenance_debt is None:
|
|
1431
|
+
details["verificationDebt"] = None
|
|
1432
|
+
details["pruneDebt"] = None
|
|
1433
|
+
status = s.codex_container_status or {}
|
|
1434
|
+
if status.get("census") == "partial":
|
|
1435
|
+
details["census"] = "partial"
|
|
1436
|
+
details["observedAt"] = status.get("observedAt")
|
|
1437
|
+
if compressed:
|
|
1438
|
+
details["decoderAvailable"] = not no_decoder
|
|
1439
|
+
pending = bool(verifications or reads or deletions)
|
|
1440
|
+
sentences = []
|
|
1441
|
+
if compressed and no_decoder:
|
|
1442
|
+
sentences.append(
|
|
1443
|
+
f"{compressed} compressed Codex rollouts can't be read on this "
|
|
1444
|
+
f"Python ({never} never ingested). Cached history was kept. "
|
|
1445
|
+
"Install `backports.zstd` or use Python 3.14+.")
|
|
1446
|
+
if unverified:
|
|
1447
|
+
sentences.append(
|
|
1448
|
+
f"{unverified} compressed Codex rollouts kept their cached history "
|
|
1449
|
+
"but couldn't be verified against it.")
|
|
1450
|
+
if pending and debt.get("olderThan24h"):
|
|
1451
|
+
sentences.append(
|
|
1452
|
+
"Codex history maintenance has been pending for more than 24 hours "
|
|
1453
|
+
f"({verifications} verifications, {reads} reads, {deletions} "
|
|
1454
|
+
"deletions).")
|
|
1455
|
+
if sentences:
|
|
1456
|
+
severity, summary = "warn", " ".join(sentences)
|
|
1457
|
+
elif pending:
|
|
1458
|
+
severity = "ok"
|
|
1459
|
+
summary = "Codex history maintenance is continuing in the background."
|
|
1460
|
+
elif maintenance_partial:
|
|
1461
|
+
severity = "warn"
|
|
1462
|
+
summary = "unavailable (compressed Codex rollout census)"
|
|
1463
|
+
else:
|
|
1464
|
+
severity = "ok"
|
|
1465
|
+
summary = "All compressed Codex rollouts are readable and verified."
|
|
1466
|
+
return CheckResult(
|
|
1467
|
+
id="data.codex_compressed_rollouts",
|
|
1468
|
+
title="Compressed Codex rollouts",
|
|
1469
|
+
severity=severity,
|
|
1470
|
+
summary=summary,
|
|
1471
|
+
remediation=None,
|
|
1472
|
+
details=details,
|
|
1473
|
+
)
|
|
1474
|
+
|
|
1475
|
+
|
|
1349
1476
|
def _check_data_codex_prune_safety(s: DoctorState) -> CheckResult:
|
|
1350
1477
|
"""Surface a fail-closed orphan-prune decision instead of silent loss."""
|
|
1351
1478
|
records = [
|
|
@@ -2916,92 +3043,119 @@ def _check_db_reclaimable(s: DoctorState) -> CheckResult:
|
|
|
2916
3043
|
|
|
2917
3044
|
|
|
2918
3045
|
def _check_db_conversations_reclaimable(s: DoctorState) -> CheckResult:
|
|
2919
|
-
"""
|
|
3046
|
+
"""Transcript free space against the paced-reclaim policy (#320, #780,
|
|
3047
|
+
#901). Read-only; judged on the OBSERVED freelist, never on a record.
|
|
3048
|
+
|
|
3049
|
+
OK for retained slack below the reclaim start threshold — reusable pages
|
|
3050
|
+
kept on purpose (operator decision Q2). WARN for an eligible backlog being
|
|
3051
|
+
paced, including one whose attempts are failing (`reclaim_failure`); a
|
|
3052
|
+
scheduled wait is not "stuck". FAIL at the hard ceiling, where transcript
|
|
3053
|
+
rebuilds are refused."""
|
|
3054
|
+
import _lib_conversation_retention as _conv_retention
|
|
3055
|
+
|
|
2920
3056
|
page_count = s.conversations_db_page_count
|
|
2921
3057
|
freelist_count = s.conversations_db_freelist_count
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
isinstance(
|
|
2925
|
-
|
|
2926
|
-
and page_count > 0
|
|
2927
|
-
and isinstance(freelist_count, int)
|
|
2928
|
-
and not isinstance(freelist_count, bool)
|
|
3058
|
+
page_size = s.conversations_db_page_size
|
|
3059
|
+
measured = (
|
|
3060
|
+
all(isinstance(v, int) and not isinstance(v, bool)
|
|
3061
|
+
for v in (page_count, freelist_count, page_size))
|
|
3062
|
+
and page_count > 0 and page_size > 0
|
|
2929
3063
|
and 0 <= freelist_count <= page_count
|
|
2930
|
-
)
|
|
2931
|
-
|
|
3064
|
+
)
|
|
3065
|
+
geometry = (
|
|
3066
|
+
_conv_retention.StoreGeometry(page_count, freelist_count, page_size)
|
|
3067
|
+
if measured else None)
|
|
3068
|
+
record = (s.conversations_reclaim_pending
|
|
3069
|
+
if isinstance(s.conversations_reclaim_pending, dict) else None)
|
|
3070
|
+
episode = bool(record and record.get("eligible") is True)
|
|
3071
|
+
failure = record.get("last_failure") if record else None
|
|
3072
|
+
if not isinstance(failure, dict):
|
|
3073
|
+
failure = None
|
|
3074
|
+
free_bytes = geometry.free_bytes if geometry is not None else None
|
|
2932
3075
|
details = {
|
|
2933
3076
|
"conversations_db_page_count": page_count,
|
|
2934
3077
|
"conversations_db_freelist_count": freelist_count,
|
|
2935
|
-
"
|
|
2936
|
-
"
|
|
3078
|
+
"conversations_db_page_size": page_size,
|
|
3079
|
+
"conversations_db_free_ratio": (
|
|
3080
|
+
freelist_count / page_count if geometry is not None else None),
|
|
3081
|
+
"reclaimable_bytes": free_bytes,
|
|
3082
|
+
"reclaim_start_bytes": _conv_retention.RECLAIM_START_BYTES,
|
|
3083
|
+
"reclaim_start_ratio": _conv_retention.RECLAIM_START_RATIO,
|
|
3084
|
+
"reclaim_stop_bytes": _conv_retention.RECLAIM_STOP_BYTES,
|
|
3085
|
+
"reclaim_stop_ratio": _conv_retention.RECLAIM_STOP_RATIO,
|
|
3086
|
+
"reclaim_ceiling_bytes": _conv_retention.RECLAIM_CEILING_BYTES,
|
|
3087
|
+
"reclaim_episode_active": episode if record is not None else None,
|
|
3088
|
+
"reclaim_failure": failure,
|
|
2937
3089
|
}
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
# — on a store with no pending record. That is every store that has never
|
|
2941
|
-
# fallen behind, so an install in the ordinary state renders exactly what
|
|
2942
|
-
# it rendered before.
|
|
2943
|
-
pending = s.conversations_reclaim_pending
|
|
2944
|
-
if pending:
|
|
2945
|
-
import _lib_conversation_retention as _conv_retention
|
|
2946
|
-
|
|
2947
|
-
backlog = _conv_retention.reclaim_backlog_bytes(pending)
|
|
2948
|
-
details["reclaim_pending"] = pending
|
|
2949
|
-
details["reclaim_backlog_bytes"] = backlog
|
|
2950
|
-
details["reclaim_ceiling_bytes"] = _conv_retention.RECLAIM_CEILING_BYTES
|
|
2951
|
-
if _conv_retention.reclaim_backlog_over_ceiling(pending):
|
|
2952
|
-
return CheckResult(
|
|
2953
|
-
id="db.conversations_reclaimable",
|
|
2954
|
-
title="Reclaimable transcript space",
|
|
2955
|
-
severity="fail",
|
|
2956
|
-
summary=(
|
|
2957
|
-
f"reclaim backlog {_gib_text(backlog)} is at or over the "
|
|
2958
|
-
f"{_gib_text(_conv_retention.RECLAIM_CEILING_BYTES)} "
|
|
2959
|
-
"ceiling; transcript rebuilds are refused"
|
|
2960
|
-
),
|
|
2961
|
-
remediation=(
|
|
2962
|
-
"Run `cctally db vacuum --db conversations` to drain the "
|
|
2963
|
-
"backlog, then `cctally db checkpoint --db conversations`."
|
|
2964
|
-
),
|
|
2965
|
-
details=details,
|
|
2966
|
-
)
|
|
2967
|
-
if _conv_retention.reclaim_backlog_escalated(pending):
|
|
2968
|
-
return CheckResult(
|
|
2969
|
-
id="db.conversations_reclaimable",
|
|
2970
|
-
title="Reclaimable transcript space",
|
|
2971
|
-
severity="warn",
|
|
2972
|
-
summary=(
|
|
2973
|
-
f"reclaim backlog {_gib_text(backlog)} is draining across "
|
|
2974
|
-
"continuation passes"
|
|
2975
|
-
),
|
|
2976
|
-
remediation=(
|
|
2977
|
-
"No action is required unless it keeps growing; "
|
|
2978
|
-
"`cctally db vacuum --db conversations` drains it now."
|
|
2979
|
-
),
|
|
2980
|
-
details=details,
|
|
2981
|
-
)
|
|
2982
|
-
if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
|
|
3090
|
+
title = "Reclaimable transcript space"
|
|
3091
|
+
if _conv_retention.backlog_over_ceiling(free_bytes):
|
|
2983
3092
|
return CheckResult(
|
|
2984
|
-
id="db.conversations_reclaimable",
|
|
2985
|
-
title="Reclaimable transcript space",
|
|
2986
|
-
severity="warn",
|
|
3093
|
+
id="db.conversations_reclaimable", title=title, severity="fail",
|
|
2987
3094
|
summary=(
|
|
2988
|
-
f"
|
|
3095
|
+
f"reclaim backlog {_gib_text(free_bytes)} is at or over the "
|
|
3096
|
+
f"{_gib_text(_conv_retention.RECLAIM_CEILING_BYTES)} "
|
|
3097
|
+
"ceiling; transcript rebuilds are refused"
|
|
2989
3098
|
),
|
|
2990
3099
|
remediation=(
|
|
2991
|
-
"Run `cctally db vacuum --db conversations` to
|
|
3100
|
+
"Run `cctally db vacuum --db conversations` to drain the "
|
|
3101
|
+
"backlog, then `cctally db checkpoint --db conversations`."
|
|
2992
3102
|
),
|
|
2993
3103
|
details=details,
|
|
2994
3104
|
)
|
|
3105
|
+
if _conv_retention.reclaim_episode_active(geometry, episode):
|
|
3106
|
+
# Q9: a paced wait is not a refused attempt. A planner refusal is an
|
|
3107
|
+
# in-memory pass record of the running dashboard (it writes nothing
|
|
3108
|
+
# durable), read from `writeIo.reclaimRefusal`; a connection-settings
|
|
3109
|
+
# or SQLite failure stays the durable `reclaim_failure`, as before.
|
|
3110
|
+
refusal = _latest_reclaim_refusal(s.dashboard_disk_writes)
|
|
3111
|
+
wait = ("refused" if refusal is not None
|
|
3112
|
+
else "failing" if failure is not None else "paced")
|
|
3113
|
+
remediation = (
|
|
3114
|
+
"No action is required unless it keeps growing; "
|
|
3115
|
+
"`cctally db vacuum --db conversations` drains it now.")
|
|
3116
|
+
if refusal is not None:
|
|
3117
|
+
remediation = (
|
|
3118
|
+
"Reclaim attempts are being refused "
|
|
3119
|
+
f"({refusal['reason'].replace('_', ' ')}); "
|
|
3120
|
+
"`cctally db vacuum --db conversations` reclaims the space "
|
|
3121
|
+
"at once.")
|
|
3122
|
+
return CheckResult(
|
|
3123
|
+
id="db.conversations_reclaimable", title=title, severity="warn",
|
|
3124
|
+
summary=(
|
|
3125
|
+
f"Reclaiming {_gib_text(free_bytes)} of free transcript "
|
|
3126
|
+
"space at a paced rate."
|
|
3127
|
+
),
|
|
3128
|
+
remediation=remediation,
|
|
3129
|
+
details={**details, "reclaim_wait": wait,
|
|
3130
|
+
"reclaim_refusal": refusal},
|
|
3131
|
+
)
|
|
2995
3132
|
return CheckResult(
|
|
2996
|
-
id="db.conversations_reclaimable",
|
|
2997
|
-
|
|
2998
|
-
severity="ok",
|
|
2999
|
-
summary="below threshold",
|
|
3000
|
-
remediation=None,
|
|
3001
|
-
details=details,
|
|
3133
|
+
id="db.conversations_reclaimable", title=title, severity="ok",
|
|
3134
|
+
summary="below threshold", remediation=None, details=details,
|
|
3002
3135
|
)
|
|
3003
3136
|
|
|
3004
3137
|
|
|
3138
|
+
def _latest_reclaim_refusal(gathered) -> "dict | None":
|
|
3139
|
+
"""The newest planner refusal any running dashboard reports in its
|
|
3140
|
+
`writeIo.reclaimRefusal`, with that dashboard's port; None when none
|
|
3141
|
+
does (or no dashboard runs)."""
|
|
3142
|
+
latest = None
|
|
3143
|
+
entries = gathered.get("instances") if isinstance(gathered, dict) else None
|
|
3144
|
+
for entry in entries or ():
|
|
3145
|
+
if not isinstance(entry, dict) or entry.get("probe") != "ok":
|
|
3146
|
+
continue
|
|
3147
|
+
refusal = (entry.get("write_io") or {}).get("reclaimRefusal")
|
|
3148
|
+
if (not isinstance(refusal, dict)
|
|
3149
|
+
or not isinstance(refusal.get("reason"), str)
|
|
3150
|
+
or not isinstance(refusal.get("at"), str)):
|
|
3151
|
+
continue
|
|
3152
|
+
candidate = {"reason": refusal["reason"], "at": refusal["at"],
|
|
3153
|
+
"port": entry.get("port")}
|
|
3154
|
+
if latest is None or candidate["at"] > latest["at"]:
|
|
3155
|
+
latest = candidate
|
|
3156
|
+
return latest
|
|
3157
|
+
|
|
3158
|
+
|
|
3005
3159
|
#: A stuck reclaim entry has persisted past this, so no pass will clear it.
|
|
3006
3160
|
_RECLAIM_STUCK_AFTER_SECONDS = 24 * 3600
|
|
3007
3161
|
|
|
@@ -3238,6 +3392,28 @@ _JOURNAL_CURSOR_LAG_WARN_BYTES = 4 * 1024 * 1024
|
|
|
3238
3392
|
_JOURNAL_HEAL_RECENT_SECONDS = 7 * 24 * 3600
|
|
3239
3393
|
|
|
3240
3394
|
|
|
3395
|
+
def _journal_unreadable_remediation(s: DoctorState) -> str:
|
|
3396
|
+
"""K9K10-R6: the remediation for a journal doctor could not read. Fix
|
|
3397
|
+
round K12 (K11R-12): it names the resolved data directory (a dev
|
|
3398
|
+
checkout or `CCTALLY_DATA_DIR` is not the default path)."""
|
|
3399
|
+
journal = f"{str(s.app_dir or '').rstrip('/')}/journal/"
|
|
3400
|
+
return (f"Check the permissions of {journal} and the disk, then run "
|
|
3401
|
+
"`cctally doctor --deep` again")
|
|
3402
|
+
|
|
3403
|
+
|
|
3404
|
+
def _journal_scan_unavailable(check_id: str, title: str, s: DoctorState,
|
|
3405
|
+
details: dict) -> CheckResult:
|
|
3406
|
+
"""A deep-scan check the scan could not complete (K9K10-R6): WARN,
|
|
3407
|
+
unavailable, with the cause."""
|
|
3408
|
+
return CheckResult(
|
|
3409
|
+
id=check_id, title=title, severity="warn",
|
|
3410
|
+
summary=f"unavailable ({s.journal_scan_error})",
|
|
3411
|
+
remediation=_journal_unreadable_remediation(s),
|
|
3412
|
+
details={**details, "scanned": True, "available": False,
|
|
3413
|
+
"error": s.journal_scan_error},
|
|
3414
|
+
)
|
|
3415
|
+
|
|
3416
|
+
|
|
3241
3417
|
def _check_journal_presence(s: DoctorState) -> CheckResult:
|
|
3242
3418
|
"""The journal directory exists and is appendable. A pre-cutover (legacy)
|
|
3243
3419
|
install has NO journal yet — that is INFO (OK), never a FAIL."""
|
|
@@ -3249,6 +3425,15 @@ def _check_journal_presence(s: DoctorState) -> CheckResult:
|
|
|
3249
3425
|
)
|
|
3250
3426
|
details = {"present": True, "appendable": s.journal_appendable,
|
|
3251
3427
|
"segments": s.journal_segment_count}
|
|
3428
|
+
if s.journal_listing_error:
|
|
3429
|
+
return CheckResult(
|
|
3430
|
+
id="journal.presence", title="Journal", severity="warn",
|
|
3431
|
+
summary=(f"segment listing unavailable "
|
|
3432
|
+
f"({s.journal_listing_error})"),
|
|
3433
|
+
remediation=_journal_unreadable_remediation(s),
|
|
3434
|
+
details={**details, "segments": None,
|
|
3435
|
+
"listingError": s.journal_listing_error},
|
|
3436
|
+
)
|
|
3252
3437
|
if s.journal_appendable is False:
|
|
3253
3438
|
return CheckResult(
|
|
3254
3439
|
id="journal.presence", title="Journal", severity="warn",
|
|
@@ -3272,6 +3457,9 @@ def _check_journal_integrity(s: DoctorState) -> CheckResult:
|
|
|
3272
3457
|
id="journal.integrity", title="Journal integrity", severity="ok",
|
|
3273
3458
|
summary="no journal", remediation=None, details={"present": False},
|
|
3274
3459
|
)
|
|
3460
|
+
if s.journal_scan_error:
|
|
3461
|
+
return _journal_scan_unavailable(
|
|
3462
|
+
"journal.integrity", "Journal integrity", s, {})
|
|
3275
3463
|
if s.journal_malformed_count is None:
|
|
3276
3464
|
return CheckResult(
|
|
3277
3465
|
id="journal.integrity", title="Journal integrity", severity="ok",
|
|
@@ -3313,6 +3501,16 @@ def _check_journal_index_freshness(s: DoctorState) -> CheckResult:
|
|
|
3313
3501
|
details = {"lag_bytes": lag, "hw_segment": s.journal_hw_segment,
|
|
3314
3502
|
"cursor_segment": s.journal_cursor_segment,
|
|
3315
3503
|
"warn_bytes": _JOURNAL_CURSOR_LAG_WARN_BYTES}
|
|
3504
|
+
if s.journal_cursor_invalid:
|
|
3505
|
+
# #946: lag is computed from the safe applied cursor, and this one is
|
|
3506
|
+
# not safe, so no amount of lag here means "caught up".
|
|
3507
|
+
details["cursor_invalid"] = True
|
|
3508
|
+
return CheckResult(
|
|
3509
|
+
id="journal.index_freshness", title="Journal index", severity="warn",
|
|
3510
|
+
summary="index cursor is not at a journal line boundary",
|
|
3511
|
+
remediation="Run `cctally db rebuild --db stats`",
|
|
3512
|
+
details=details,
|
|
3513
|
+
)
|
|
3316
3514
|
if lag == 0:
|
|
3317
3515
|
return CheckResult(
|
|
3318
3516
|
id="journal.index_freshness", title="Journal index", severity="ok",
|
|
@@ -3547,6 +3745,9 @@ def _check_journal_conflicts(s: DoctorState) -> CheckResult:
|
|
|
3547
3745
|
remediation=None,
|
|
3548
3746
|
details={"scanned": True, "available": False, "conflicts": None},
|
|
3549
3747
|
)
|
|
3748
|
+
if s.journal_scan_error:
|
|
3749
|
+
return _journal_scan_unavailable(
|
|
3750
|
+
"journal.conflicts", "Journal conflicts", s, {"conflicts": None})
|
|
3550
3751
|
if s.journal_conflicts is None:
|
|
3551
3752
|
return CheckResult(
|
|
3552
3753
|
id="journal.conflicts", title="Journal conflicts", severity="ok",
|
|
@@ -3585,6 +3786,283 @@ def _check_journal_conflicts(s: DoctorState) -> CheckResult:
|
|
|
3585
3786
|
)
|
|
3586
3787
|
|
|
3587
3788
|
|
|
3789
|
+
def _check_journal_legacy_writer(s: DoctorState) -> CheckResult:
|
|
3790
|
+
"""#825/#832 (spec section 8, F9, operator Q4): records an older cctally
|
|
3791
|
+
process wrote after the account-bound cutover are excluded from current
|
|
3792
|
+
accounting. WARN while the latest excluded population was first observed
|
|
3793
|
+
under seven days ago; OK with a historical note after seven quiet days.
|
|
3794
|
+
Never asserts that an old writer is alive, and never reports a missing
|
|
3795
|
+
summary as zero. An incomplete legacy spool inventory is also a WARN,
|
|
3796
|
+
independently of the excluded-record count (spec revision 5, section
|
|
3797
|
+
8; `_with_legacy_spool_inventory`)."""
|
|
3798
|
+
result = _legacy_writer_count_result(s)
|
|
3799
|
+
inventory = (s.legacy_writer or {}).get("legacySpoolInventory")
|
|
3800
|
+
if not inventory or inventory.get("available", False):
|
|
3801
|
+
return result
|
|
3802
|
+
return _with_legacy_spool_inventory(result, inventory)
|
|
3803
|
+
|
|
3804
|
+
|
|
3805
|
+
def _with_legacy_spool_inventory(result: CheckResult,
|
|
3806
|
+
inventory: dict) -> CheckResult:
|
|
3807
|
+
"""Spec revision 5 (section 8): an incomplete legacy spool inventory is a
|
|
3808
|
+
WARN independently of the excluded-record count. It names the path, the
|
|
3809
|
+
failed operation and the error, never claims an old writer exists, and
|
|
3810
|
+
says detection is paused only when the validated index holds the
|
|
3811
|
+
durable `inventory-incomplete` marker. An inventory that failed for any
|
|
3812
|
+
other reason (fix round K12, K11R-13) names no file: it is reported
|
|
3813
|
+
unavailable with its cause."""
|
|
3814
|
+
if inventory.get("unavailableCause") and not inventory.get("path"):
|
|
3815
|
+
return _with_unavailable_spool_inventory(
|
|
3816
|
+
result, inventory, inventory["unavailableCause"])
|
|
3817
|
+
text = LEGACY_SPOOL_INVENTORY_TEXT.format(
|
|
3818
|
+
path=inventory.get("path"), operation=inventory.get("operation"),
|
|
3819
|
+
error=inventory.get("error"))
|
|
3820
|
+
if inventory.get("paused"):
|
|
3821
|
+
text += " " + LEGACY_SPOOL_INVENTORY_PAUSED_TEXT
|
|
3822
|
+
note = "legacy status-line inventory unavailable"
|
|
3823
|
+
if result.severity == "warn":
|
|
3824
|
+
summary = f"{result.summary}; {note}"
|
|
3825
|
+
remediation = (f"{result.remediation} {text}" if result.remediation
|
|
3826
|
+
else text)
|
|
3827
|
+
else:
|
|
3828
|
+
summary = (f"{note} ({inventory.get('operation')} "
|
|
3829
|
+
f"{inventory.get('path')}: {inventory.get('error')})")
|
|
3830
|
+
remediation = text
|
|
3831
|
+
return CheckResult(
|
|
3832
|
+
id=result.id, title=result.title, severity="warn", summary=summary,
|
|
3833
|
+
remediation=remediation,
|
|
3834
|
+
details={**(result.details or {}),
|
|
3835
|
+
"legacySpoolInventory": dict(inventory)},
|
|
3836
|
+
)
|
|
3837
|
+
|
|
3838
|
+
|
|
3839
|
+
def _with_unavailable_spool_inventory(result: CheckResult, inventory: dict,
|
|
3840
|
+
cause: str) -> CheckResult:
|
|
3841
|
+
"""`_with_legacy_spool_inventory` for an inventory that names no file:
|
|
3842
|
+
a WARN, "legacy status-line inventory unavailable ({cause})", which keeps
|
|
3843
|
+
the excluded-record half's own summary and remediation."""
|
|
3844
|
+
note = f"legacy status-line inventory unavailable ({cause})"
|
|
3845
|
+
if result.severity == "warn":
|
|
3846
|
+
summary, remediation = f"{result.summary}; {note}", result.remediation
|
|
3847
|
+
else:
|
|
3848
|
+
summary, remediation = note, None
|
|
3849
|
+
return CheckResult(
|
|
3850
|
+
id=result.id, title=result.title, severity="warn", summary=summary,
|
|
3851
|
+
remediation=remediation,
|
|
3852
|
+
details={**(result.details or {}),
|
|
3853
|
+
"legacySpoolInventory": dict(inventory)},
|
|
3854
|
+
)
|
|
3855
|
+
|
|
3856
|
+
|
|
3857
|
+
def _legacy_writer_count_result(s: DoctorState) -> CheckResult:
|
|
3858
|
+
"""The excluded-record half of `journal.legacy_writer`."""
|
|
3859
|
+
import _lib_legacy_writer_evidence as _lw
|
|
3860
|
+
|
|
3861
|
+
data = s.legacy_writer
|
|
3862
|
+
if data and data.get("unavailableCause"):
|
|
3863
|
+
# K9K10-R6/R11: the deep recomputation could not complete. The count
|
|
3864
|
+
# is unavailable — never zero, never the stale summary.
|
|
3865
|
+
return CheckResult(
|
|
3866
|
+
id="journal.legacy_writer", title="Legacy writer",
|
|
3867
|
+
severity="warn",
|
|
3868
|
+
summary=f"unavailable ({data['unavailableCause']})",
|
|
3869
|
+
remediation=_journal_unreadable_remediation(s),
|
|
3870
|
+
details={"available": False,
|
|
3871
|
+
"unavailableCause": data["unavailableCause"]},
|
|
3872
|
+
)
|
|
3873
|
+
if not data or not data.get("available"):
|
|
3874
|
+
# No current summary (for example an older process republished its
|
|
3875
|
+
# own epoch). The durable sidecar alone still proves an excluded
|
|
3876
|
+
# population and when it was first observed, so it decides the leg;
|
|
3877
|
+
# without one the leg is unavailable, never zero.
|
|
3878
|
+
sidecar = _lw.from_document((data or {}).get("evidence"))
|
|
3879
|
+
if sidecar is None or sidecar.excluded_count <= 0:
|
|
3880
|
+
return CheckResult(
|
|
3881
|
+
id="journal.legacy_writer", title="Legacy writer",
|
|
3882
|
+
severity="ok",
|
|
3883
|
+
summary="unavailable (no current admission summary)",
|
|
3884
|
+
remediation=None,
|
|
3885
|
+
details={"available": False},
|
|
3886
|
+
)
|
|
3887
|
+
verdict = _lw.classify(sidecar, excluded_count=sidecar.excluded_count,
|
|
3888
|
+
now=s.now_utc)
|
|
3889
|
+
return _legacy_writer_verdict_result(
|
|
3890
|
+
verdict, count=sidecar.excluded_count,
|
|
3891
|
+
details={"available": False, "source": "evidence-sidecar",
|
|
3892
|
+
"cutoverId": sidecar.cutover_id,
|
|
3893
|
+
"excludedCount": sidecar.excluded_count,
|
|
3894
|
+
"state": verdict.state,
|
|
3895
|
+
"lastSeenAt": verdict.last_seen_at})
|
|
3896
|
+
cutover_id = data.get("cutoverId")
|
|
3897
|
+
count = int(data.get("excludedCount") or 0)
|
|
3898
|
+
if cutover_id is None:
|
|
3899
|
+
return CheckResult(
|
|
3900
|
+
id="journal.legacy_writer", title="Legacy writer", severity="ok",
|
|
3901
|
+
summary="no account-bound cutover yet", remediation=None,
|
|
3902
|
+
details={"available": True, "cutoverId": None,
|
|
3903
|
+
"excludedCount": count, "state": "none",
|
|
3904
|
+
"lastSeenAt": None},
|
|
3905
|
+
)
|
|
3906
|
+
# A missing or stale sidecar reads as "first observed now", which can only
|
|
3907
|
+
# lengthen a warning (spec section 8).
|
|
3908
|
+
evidence = _lw.observe(
|
|
3909
|
+
_lw.from_document(data.get("evidence")), cutover_id=cutover_id,
|
|
3910
|
+
excluded_count=count, coordinate_digest=data.get("coordinateDigest"),
|
|
3911
|
+
now=s.now_utc)
|
|
3912
|
+
verdict = _lw.classify(evidence, excluded_count=count, now=s.now_utc)
|
|
3913
|
+
details = {"available": True, "cutoverId": cutover_id,
|
|
3914
|
+
"excludedCount": count, "state": verdict.state,
|
|
3915
|
+
"lastSeenAt": verdict.last_seen_at}
|
|
3916
|
+
if verdict.state == "none":
|
|
3917
|
+
return CheckResult(
|
|
3918
|
+
id="journal.legacy_writer", title="Legacy writer", severity="ok",
|
|
3919
|
+
summary="no legacy writes since the account-bound cutover",
|
|
3920
|
+
remediation=None, details=details,
|
|
3921
|
+
)
|
|
3922
|
+
return _legacy_writer_verdict_result(verdict, count=count, details=details)
|
|
3923
|
+
|
|
3924
|
+
|
|
3925
|
+
def _legacy_writer_verdict_result(verdict, *, count: int,
|
|
3926
|
+
details: dict) -> CheckResult:
|
|
3927
|
+
"""The WARN (recent) or historical OK (quiet) result in the approved
|
|
3928
|
+
spec-section-8 content: the count, the last-seen time and the remedy."""
|
|
3929
|
+
import _lib_legacy_writer_evidence as _lw
|
|
3930
|
+
|
|
3931
|
+
if verdict.state == "recent":
|
|
3932
|
+
return CheckResult(
|
|
3933
|
+
id="journal.legacy_writer", title="Legacy writer", severity="warn",
|
|
3934
|
+
summary=(f"{count} record(s) an older cctally process wrote after "
|
|
3935
|
+
"the upgrade were excluded from accounting (last seen "
|
|
3936
|
+
f"{verdict.last_seen_at})"),
|
|
3937
|
+
remediation=_lw.REMEDY, details=details,
|
|
3938
|
+
)
|
|
3939
|
+
return CheckResult(
|
|
3940
|
+
id="journal.legacy_writer", title="Legacy writer", severity="ok",
|
|
3941
|
+
summary=(f"no legacy writes in the last {_lw.QUIET_DAYS} days "
|
|
3942
|
+
f"({count} excluded in total, last seen "
|
|
3943
|
+
f"{verdict.last_seen_at})"),
|
|
3944
|
+
remediation=None, details=details,
|
|
3945
|
+
)
|
|
3946
|
+
|
|
3947
|
+
|
|
3948
|
+
#: Spec revision 5 (section 8), operator-owned wording, verbatim: an
|
|
3949
|
+
#: incomplete legacy spool inventory, and the sentence added when the
|
|
3950
|
+
#: validated index holds its durable `inventory-incomplete` marker.
|
|
3951
|
+
LEGACY_SPOOL_INVENTORY_TEXT = (
|
|
3952
|
+
"cctally cannot read {path} ({operation}: {error}), one of the "
|
|
3953
|
+
"status-line files an older cctally release used. Until it can, cctally "
|
|
3954
|
+
"cannot tell whether an older process changed it. Check the file's "
|
|
3955
|
+
"permissions and the disk.")
|
|
3956
|
+
LEGACY_SPOOL_INVENTORY_PAUSED_TEXT = (
|
|
3957
|
+
"Automatic credit and reset detection is paused because of this, and "
|
|
3958
|
+
"making the file readable again does not end the pause (see "
|
|
3959
|
+
"`accounting.outcome_quarantine`); usage recording continues.")
|
|
3960
|
+
|
|
3961
|
+
|
|
3962
|
+
#: Operator-approved wording (#825/#832 spec section 8), verbatim.
|
|
3963
|
+
OUTCOME_QUARANTINE_ACCOUNTING_TEXT = (
|
|
3964
|
+
"Automatic credit and reset detection is paused for {account} since "
|
|
3965
|
+
"{time} because an older cctally process wrote data after the upgrade. It "
|
|
3966
|
+
"resumes only after cctally verifies a clean reading for this account, "
|
|
3967
|
+
"typically the first one after an idle period spanning a weekly reset; "
|
|
3968
|
+
"there is no guaranteed time. If your usage was reset in place meanwhile, "
|
|
3969
|
+
"record it with `cctally record-credit`.")
|
|
3970
|
+
OUTCOME_QUARANTINE_CONTROL_TEXT = (
|
|
3971
|
+
"Status-line drop detection also waits for a successful usage refresh for "
|
|
3972
|
+
"this account.")
|
|
3973
|
+
#: Spec revision 3 (section 8; operator decision dec-2b5bcf910816), verbatim:
|
|
3974
|
+
#: an account paused by an unresolved spool marker. It replaces the "resumes
|
|
3975
|
+
#: only after..." sentence and already names drop detection, so no control
|
|
3976
|
+
#: sentence is appended.
|
|
3977
|
+
OUTCOME_QUARANTINE_SPOOL_MARKER_TEXT = (
|
|
3978
|
+
"Automatic credit, reset and status-line drop detection is paused for "
|
|
3979
|
+
"{account} since {time} because an older cctally process wrote "
|
|
3980
|
+
"status-line data after the upgrade. This version of cctally cannot yet "
|
|
3981
|
+
"confirm a safe point to resume, so the pause does not end on its own; "
|
|
3982
|
+
"usage recording continues. If your usage was reset in place meanwhile, "
|
|
3983
|
+
"record it with `cctally record-credit`.")
|
|
3984
|
+
#: Spec revision 5 (section 8), operator-owned wording, verbatim: an account
|
|
3985
|
+
#: paused by a marker an incomplete legacy spool inventory left.
|
|
3986
|
+
OUTCOME_QUARANTINE_INVENTORY_MARKER_TEXT = (
|
|
3987
|
+
"Automatic credit, reset and status-line drop detection is paused for "
|
|
3988
|
+
"{account} since {time} because cctally could not read the status-line "
|
|
3989
|
+
"files an older cctally release used, so it cannot rule out that an "
|
|
3990
|
+
"older process changed them. This version of cctally cannot yet confirm "
|
|
3991
|
+
"a safe point to resume, so the pause does not end on its own; usage "
|
|
3992
|
+
"recording continues. If your usage was reset in place meanwhile, record "
|
|
3993
|
+
"it with `cctally record-credit`.")
|
|
3994
|
+
#: Operator decision dec-5d2a9c2b004a (escalation esc-8a386c207966): the
|
|
3995
|
+
#: standalone sentence for an account missing only its control certificate.
|
|
3996
|
+
#: The approved control sentence above opens with "also", so it reads only as
|
|
3997
|
+
#: an addendum to the accounting one.
|
|
3998
|
+
OUTCOME_QUARANTINE_CONTROL_ONLY_TEXT = (
|
|
3999
|
+
"Status-line drop detection is paused for {account} since {time}. It "
|
|
4000
|
+
"resumes after a successful usage refresh for this account.")
|
|
4001
|
+
|
|
4002
|
+
|
|
4003
|
+
def _outcome_quarantine_message(row: dict) -> str:
|
|
4004
|
+
"""One account's paused outcome classes, in the approved wording. A
|
|
4005
|
+
missing control certificate appends its sentence to the accounting one;
|
|
4006
|
+
alone, it gets its own operator-approved standalone sentence."""
|
|
4007
|
+
label = row.get("accountLabel") or row.get("account")
|
|
4008
|
+
if row.get("unresolvedSpoolMarker"):
|
|
4009
|
+
text = (OUTCOME_QUARANTINE_INVENTORY_MARKER_TEXT
|
|
4010
|
+
if row.get("spoolMarkerCause") == "inventory-incomplete"
|
|
4011
|
+
else OUTCOME_QUARANTINE_SPOOL_MARKER_TEXT)
|
|
4012
|
+
return text.format(
|
|
4013
|
+
account=label, time=row.get("accountingSince") or "an unknown time")
|
|
4014
|
+
if row.get("accountingPaused"):
|
|
4015
|
+
text = OUTCOME_QUARANTINE_ACCOUNTING_TEXT.format(
|
|
4016
|
+
account=label, time=row.get("accountingSince") or "an unknown time")
|
|
4017
|
+
if row.get("controlPaused"):
|
|
4018
|
+
text += " " + OUTCOME_QUARANTINE_CONTROL_TEXT
|
|
4019
|
+
return text
|
|
4020
|
+
return OUTCOME_QUARANTINE_CONTROL_ONLY_TEXT.format(
|
|
4021
|
+
account=label, time=row.get("controlSince") or "an unknown time")
|
|
4022
|
+
|
|
4023
|
+
|
|
4024
|
+
def _check_accounting_outcome_quarantine(s: DoctorState) -> CheckResult:
|
|
4025
|
+
"""#825/#832 (spec section 8, F16, operator Q9): after a post-cutover
|
|
4026
|
+
legacy write or legacy spool change, automatic outcomes are withheld per
|
|
4027
|
+
account until certified recovery. OK when no account is quarantined; WARN
|
|
4028
|
+
naming per account what is paused and since when; unavailable — WARN,
|
|
4029
|
+
never OK — on an unavailable or unvalidated index. Separate from
|
|
4030
|
+
`journal.legacy_writer`: seven quiet days do not mean outcomes resumed.
|
|
4031
|
+
Never claims when recovery will happen. While a spool marker is
|
|
4032
|
+
unresolved (spec revision 3) every row uses the marker wording and the
|
|
4033
|
+
details carry the markers."""
|
|
4034
|
+
data = s.outcome_quarantine
|
|
4035
|
+
title = "Outcome quarantine"
|
|
4036
|
+
if not data or not data.get("available"):
|
|
4037
|
+
return CheckResult(
|
|
4038
|
+
id="accounting.outcome_quarantine", title=title, severity="warn",
|
|
4039
|
+
summary="unavailable (no validated stats index)",
|
|
4040
|
+
remediation=("Run any cctally command to bring the stats index "
|
|
4041
|
+
"up to date, then run `cctally doctor` again."),
|
|
4042
|
+
details={"available": False, "accounts": []},
|
|
4043
|
+
)
|
|
4044
|
+
rows = []
|
|
4045
|
+
for row in data.get("accounts") or []:
|
|
4046
|
+
rows.append({**row, "message": _outcome_quarantine_message(row)})
|
|
4047
|
+
details = {"available": True, "triggered": bool(data.get("triggered")),
|
|
4048
|
+
"accounts": rows}
|
|
4049
|
+
if data.get("unresolvedSpoolMarkers"):
|
|
4050
|
+
details["unresolvedSpoolMarkers"] = list(data["unresolvedSpoolMarkers"])
|
|
4051
|
+
if not rows:
|
|
4052
|
+
return CheckResult(
|
|
4053
|
+
id="accounting.outcome_quarantine", title=title, severity="ok",
|
|
4054
|
+
summary=("no account is quarantined" if data.get("triggered")
|
|
4055
|
+
else "no legacy write or spool change since the upgrade"),
|
|
4056
|
+
remediation=None, details=details,
|
|
4057
|
+
)
|
|
4058
|
+
return CheckResult(
|
|
4059
|
+
id="accounting.outcome_quarantine", title=title, severity="warn",
|
|
4060
|
+
summary=(f"automatic outcomes paused for {len(rows)} account(s)"),
|
|
4061
|
+
remediation=" ".join(row["message"] for row in rows),
|
|
4062
|
+
details=details,
|
|
4063
|
+
)
|
|
4064
|
+
|
|
4065
|
+
|
|
3588
4066
|
def _check_journal_protocol(s: DoctorState) -> CheckResult:
|
|
3589
4067
|
"""Report selector failure or whole-batch tainting without false health."""
|
|
3590
4068
|
if (
|
|
@@ -4086,6 +4564,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
4086
4564
|
("journal.writer_guard", "_check_journal_writer_guard"),
|
|
4087
4565
|
("journal.conflicts", "_check_journal_conflicts"),
|
|
4088
4566
|
("journal.protocol", "_check_journal_protocol"),
|
|
4567
|
+
("journal.legacy_writer", "_check_journal_legacy_writer"),
|
|
4089
4568
|
("journal.quota_projection", "_check_journal_quota_projection"),
|
|
4090
4569
|
)),
|
|
4091
4570
|
("data", "Data", (
|
|
@@ -4099,6 +4578,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
4099
4578
|
# after the pair rather than between them.
|
|
4100
4579
|
("data.codex_project_metadata", "_check_data_codex_project_metadata"),
|
|
4101
4580
|
("data.codex_prune_safety", "_check_data_codex_prune_safety"),
|
|
4581
|
+
("data.codex_compressed_rollouts", "_check_data_codex_compressed_rollouts"),
|
|
4102
4582
|
("data.codex_replay", "_check_data_codex_replay"),
|
|
4103
4583
|
("data.codex_ingest_backlog", "_check_data_codex_ingest_backlog"),
|
|
4104
4584
|
("data.codex_quota", "_check_data_codex_quota"),
|
|
@@ -4120,6 +4600,12 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
4120
4600
|
("accounts.codex_window_attribution",
|
|
4121
4601
|
"_check_accounts_codex_window_attribution"),
|
|
4122
4602
|
)),
|
|
4603
|
+
# #825/#832 spec section 8: reported separately from the seven-day
|
|
4604
|
+
# `journal.legacy_writer` recency, because quiet time never decides
|
|
4605
|
+
# recovery.
|
|
4606
|
+
("accounting", "Accounting", (
|
|
4607
|
+
("accounting.outcome_quarantine", "_check_accounting_outcome_quarantine"),
|
|
4608
|
+
)),
|
|
4123
4609
|
("pricing", "Pricing", (
|
|
4124
4610
|
("pricing.coverage", "_check_pricing_coverage"),
|
|
4125
4611
|
# #705. Cannot FAIL, so `doctor`'s exit code is unaffected — the same
|
|
@@ -4149,6 +4635,12 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
4149
4635
|
("telemetry", "Telemetry", (
|
|
4150
4636
|
("telemetry.state", "_check_telemetry"),
|
|
4151
4637
|
)),
|
|
4638
|
+
# #901 §5.6. Never FAILs, so `doctor`'s exit code is unaffected. Last, so
|
|
4639
|
+
# every existing check keeps its order.
|
|
4640
|
+
("performance", "Performance", (
|
|
4641
|
+
("performance.dashboard_disk_writes",
|
|
4642
|
+
"_check_performance_dashboard_disk_writes"),
|
|
4643
|
+
)),
|
|
4152
4644
|
)
|
|
4153
4645
|
|
|
4154
4646
|
|
|
@@ -4322,6 +4814,159 @@ def _check_quota_meter_drift(s: DoctorState) -> CheckResult:
|
|
|
4322
4814
|
)
|
|
4323
4815
|
|
|
4324
4816
|
|
|
4817
|
+
_DISK_WRITES_ID = "performance.dashboard_disk_writes"
|
|
4818
|
+
_DISK_WRITES_TITLE = "Dashboard disk writes"
|
|
4819
|
+
_DISK_WRITES_REMEDIATION = (
|
|
4820
|
+
"Run `cctally dashboard-perf` for the breakdown, and report it if it "
|
|
4821
|
+
"persists."
|
|
4822
|
+
)
|
|
4823
|
+
|
|
4824
|
+
|
|
4825
|
+
def _words(reason) -> str:
|
|
4826
|
+
return str(reason or "unavailable").replace("_", " ")
|
|
4827
|
+
|
|
4828
|
+
|
|
4829
|
+
#: PR-9: probe failures doctor cannot overcome from here (it sends no token
|
|
4830
|
+
#: and probes over loopback only), each with its sentence fragment.
|
|
4831
|
+
_NOT_MEASURABLE_FROM_HERE = {
|
|
4832
|
+
"authentication_required": "needs its access token",
|
|
4833
|
+
"not_loopback_reachable": "is not reachable over loopback",
|
|
4834
|
+
}
|
|
4835
|
+
|
|
4836
|
+
|
|
4837
|
+
def _check_performance_dashboard_disk_writes(s: DoctorState) -> CheckResult:
|
|
4838
|
+
"""Running dashboards' disk writes against the write budget, and
|
|
4839
|
+
transcript maintenance against its allowance (#901 §5.6, §5.7).
|
|
4840
|
+
|
|
4841
|
+
WARN when a measured running dashboard is over budget, when a live
|
|
4842
|
+
instance cannot supply valid telemetry, or when the durable maintenance
|
|
4843
|
+
ledger is over its allowance (so that leg needs no running dashboard).
|
|
4844
|
+
OK otherwise: none running, within budget, a statistic that has no
|
|
4845
|
+
verdict yet (named by its reason), a dashboard doctor cannot measure from
|
|
4846
|
+
here (a token-mode or non-loopback bind), or a platform with no counter.
|
|
4847
|
+
Never FAIL."""
|
|
4848
|
+
import _lib_write_budget as wb
|
|
4849
|
+
|
|
4850
|
+
gathered = (s.dashboard_disk_writes
|
|
4851
|
+
if isinstance(s.dashboard_disk_writes, dict)
|
|
4852
|
+
else {"mode": "error", "instances": []})
|
|
4853
|
+
record = (s.conversations_reclaim_pending
|
|
4854
|
+
if isinstance(s.conversations_reclaim_pending, dict) else {})
|
|
4855
|
+
maintenance = wb.maintenance_statistic(record.get("ledger"), s.now_utc)
|
|
4856
|
+
rows, over, cannot, within, warming, platform = [], [], [], [], [], []
|
|
4857
|
+
unmeasured = []
|
|
4858
|
+
for entry in gathered.get("instances") or ():
|
|
4859
|
+
if not isinstance(entry, dict):
|
|
4860
|
+
continue
|
|
4861
|
+
port = entry.get("port")
|
|
4862
|
+
write_io = entry.get("write_io") if entry.get("probe") == "ok" else None
|
|
4863
|
+
budget = (write_io or {}).get("budget") or {}
|
|
4864
|
+
if write_io is None:
|
|
4865
|
+
verdict, reason = "unavailable", entry.get("reason") or "unreachable"
|
|
4866
|
+
if reason in _NOT_MEASURABLE_FROM_HERE:
|
|
4867
|
+
# PR-9: doctor sends no token and probes over loopback only,
|
|
4868
|
+
# so these never answer it: a note, not a warning.
|
|
4869
|
+
unmeasured.append((port, reason))
|
|
4870
|
+
else:
|
|
4871
|
+
cannot.append((port, reason))
|
|
4872
|
+
elif write_io.get("status") != "ok":
|
|
4873
|
+
verdict = "unavailable"
|
|
4874
|
+
reason = write_io.get("reason") or "counter_error"
|
|
4875
|
+
if reason == "unsupported_platform":
|
|
4876
|
+
platform.append((port, reason))
|
|
4877
|
+
else:
|
|
4878
|
+
cannot.append((port, reason))
|
|
4879
|
+
else:
|
|
4880
|
+
verdict, reason = budget.get("verdict"), None
|
|
4881
|
+
if verdict == "over":
|
|
4882
|
+
over.append((port, write_io, budget))
|
|
4883
|
+
elif verdict == "ok":
|
|
4884
|
+
within.append((port, write_io))
|
|
4885
|
+
else:
|
|
4886
|
+
verdict = "insufficient"
|
|
4887
|
+
reason = (budget.get("reasons") or ["warming_up"])[0]
|
|
4888
|
+
warming.append((port, reason))
|
|
4889
|
+
rows.append({
|
|
4890
|
+
"instance_id": entry.get("instance_id"),
|
|
4891
|
+
"port": port,
|
|
4892
|
+
"pid": entry.get("pid"),
|
|
4893
|
+
"verdict": verdict,
|
|
4894
|
+
"reason": reason,
|
|
4895
|
+
"bytes_per_minute": (write_io or {}).get("bytesPerMinute"),
|
|
4896
|
+
"mean_bytes_per_publication": (write_io or {}).get(
|
|
4897
|
+
"meanBytesPerTick"),
|
|
4898
|
+
"bytes_per_minute_limit": budget.get("bytesPerMinuteLimit"),
|
|
4899
|
+
"mean_bytes_per_publication_limit": budget.get(
|
|
4900
|
+
"meanBytesPerTickLimit"),
|
|
4901
|
+
})
|
|
4902
|
+
rows.sort(key=lambda row: (row["port"] is None, row["port"] or 0))
|
|
4903
|
+
details = {
|
|
4904
|
+
"policy_version": wb.POLICY_VERSION,
|
|
4905
|
+
"mode": gathered.get("mode"),
|
|
4906
|
+
"instances": rows,
|
|
4907
|
+
"maintenance": maintenance.as_details(),
|
|
4908
|
+
}
|
|
4909
|
+
warn = []
|
|
4910
|
+
for port, write_io, budget in sorted(over, key=lambda item: item[0] or 0):
|
|
4911
|
+
reasons = budget.get("reasons") or []
|
|
4912
|
+
if ("publication_over_limit" in reasons
|
|
4913
|
+
and "rate_over_limit" not in reasons):
|
|
4914
|
+
warn.append(
|
|
4915
|
+
f"The running dashboard (port {port}) is writing "
|
|
4916
|
+
f"{wb.format_mib(write_io.get('meanBytesPerTick') or 0)} per "
|
|
4917
|
+
f"publication, over its "
|
|
4918
|
+
f"{wb.format_mib(budget.get('meanBytesPerTickLimit') or 0)} "
|
|
4919
|
+
f"per publication budget.")
|
|
4920
|
+
else:
|
|
4921
|
+
warn.append(
|
|
4922
|
+
f"The running dashboard (port {port}) is writing "
|
|
4923
|
+
f"{wb.format_mib(write_io.get('bytesPerMinute') or 0)}/min, "
|
|
4924
|
+
f"over its "
|
|
4925
|
+
f"{wb.format_mib(budget.get('bytesPerMinuteLimit') or 0)}/min "
|
|
4926
|
+
f"budget.")
|
|
4927
|
+
for port, reason in sorted(cannot, key=lambda item: item[0] or 0):
|
|
4928
|
+
warn.append(f"The running dashboard (port {port}) cannot report its "
|
|
4929
|
+
f"disk writes ({_words(reason)}).")
|
|
4930
|
+
if maintenance.verdict == "over":
|
|
4931
|
+
warn.append(
|
|
4932
|
+
f"Transcript maintenance was charged "
|
|
4933
|
+
f"{wb.format_size(maintenance.charged_bytes)} over the last "
|
|
4934
|
+
f"{wb.format_window_hm(maintenance.window_minutes)}, over its "
|
|
4935
|
+
f"{wb.format_size(maintenance.allowance_bytes)} allowance.")
|
|
4936
|
+
if warn:
|
|
4937
|
+
return CheckResult(
|
|
4938
|
+
id=_DISK_WRITES_ID, title=_DISK_WRITES_TITLE, severity="warn",
|
|
4939
|
+
summary=" ".join(warn), remediation=_DISK_WRITES_REMEDIATION,
|
|
4940
|
+
details=details)
|
|
4941
|
+
if within:
|
|
4942
|
+
port, write_io = sorted(within, key=lambda item: item[0] or 0)[0]
|
|
4943
|
+
summary = (f"Within budget: "
|
|
4944
|
+
f"{wb.format_mib(write_io.get('bytesPerMinute') or 0)}/min "
|
|
4945
|
+
f"over the last 5 minutes.")
|
|
4946
|
+
elif warming:
|
|
4947
|
+
# PR-8: the first instance's own reason; only `warming_up` is
|
|
4948
|
+
# "no samples yet".
|
|
4949
|
+
_port, reason = sorted(warming, key=lambda item: item[0] or 0)[0]
|
|
4950
|
+
summary = f"Disk writes: {wb.statistic_reason_phrase(reason)}."
|
|
4951
|
+
elif unmeasured:
|
|
4952
|
+
port, reason = sorted(unmeasured, key=lambda item: item[0] or 0)[0]
|
|
4953
|
+
summary = (f"Disk writes: the dashboard on port {port} "
|
|
4954
|
+
f"{_NOT_MEASURABLE_FROM_HERE[reason]}, so it is not "
|
|
4955
|
+
"measured from here.")
|
|
4956
|
+
elif platform:
|
|
4957
|
+
summary = f"Disk writes: unavailable ({_words(platform[0][1])})."
|
|
4958
|
+
elif gathered.get("mode") == "error":
|
|
4959
|
+
summary = "Disk writes: unavailable (gather failed)."
|
|
4960
|
+
elif gathered.get("mode") == "not_probed":
|
|
4961
|
+
summary = ("Disk writes: not measured here; run `cctally doctor` to "
|
|
4962
|
+
"check running dashboards.")
|
|
4963
|
+
else:
|
|
4964
|
+
summary = "No dashboard is running."
|
|
4965
|
+
return CheckResult(
|
|
4966
|
+
id=_DISK_WRITES_ID, title=_DISK_WRITES_TITLE, severity="ok",
|
|
4967
|
+
summary=summary, remediation=None, details=details)
|
|
4968
|
+
|
|
4969
|
+
|
|
4325
4970
|
def _evaluate_one(check_id: str, check_fn_name: str,
|
|
4326
4971
|
state: DoctorState) -> CheckResult:
|
|
4327
4972
|
"""Invoke a single check evaluator by name, catching any exception so
|