cctally 1.114.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +4394 -279
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +178 -2
- package/bin/_cctally_dashboard.py +2 -2
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_sources.py +18 -9
- package/bin/_cctally_db.py +316 -5
- package/bin/_cctally_doctor.py +436 -27
- package/bin/_cctally_five_hour.py +82 -31
- package/bin/_cctally_forecast.py +81 -9
- package/bin/_cctally_journal.py +3884 -112
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +26 -4
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +677 -149
- package/bin/_cctally_rederive.py +297 -32
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +23 -2
- package/bin/_cctally_statusline.py +6137 -517
- package/bin/_cctally_store.py +287 -48
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +57 -53
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation_query.py +133 -19
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +11 -3
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +452 -1
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +29 -302
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_pricing.py +7 -2
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_rederive.py +94 -5
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/cctally +23 -1
- package/package.json +14 -1
package/bin/_lib_doctor.py
CHANGED
|
@@ -232,6 +232,10 @@ class DoctorState:
|
|
|
232
232
|
# evidence. None is the normal state; each record contains counts/reasons
|
|
233
233
|
# but never configured paths or provider identifiers.
|
|
234
234
|
codex_prune_refusals: Optional[list[dict]] = None
|
|
235
|
+
codex_compressed_census: Optional[dict] = None
|
|
236
|
+
codex_decoder_capability: Optional[str] = None
|
|
237
|
+
codex_container_status: Optional[dict] = None
|
|
238
|
+
codex_maintenance_debt: Optional[dict] = None
|
|
235
239
|
# #705: the conversations store's refused-pricing-write record, read
|
|
236
240
|
# verbatim from cache_meta. None means no refusal is recorded. A record
|
|
237
241
|
# that exists but cannot be parsed arrives as {"__malformed__": True}
|
|
@@ -343,6 +347,18 @@ class DoctorState:
|
|
|
343
347
|
# retained corpus from unreadable metadata.
|
|
344
348
|
codex_project_metadata_health: Optional[dict] = None
|
|
345
349
|
codex_project_metadata_error: Optional[str] = None
|
|
350
|
+
# #825/#832 (spec section 8, F9): the account-bound legacy-writer evidence.
|
|
351
|
+
# `{"available": bool, "cutoverId", "excludedCount", "coordinateDigest",
|
|
352
|
+
# "evidence": <sidecar document or None>}` from the validated admission
|
|
353
|
+
# summary (a deep run substitutes its own whole-journal recomputation).
|
|
354
|
+
# None / available=False is reported as unavailable, never as zero.
|
|
355
|
+
legacy_writer: Optional[dict] = None
|
|
356
|
+
# #825/#832 (spec section 8, F16): the outcome quarantine, read from the
|
|
357
|
+
# validated index. `{"available": bool, "triggered": bool, "accounts":
|
|
358
|
+
# [{"account", "accountLabel", "accountingPaused", "controlPaused",
|
|
359
|
+
# "accountingSince", "controlSince"}]}`. None / available=False is
|
|
360
|
+
# reported as unavailable, never as OK.
|
|
361
|
+
outcome_quarantine: Optional[dict] = None
|
|
346
362
|
# Beta-channel (spec 2026-07-21 §3): the configured RELEASE channel
|
|
347
363
|
# (stable|beta) that `cctally update` tracks — DISTINCT from the preview
|
|
348
364
|
# `channel` (prod|preview) above. Populated by doctor_gather_state from a
|
|
@@ -391,6 +407,11 @@ class DoctorState:
|
|
|
391
407
|
journal_cursor_lag_bytes: Optional[int] = None
|
|
392
408
|
journal_hw_segment: Optional[str] = None
|
|
393
409
|
journal_cursor_segment: Optional[str] = None
|
|
410
|
+
# * journal_cursor_invalid (#946) — the applied cursor names no safe
|
|
411
|
+
# endpoint: past its segment's size, or inside a line. The lag beside
|
|
412
|
+
# it is then not a measure of anything consumed, so it is never
|
|
413
|
+
# reported as caught up.
|
|
414
|
+
journal_cursor_invalid: bool = False
|
|
394
415
|
journal_heal_incidents: Optional[list] = None
|
|
395
416
|
journal_heal_detections: Optional[list] = None
|
|
396
417
|
# * retained_artifacts (#496 S6 §7.3) — the read-only retention scan
|
|
@@ -423,6 +444,15 @@ class DoctorState:
|
|
|
423
444
|
journal_protocol_violations: Optional[list] = None
|
|
424
445
|
journal_protocol_acknowledged: Optional[list] = None
|
|
425
446
|
journal_protocol_error: Optional[str] = None
|
|
447
|
+
# * journal_listing_error (#825/#832 fix round K11, K9K10-R6) — why the
|
|
448
|
+
# segment listing failed. An unlistable journal is never an empty one.
|
|
449
|
+
# * journal_scan_error (K9K10-R6/R11) — why the deep scan could not
|
|
450
|
+
# complete: an unreadable segment, a raising admission-boundary lookup
|
|
451
|
+
# or an unreadable excluded population. The scan's checks are then
|
|
452
|
+
# unavailable — never zero excluded records, never an unfiltered
|
|
453
|
+
# conflict scan.
|
|
454
|
+
journal_listing_error: Optional[str] = None
|
|
455
|
+
journal_scan_error: Optional[str] = None
|
|
426
456
|
# Multi-account attribution (#341). A dict (None when unavailable) with:
|
|
427
457
|
# * claude_identity_status — "identified" | "stably_absent" | "torn"
|
|
428
458
|
# * claude_email — the active Claude email (or None)
|
|
@@ -1019,7 +1049,7 @@ def _check_db_version_ahead(s: DoctorState) -> CheckResult:
|
|
|
1019
1049
|
# current index as a mismatch; keep this in lockstep with core
|
|
1020
1050
|
# (#496 S5b §6.1). It stays a literal because this kernel is pure and
|
|
1021
1051
|
# must not import `_cctally_core`.
|
|
1022
|
-
epoch =
|
|
1052
|
+
epoch = 1018
|
|
1023
1053
|
mismatch = uv > legacy_head and uv != epoch
|
|
1024
1054
|
return {"user_version": uv, "legacy_head": legacy_head, "epoch": epoch,
|
|
1025
1055
|
"mismatch": mismatch}
|
|
@@ -1354,6 +1384,95 @@ def _check_data_codex_replay(s: DoctorState) -> CheckResult:
|
|
|
1354
1384
|
)
|
|
1355
1385
|
|
|
1356
1386
|
|
|
1387
|
+
def _check_data_codex_compressed_rollouts(s: DoctorState) -> CheckResult:
|
|
1388
|
+
"""#968 5.6 with operator clarifications D2, D3 and D6.
|
|
1389
|
+
|
|
1390
|
+
`warn` when any of three conditions holds: compressed rollouts exist and
|
|
1391
|
+
no decoder is available; a verification ended `unverified` (D3, at once,
|
|
1392
|
+
never debt); verification, reading or deletion work has been pending for
|
|
1393
|
+
more than 24 hours (D2, D6). The summary states each true condition's
|
|
1394
|
+
sentence once, joined by a single space, in that order. Otherwise pending
|
|
1395
|
+
work is `ok` "continuing", and the all-clear sentence is used only when
|
|
1396
|
+
no compressed rollout is unread, pending or unverified. Never FAIL: a
|
|
1397
|
+
missing optional decoder is an environment limitation.
|
|
1398
|
+
"""
|
|
1399
|
+
if s.codex_compressed_census is None:
|
|
1400
|
+
return CheckResult(
|
|
1401
|
+
id="data.codex_compressed_rollouts", title="Compressed Codex rollouts",
|
|
1402
|
+
severity="warn", summary="unavailable (compressed Codex rollout census)",
|
|
1403
|
+
remediation=None, details={"available": False})
|
|
1404
|
+
census = s.codex_compressed_census
|
|
1405
|
+
compressed = int(census.get("compressedOnly", 0) or 0)
|
|
1406
|
+
never = int(census.get("neverIngested", 0) or 0)
|
|
1407
|
+
incomplete = int(census.get("retainedIncomplete", 0) or 0)
|
|
1408
|
+
unverified = int(census.get("unverified", 0) or 0)
|
|
1409
|
+
debt = s.codex_maintenance_debt or {}
|
|
1410
|
+
maintenance_partial = (s.codex_maintenance_debt is None
|
|
1411
|
+
or debt.get("coverage") == "partial")
|
|
1412
|
+
verifications = int(debt.get("verifications", 0) or 0)
|
|
1413
|
+
deletions = int(debt.get("deletions", 0) or 0)
|
|
1414
|
+
no_decoder = s.codex_decoder_capability == "unavailable"
|
|
1415
|
+
# D6: with a decoder, an unread or partly read compressed rollout is
|
|
1416
|
+
# pending reading work; without one it is the no-decoder condition.
|
|
1417
|
+
reads = 0 if no_decoder else never + incomplete
|
|
1418
|
+
details = {
|
|
1419
|
+
"compressedOnly": compressed,
|
|
1420
|
+
"retainedComplete": int(census.get("retainedComplete", 0) or 0),
|
|
1421
|
+
"retainedIncomplete": incomplete,
|
|
1422
|
+
"neverIngested": never,
|
|
1423
|
+
"unverified": unverified,
|
|
1424
|
+
"verificationDebt": verifications,
|
|
1425
|
+
"readDebt": reads,
|
|
1426
|
+
"pruneDebt": deletions,
|
|
1427
|
+
}
|
|
1428
|
+
if maintenance_partial:
|
|
1429
|
+
details["maintenanceCoverage"] = "partial"
|
|
1430
|
+
if s.codex_maintenance_debt is None:
|
|
1431
|
+
details["verificationDebt"] = None
|
|
1432
|
+
details["pruneDebt"] = None
|
|
1433
|
+
status = s.codex_container_status or {}
|
|
1434
|
+
if status.get("census") == "partial":
|
|
1435
|
+
details["census"] = "partial"
|
|
1436
|
+
details["observedAt"] = status.get("observedAt")
|
|
1437
|
+
if compressed:
|
|
1438
|
+
details["decoderAvailable"] = not no_decoder
|
|
1439
|
+
pending = bool(verifications or reads or deletions)
|
|
1440
|
+
sentences = []
|
|
1441
|
+
if compressed and no_decoder:
|
|
1442
|
+
sentences.append(
|
|
1443
|
+
f"{compressed} compressed Codex rollouts can't be read on this "
|
|
1444
|
+
f"Python ({never} never ingested). Cached history was kept. "
|
|
1445
|
+
"Install `backports.zstd` or use Python 3.14+.")
|
|
1446
|
+
if unverified:
|
|
1447
|
+
sentences.append(
|
|
1448
|
+
f"{unverified} compressed Codex rollouts kept their cached history "
|
|
1449
|
+
"but couldn't be verified against it.")
|
|
1450
|
+
if pending and debt.get("olderThan24h"):
|
|
1451
|
+
sentences.append(
|
|
1452
|
+
"Codex history maintenance has been pending for more than 24 hours "
|
|
1453
|
+
f"({verifications} verifications, {reads} reads, {deletions} "
|
|
1454
|
+
"deletions).")
|
|
1455
|
+
if sentences:
|
|
1456
|
+
severity, summary = "warn", " ".join(sentences)
|
|
1457
|
+
elif pending:
|
|
1458
|
+
severity = "ok"
|
|
1459
|
+
summary = "Codex history maintenance is continuing in the background."
|
|
1460
|
+
elif maintenance_partial:
|
|
1461
|
+
severity = "warn"
|
|
1462
|
+
summary = "unavailable (compressed Codex rollout census)"
|
|
1463
|
+
else:
|
|
1464
|
+
severity = "ok"
|
|
1465
|
+
summary = "All compressed Codex rollouts are readable and verified."
|
|
1466
|
+
return CheckResult(
|
|
1467
|
+
id="data.codex_compressed_rollouts",
|
|
1468
|
+
title="Compressed Codex rollouts",
|
|
1469
|
+
severity=severity,
|
|
1470
|
+
summary=summary,
|
|
1471
|
+
remediation=None,
|
|
1472
|
+
details=details,
|
|
1473
|
+
)
|
|
1474
|
+
|
|
1475
|
+
|
|
1357
1476
|
def _check_data_codex_prune_safety(s: DoctorState) -> CheckResult:
|
|
1358
1477
|
"""Surface a fail-closed orphan-prune decision instead of silent loss."""
|
|
1359
1478
|
records = [
|
|
@@ -3273,6 +3392,28 @@ _JOURNAL_CURSOR_LAG_WARN_BYTES = 4 * 1024 * 1024
|
|
|
3273
3392
|
_JOURNAL_HEAL_RECENT_SECONDS = 7 * 24 * 3600
|
|
3274
3393
|
|
|
3275
3394
|
|
|
3395
|
+
def _journal_unreadable_remediation(s: DoctorState) -> str:
|
|
3396
|
+
"""K9K10-R6: the remediation for a journal doctor could not read. Fix
|
|
3397
|
+
round K12 (K11R-12): it names the resolved data directory (a dev
|
|
3398
|
+
checkout or `CCTALLY_DATA_DIR` is not the default path)."""
|
|
3399
|
+
journal = f"{str(s.app_dir or '').rstrip('/')}/journal/"
|
|
3400
|
+
return (f"Check the permissions of {journal} and the disk, then run "
|
|
3401
|
+
"`cctally doctor --deep` again")
|
|
3402
|
+
|
|
3403
|
+
|
|
3404
|
+
def _journal_scan_unavailable(check_id: str, title: str, s: DoctorState,
|
|
3405
|
+
details: dict) -> CheckResult:
|
|
3406
|
+
"""A deep-scan check the scan could not complete (K9K10-R6): WARN,
|
|
3407
|
+
unavailable, with the cause."""
|
|
3408
|
+
return CheckResult(
|
|
3409
|
+
id=check_id, title=title, severity="warn",
|
|
3410
|
+
summary=f"unavailable ({s.journal_scan_error})",
|
|
3411
|
+
remediation=_journal_unreadable_remediation(s),
|
|
3412
|
+
details={**details, "scanned": True, "available": False,
|
|
3413
|
+
"error": s.journal_scan_error},
|
|
3414
|
+
)
|
|
3415
|
+
|
|
3416
|
+
|
|
3276
3417
|
def _check_journal_presence(s: DoctorState) -> CheckResult:
|
|
3277
3418
|
"""The journal directory exists and is appendable. A pre-cutover (legacy)
|
|
3278
3419
|
install has NO journal yet — that is INFO (OK), never a FAIL."""
|
|
@@ -3284,6 +3425,15 @@ def _check_journal_presence(s: DoctorState) -> CheckResult:
|
|
|
3284
3425
|
)
|
|
3285
3426
|
details = {"present": True, "appendable": s.journal_appendable,
|
|
3286
3427
|
"segments": s.journal_segment_count}
|
|
3428
|
+
if s.journal_listing_error:
|
|
3429
|
+
return CheckResult(
|
|
3430
|
+
id="journal.presence", title="Journal", severity="warn",
|
|
3431
|
+
summary=(f"segment listing unavailable "
|
|
3432
|
+
f"({s.journal_listing_error})"),
|
|
3433
|
+
remediation=_journal_unreadable_remediation(s),
|
|
3434
|
+
details={**details, "segments": None,
|
|
3435
|
+
"listingError": s.journal_listing_error},
|
|
3436
|
+
)
|
|
3287
3437
|
if s.journal_appendable is False:
|
|
3288
3438
|
return CheckResult(
|
|
3289
3439
|
id="journal.presence", title="Journal", severity="warn",
|
|
@@ -3307,6 +3457,9 @@ def _check_journal_integrity(s: DoctorState) -> CheckResult:
|
|
|
3307
3457
|
id="journal.integrity", title="Journal integrity", severity="ok",
|
|
3308
3458
|
summary="no journal", remediation=None, details={"present": False},
|
|
3309
3459
|
)
|
|
3460
|
+
if s.journal_scan_error:
|
|
3461
|
+
return _journal_scan_unavailable(
|
|
3462
|
+
"journal.integrity", "Journal integrity", s, {})
|
|
3310
3463
|
if s.journal_malformed_count is None:
|
|
3311
3464
|
return CheckResult(
|
|
3312
3465
|
id="journal.integrity", title="Journal integrity", severity="ok",
|
|
@@ -3348,6 +3501,16 @@ def _check_journal_index_freshness(s: DoctorState) -> CheckResult:
|
|
|
3348
3501
|
details = {"lag_bytes": lag, "hw_segment": s.journal_hw_segment,
|
|
3349
3502
|
"cursor_segment": s.journal_cursor_segment,
|
|
3350
3503
|
"warn_bytes": _JOURNAL_CURSOR_LAG_WARN_BYTES}
|
|
3504
|
+
if s.journal_cursor_invalid:
|
|
3505
|
+
# #946: lag is computed from the safe applied cursor, and this one is
|
|
3506
|
+
# not safe, so no amount of lag here means "caught up".
|
|
3507
|
+
details["cursor_invalid"] = True
|
|
3508
|
+
return CheckResult(
|
|
3509
|
+
id="journal.index_freshness", title="Journal index", severity="warn",
|
|
3510
|
+
summary="index cursor is not at a journal line boundary",
|
|
3511
|
+
remediation="Run `cctally db rebuild --db stats`",
|
|
3512
|
+
details=details,
|
|
3513
|
+
)
|
|
3351
3514
|
if lag == 0:
|
|
3352
3515
|
return CheckResult(
|
|
3353
3516
|
id="journal.index_freshness", title="Journal index", severity="ok",
|
|
@@ -3582,6 +3745,9 @@ def _check_journal_conflicts(s: DoctorState) -> CheckResult:
|
|
|
3582
3745
|
remediation=None,
|
|
3583
3746
|
details={"scanned": True, "available": False, "conflicts": None},
|
|
3584
3747
|
)
|
|
3748
|
+
if s.journal_scan_error:
|
|
3749
|
+
return _journal_scan_unavailable(
|
|
3750
|
+
"journal.conflicts", "Journal conflicts", s, {"conflicts": None})
|
|
3585
3751
|
if s.journal_conflicts is None:
|
|
3586
3752
|
return CheckResult(
|
|
3587
3753
|
id="journal.conflicts", title="Journal conflicts", severity="ok",
|
|
@@ -3620,6 +3786,283 @@ def _check_journal_conflicts(s: DoctorState) -> CheckResult:
|
|
|
3620
3786
|
)
|
|
3621
3787
|
|
|
3622
3788
|
|
|
3789
|
+
def _check_journal_legacy_writer(s: DoctorState) -> CheckResult:
|
|
3790
|
+
"""#825/#832 (spec section 8, F9, operator Q4): records an older cctally
|
|
3791
|
+
process wrote after the account-bound cutover are excluded from current
|
|
3792
|
+
accounting. WARN while the latest excluded population was first observed
|
|
3793
|
+
under seven days ago; OK with a historical note after seven quiet days.
|
|
3794
|
+
Never asserts that an old writer is alive, and never reports a missing
|
|
3795
|
+
summary as zero. An incomplete legacy spool inventory is also a WARN,
|
|
3796
|
+
independently of the excluded-record count (spec revision 5, section
|
|
3797
|
+
8; `_with_legacy_spool_inventory`)."""
|
|
3798
|
+
result = _legacy_writer_count_result(s)
|
|
3799
|
+
inventory = (s.legacy_writer or {}).get("legacySpoolInventory")
|
|
3800
|
+
if not inventory or inventory.get("available", False):
|
|
3801
|
+
return result
|
|
3802
|
+
return _with_legacy_spool_inventory(result, inventory)
|
|
3803
|
+
|
|
3804
|
+
|
|
3805
|
+
def _with_legacy_spool_inventory(result: CheckResult,
|
|
3806
|
+
inventory: dict) -> CheckResult:
|
|
3807
|
+
"""Spec revision 5 (section 8): an incomplete legacy spool inventory is a
|
|
3808
|
+
WARN independently of the excluded-record count. It names the path, the
|
|
3809
|
+
failed operation and the error, never claims an old writer exists, and
|
|
3810
|
+
says detection is paused only when the validated index holds the
|
|
3811
|
+
durable `inventory-incomplete` marker. An inventory that failed for any
|
|
3812
|
+
other reason (fix round K12, K11R-13) names no file: it is reported
|
|
3813
|
+
unavailable with its cause."""
|
|
3814
|
+
if inventory.get("unavailableCause") and not inventory.get("path"):
|
|
3815
|
+
return _with_unavailable_spool_inventory(
|
|
3816
|
+
result, inventory, inventory["unavailableCause"])
|
|
3817
|
+
text = LEGACY_SPOOL_INVENTORY_TEXT.format(
|
|
3818
|
+
path=inventory.get("path"), operation=inventory.get("operation"),
|
|
3819
|
+
error=inventory.get("error"))
|
|
3820
|
+
if inventory.get("paused"):
|
|
3821
|
+
text += " " + LEGACY_SPOOL_INVENTORY_PAUSED_TEXT
|
|
3822
|
+
note = "legacy status-line inventory unavailable"
|
|
3823
|
+
if result.severity == "warn":
|
|
3824
|
+
summary = f"{result.summary}; {note}"
|
|
3825
|
+
remediation = (f"{result.remediation} {text}" if result.remediation
|
|
3826
|
+
else text)
|
|
3827
|
+
else:
|
|
3828
|
+
summary = (f"{note} ({inventory.get('operation')} "
|
|
3829
|
+
f"{inventory.get('path')}: {inventory.get('error')})")
|
|
3830
|
+
remediation = text
|
|
3831
|
+
return CheckResult(
|
|
3832
|
+
id=result.id, title=result.title, severity="warn", summary=summary,
|
|
3833
|
+
remediation=remediation,
|
|
3834
|
+
details={**(result.details or {}),
|
|
3835
|
+
"legacySpoolInventory": dict(inventory)},
|
|
3836
|
+
)
|
|
3837
|
+
|
|
3838
|
+
|
|
3839
|
+
def _with_unavailable_spool_inventory(result: CheckResult, inventory: dict,
|
|
3840
|
+
cause: str) -> CheckResult:
|
|
3841
|
+
"""`_with_legacy_spool_inventory` for an inventory that names no file:
|
|
3842
|
+
a WARN, "legacy status-line inventory unavailable ({cause})", which keeps
|
|
3843
|
+
the excluded-record half's own summary and remediation."""
|
|
3844
|
+
note = f"legacy status-line inventory unavailable ({cause})"
|
|
3845
|
+
if result.severity == "warn":
|
|
3846
|
+
summary, remediation = f"{result.summary}; {note}", result.remediation
|
|
3847
|
+
else:
|
|
3848
|
+
summary, remediation = note, None
|
|
3849
|
+
return CheckResult(
|
|
3850
|
+
id=result.id, title=result.title, severity="warn", summary=summary,
|
|
3851
|
+
remediation=remediation,
|
|
3852
|
+
details={**(result.details or {}),
|
|
3853
|
+
"legacySpoolInventory": dict(inventory)},
|
|
3854
|
+
)
|
|
3855
|
+
|
|
3856
|
+
|
|
3857
|
+
def _legacy_writer_count_result(s: DoctorState) -> CheckResult:
|
|
3858
|
+
"""The excluded-record half of `journal.legacy_writer`."""
|
|
3859
|
+
import _lib_legacy_writer_evidence as _lw
|
|
3860
|
+
|
|
3861
|
+
data = s.legacy_writer
|
|
3862
|
+
if data and data.get("unavailableCause"):
|
|
3863
|
+
# K9K10-R6/R11: the deep recomputation could not complete. The count
|
|
3864
|
+
# is unavailable — never zero, never the stale summary.
|
|
3865
|
+
return CheckResult(
|
|
3866
|
+
id="journal.legacy_writer", title="Legacy writer",
|
|
3867
|
+
severity="warn",
|
|
3868
|
+
summary=f"unavailable ({data['unavailableCause']})",
|
|
3869
|
+
remediation=_journal_unreadable_remediation(s),
|
|
3870
|
+
details={"available": False,
|
|
3871
|
+
"unavailableCause": data["unavailableCause"]},
|
|
3872
|
+
)
|
|
3873
|
+
if not data or not data.get("available"):
|
|
3874
|
+
# No current summary (for example an older process republished its
|
|
3875
|
+
# own epoch). The durable sidecar alone still proves an excluded
|
|
3876
|
+
# population and when it was first observed, so it decides the leg;
|
|
3877
|
+
# without one the leg is unavailable, never zero.
|
|
3878
|
+
sidecar = _lw.from_document((data or {}).get("evidence"))
|
|
3879
|
+
if sidecar is None or sidecar.excluded_count <= 0:
|
|
3880
|
+
return CheckResult(
|
|
3881
|
+
id="journal.legacy_writer", title="Legacy writer",
|
|
3882
|
+
severity="ok",
|
|
3883
|
+
summary="unavailable (no current admission summary)",
|
|
3884
|
+
remediation=None,
|
|
3885
|
+
details={"available": False},
|
|
3886
|
+
)
|
|
3887
|
+
verdict = _lw.classify(sidecar, excluded_count=sidecar.excluded_count,
|
|
3888
|
+
now=s.now_utc)
|
|
3889
|
+
return _legacy_writer_verdict_result(
|
|
3890
|
+
verdict, count=sidecar.excluded_count,
|
|
3891
|
+
details={"available": False, "source": "evidence-sidecar",
|
|
3892
|
+
"cutoverId": sidecar.cutover_id,
|
|
3893
|
+
"excludedCount": sidecar.excluded_count,
|
|
3894
|
+
"state": verdict.state,
|
|
3895
|
+
"lastSeenAt": verdict.last_seen_at})
|
|
3896
|
+
cutover_id = data.get("cutoverId")
|
|
3897
|
+
count = int(data.get("excludedCount") or 0)
|
|
3898
|
+
if cutover_id is None:
|
|
3899
|
+
return CheckResult(
|
|
3900
|
+
id="journal.legacy_writer", title="Legacy writer", severity="ok",
|
|
3901
|
+
summary="no account-bound cutover yet", remediation=None,
|
|
3902
|
+
details={"available": True, "cutoverId": None,
|
|
3903
|
+
"excludedCount": count, "state": "none",
|
|
3904
|
+
"lastSeenAt": None},
|
|
3905
|
+
)
|
|
3906
|
+
# A missing or stale sidecar reads as "first observed now", which can only
|
|
3907
|
+
# lengthen a warning (spec section 8).
|
|
3908
|
+
evidence = _lw.observe(
|
|
3909
|
+
_lw.from_document(data.get("evidence")), cutover_id=cutover_id,
|
|
3910
|
+
excluded_count=count, coordinate_digest=data.get("coordinateDigest"),
|
|
3911
|
+
now=s.now_utc)
|
|
3912
|
+
verdict = _lw.classify(evidence, excluded_count=count, now=s.now_utc)
|
|
3913
|
+
details = {"available": True, "cutoverId": cutover_id,
|
|
3914
|
+
"excludedCount": count, "state": verdict.state,
|
|
3915
|
+
"lastSeenAt": verdict.last_seen_at}
|
|
3916
|
+
if verdict.state == "none":
|
|
3917
|
+
return CheckResult(
|
|
3918
|
+
id="journal.legacy_writer", title="Legacy writer", severity="ok",
|
|
3919
|
+
summary="no legacy writes since the account-bound cutover",
|
|
3920
|
+
remediation=None, details=details,
|
|
3921
|
+
)
|
|
3922
|
+
return _legacy_writer_verdict_result(verdict, count=count, details=details)
|
|
3923
|
+
|
|
3924
|
+
|
|
3925
|
+
def _legacy_writer_verdict_result(verdict, *, count: int,
|
|
3926
|
+
details: dict) -> CheckResult:
|
|
3927
|
+
"""The WARN (recent) or historical OK (quiet) result in the approved
|
|
3928
|
+
spec-section-8 content: the count, the last-seen time and the remedy."""
|
|
3929
|
+
import _lib_legacy_writer_evidence as _lw
|
|
3930
|
+
|
|
3931
|
+
if verdict.state == "recent":
|
|
3932
|
+
return CheckResult(
|
|
3933
|
+
id="journal.legacy_writer", title="Legacy writer", severity="warn",
|
|
3934
|
+
summary=(f"{count} record(s) an older cctally process wrote after "
|
|
3935
|
+
"the upgrade were excluded from accounting (last seen "
|
|
3936
|
+
f"{verdict.last_seen_at})"),
|
|
3937
|
+
remediation=_lw.REMEDY, details=details,
|
|
3938
|
+
)
|
|
3939
|
+
return CheckResult(
|
|
3940
|
+
id="journal.legacy_writer", title="Legacy writer", severity="ok",
|
|
3941
|
+
summary=(f"no legacy writes in the last {_lw.QUIET_DAYS} days "
|
|
3942
|
+
f"({count} excluded in total, last seen "
|
|
3943
|
+
f"{verdict.last_seen_at})"),
|
|
3944
|
+
remediation=None, details=details,
|
|
3945
|
+
)
|
|
3946
|
+
|
|
3947
|
+
|
|
3948
|
+
#: Spec revision 5 (section 8), operator-owned wording, verbatim: an
|
|
3949
|
+
#: incomplete legacy spool inventory, and the sentence added when the
|
|
3950
|
+
#: validated index holds its durable `inventory-incomplete` marker.
|
|
3951
|
+
LEGACY_SPOOL_INVENTORY_TEXT = (
|
|
3952
|
+
"cctally cannot read {path} ({operation}: {error}), one of the "
|
|
3953
|
+
"status-line files an older cctally release used. Until it can, cctally "
|
|
3954
|
+
"cannot tell whether an older process changed it. Check the file's "
|
|
3955
|
+
"permissions and the disk.")
|
|
3956
|
+
LEGACY_SPOOL_INVENTORY_PAUSED_TEXT = (
|
|
3957
|
+
"Automatic credit and reset detection is paused because of this, and "
|
|
3958
|
+
"making the file readable again does not end the pause (see "
|
|
3959
|
+
"`accounting.outcome_quarantine`); usage recording continues.")
|
|
3960
|
+
|
|
3961
|
+
|
|
3962
|
+
#: Operator-approved wording (#825/#832 spec section 8), verbatim.
|
|
3963
|
+
OUTCOME_QUARANTINE_ACCOUNTING_TEXT = (
|
|
3964
|
+
"Automatic credit and reset detection is paused for {account} since "
|
|
3965
|
+
"{time} because an older cctally process wrote data after the upgrade. It "
|
|
3966
|
+
"resumes only after cctally verifies a clean reading for this account, "
|
|
3967
|
+
"typically the first one after an idle period spanning a weekly reset; "
|
|
3968
|
+
"there is no guaranteed time. If your usage was reset in place meanwhile, "
|
|
3969
|
+
"record it with `cctally record-credit`.")
|
|
3970
|
+
OUTCOME_QUARANTINE_CONTROL_TEXT = (
|
|
3971
|
+
"Status-line drop detection also waits for a successful usage refresh for "
|
|
3972
|
+
"this account.")
|
|
3973
|
+
#: Spec revision 3 (section 8; operator decision dec-2b5bcf910816), verbatim:
|
|
3974
|
+
#: an account paused by an unresolved spool marker. It replaces the "resumes
|
|
3975
|
+
#: only after..." sentence and already names drop detection, so no control
|
|
3976
|
+
#: sentence is appended.
|
|
3977
|
+
OUTCOME_QUARANTINE_SPOOL_MARKER_TEXT = (
|
|
3978
|
+
"Automatic credit, reset and status-line drop detection is paused for "
|
|
3979
|
+
"{account} since {time} because an older cctally process wrote "
|
|
3980
|
+
"status-line data after the upgrade. This version of cctally cannot yet "
|
|
3981
|
+
"confirm a safe point to resume, so the pause does not end on its own; "
|
|
3982
|
+
"usage recording continues. If your usage was reset in place meanwhile, "
|
|
3983
|
+
"record it with `cctally record-credit`.")
|
|
3984
|
+
#: Spec revision 5 (section 8), operator-owned wording, verbatim: an account
|
|
3985
|
+
#: paused by a marker an incomplete legacy spool inventory left.
|
|
3986
|
+
OUTCOME_QUARANTINE_INVENTORY_MARKER_TEXT = (
|
|
3987
|
+
"Automatic credit, reset and status-line drop detection is paused for "
|
|
3988
|
+
"{account} since {time} because cctally could not read the status-line "
|
|
3989
|
+
"files an older cctally release used, so it cannot rule out that an "
|
|
3990
|
+
"older process changed them. This version of cctally cannot yet confirm "
|
|
3991
|
+
"a safe point to resume, so the pause does not end on its own; usage "
|
|
3992
|
+
"recording continues. If your usage was reset in place meanwhile, record "
|
|
3993
|
+
"it with `cctally record-credit`.")
|
|
3994
|
+
#: Operator decision dec-5d2a9c2b004a (escalation esc-8a386c207966): the
|
|
3995
|
+
#: standalone sentence for an account missing only its control certificate.
|
|
3996
|
+
#: The approved control sentence above opens with "also", so it reads only as
|
|
3997
|
+
#: an addendum to the accounting one.
|
|
3998
|
+
OUTCOME_QUARANTINE_CONTROL_ONLY_TEXT = (
|
|
3999
|
+
"Status-line drop detection is paused for {account} since {time}. It "
|
|
4000
|
+
"resumes after a successful usage refresh for this account.")
|
|
4001
|
+
|
|
4002
|
+
|
|
4003
|
+
def _outcome_quarantine_message(row: dict) -> str:
|
|
4004
|
+
"""One account's paused outcome classes, in the approved wording. A
|
|
4005
|
+
missing control certificate appends its sentence to the accounting one;
|
|
4006
|
+
alone, it gets its own operator-approved standalone sentence."""
|
|
4007
|
+
label = row.get("accountLabel") or row.get("account")
|
|
4008
|
+
if row.get("unresolvedSpoolMarker"):
|
|
4009
|
+
text = (OUTCOME_QUARANTINE_INVENTORY_MARKER_TEXT
|
|
4010
|
+
if row.get("spoolMarkerCause") == "inventory-incomplete"
|
|
4011
|
+
else OUTCOME_QUARANTINE_SPOOL_MARKER_TEXT)
|
|
4012
|
+
return text.format(
|
|
4013
|
+
account=label, time=row.get("accountingSince") or "an unknown time")
|
|
4014
|
+
if row.get("accountingPaused"):
|
|
4015
|
+
text = OUTCOME_QUARANTINE_ACCOUNTING_TEXT.format(
|
|
4016
|
+
account=label, time=row.get("accountingSince") or "an unknown time")
|
|
4017
|
+
if row.get("controlPaused"):
|
|
4018
|
+
text += " " + OUTCOME_QUARANTINE_CONTROL_TEXT
|
|
4019
|
+
return text
|
|
4020
|
+
return OUTCOME_QUARANTINE_CONTROL_ONLY_TEXT.format(
|
|
4021
|
+
account=label, time=row.get("controlSince") or "an unknown time")
|
|
4022
|
+
|
|
4023
|
+
|
|
4024
|
+
def _check_accounting_outcome_quarantine(s: DoctorState) -> CheckResult:
|
|
4025
|
+
"""#825/#832 (spec section 8, F16, operator Q9): after a post-cutover
|
|
4026
|
+
legacy write or legacy spool change, automatic outcomes are withheld per
|
|
4027
|
+
account until certified recovery. OK when no account is quarantined; WARN
|
|
4028
|
+
naming per account what is paused and since when; unavailable — WARN,
|
|
4029
|
+
never OK — on an unavailable or unvalidated index. Separate from
|
|
4030
|
+
`journal.legacy_writer`: seven quiet days do not mean outcomes resumed.
|
|
4031
|
+
Never claims when recovery will happen. While a spool marker is
|
|
4032
|
+
unresolved (spec revision 3) every row uses the marker wording and the
|
|
4033
|
+
details carry the markers."""
|
|
4034
|
+
data = s.outcome_quarantine
|
|
4035
|
+
title = "Outcome quarantine"
|
|
4036
|
+
if not data or not data.get("available"):
|
|
4037
|
+
return CheckResult(
|
|
4038
|
+
id="accounting.outcome_quarantine", title=title, severity="warn",
|
|
4039
|
+
summary="unavailable (no validated stats index)",
|
|
4040
|
+
remediation=("Run any cctally command to bring the stats index "
|
|
4041
|
+
"up to date, then run `cctally doctor` again."),
|
|
4042
|
+
details={"available": False, "accounts": []},
|
|
4043
|
+
)
|
|
4044
|
+
rows = []
|
|
4045
|
+
for row in data.get("accounts") or []:
|
|
4046
|
+
rows.append({**row, "message": _outcome_quarantine_message(row)})
|
|
4047
|
+
details = {"available": True, "triggered": bool(data.get("triggered")),
|
|
4048
|
+
"accounts": rows}
|
|
4049
|
+
if data.get("unresolvedSpoolMarkers"):
|
|
4050
|
+
details["unresolvedSpoolMarkers"] = list(data["unresolvedSpoolMarkers"])
|
|
4051
|
+
if not rows:
|
|
4052
|
+
return CheckResult(
|
|
4053
|
+
id="accounting.outcome_quarantine", title=title, severity="ok",
|
|
4054
|
+
summary=("no account is quarantined" if data.get("triggered")
|
|
4055
|
+
else "no legacy write or spool change since the upgrade"),
|
|
4056
|
+
remediation=None, details=details,
|
|
4057
|
+
)
|
|
4058
|
+
return CheckResult(
|
|
4059
|
+
id="accounting.outcome_quarantine", title=title, severity="warn",
|
|
4060
|
+
summary=(f"automatic outcomes paused for {len(rows)} account(s)"),
|
|
4061
|
+
remediation=" ".join(row["message"] for row in rows),
|
|
4062
|
+
details=details,
|
|
4063
|
+
)
|
|
4064
|
+
|
|
4065
|
+
|
|
3623
4066
|
def _check_journal_protocol(s: DoctorState) -> CheckResult:
|
|
3624
4067
|
"""Report selector failure or whole-batch tainting without false health."""
|
|
3625
4068
|
if (
|
|
@@ -4121,6 +4564,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
4121
4564
|
("journal.writer_guard", "_check_journal_writer_guard"),
|
|
4122
4565
|
("journal.conflicts", "_check_journal_conflicts"),
|
|
4123
4566
|
("journal.protocol", "_check_journal_protocol"),
|
|
4567
|
+
("journal.legacy_writer", "_check_journal_legacy_writer"),
|
|
4124
4568
|
("journal.quota_projection", "_check_journal_quota_projection"),
|
|
4125
4569
|
)),
|
|
4126
4570
|
("data", "Data", (
|
|
@@ -4134,6 +4578,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
4134
4578
|
# after the pair rather than between them.
|
|
4135
4579
|
("data.codex_project_metadata", "_check_data_codex_project_metadata"),
|
|
4136
4580
|
("data.codex_prune_safety", "_check_data_codex_prune_safety"),
|
|
4581
|
+
("data.codex_compressed_rollouts", "_check_data_codex_compressed_rollouts"),
|
|
4137
4582
|
("data.codex_replay", "_check_data_codex_replay"),
|
|
4138
4583
|
("data.codex_ingest_backlog", "_check_data_codex_ingest_backlog"),
|
|
4139
4584
|
("data.codex_quota", "_check_data_codex_quota"),
|
|
@@ -4155,6 +4600,12 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
4155
4600
|
("accounts.codex_window_attribution",
|
|
4156
4601
|
"_check_accounts_codex_window_attribution"),
|
|
4157
4602
|
)),
|
|
4603
|
+
# #825/#832 spec section 8: reported separately from the seven-day
|
|
4604
|
+
# `journal.legacy_writer` recency, because quiet time never decides
|
|
4605
|
+
# recovery.
|
|
4606
|
+
("accounting", "Accounting", (
|
|
4607
|
+
("accounting.outcome_quarantine", "_check_accounting_outcome_quarantine"),
|
|
4608
|
+
)),
|
|
4158
4609
|
("pricing", "Pricing", (
|
|
4159
4610
|
("pricing.coverage", "_check_pricing_coverage"),
|
|
4160
4611
|
# #705. Cannot FAIL, so `doctor`'s exit code is unaffected — the same
|