cctally 1.114.0 → 1.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/bin/_cctally_account.py +32 -8
  3. package/bin/_cctally_cache.py +4394 -279
  4. package/bin/_cctally_completion.py +1287 -0
  5. package/bin/_cctally_config.py +37 -0
  6. package/bin/_cctally_core.py +178 -2
  7. package/bin/_cctally_dashboard.py +2 -2
  8. package/bin/_cctally_dashboard_conversation.py +292 -23
  9. package/bin/_cctally_dashboard_envelope.py +16 -3
  10. package/bin/_cctally_dashboard_sources.py +18 -9
  11. package/bin/_cctally_db.py +316 -5
  12. package/bin/_cctally_doctor.py +436 -27
  13. package/bin/_cctally_five_hour.py +82 -31
  14. package/bin/_cctally_forecast.py +81 -9
  15. package/bin/_cctally_journal.py +3884 -112
  16. package/bin/_cctally_journal_repair.py +101 -12
  17. package/bin/_cctally_parser.py +42 -0
  18. package/bin/_cctally_project.py +8 -2
  19. package/bin/_cctally_quarantine.py +652 -0
  20. package/bin/_cctally_quota.py +26 -4
  21. package/bin/_cctally_quota_model.py +360 -83
  22. package/bin/_cctally_record.py +677 -149
  23. package/bin/_cctally_rederive.py +297 -32
  24. package/bin/_cctally_refresh.py +84 -13
  25. package/bin/_cctally_source_analytics.py +23 -2
  26. package/bin/_cctally_statusline.py +6137 -517
  27. package/bin/_cctally_store.py +287 -48
  28. package/bin/_cctally_transcript.py +8 -7
  29. package/bin/_cctally_tui.py +57 -53
  30. package/bin/_lib_account_binding.py +152 -0
  31. package/bin/_lib_account_protocol.py +386 -0
  32. package/bin/_lib_aggregators.py +4 -0
  33. package/bin/_lib_cache_coverage.py +13 -1
  34. package/bin/_lib_cache_writer_lock.py +19 -3
  35. package/bin/_lib_clone_copy.py +142 -0
  36. package/bin/_lib_codex_conversation_query.py +133 -19
  37. package/bin/_lib_codex_conversation_watch.py +59 -14
  38. package/bin/_lib_codex_drain_admission.py +214 -0
  39. package/bin/_lib_codex_metadata.py +18 -1
  40. package/bin/_lib_codex_publication_barrier.py +277 -0
  41. package/bin/_lib_codex_rollout_container.py +502 -0
  42. package/bin/_lib_conversation_dispatch.py +3 -0
  43. package/bin/_lib_conversation_query.py +11 -3
  44. package/bin/_lib_detector_access_registry.py +3817 -0
  45. package/bin/_lib_doctor.py +452 -1
  46. package/bin/_lib_five_hour.py +126 -0
  47. package/bin/_lib_ingest_frontier.py +414 -20
  48. package/bin/_lib_journal.py +141 -10
  49. package/bin/_lib_jsonl.py +5 -4
  50. package/bin/_lib_legacy_spool.py +126 -0
  51. package/bin/_lib_legacy_writer_evidence.py +136 -0
  52. package/bin/_lib_merge_gate.py +29 -302
  53. package/bin/_lib_meter_rate_change.py +10 -0
  54. package/bin/_lib_outcome_quarantine.py +690 -0
  55. package/bin/_lib_pricing.py +7 -2
  56. package/bin/_lib_quarantine.py +154 -0
  57. package/bin/_lib_quota_calibration.py +53 -8
  58. package/bin/_lib_quota_copy.py +2 -1
  59. package/bin/_lib_quota_model.py +288 -46
  60. package/bin/_lib_rederive.py +94 -5
  61. package/bin/_lib_segment_summary.py +35 -2
  62. package/bin/_lib_statusline_candidates.py +292 -23
  63. package/bin/cctally +23 -1
  64. package/package.json +14 -1
@@ -542,6 +542,190 @@ def _gather_dashboard_disk_writes(
542
542
  return {"mode": "discovery", "instances": instances}
543
543
 
544
544
 
545
+ def _gather_legacy_writer(deep_excluded, deep_boundary,
546
+ deep_error=None) -> dict:
547
+ """The `journal.legacy_writer` evidence (#825/#832 spec section 8).
548
+
549
+ Reads the admission summary on a validated current snapshot (an
550
+ unvalidated or missing summary is unavailable, never zero — the durable
551
+ evidence sidecar is still returned beside it, for the leg to report from)
552
+ and the durable evidence sidecar. A deep run substitutes its own whole-journal
553
+ recomputation of the excluded population; ``deep_error`` names why that
554
+ recomputation could not complete (fix round K11, K9K10-R6/R11), and then
555
+ the count is unavailable — never zero.
556
+
557
+ Spec revision 5: the leg also reports an incomplete legacy spool
558
+ inventory (``legacySpoolInventory``), independently of the count, and
559
+ says the detection pause exists only when the validated index holds the
560
+ durable `inventory-incomplete` marker. Read-only: the sidecar is never
561
+ written here, the spool is only read, and nothing is journaled."""
562
+ import _cctally_store
563
+ import _lib_account_protocol
564
+
565
+ out: dict = {"available": False}
566
+ row = None
567
+ inventory_marker = False
568
+ try:
569
+ if _cctally_core.DB_PATH.exists():
570
+ pc = _stats_ro_guarded()
571
+ try:
572
+ with _cctally_store.current_stats_snapshot(pc):
573
+ row = pc.execute(
574
+ "SELECT cutover_id, excluded_total, coordinate_digest "
575
+ "FROM journal_admission_summary WHERE id = 1"
576
+ ).fetchone()
577
+ inventory_marker = _durable_inventory_marker(pc)
578
+ finally:
579
+ pc.close()
580
+ except Exception: # noqa: BLE001 — a diagnostic read degrades to unavailable
581
+ row = None
582
+ try:
583
+ evidence = json.loads(
584
+ _cctally_core.LEGACY_WRITER_EVIDENCE_PATH.read_bytes())
585
+ except (OSError, ValueError):
586
+ evidence = None
587
+ inventory = _legacy_spool_inventory_probe(inventory_marker)
588
+ if inventory is not None:
589
+ out["legacySpoolInventory"] = inventory
590
+ if deep_error is not None:
591
+ return {**out, "unavailableCause": str(deep_error),
592
+ "evidence": evidence}
593
+ if deep_excluded is not None:
594
+ cutover_id = (None if deep_boundary is None
595
+ else _lib_account_protocol.CUTOVER_OP_ID)
596
+ count = len(deep_excluded)
597
+ digest = (_lib_account_protocol.coordinate_population_digest(
598
+ item.coordinate for item in deep_excluded) if count else None)
599
+ elif row is not None:
600
+ cutover_id, count, digest = row[0], int(row[1] or 0), row[2]
601
+ else:
602
+ # No current summary — exactly what an older process's epoch
603
+ # republish leaves behind. The durable sidecar still carries the
604
+ # excluded population and when it was first observed (#825/#832 fix
605
+ # round B, P2-2), so the leg reports from it rather than "unavailable".
606
+ return {**out, "evidence": evidence}
607
+ return {**out, "available": True, "cutoverId": cutover_id,
608
+ "excludedCount": count, "coordinateDigest": digest,
609
+ "evidence": evidence}
610
+
611
+
612
+ def _scan_cause(exc: BaseException) -> str:
613
+ """A journal read failure as a doctor cause (K9K10-R6): the message and,
614
+ for an OS error, its errno name — never an empty string."""
615
+ import errno as _errno
616
+ name = _errno.errorcode.get(getattr(exc, "errno", None) or 0)
617
+ text = getattr(exc, "strerror", None) or str(exc) or type(exc).__name__
618
+ filename = getattr(exc, "filename", None)
619
+ if filename:
620
+ text = f"{text}: {filename}"
621
+ return f"{text} [{name}]" if name else text
622
+
623
+
624
+ def _durable_inventory_marker(conn) -> bool:
625
+ """Whether the validated index holds an unresolved spool marker left by
626
+ an incomplete inventory (spec revision 5, section 8). Read inside the
627
+ caller's validated snapshot; a missing quarantine table is no marker."""
628
+ import _cctally_quarantine
629
+ import _lib_account_protocol
630
+ try:
631
+ state = _cctally_quarantine.load_state(conn)
632
+ except Exception: # noqa: BLE001 — no readable marker claims no pause
633
+ return False
634
+ return any(marker.get("reason")
635
+ == _lib_account_protocol.INVENTORY_INCOMPLETE_REASON
636
+ for marker in state.spool_markers)
637
+
638
+
639
+ def _legacy_spool_inventory_probe(paused: bool):
640
+ """The legacy spool inventory doctor itself takes (spec revision 5,
641
+ section 8): None when it completes, else ``{"available": False, "path",
642
+ "operation", "error", "paused"}``. Doctor can be the first process to
643
+ meet the error, so it probes rather than reading what the reducer saw.
644
+ Read-only: the spool is listed and read to digest it, never changed.
645
+
646
+ Fix round K12 (K11R-13): any other failure leaves the inventory
647
+ unavailable with its cause (``{"available": False, "unavailableCause",
648
+ "paused"}``), never complete."""
649
+ import errno as _errno
650
+ import _cctally_journal
651
+ try:
652
+ _cctally_journal.capture_legacy_spool_state()
653
+ except _cctally_journal.LegacySpoolInventoryIncomplete as exc:
654
+ code = exc.errno or _errno.EIO
655
+ return {"available": False,
656
+ "path": exc.failed_path or exc.rel,
657
+ "operation": exc.operation,
658
+ "error": f"{os.strerror(code)} [{exc.error}]",
659
+ "paused": bool(paused)}
660
+ except Exception as exc: # noqa: BLE001 — a diagnostic probe never crashes doctor
661
+ return {"available": False, "unavailableCause": _scan_cause(exc),
662
+ "paused": bool(paused)}
663
+ return None
664
+
665
+
666
+ def _statusline_authority_commits() -> dict:
667
+ """Each account partition's reconciled `authorityCommit` (#825/#832 spec
668
+ 5.6 item 7, fix-round-1 Q6): control recovery is complete only when the
669
+ account's statusline control transition records the commit of the
670
+ observation witnessing its control certificate. Read-only; a missing or
671
+ unreadable transition contributes nothing, so its account stays paused."""
672
+ import _cctally_statusline
673
+
674
+ now_epoch = int(dt.datetime.now(dt.timezone.utc).timestamp())
675
+ commits = {}
676
+ for account_key in _cctally_statusline._account_keys_on_disk():
677
+ try:
678
+ transition = _cctally_statusline._read_transition(
679
+ account_key, now_epoch=now_epoch)
680
+ except Exception: # noqa: BLE001 — unreadable state stays paused
681
+ continue
682
+ quarantine = None if transition is None else transition.quarantine
683
+ commit = None if quarantine is None else quarantine.get(
684
+ "authorityCommit")
685
+ if commit is not None:
686
+ commits[account_key] = commit
687
+ return commits
688
+
689
+
690
+ def _gather_outcome_quarantine() -> dict:
691
+ """The `accounting.outcome_quarantine` evidence (#825/#832 spec 8, 5.6).
692
+
693
+ Read from ONE snapshot proven current (epoch and protocol); an absent,
694
+ unreadable or unvalidated index is unavailable, never "no quarantine".
695
+ Read-only."""
696
+ import _cctally_quarantine
697
+ import _cctally_store
698
+
699
+ out: dict = {"available": False, "accounts": []}
700
+ try:
701
+ if not _cctally_core.DB_PATH.exists():
702
+ return out
703
+ commits = _statusline_authority_commits()
704
+ pc = _stats_ro_guarded()
705
+ try:
706
+ with _cctally_store.current_stats_snapshot(pc):
707
+ report = _cctally_quarantine.report(
708
+ pc, authority_commits=commits)
709
+ labels = {}
710
+ for row in pc.execute(
711
+ "SELECT account_key, label, email FROM accounts"):
712
+ labels[row[0]] = row[1] or row[2] or row[0]
713
+ finally:
714
+ pc.close()
715
+ except Exception: # noqa: BLE001 — a diagnostic read degrades to unavailable
716
+ return out
717
+ accounts = []
718
+ for row in report["accounts"]:
719
+ accounts.append({**row, "accountLabel": labels.get(
720
+ row["account"], row["account"])})
721
+ out = {"available": True, "triggered": report["triggered"],
722
+ "accounts": accounts}
723
+ # Spec revision 3: the unresolved spool markers, only while one exists.
724
+ if report.get("unresolvedSpoolMarkers"):
725
+ out["unresolvedSpoolMarkers"] = report["unresolvedSpoolMarkers"]
726
+ return out
727
+
728
+
545
729
  def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
546
730
  """Read the #318 statusline pipeline without creating or pruning files."""
547
731
  now_epoch = int(now_utc.timestamp())
@@ -586,21 +770,12 @@ def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
586
770
  )
587
771
  except Exception:
588
772
  pass
589
- for axis, path in (
590
- ("fiveHour", _cctally_core.STATUSLINE_AUTHORITATIVE_5H_PATH),
591
- ("sevenDay", _cctally_core.STATUSLINE_AUTHORITATIVE_7D_PATH),
592
- ):
593
- try:
594
- if not path.exists():
595
- continue
596
- tombstone = c._read_tombstone(
597
- axis, now_epoch=now_epoch, fail_closed=False
598
- )
599
- result["tombstones"][axis] = (
600
- tombstone.state if tombstone is not None else "invalid"
601
- )
602
- except Exception:
603
- result["tombstones"][axis] = "invalid"
773
+ # #825/#832 section 7: the current protocol's fences are per exact
774
+ # account and window under the account-bound root; only live windows count.
775
+ try:
776
+ result["tombstones"] = c._statusline_tombstone_states(now_epoch=now_epoch)
777
+ except Exception:
778
+ result["tombstones"] = {"fiveHour": "invalid", "sevenDay": "invalid"}
604
779
  return result
605
780
 
606
781
 
@@ -1436,6 +1611,132 @@ def doctor_gather_state(
1436
1611
  cache_lock.close()
1437
1612
 
1438
1613
 
1614
+ def _gather_codex_container_state(c, *, now_utc, cache_probe_allowed):
1615
+ """Live, read-only #968 census; never opens/migrates a writable store.
1616
+
1617
+ Presence is independent of decoder availability. Missing pre-migration
1618
+ tables mean no retained evidence; an unreadable probe means unknown, not
1619
+ an invented zero population. No decoding or detached work runs here.
1620
+ """
1621
+ import _cctally_cache as cache
1622
+ import _lib_codex_rollout_container as rc
1623
+
1624
+ def table_exists(conn, name):
1625
+ return conn.execute(
1626
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
1627
+ (name,)).fetchone() is not None
1628
+
1629
+ def deletion_debt(conn, table):
1630
+ if not table_exists(conn, table):
1631
+ return 0, None
1632
+ row = conn.execute(
1633
+ f"SELECT COUNT(*), MIN(recorded_at_utc) FROM {table}").fetchone()
1634
+ return int(row[0]), row[1]
1635
+
1636
+ try:
1637
+ decoder = rc.decoder_capability()
1638
+ selected = [item for item in cache._iter_codex_logical_containers(
1639
+ c._codex_session_roots()) if item.kind == rc.ZSTD]
1640
+ census = dict(compressedOnly=len(selected), retainedComplete=0,
1641
+ retainedIncomplete=0, neverIngested=0, unverified=0)
1642
+ status = None
1643
+ verifications, deletions = 0, 0
1644
+ deletion_starts = []
1645
+ path = _cctally_core.CACHE_DB_PATH
1646
+ if path.exists():
1647
+ if not cache_probe_allowed:
1648
+ return None, decoder, None, None
1649
+ conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=0.0)
1650
+ try:
1651
+ conn.row_factory = sqlite3.Row
1652
+ cursor_exists = table_exists(conn, "codex_session_files")
1653
+ columns = ({row[1] for row in conn.execute(
1654
+ "PRAGMA table_info(codex_session_files)")} if cursor_exists else set())
1655
+ if "container_state" in columns:
1656
+ verifications = int(conn.execute(
1657
+ "SELECT COUNT(*) FROM codex_session_files "
1658
+ "WHERE container_state='pending'").fetchone()[0])
1659
+ for item in selected:
1660
+ row = (conn.execute("SELECT * FROM codex_session_files WHERE path=?",
1661
+ (str(item.logical),)).fetchone()
1662
+ if cursor_exists else None)
1663
+ if row is None:
1664
+ census["neverIngested"] += 1
1665
+ continue
1666
+ row = dict(row)
1667
+ state = row.get("container_state")
1668
+ if state == "unverified":
1669
+ census["unverified"] += 1
1670
+ elif state == "pending":
1671
+ continue # verification work, never read debt (D3)
1672
+ elif (row.get("container_kind") == rc.ZSTD
1673
+ and state == "accepted"
1674
+ and row.get("ingest_complete") == 1
1675
+ and row.get("size_bytes") == row.get("decoded_size_bytes")
1676
+ and all(row.get(key) == value for key, value in (
1677
+ ("container_size_bytes", item.fingerprint.size),
1678
+ ("mtime_ns", item.fingerprint.mtime_ns),
1679
+ ("device_id", item.fingerprint.device),
1680
+ ("inode", item.fingerprint.inode)))):
1681
+ census["retainedComplete"] += 1
1682
+ else:
1683
+ census["retainedIncomplete"] += 1
1684
+ if table_exists(conn, "cache_meta"):
1685
+ row = conn.execute("SELECT value FROM cache_meta WHERE key=?",
1686
+ (cache.CODEX_CONTAINER_STATUS_KEY,)).fetchone()
1687
+ if row:
1688
+ parsed = json.loads(row[0])
1689
+ status = parsed if isinstance(parsed, dict) else None
1690
+ count, since = deletion_debt(conn, "codex_prune_debt")
1691
+ deletions += count
1692
+ if count:
1693
+ deletion_starts.append(since)
1694
+ finally:
1695
+ conn.close()
1696
+ else:
1697
+ census["neverIngested"] = len(selected)
1698
+ maintenance_coverage = "complete"
1699
+ if _cctally_core.CONVERSATIONS_DB_PATH.exists():
1700
+ # An unavailable second store cannot erase the first store's known
1701
+ # counts or ages. Its missing contribution remains explicitly unknown.
1702
+ try:
1703
+ with _conversation_ro_guarded(timeout=0.0) as conn:
1704
+ if conn is None:
1705
+ maintenance_coverage = "partial"
1706
+ else:
1707
+ count, since = deletion_debt(conn, "codex_conversation_prune_debt")
1708
+ deletions += count
1709
+ if count:
1710
+ deletion_starts.append(since)
1711
+ except Exception:
1712
+ maintenance_coverage = "partial"
1713
+ record = status or {}
1714
+ starts = list(deletion_starts)
1715
+ if verifications:
1716
+ starts.append(record.get("verificationPendingSince"))
1717
+ reads = (census["neverIngested"] + census["retainedIncomplete"]
1718
+ if decoder != "unavailable" else 0)
1719
+ if reads:
1720
+ starts.append(record.get("readsPendingSince"))
1721
+ old = False
1722
+ for stamp in starts:
1723
+ if not stamp:
1724
+ continue
1725
+ try:
1726
+ age = (now_utc - parse_iso_datetime(
1727
+ stamp, "codex maintenance start")).total_seconds()
1728
+ old = old or age > 86400
1729
+ except (TypeError, ValueError):
1730
+ pass
1731
+ debt = dict(verifications=verifications, deletions=deletions,
1732
+ olderThan24h=old)
1733
+ if maintenance_coverage != "complete":
1734
+ debt["coverage"] = maintenance_coverage
1735
+ return census, decoder, status, debt
1736
+ except Exception:
1737
+ return None, None, None, None
1738
+
1739
+
1439
1740
  def _doctor_gather_state_impl(
1440
1741
  *,
1441
1742
  now_utc: "dt.datetime | None" = None,
@@ -2032,12 +2333,18 @@ def _doctor_gather_state_impl(
2032
2333
  codex_jsonl_present = False
2033
2334
  try:
2034
2335
  for codex_dir in c._codex_session_roots():
2035
- if next(codex_dir.glob("**/*.jsonl"), None) is not None:
2336
+ if (next(codex_dir.glob("**/*.jsonl"), None) is not None
2337
+ or next(codex_dir.glob("**/*.jsonl.zst"), None) is not None):
2036
2338
  codex_jsonl_present = True
2037
2339
  break
2038
2340
  except Exception:
2039
2341
  pass
2040
2342
 
2343
+ with _lib_perf.phase("doctor.codex_containers"):
2344
+ (codex_compressed_census, codex_decoder_capability,
2345
+ codex_container_status, codex_maintenance_debt) = _gather_codex_container_state(
2346
+ c, now_utc=now_utc, cache_probe_allowed=_cache_probe_allowed)
2347
+
2041
2348
  with _lib_perf.phase("doctor.quota_summary"):
2042
2349
  # ── Codex quota lifecycle (#294 S2) ──────────────────────────────
2043
2350
  # All three probes are read-only and root-qualified. The physical cache
@@ -2496,10 +2803,17 @@ def _doctor_gather_state_impl(
2496
2803
  journal_cursor_lag_bytes = None
2497
2804
  journal_hw_segment = None
2498
2805
  journal_cursor_segment = None
2806
+ journal_cursor_invalid = False
2499
2807
  journal_conflicts = None
2500
2808
  journal_protocol_violations = None
2501
2809
  journal_protocol_acknowledged = None
2502
2810
  journal_protocol_error = None
2811
+ # #825/#832 spec 8: the deep scan's whole-journal excluded population.
2812
+ deep_excluded = None
2813
+ deep_admission_boundary = None
2814
+ # K9K10-R6/R11: why the listing or the deep scan could not complete.
2815
+ journal_listing_error = None
2816
+ journal_scan_error = None
2503
2817
  try:
2504
2818
  jdir = _cctally_core.JOURNAL_DIR
2505
2819
  journal_present = jdir.exists()
@@ -2511,8 +2825,14 @@ def _doctor_gather_state_impl(
2511
2825
  import _cctally_journal as _jr
2512
2826
  try:
2513
2827
  segs = _jr.list_segments() # canonical (segment) order
2514
- except Exception:
2828
+ except Exception as exc: # noqa: BLE001 — reported, never empty
2829
+ # K9K10-R6: an unlistable journal is unreadable, not an
2830
+ # empty one; the presence leg names the cause.
2515
2831
  segs = []
2832
+ journal_listing_error = _scan_cause(exc)
2833
+ if deep:
2834
+ # The deep recomputation cannot run either.
2835
+ journal_scan_error = journal_listing_error
2516
2836
  journal_segment_count = len(segs)
2517
2837
  # #402: the disposable stats index persists the most recent complete
2518
2838
  # selector result. Shallow Dashboard/TUI gathers read that bounded
@@ -2556,13 +2876,28 @@ def _doctor_gather_state_impl(
2556
2876
  torn = 0
2557
2877
  decoded_records: list = []
2558
2878
  protocol_evidence = []
2879
+ protocol_evidence_owners = []
2559
2880
  prior_high_water = None
2560
2881
  cutover_value = None
2882
+ # #825/#832 (spec 5.3, 8): the deep scan applies the same
2883
+ # admission predicate as ingest and rebuild — resolved
2884
+ # within this one pass and applied after it — and
2885
+ # recomputes the excluded population the legacy-writer leg
2886
+ # compares.
2887
+ pass_boundary = _jr._PassBoundary()
2888
+ deep_coordinates: list = []
2889
+ cutover_candidates: list = []
2890
+ deep_excluded: list = []
2561
2891
  for seg in segs:
2562
2892
  try:
2563
2893
  data = (jdir / seg).read_bytes()
2564
- except OSError:
2894
+ except FileNotFoundError:
2565
2895
  continue
2896
+ except OSError as exc:
2897
+ # K9K10-R6: a skipped segment would let every
2898
+ # check below report a partial journal as whole.
2899
+ journal_scan_error = _scan_cause(exc)
2900
+ break
2566
2901
  if not data:
2567
2902
  continue
2568
2903
  if not data.endswith(b"\n"):
@@ -2585,20 +2920,22 @@ def _doctor_gather_state_impl(
2585
2920
  )
2586
2921
  offset += len(raw) + 1
2587
2922
  continue
2923
+ pass_boundary.observe(record)
2588
2924
  _jr._capture_protocol_prefix_evidence(
2589
2925
  record,
2590
2926
  prior_high_water,
2591
2927
  protocol_evidence,
2592
2928
  )
2929
+ if (len(protocol_evidence)
2930
+ > len(protocol_evidence_owners)):
2931
+ protocol_evidence_owners.append(
2932
+ len(decoded_records))
2593
2933
  # first cutover op wins, exactly as
2594
2934
  # `find_accounts_cutover_op` scans — captured here so the
2595
2935
  # conflict scan does not decode the whole journal twice.
2596
- if (cutover_value is None
2597
- and record.get("id") == _jr.CUTOVER_OP_ID):
2598
- payload = record.get("payload")
2599
- if isinstance(payload, dict):
2600
- cutover_value = payload.get(
2601
- "claude_legacy_account")
2936
+ if record.get("id") == _jr.CUTOVER_OP_ID:
2937
+ cutover_candidates.append(
2938
+ (record, seg, offset))
2602
2939
  # RETAIN ONLY what the selector consumes. `obs` lines are
2603
2940
  # ~97% of a real journal (984k of 1.02M) and
2604
2941
  # `resolve_effective_events` ignores them entirely —
@@ -2609,13 +2946,56 @@ def _doctor_gather_state_impl(
2609
2946
  decoded_records.append(
2610
2947
  _lib_journal_router.selector_slot(record)
2611
2948
  )
2949
+ deep_coordinates.append((seg, offset))
2612
2950
  prior_high_water = (
2613
2951
  seg,
2614
2952
  offset + len(raw) + 1,
2615
2953
  )
2616
2954
  offset += len(raw) + 1
2617
- journal_malformed_count = malformed
2618
- journal_torn_tail_count = torn
2955
+ admission_boundary = None
2956
+ if journal_scan_error is None:
2957
+ try:
2958
+ admission_boundary = pass_boundary.resolve(
2959
+ (segs[-1], sizes.get(segs[-1], 0)))
2960
+ except Exception as exc: # noqa: BLE001
2961
+ # K9K10-R6: a None boundary admits everything:
2962
+ # zero excluded records and an unfiltered scan.
2963
+ journal_scan_error = _scan_cause(exc)
2964
+ if journal_scan_error is None:
2965
+ try:
2966
+ deep_excluded = _jr.excluded_population(
2967
+ admission_boundary)
2968
+ except Exception as exc: # noqa: BLE001
2969
+ # K9K10-R11: an unreadable tail is never a
2970
+ # smaller population.
2971
+ journal_scan_error = _scan_cause(exc)
2972
+ if journal_scan_error is not None:
2973
+ deep_excluded = None
2974
+ journal_conflicts = None
2975
+ journal_protocol_error = None
2976
+ else:
2977
+ journal_malformed_count = malformed
2978
+ journal_torn_tail_count = torn
2979
+ if deep and segs and journal_scan_error is None:
2980
+ _jr._apply_pass_admission(
2981
+ decoded_records, deep_coordinates, admission_boundary)
2982
+ # #825/#832 fix round C (CD2-005): an excluded
2983
+ # acknowledgement's evidence is excluded with it.
2984
+ protocol_evidence = _jr._admitted_prefix_evidence(
2985
+ protocol_evidence, protocol_evidence_owners,
2986
+ decoded_records)
2987
+ # Every excluded record counts, obs included, so the
2988
+ # population matches what ingest and rebuild record. Only
2989
+ # the post-boundary byte ranges can hold one.
2990
+ deep_admission_boundary = admission_boundary
2991
+ for candidate, cand_seg, cand_off in cutover_candidates:
2992
+ if _jr._is_admitted(candidate, cand_seg, cand_off,
2993
+ admission_boundary):
2994
+ payload = candidate.get("payload")
2995
+ if isinstance(payload, dict):
2996
+ cutover_value = payload.get(
2997
+ "claude_legacy_account")
2998
+ break
2619
2999
  # #374: same-revision quarantine, via the SHARED selector over
2620
3000
  # rebuild-equivalent input. Raw `(id, rev)` grouping would report
2621
3001
  # lower-revision groups a completed rev-1 batch legitimately
@@ -2704,7 +3084,23 @@ def _doctor_gather_state_impl(
2704
3084
  order = {s: i for i, s in enumerate(segs)}
2705
3085
  if cseg in order:
2706
3086
  ci = order[cseg]
2707
- lag = max(0, sizes.get(segs[ci], 0) - coff)
3087
+ # #946: the lag of the SAFE applied cursor. A torn
3088
+ # suffix past it stays outstanding until a repair, and
3089
+ # a cursor past its segment or inside a line is no
3090
+ # safe endpoint at all.
3091
+ csize = sizes.get(segs[ci], 0)
3092
+ if coff > csize:
3093
+ journal_cursor_invalid = True
3094
+ elif coff > 0:
3095
+ try:
3096
+ with open(_cctally_core.JOURNAL_DIR / cseg,
3097
+ "rb") as _seg_fh:
3098
+ _seg_fh.seek(coff - 1)
3099
+ if _seg_fh.read(1) != b"\n":
3100
+ journal_cursor_invalid = True
3101
+ except OSError:
3102
+ journal_cursor_invalid = True
3103
+ lag = max(0, csize - coff)
2708
3104
  for s in segs[ci + 1:]:
2709
3105
  lag += sizes.get(s, 0)
2710
3106
  journal_cursor_lag_bytes = lag
@@ -2881,6 +3277,10 @@ def _doctor_gather_state_impl(
2881
3277
  codex_replay_blocked=codex_replay_blocked,
2882
3278
  codex_replay_deferred=codex_replay_deferred,
2883
3279
  codex_prune_refusals=codex_prune_refusals or None,
3280
+ codex_compressed_census=codex_compressed_census,
3281
+ codex_decoder_capability=codex_decoder_capability,
3282
+ codex_container_status=codex_container_status,
3283
+ codex_maintenance_debt=codex_maintenance_debt,
2884
3284
  conversation_rollup_pricing_refusal=(
2885
3285
  conversation_rollup_pricing_refusal),
2886
3286
  conversation_rollup_pricing_fp=conversation_rollup_pricing_fp,
@@ -2898,9 +3298,17 @@ def _doctor_gather_state_impl(
2898
3298
  conversations_db_wal_bytes=conversations_db_wal_bytes,
2899
3299
  # #374: quarantined same-revision groups + structural protocol violation.
2900
3300
  journal_conflicts=journal_conflicts,
3301
+ # #825/#832 spec 8: the legacy-writer evidence.
3302
+ legacy_writer=_gather_legacy_writer(
3303
+ deep_excluded, deep_admission_boundary,
3304
+ deep_error=journal_scan_error),
3305
+ # #825/#832 spec 8: the per-account outcome quarantine.
3306
+ outcome_quarantine=_gather_outcome_quarantine(),
2901
3307
  journal_protocol_violations=journal_protocol_violations,
2902
3308
  journal_protocol_acknowledged=journal_protocol_acknowledged,
2903
3309
  journal_protocol_error=journal_protocol_error,
3310
+ journal_listing_error=journal_listing_error,
3311
+ journal_scan_error=journal_scan_error,
2904
3312
  # #496 S5b: the published generation's incomplete-quota-projection flag.
2905
3313
  stats_quota_projection_incomplete=stats_quota_projection_incomplete,
2906
3314
  # #315: read-only cache free-page evidence for the reclaim hint.
@@ -2927,6 +3335,7 @@ def _doctor_gather_state_impl(
2927
3335
  journal_malformed_count=journal_malformed_count,
2928
3336
  journal_torn_tail_count=journal_torn_tail_count,
2929
3337
  journal_cursor_lag_bytes=journal_cursor_lag_bytes,
3338
+ journal_cursor_invalid=journal_cursor_invalid,
2930
3339
  journal_hw_segment=journal_hw_segment,
2931
3340
  journal_cursor_segment=journal_cursor_segment,
2932
3341
  journal_heal_incidents=journal_heal_incidents,