cctally 1.114.0 → 1.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/bin/_cctally_account.py +32 -8
  3. package/bin/_cctally_cache.py +4491 -295
  4. package/bin/_cctally_completion.py +1355 -0
  5. package/bin/_cctally_config.py +37 -0
  6. package/bin/_cctally_core.py +282 -9
  7. package/bin/_cctally_dashboard.py +8 -11
  8. package/bin/_cctally_dashboard_conversation.py +292 -23
  9. package/bin/_cctally_dashboard_envelope.py +42 -35
  10. package/bin/_cctally_dashboard_sources.py +18 -9
  11. package/bin/_cctally_db.py +401 -9
  12. package/bin/_cctally_doctor.py +469 -27
  13. package/bin/_cctally_five_hour.py +166 -1049
  14. package/bin/_cctally_forecast.py +81 -9
  15. package/bin/_cctally_journal.py +5076 -177
  16. package/bin/_cctally_journal_repair.py +110 -19
  17. package/bin/_cctally_oauth_samples.py +280 -0
  18. package/bin/_cctally_parser.py +47 -0
  19. package/bin/_cctally_project.py +8 -2
  20. package/bin/_cctally_quarantine.py +662 -0
  21. package/bin/_cctally_quota.py +96 -12
  22. package/bin/_cctally_quota_model.py +391 -83
  23. package/bin/_cctally_record.py +1088 -269
  24. package/bin/_cctally_rederive.py +854 -192
  25. package/bin/_cctally_refresh.py +113 -13
  26. package/bin/_cctally_source_analytics.py +23 -2
  27. package/bin/_cctally_statusline.py +7062 -505
  28. package/bin/_cctally_store.py +1244 -91
  29. package/bin/_cctally_transcript.py +8 -7
  30. package/bin/_cctally_tui.py +119 -60
  31. package/bin/_cctally_update.py +25 -0
  32. package/bin/_lib_account_binding.py +152 -0
  33. package/bin/_lib_account_protocol.py +569 -0
  34. package/bin/_lib_aggregators.py +4 -0
  35. package/bin/_lib_cache_coverage.py +13 -1
  36. package/bin/_lib_cache_writer_lock.py +19 -3
  37. package/bin/_lib_clone_copy.py +177 -0
  38. package/bin/_lib_codex_conversation_query.py +133 -19
  39. package/bin/_lib_codex_conversation_watch.py +59 -14
  40. package/bin/_lib_codex_drain_admission.py +214 -0
  41. package/bin/_lib_codex_metadata.py +18 -1
  42. package/bin/_lib_codex_publication_barrier.py +277 -0
  43. package/bin/_lib_codex_rollout_container.py +502 -0
  44. package/bin/_lib_conversation_dispatch.py +3 -0
  45. package/bin/_lib_conversation_query.py +11 -3
  46. package/bin/_lib_dashboard_sources.py +57 -15
  47. package/bin/_lib_detector_access_registry.py +4038 -0
  48. package/bin/_lib_doctor.py +753 -1
  49. package/bin/_lib_five_hour.py +126 -0
  50. package/bin/_lib_ingest_frontier.py +414 -20
  51. package/bin/_lib_journal.py +261 -10
  52. package/bin/_lib_journal_router.py +166 -48
  53. package/bin/_lib_jsonl.py +5 -4
  54. package/bin/_lib_legacy_spool.py +126 -0
  55. package/bin/_lib_legacy_writer_evidence.py +136 -0
  56. package/bin/_lib_meter_rate_change.py +10 -0
  57. package/bin/_lib_oauth_samples.py +238 -0
  58. package/bin/_lib_outcome_quarantine.py +690 -0
  59. package/bin/_lib_pricing.py +7 -2
  60. package/bin/_lib_pricing_check.py +0 -12
  61. package/bin/_lib_provider_activity.py +770 -0
  62. package/bin/_lib_quarantine.py +155 -0
  63. package/bin/_lib_quota_calibration.py +53 -8
  64. package/bin/_lib_quota_copy.py +2 -1
  65. package/bin/_lib_quota_model.py +384 -52
  66. package/bin/_lib_rebuild_progress.py +146 -0
  67. package/bin/_lib_record.py +22 -0
  68. package/bin/_lib_rederive.py +393 -8
  69. package/bin/_lib_segment_summary.py +35 -2
  70. package/bin/_lib_snapshot_cache.py +43 -1
  71. package/bin/_lib_stale_proof.py +708 -0
  72. package/bin/_lib_statusline_candidates.py +356 -27
  73. package/bin/_lib_worker_launches.py +315 -0
  74. package/bin/cctally +29 -12
  75. package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
  76. package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
  77. package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
  78. package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
  79. package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
  80. package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
  81. package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
  82. package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
  83. package/dashboard/static/dashboard.html +1 -1
  84. package/package.json +20 -2
  85. package/bin/_lib_merge_gate.py +0 -725
  86. package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
@@ -30,8 +30,11 @@ from __future__ import annotations
30
30
  import datetime as dt
31
31
  import hashlib
32
32
  import json
33
+ import sys
33
34
  from dataclasses import dataclass
34
35
 
36
+ import _lib_account_protocol
37
+
35
38
  LINE_VERSION = 1
36
39
 
37
40
  SEGMENT_PREFIX = "observations-"
@@ -220,6 +223,47 @@ def decode_line(raw: bytes) -> dict | None:
220
223
  return obj
221
224
 
222
225
 
226
+ #: Strings up to this many characters are interned by `compact_retained`.
227
+ #: Keys, kinds, sources, account keys, ids and timestamps fall under it; a
228
+ #: long free-text value is rare enough that sharing it would buy nothing.
229
+ COMPACT_INTERN_MAX_CHARS = 64
230
+
231
+
232
+ def compact_retained(value):
233
+ """An equal copy of a decoded record whose keys and string values of at
234
+ most `COMPACT_INTERN_MAX_CHARS` characters are ``sys.intern``ed (#997 D15).
235
+
236
+ CPython's JSON decoder shares keys only within one ``loads`` call, so
237
+ every record a pass retains carries its own copy of every key and every
238
+ repeated short value: 381 k retained records took 0.83 GB decoded and
239
+ 0.375 GB compacted on the maintainer's copy. Recursive through ``dict``
240
+ and ``list``; every other value (numbers, booleans, ``None``, long
241
+ strings) is shared unchanged. The result is a NEW record with new
242
+ containers, so equality, hashing, canonical encoding and mutation behave
243
+ exactly as on the plain record; only the object identity of equal strings
244
+ changes, and no consumer compares strings by identity.
245
+
246
+ Apply it immediately after decoding, before any consumer holds a
247
+ reference, so an identity check such as the rederive reader's
248
+ ``records[index] is record`` only ever sees the compacted object.
249
+ """
250
+ return _compact(value)
251
+
252
+
253
+ def _compact(value, _intern=sys.intern):
254
+ kind = type(value)
255
+ if kind is dict:
256
+ return {
257
+ (_intern(key) if type(key) is str else key): _compact(item)
258
+ for key, item in value.items()
259
+ }
260
+ if kind is list:
261
+ return [_compact(item) for item in value]
262
+ if kind is str and len(value) <= COMPACT_INTERN_MAX_CHARS:
263
+ return _intern(value)
264
+ return value
265
+
266
+
223
267
  # --------------------------------------------------------------------------
224
268
  # identity
225
269
  # --------------------------------------------------------------------------
@@ -379,8 +423,31 @@ def effects_payload_digest(payload: dict, refs: dict | None = None) -> str:
379
423
  # fully-formed line records
380
424
  # --------------------------------------------------------------------------
381
425
 
426
+ _WRITER_GENERATION = _lib_account_protocol.PROTOCOL_GENERATION
427
+ _WPG = _lib_account_protocol.WRITER_GENERATION_KEY
428
+ #: #797/#823/#840 (spec 4.6 B1): the protocol revision stamp, transport
429
+ #: metadata exactly like ``wpg``.
430
+ _WRITER_REVISION = _lib_account_protocol.PROTOCOL_REVISION
431
+ _WPR = _lib_account_protocol.REVISION_KEY
432
+ #: ``writer_revision`` default: the current revision when the record carries
433
+ #: the current writer generation, none for a previous release's shape.
434
+ _REVISION_DEFAULT = object()
435
+
436
+
437
+ def _writer_core(core: dict, writer_generation, writer_revision) -> None:
438
+ if writer_generation is not None:
439
+ core[_WPG] = writer_generation
440
+ if writer_revision is _REVISION_DEFAULT:
441
+ writer_revision = (None if writer_generation is None
442
+ else _WRITER_REVISION)
443
+ if writer_revision is not None:
444
+ core[_WPR] = writer_revision
445
+
446
+
382
447
  def make_obs(at: str, src: str, provider: str, payload: dict,
383
- account: str | None = None) -> dict:
448
+ account: str | None = None, *,
449
+ writer_generation: str | None = _WRITER_GENERATION,
450
+ writer_revision=_REVISION_DEFAULT) -> dict:
384
451
  """Build a complete ``obs`` line record (raw capture; canonicalization is
385
452
  derivation-time, never baked into the stored line — spec §4.2).
386
453
 
@@ -388,20 +455,159 @@ def make_obs(at: str, src: str, provider: str, payload: dict,
388
455
  ``provider``. It is emitted ONLY when supplied, so a pre-epic / single-account
389
456
  writer that passes nothing produces a byte-identical line (and a byte-stable
390
457
  content id). When supplied it participates in the content id, so two obs that
391
- differ only by account are distinct records."""
458
+ differ only by account are distinct records.
459
+
460
+ ``writer_generation`` (#825/#832) is the top-level ``wpg`` stamp naming the
461
+ writing protocol. It is part of the content-id core, so a current record and
462
+ a previous-release record of the same capture are distinct lines. ``None``
463
+ builds the previous release's exact shape (a validator re-deriving a
464
+ retained pre-stamp record, or a test simulating an old writer).
465
+
466
+ ``writer_revision`` (#797/#823/#840 spec 4.6 B1) is the top-level ``wpr``
467
+ protocol revision, also part of the content-id core. It defaults to the
468
+ current revision for a current-generation record and to none for a
469
+ previous release's shape; ``None`` with the current generation builds
470
+ the revision-1 (#825) shape."""
392
471
  core = {"t": "obs", "at": at, "src": src, "provider": provider, "payload": payload}
393
472
  if account is not None:
394
473
  core["account"] = account
474
+ _writer_core(core, writer_generation, writer_revision)
395
475
  return {"v": LINE_VERSION, **core, "id": content_id(core)}
396
476
 
397
477
 
398
- def make_op(at: str, src: str, payload: dict) -> dict:
399
- """Build a complete ``op`` (operator record) line — no ``provider``."""
478
+ def make_op(at: str, src: str, payload: dict, *,
479
+ writer_generation: str | None = _WRITER_GENERATION,
480
+ writer_revision=_REVISION_DEFAULT) -> dict:
481
+ """Build a complete ``op`` (operator record) line — no ``provider``.
482
+ ``writer_generation`` and ``writer_revision`` as in ``make_obs``."""
400
483
  core = {"t": "op", "at": at, "src": src, "payload": payload}
484
+ _writer_core(core, writer_generation, writer_revision)
401
485
  return {"v": LINE_VERSION, **core, "id": content_id(core)}
402
486
 
403
487
 
488
+ def prepare_record(record: dict) -> dict:
489
+ """The single shared preparation step both append APIs apply (#825/#832
490
+ spec 5.1): return ``record`` carrying the current writer generation and,
491
+ since #797/#823/#840 (spec 4.6 B1), the current protocol revision.
492
+
493
+ A content-id record (``id`` starting ``o:``) that is not yet fully
494
+ stamped and whose id verifies over its current core is re-identified over
495
+ the stamped core, exactly as the builders compute it. Every other id —
496
+ semantic event ids, bootstrap ids, correction batch ids, a content id
497
+ that does not verify — is preserved. Idempotent: a record carrying both
498
+ stamps is returned unchanged. Only records about to be appended pass
499
+ through here, so a retained record is never restamped."""
500
+ if (_lib_account_protocol.is_stamped(record)
501
+ and _lib_account_protocol.has_current_revision(record)):
502
+ return record
503
+ stamped = _lib_account_protocol.stamp(record)
504
+ rid = record.get("id")
505
+ if (record.get("t") in ("obs", "op") and isinstance(rid, str)
506
+ and rid.startswith("o:")):
507
+ core = {k: v for k, v in record.items() if k not in ("v", "id")}
508
+ if content_id(core) == rid:
509
+ stamped_core = {k: v for k, v in stamped.items()
510
+ if k not in ("v", "id")}
511
+ stamped["id"] = content_id(stamped_core)
512
+ return stamped
513
+
514
+
515
+ def unstamped_record(record: dict) -> dict:
516
+ """The previous release's shape of ``record``: the inverse of
517
+ ``prepare_record``. Drops the writer-generation stamp and, for a content-id
518
+ record whose id verifies over its stamped core, re-identifies it over the
519
+ unstamped core. Any other id is preserved. For fixtures and tests that
520
+ simulate a pre-stamp writer; never used on a production write path.
521
+ The protocol revision (``wpr``) goes with the stamp."""
522
+ if not isinstance(record, dict) or (
523
+ _WPG not in record and _WPR not in record):
524
+ return record
525
+ out = {k: v for k, v in record.items() if k not in (_WPG, _WPR)}
526
+ rid = record.get("id")
527
+ if (record.get("t") in ("obs", "op") and isinstance(rid, str)
528
+ and rid.startswith("o:")):
529
+ core = {k: v for k, v in record.items() if k not in ("v", "id")}
530
+ if content_id(core) == rid:
531
+ out["id"] = content_id(
532
+ {k: v for k, v in out.items() if k not in ("v", "id")})
533
+ return out
534
+
535
+
536
+ def prerevision_record(record: dict) -> dict:
537
+ """The revision-1 (#825 release) shape of ``record``: the current writer
538
+ generation without the protocol revision, re-identified like
539
+ ``unstamped_record``. For fixtures and tests that simulate the previous
540
+ release; never used on a production write path."""
541
+ if not isinstance(record, dict) or _WPR not in record:
542
+ return record
543
+ out = {k: v for k, v in record.items() if k != _WPR}
544
+ rid = record.get("id")
545
+ if (record.get("t") in ("obs", "op") and isinstance(rid, str)
546
+ and rid.startswith("o:")):
547
+ core = {k: v for k, v in record.items() if k not in ("v", "id")}
548
+ if content_id(core) == rid:
549
+ out["id"] = content_id(
550
+ {k: v for k, v in out.items() if k not in ("v", "id")})
551
+ return out
552
+
553
+
554
+ def legacy_writer_view():
555
+ """A namespace exposing this module with every ``make_*`` builder wrapped by
556
+ ``unstamped_record``, so a fixture builder written against the module
557
+ produces the previous release's exact bytes (cross-references between
558
+ records stay consistent because every id is the unstamped one)."""
559
+ import types
560
+
561
+ def _wrap(fn):
562
+ def build(*args, **kwargs):
563
+ out = fn(*args, **kwargs)
564
+ if isinstance(out, list):
565
+ return [unstamped_record(item) for item in out]
566
+ return unstamped_record(out)
567
+ build.__name__ = fn.__name__
568
+ build.__doc__ = fn.__doc__
569
+ return build
570
+
571
+ def _legacy_protocol_resolution(**kwargs):
572
+ # The previous release's acknowledgement kind, which the current
573
+ # binary never emits (#825/#832 spec 5.1).
574
+ current = make_protocol_resolution(**kwargs)
575
+ return make_op(
576
+ at=current["at"], src=current["src"],
577
+ payload={**current["payload"], "kind": _PROTOCOL_RESOLUTION_KIND},
578
+ writer_generation=None)
579
+
580
+ attrs = {}
581
+ for name, value in globals().items():
582
+ if name.startswith("__"):
583
+ continue
584
+ if name.startswith("make_") and callable(value):
585
+ value = _wrap(value)
586
+ attrs[name] = value
587
+ attrs["make_protocol_resolution"] = _legacy_protocol_resolution
588
+ return types.SimpleNamespace(**attrs)
589
+
590
+
591
+ def event_content_hash(record: dict) -> str:
592
+ """The semantic content hash of an evt line. The writer-generation stamp is
593
+ transport metadata, so it is stripped before hashing: a current emission
594
+ that differs from a retained pre-stamp event only by ``wpg`` is the same
595
+ event (a duplicate), never a same-revision conflict."""
596
+ return _sha256_canonical(_lib_account_protocol.strip_stamp(record))
597
+
598
+
599
+ #: The previous release's exact-key repair acknowledgement. The current binary
600
+ #: never emits it: its validator would reject the writer-generation stamp, so
601
+ #: current acknowledgements use the versioned kind the previous release ignores
602
+ #: as an unknown op (#825/#832 spec 5.1). Current readers accept every kind.
404
603
  _PROTOCOL_RESOLUTION_KIND = "journal_protocol_resolution"
604
+ _PROTOCOL_RESOLUTION_V2_KIND = _lib_account_protocol.PROTOCOL_RESOLUTION_V2_KIND
605
+ #: #797/#823/#840 (spec 4.6 B5): the previous release's v2 validator rejects
606
+ #: the protocol revision stamp, so the current binary emits this successor.
607
+ PROTOCOL_RESOLUTION_V3_KIND = _lib_account_protocol.PROTOCOL_RESOLUTION_V3_KIND
608
+ PROTOCOL_RESOLUTION_KINDS = frozenset((
609
+ _PROTOCOL_RESOLUTION_KIND, _PROTOCOL_RESOLUTION_V2_KIND,
610
+ PROTOCOL_RESOLUTION_V3_KIND))
405
611
  _PROTOCOL_RESOLUTION_SRC = "journal-repair"
406
612
 
407
613
 
@@ -441,7 +647,7 @@ def make_protocol_resolution(
441
647
  at=at,
442
648
  src=_PROTOCOL_RESOLUTION_SRC,
443
649
  payload={
444
- "kind": _PROTOCOL_RESOLUTION_KIND,
650
+ "kind": PROTOCOL_RESOLUTION_V3_KIND,
445
651
  "violations": refs,
446
652
  "journal_high_water": {
447
653
  "segment": segment,
@@ -452,6 +658,17 @@ def make_protocol_resolution(
452
658
  )
453
659
 
454
660
 
661
+ def is_protocol_resolution(record) -> bool:
662
+ """Whether ``record`` is a journal-repair acknowledgement of either
663
+ version."""
664
+ return (
665
+ isinstance(record, dict)
666
+ and record.get("t") == "op"
667
+ and isinstance(record.get("payload"), dict)
668
+ and record["payload"].get("kind") in PROTOCOL_RESOLUTION_KINDS
669
+ )
670
+
671
+
455
672
  def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
456
673
  """Build a complete ``evt`` (derived) line record.
457
674
 
@@ -463,7 +680,8 @@ def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
463
680
  body = dict(payload)
464
681
  body["kind"] = kind
465
682
  return {"v": LINE_VERSION, "t": "evt", "id": id, "rev": rev,
466
- "at": at, "src": "ingest", "payload": body}
683
+ "at": at, "src": "ingest", "payload": body,
684
+ _WPG: _WRITER_GENERATION, _WPR: _WRITER_REVISION}
467
685
 
468
686
 
469
687
  # --------------------------------------------------------------------------
@@ -501,7 +719,11 @@ def _protocol_violation(
501
719
 
502
720
  def _validate_protocol_resolution(record: dict) -> dict:
503
721
  required = {"v", "t", "id", "at", "src", "payload"}
504
- if set(record) != required:
722
+ # #825/#832: a current record also carries the writer-generation stamp,
723
+ # which is part of its content-id core; #797/#823/#840 adds the protocol
724
+ # revision (spec 4.6 B5).
725
+ if set(record) not in (required, required | {_WPG},
726
+ required | {_WPG, _WPR}):
505
727
  raise JournalProtocolError(
506
728
  "journal protocol resolution record shape is invalid"
507
729
  )
@@ -521,6 +743,10 @@ def _validate_protocol_resolution(record: dict) -> dict:
521
743
  "src": record["src"],
522
744
  "payload": record["payload"],
523
745
  }
746
+ if _WPG in record:
747
+ core[_WPG] = record[_WPG]
748
+ if _WPR in record:
749
+ core[_WPR] = record[_WPR]
524
750
  if record.get("id") != content_id(core):
525
751
  raise JournalProtocolError(
526
752
  "journal protocol resolution record id does not match its content"
@@ -535,7 +761,7 @@ def _validate_protocol_resolution(record: dict) -> dict:
535
761
  raise JournalProtocolError(
536
762
  "journal protocol resolution payload shape is invalid"
537
763
  )
538
- if payload.get("kind") != _PROTOCOL_RESOLUTION_KIND:
764
+ if payload.get("kind") not in PROTOCOL_RESOLUTION_KINDS:
539
765
  raise JournalProtocolError(
540
766
  "journal protocol resolution kind is invalid"
541
767
  )
@@ -672,6 +898,8 @@ def make_correction_batch(
672
898
  "family": family,
673
899
  "action_count": len(cores),
674
900
  "actions_hash": actions_hash,
901
+ _WPG: _WRITER_GENERATION,
902
+ _WPR: _WRITER_REVISION,
675
903
  }
676
904
  records = [{**marker, "phase": "begin"}]
677
905
  for seq, core in enumerate(cores):
@@ -682,12 +910,35 @@ def make_correction_batch(
682
910
  **core,
683
911
  "batch": batch_id,
684
912
  "seq": seq,
913
+ _WPG: _WRITER_GENERATION,
914
+ _WPR: _WRITER_REVISION,
685
915
  }
686
916
  )
687
917
  records.append({**marker, "phase": "commit"})
688
918
  return records
689
919
 
690
920
 
921
+ def revision_batch_id(batch_id: str, begins) -> str:
922
+ """The batch id a retry of ``batch_id`` appends under (#797/#823/#840
923
+ spec 4.6 B1).
924
+
925
+ ``begins`` are the retained begin markers already journalled under
926
+ ``batch_id``. A retry reuses the id only when no begin exists or one
927
+ carries the current revision stamp. An interrupted batch an earlier
928
+ writer began without that stamp is never restamped: a revision-2 copy of
929
+ its lines would be a marker conflict under the original id (the fold
930
+ hashes markers byte-exactly), and a copy without the stamp would land
931
+ after the revision boundary, where admission excludes it. Its plan is
932
+ therefore authored afresh under a derived id, and the original begin
933
+ stays the inert, incomplete batch it was."""
934
+ begins = list(begins)
935
+ if not begins or any(
936
+ _lib_account_protocol.has_current_revision(begin)
937
+ for begin in begins):
938
+ return batch_id
939
+ return f"{batch_id}:wpr{_lib_account_protocol.PROTOCOL_REVISION}"
940
+
941
+
691
942
  def _validate_batch_marker(record: dict) -> dict:
692
943
  batch_id = record.get("id")
693
944
  if not isinstance(batch_id, str) or not batch_id:
@@ -769,7 +1020,7 @@ def _candidate_from_evt(record: dict, sequence: int) -> EffectiveEvent:
769
1020
  )
770
1021
  if not isinstance(record.get("payload"), dict):
771
1022
  raise JournalProtocolError("event payload must be an object")
772
- digest = _sha256_canonical(record)
1023
+ digest = event_content_hash(record)
773
1024
  return EffectiveEvent(
774
1025
  event_id=event_id,
775
1026
  rev=rev,
@@ -919,7 +1170,7 @@ def _fold_one(fold: SelectorFold, record, sequence: int) -> None:
919
1170
  if (
920
1171
  record_type == "op"
921
1172
  and isinstance(record.get("payload"), dict)
922
- and record["payload"].get("kind") == _PROTOCOL_RESOLUTION_KIND
1173
+ and record["payload"].get("kind") in PROTOCOL_RESOLUTION_KINDS
923
1174
  ):
924
1175
  resolutions.append(
925
1176
  (sequence, _validate_protocol_resolution(record))
@@ -155,12 +155,37 @@ class PrefixHashAccumulator:
155
155
 
156
156
  The framing is `journal_prefix_hash`'s, verbatim: per segment, the 4-byte
157
157
  big-endian name length, the name, the 8-byte big-endian data length, then
158
- the data. Completed segments are absorbed into a running `sha256`; only the
159
- segment currently being streamed is buffered, so residency is bounded by one
160
- segment. At a segment transition the caller passes the boundary the next
161
- record's `prior_high_water` will name, which is the ONE offset in the
162
- outgoing segment that can still be asked for; its digest is precomputed
163
- before the buffer is released.
158
+ the data. The data length precedes the data, so `begin_segment` takes the
159
+ exact byte count the pass will feed (`size`) and frames the segment header
160
+ with it; every chunk is then hashed as it arrives and NO segment bytes are
161
+ retained (#997 D12). The buffered form held the whole current segment in a
162
+ `bytearray` grown by `extend` — 410 MB on the maintainer's largest segment
163
+ — and the freed growth steps stayed resident in the macOS allocator.
164
+
165
+ A digest at a fully fed segment end (every transition, and every boundary
166
+ not short of a torn tail) comes from the streamed state. A digest at any
167
+ other offset — an evidence point inside the current segment, a boundary
168
+ short of a torn trailing line, a feed that ended short of its declared
169
+ size — re-reads the segment through `open_segment(name)` in 1 MiB chunks,
170
+ which the production callers bind to the journal's one segment-read seam,
171
+ so an evidence re-read is visible to it.
172
+
173
+ Consumed extent (#997 D12, revision 10). Per segment the accumulator also
174
+ tracks the bytes actually fed (`consumed`) and a raw SHA-256 of those bytes
175
+ alone (the witness). An offset beyond `consumed` raises. Every re-read reads
176
+ through `consumed`, not only through the requested offset: one pass frames
177
+ the prefix digest at the offset and hashes `[0, consumed)` raw, and the
178
+ digest is returned only when the read was not short and that raw hash
179
+ equals the witness. A torn tail an appender truncated and replaced after
180
+ the pass read it, or a line the reader assembled from old and replacement
181
+ bytes across two reads, can therefore never be certified. A registered
182
+ boundary keeps its segment's `consumed` and witness with its saved hash
183
+ state. A segment fed short of its declared size is re-framed from its
184
+ consumed bytes through the same verified re-read at the next
185
+ `begin_segment`; when that re-read is unavailable or fails verification
186
+ the accumulator is poisoned and every later digest raises. A missing
187
+ opener, a short re-read or a failed verification raises
188
+ `PrefixEvidenceUnavailable`.
164
189
  """
165
190
 
166
191
  # A transition can only ever register one boundary, and consecutive empty
@@ -176,10 +201,28 @@ class PrefixHashAccumulator:
176
201
  # entry that is still live.
177
202
  _MAX_BOUNDARIES = 16
178
203
 
179
- def __init__(self) -> None:
204
+ #: Chunk size of an evidence re-read through the opener.
205
+ _REREAD_CHUNK = 1 << 20
206
+
207
+ def __init__(self, *, open_segment=None) -> None:
208
+ self._open_segment = open_segment
209
+ #: The hash of every segment before the current one, absorbed.
180
210
  self._running = hashlib.sha256()
181
211
  self._current_name = None
182
- self._current = bytearray()
212
+ #: `_running` plus the current segment's header (framed with its
213
+ #: declared size) and every byte fed so far.
214
+ self._segment = None
215
+ self._declared = 0
216
+ #: Bytes of the current segment actually fed (its consumed extent),
217
+ #: and the raw SHA-256 of exactly those bytes (the witness).
218
+ self._fed = 0
219
+ self._witness = hashlib.sha256()
220
+ #: Set when a short segment could not be re-framed from verified
221
+ #: bytes: the running state no longer describes the journal.
222
+ self._poisoned = None
223
+ #: `(segment, offset) -> (state before that segment, final digest or
224
+ #: None, consumed, witness digest)`; a `None` final digest is re-read
225
+ #: through the opener, verified against the witness, when read.
183
226
  self._boundaries: dict = {}
184
227
  self._boundary_order: list = []
185
228
  #: Bytes fed into an evidence digest, reported as the `protocol_evidence`
@@ -189,55 +232,92 @@ class PrefixHashAccumulator:
189
232
 
190
233
  # -- feeding ----------------------------------------------------------
191
234
 
192
- def begin_segment(self, name: str, boundary=None) -> None:
193
- """Start `name`, absorbing whatever segment was open before it.
235
+ def begin_segment(self, name: str, boundary=None, *, size: int) -> None:
236
+ """Start `name`, which the pass will feed `size` bytes of, absorbing
237
+ whatever segment was open before it.
194
238
 
195
239
  `boundary` is the `(segment, offset)` the next record's evidence would
196
- name — i.e. the streaming loop's current `prior_high_water`. It is
197
- resolved and cached HERE because the outgoing segment's bytes are gone
198
- immediately afterwards.
240
+ name — i.e. the streaming loop's current `prior_high_water`. Its hash
241
+ state is kept HERE because the outgoing segment's state is folded into
242
+ the running hash immediately afterwards.
199
243
  """
200
244
  if boundary is not None:
201
245
  self._register_boundary(boundary)
202
246
  if self._current_name is not None:
203
- # `hashlib.update` accepts the buffer directly, so the outgoing
204
- # segment is framed WITHOUT a copy of itself. The maintainer's
205
- # largest segment is 410 MB, and copying it here briefly doubled
206
- # that at every transition.
207
- with memoryview(self._current) as data:
208
- self._absorb(self._current_name, data)
247
+ self._running = self._completed_segment()
209
248
  self._current_name = name
210
- self._current = bytearray()
211
-
212
- def extend(self, data: bytes) -> None:
213
- """Absorb raw bytes read from the segment currently open."""
214
- self._current.extend(data)
249
+ self._declared = int(size)
250
+ self._fed = 0
251
+ self._witness = hashlib.sha256()
252
+ self._segment = self._running.copy()
253
+ self._header(self._segment, name, self._declared)
254
+
255
+ def extend(self, data) -> None:
256
+ """Hash raw bytes read from the segment currently open."""
257
+ self._segment.update(data)
258
+ self._witness.update(data)
259
+ self._fed += len(data)
215
260
 
216
261
  # -- reading ----------------------------------------------------------
217
262
 
218
263
  def digest_at(self, high_water) -> "str | None":
219
- """`journal_prefix_hash(high_water)`, from the streamed bytes."""
264
+ """`journal_prefix_hash(high_water)`, from the streamed state."""
220
265
  if high_water is None:
221
266
  return None
222
267
  segment, offset = high_water
223
- if segment == self._current_name:
224
- digest = self._running.copy()
225
- with memoryview(self._current)[:offset] as data:
226
- self._frame(digest, segment, data)
227
- self.bytes_hashed += len(data)
228
- self.digests_computed += 1
229
- return "sha256:" + digest.hexdigest()
230
- cached = self._boundaries.get((segment, offset))
231
- if cached is None:
268
+ if self._poisoned is not None:
232
269
  raise PrefixEvidenceUnavailable(
233
- f"no streamed prefix evidence for {segment}@{offset}"
270
+ f"no verified prefix evidence for {segment}@{offset}: "
271
+ f"{self._poisoned}"
234
272
  )
235
- self.bytes_hashed += cached[1]
273
+ if segment == self._current_name:
274
+ if self._streamed_end(offset):
275
+ digest = self._segment.copy()
276
+ else:
277
+ digest = self._verified_rehash(
278
+ self._running, segment, offset, self._fed,
279
+ self._witness.digest())
280
+ else:
281
+ cached = self._boundaries.get((segment, offset))
282
+ if cached is None:
283
+ raise PrefixEvidenceUnavailable(
284
+ f"no streamed prefix evidence for {segment}@{offset}"
285
+ )
286
+ before, final, consumed, witness = cached
287
+ digest = (final.copy() if final is not None
288
+ else self._verified_rehash(
289
+ before, segment, offset, consumed, witness))
290
+ self.bytes_hashed += offset
236
291
  self.digests_computed += 1
237
- return cached[0]
292
+ return "sha256:" + digest.hexdigest()
238
293
 
239
294
  # -- internals --------------------------------------------------------
240
295
 
296
+ def _streamed_end(self, offset) -> bool:
297
+ """Whether the streamed state IS the digest at `offset`: the current
298
+ segment was fed exactly its declared size and `offset` is its end."""
299
+ return offset == self._fed == self._declared
300
+
301
+ def _completed_segment(self):
302
+ """The running hash with the current segment absorbed as fed."""
303
+ if self._fed == self._declared or self._poisoned is not None:
304
+ return self._segment
305
+ # The feed ended short of the declared size (the segment shrank under
306
+ # the pass): frame the bytes actually fed, as the buffered form did,
307
+ # from a re-read verified against them. Bytes that cannot be verified
308
+ # leave no running state that describes the journal, so every later
309
+ # digest raises rather than certifying one.
310
+ try:
311
+ return self._verified_rehash(
312
+ self._running, self._current_name, self._fed, self._fed,
313
+ self._witness.digest())
314
+ except (PrefixEvidenceUnavailable, OSError) as exc:
315
+ self._poisoned = (
316
+ f"short segment {self._current_name} could not be re-framed "
317
+ f"from its consumed bytes ({exc})"
318
+ )
319
+ return self._segment
320
+
241
321
  def _register_boundary(self, boundary) -> None:
242
322
  segment, offset = boundary
243
323
  if segment != self._current_name:
@@ -247,23 +327,61 @@ class PrefixHashAccumulator:
247
327
  key = (segment, offset)
248
328
  if key in self._boundaries:
249
329
  return
250
- digest = self._running.copy()
251
- with memoryview(self._current)[:offset] as data:
252
- self._frame(digest, segment, data)
253
- size = len(data)
254
- self._boundaries[key] = ("sha256:" + digest.hexdigest(), size)
330
+ final = (self._segment.copy() if self._streamed_end(offset)
331
+ else None)
332
+ self._boundaries[key] = (self._running.copy(), final, self._fed,
333
+ self._witness.digest())
255
334
  self._boundary_order.append(key)
256
335
  while len(self._boundary_order) > self._MAX_BOUNDARIES:
257
336
  self._boundaries.pop(self._boundary_order.pop(0), None)
258
337
 
259
- def _absorb(self, name, data) -> None:
260
- self._frame(self._running, name, data)
338
+ def _verified_rehash(self, before, name: str, offset: int,
339
+ consumed: int, witness: bytes):
340
+ """`before` plus `name`'s first `offset` bytes, re-read from the
341
+ segment through the opener in `_REREAD_CHUNK` pieces.
342
+
343
+ The read always runs through `consumed`, the bytes the pass fed, and
344
+ its raw SHA-256 must equal `witness`; only then are the bytes the
345
+ digest frames the bytes the pass read."""
346
+ if offset > consumed:
347
+ raise PrefixEvidenceUnavailable(
348
+ f"prefix evidence {name}@{offset} lies beyond the "
349
+ f"{consumed} bytes the pass consumed"
350
+ )
351
+ digest = before.copy()
352
+ self._header(digest, name, offset)
353
+ if consumed <= 0:
354
+ return digest
355
+ if self._open_segment is None:
356
+ raise PrefixEvidenceUnavailable(
357
+ f"no segment opener to re-read prefix evidence for "
358
+ f"{name}@{offset}"
359
+ )
360
+ raw = hashlib.sha256()
361
+ pos = 0
362
+ with self._open_segment(name) as handle:
363
+ while pos < consumed:
364
+ chunk = handle.read(min(self._REREAD_CHUNK, consumed - pos))
365
+ if not chunk:
366
+ raise PrefixEvidenceUnavailable(
367
+ f"short re-read of prefix evidence for {name}@{offset}"
368
+ )
369
+ raw.update(chunk)
370
+ if pos < offset:
371
+ with memoryview(chunk)[:offset - pos] as framed:
372
+ digest.update(framed)
373
+ pos += len(chunk)
374
+ if raw.digest() != witness:
375
+ raise PrefixEvidenceUnavailable(
376
+ f"segment {name} no longer holds the {consumed} bytes the "
377
+ f"pass consumed; prefix evidence {name}@{offset} is not "
378
+ f"certified"
379
+ )
380
+ return digest
261
381
 
262
382
  @staticmethod
263
- def _frame(digest, name: str, data) -> None:
264
- """`data` is any bytes-like buffer; `hashlib` consumes it in place."""
383
+ def _header(digest, name: str, length: int) -> None:
265
384
  encoded = name.encode("utf-8")
266
385
  digest.update(len(encoded).to_bytes(4, "big"))
267
386
  digest.update(encoded)
268
- digest.update(len(data).to_bytes(8, "big"))
269
- digest.update(data)
387
+ digest.update(int(length).to_bytes(8, "big"))