cctally 1.82.0 → 1.83.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +52 -74
  3. package/bin/_cctally_alerts.py +8 -1
  4. package/bin/_cctally_cache.py +963 -149
  5. package/bin/_cctally_config.py +43 -4
  6. package/bin/_cctally_core.py +933 -759
  7. package/bin/_cctally_dashboard.py +157 -47
  8. package/bin/_cctally_dashboard_cache_report.py +13 -6
  9. package/bin/_cctally_dashboard_conversation.py +1 -0
  10. package/bin/_cctally_dashboard_envelope.py +186 -8
  11. package/bin/_cctally_dashboard_share.py +60 -20
  12. package/bin/_cctally_dashboard_sources.py +427 -128
  13. package/bin/_cctally_db.py +605 -128
  14. package/bin/_cctally_doctor.py +413 -28
  15. package/bin/_cctally_five_hour.py +12 -5
  16. package/bin/_cctally_journal.py +2050 -156
  17. package/bin/_cctally_journal_repair.py +519 -0
  18. package/bin/_cctally_milestone_history.py +142 -56
  19. package/bin/_cctally_milestones.py +179 -111
  20. package/bin/_cctally_parser.py +42 -0
  21. package/bin/_cctally_project.py +24 -18
  22. package/bin/_cctally_quota.py +139 -25
  23. package/bin/_cctally_record.py +279 -108
  24. package/bin/_cctally_rederive.py +1052 -0
  25. package/bin/_cctally_reporting.py +58 -53
  26. package/bin/_cctally_setup.py +1 -0
  27. package/bin/_cctally_source_analytics.py +4 -1
  28. package/bin/_cctally_statusline.py +11 -11
  29. package/bin/_cctally_store.py +1039 -31
  30. package/bin/_cctally_sync_week.py +17 -8
  31. package/bin/_cctally_tui.py +421 -54
  32. package/bin/_cctally_update.py +133 -8
  33. package/bin/_cctally_weekrefs.py +14 -0
  34. package/bin/_lib_aggregators.py +10 -6
  35. package/bin/_lib_cache_report.py +101 -9
  36. package/bin/_lib_codex_pools.py +82 -0
  37. package/bin/_lib_conversation_query.py +126 -33
  38. package/bin/_lib_dashboard_sources.py +126 -1
  39. package/bin/_lib_diff_kernel.py +28 -15
  40. package/bin/_lib_doctor.py +342 -4
  41. package/bin/_lib_journal.py +924 -2
  42. package/bin/_lib_jsonl.py +43 -14
  43. package/bin/_lib_pricing.py +140 -21
  44. package/bin/_lib_readme_refresh.py +401 -0
  45. package/bin/_lib_rederive.py +395 -0
  46. package/bin/_lib_share.py +58 -2
  47. package/bin/cctally +56 -8
  48. package/dashboard/static/assets/{index-DJP4gEB7.js → index-3bgCMVHb.js} +52 -52
  49. package/dashboard/static/assets/index-D27EIHEI.css +1 -0
  50. package/dashboard/static/dashboard.html +2 -2
  51. package/package.json +6 -1
  52. package/dashboard/static/assets/index-Dk1nplOz.css +0 -1
@@ -22,14 +22,15 @@ and unknown `t` values:
22
22
  - `id`: obs/op carry a content digest over (t, at, src[, provider], payload);
23
23
  bootstrap-exported lines use `b:<table>:<rowid>`; evt lines carry their target
24
24
  table's full natural key with logical-id FK refs (spec §4.2 FK rule).
25
- - `rev`: evt revision, default 0; the fold keeps the highest rev per id
26
- (reserved for the deferred correction subsystem, spec §5.5).
25
+ - `rev`: evt revision, default 0; completed correction batches select the
26
+ highest non-conflicting revision per id before fold (#372 Task A).
27
27
  """
28
28
  from __future__ import annotations
29
29
 
30
30
  import datetime as dt
31
31
  import hashlib
32
32
  import json
33
+ from dataclasses import dataclass
33
34
 
34
35
  LINE_VERSION = 1
35
36
 
@@ -37,6 +38,154 @@ SEGMENT_PREFIX = "observations-"
37
38
  BOOTSTRAP_PREFIX = "bootstrap-"
38
39
 
39
40
 
41
+ class JournalProtocolError(ValueError):
42
+ """A known journal record violates the revision/correction protocol."""
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class EffectiveEvent:
47
+ """The selected state for one opaque logical event id."""
48
+
49
+ event_id: str
50
+ rev: int
51
+ status: str
52
+ content_hash: str
53
+ batch_id: str | None
54
+ record: dict | None
55
+ sequence: int
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class EventConflict:
60
+ """A divergent same-revision group, quarantined rather than fatal (#374).
61
+
62
+ Two lines carrying the SAME `(id, rev)` but different content are a protocol
63
+ violation the append-only journal cannot un-write: a live cycle that appends
64
+ its evt and then aborts before the transactional `journal_id` stamp leaves
65
+ the divergent line on disk forever. Raising on read wedged every subsequent
66
+ `rebuild_stats_index`, which is the epoch-1002 release blocker. The selector
67
+ now records the group here and falls through to the lowest-sequence
68
+ provisional winner, so the index is complete and usable while the ambiguity
69
+ is REPORTED rather than silently guessed at. `db rederive` resolves it by
70
+ superseding the group at `rev + 1`.
71
+ """
72
+
73
+ event_id: str
74
+ rev: int
75
+ content_hashes: tuple # every distinct hash in the group, sorted
76
+ selected_hash: str # the provisional (lowest-sequence) winner
77
+
78
+ def to_dict(self) -> dict:
79
+ """camelCase, JSON-serializable — the `journalConflicts` wire shape."""
80
+ return {
81
+ "eventId": self.event_id,
82
+ "revision": self.rev,
83
+ "contentHashes": list(self.content_hashes),
84
+ "selectedHash": self.selected_hash,
85
+ }
86
+
87
+
88
+ @dataclass(frozen=True)
89
+ class ProtocolViolation:
90
+ """One structural correction-batch violation with bounded identity evidence.
91
+
92
+ Unlike an :class:`EventConflict`, no provisional action is selected. The
93
+ whole batch is tainted and omitted. ``evidence`` is an immutable tuple of at
94
+ most eight scalar key/value pairs so diagnostics can identify the exact
95
+ journal state without echoing arbitrary payloads.
96
+ """
97
+
98
+ batch_id: str
99
+ kind: str
100
+ evidence: tuple[tuple[str, object], ...]
101
+
102
+ def __post_init__(self) -> None:
103
+ if (
104
+ not self.batch_id
105
+ or not self.kind
106
+ or not 1 <= len(self.evidence) <= 8
107
+ or len({key for key, _value in self.evidence}) != len(self.evidence)
108
+ or any(
109
+ not isinstance(key, str)
110
+ or not key
111
+ or type(value) not in {int, str}
112
+ for key, value in self.evidence
113
+ )
114
+ ):
115
+ raise ValueError("protocol violation evidence must be bounded scalars")
116
+
117
+ @property
118
+ def fingerprint(self) -> str:
119
+ """Stable identity for Task B's exact append-only acknowledgement."""
120
+ return _sha256_canonical(
121
+ {
122
+ "batchId": self.batch_id,
123
+ "kind": self.kind,
124
+ "evidence": dict(self.evidence),
125
+ }
126
+ )
127
+
128
+ def to_dict(self) -> dict:
129
+ """Stable camelCase JSON wire shape."""
130
+ return {
131
+ "batchId": self.batch_id,
132
+ "kind": self.kind,
133
+ "evidence": dict(self.evidence),
134
+ "fingerprint": self.fingerprint,
135
+ }
136
+
137
+
138
+ @dataclass(frozen=True)
139
+ class AcknowledgedProtocolViolation:
140
+ """One exact tainted-batch violation plus its durable operator audit."""
141
+
142
+ violation: ProtocolViolation
143
+ audit_id: str
144
+ journal_high_water: tuple[str, int]
145
+ journal_prefix_hash: str
146
+
147
+ @property
148
+ def batch_id(self) -> str:
149
+ return self.violation.batch_id
150
+
151
+ @property
152
+ def kind(self) -> str:
153
+ return self.violation.kind
154
+
155
+ @property
156
+ def fingerprint(self) -> str:
157
+ return self.violation.fingerprint
158
+
159
+ def to_dict(self) -> dict:
160
+ return {
161
+ **self.violation.to_dict(),
162
+ "auditId": self.audit_id,
163
+ "journalHighWater": {
164
+ "segment": self.journal_high_water[0],
165
+ "offset": self.journal_high_water[1],
166
+ },
167
+ "journalPrefixHash": self.journal_prefix_hash,
168
+ }
169
+
170
+
171
+ @dataclass(frozen=True)
172
+ class EffectiveSelection:
173
+ """Active fold records plus active/tombstoned metadata keyed by event id.
174
+
175
+ `conflicts` (#374) is additive and defaults to empty: the quarantined
176
+ same-revision groups whose `rev` equals the WINNING revision for their event
177
+ id. A group a completed correction batch has superseded at a higher revision
178
+ reports nothing — that filter is what makes `db rederive` a real remedy.
179
+ """
180
+
181
+ active: list[dict]
182
+ by_id: dict[str, EffectiveEvent]
183
+ completed_batches: frozenset[str]
184
+ conflicts: tuple = ()
185
+ protocol_violations: tuple = ()
186
+ acknowledged_protocol_violations: tuple = ()
187
+
188
+
40
189
  # --------------------------------------------------------------------------
41
190
  # line codec
42
191
  # --------------------------------------------------------------------------
@@ -129,6 +278,57 @@ def make_op(at: str, src: str, payload: dict) -> dict:
129
278
  return {"v": LINE_VERSION, **core, "id": content_id(core)}
130
279
 
131
280
 
281
+ _PROTOCOL_RESOLUTION_KIND = "journal_protocol_resolution"
282
+ _PROTOCOL_RESOLUTION_SRC = "journal-repair"
283
+
284
+
285
+ def make_protocol_resolution(
286
+ *,
287
+ at: str,
288
+ violations: list[ProtocolViolation],
289
+ journal_high_water: tuple[str, int],
290
+ journal_prefix_hash: str,
291
+ ) -> dict:
292
+ """Build one deterministic audit decision over exact violation identities."""
293
+ if not isinstance(at, str) or not at:
294
+ raise ValueError("protocol resolution at must be a non-empty string")
295
+ if not isinstance(violations, list) or not violations:
296
+ raise ValueError("protocol resolution requires at least one violation")
297
+ refs = [
298
+ {
299
+ "batch_id": violation.batch_id,
300
+ "kind": violation.kind,
301
+ "fingerprint": violation.fingerprint,
302
+ }
303
+ for violation in sorted(violations, key=lambda item: item.fingerprint)
304
+ ]
305
+ if len({item["fingerprint"] for item in refs}) != len(refs):
306
+ raise ValueError("protocol resolution violations must be unique")
307
+ segment, offset = journal_high_water
308
+ if not isinstance(segment, str) or not segment or type(offset) is not int or offset < 0:
309
+ raise ValueError("protocol resolution high-water is invalid")
310
+ if (
311
+ not isinstance(journal_prefix_hash, str)
312
+ or not journal_prefix_hash.startswith("sha256:")
313
+ or len(journal_prefix_hash) != 71
314
+ or any(ch not in "0123456789abcdef" for ch in journal_prefix_hash[7:])
315
+ ):
316
+ raise ValueError("protocol resolution prefix hash is invalid")
317
+ return make_op(
318
+ at=at,
319
+ src=_PROTOCOL_RESOLUTION_SRC,
320
+ payload={
321
+ "kind": _PROTOCOL_RESOLUTION_KIND,
322
+ "violations": refs,
323
+ "journal_high_water": {
324
+ "segment": segment,
325
+ "offset": offset,
326
+ },
327
+ "journal_prefix_hash": journal_prefix_hash,
328
+ },
329
+ )
330
+
331
+
132
332
  def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
133
333
  """Build a complete ``evt`` (derived) line record.
134
334
 
@@ -136,12 +336,734 @@ def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
136
336
  fold-dispatch family written into ``payload["kind"]`` (spec §5.3). The
137
337
  caller's ``payload`` dict is not mutated. ``src`` is always ``"ingest"`` —
138
338
  evt lines exist only because the ingester derived them."""
339
+ _validate_revision(rev)
139
340
  body = dict(payload)
140
341
  body["kind"] = kind
141
342
  return {"v": LINE_VERSION, "t": "evt", "id": id, "rev": rev,
142
343
  "at": at, "src": "ingest", "payload": body}
143
344
 
144
345
 
346
+ # --------------------------------------------------------------------------
347
+ # effective revisions + crash-safe correction batches (#372 Task A)
348
+ # --------------------------------------------------------------------------
349
+
350
+ def _validate_revision(rev: object) -> int:
351
+ if type(rev) is not int or rev < 0:
352
+ raise JournalProtocolError("event rev must be a non-negative integer")
353
+ return rev
354
+
355
+
356
+ def event_revision(record: dict) -> int:
357
+ """Return a strict event revision, defaulting an absent ``rev`` to zero."""
358
+ return _validate_revision(record.get("rev", 0))
359
+
360
+
361
+ def _sha256_canonical(value: object) -> str:
362
+ raw = json.dumps(
363
+ value, separators=(",", ":"), sort_keys=True, ensure_ascii=False
364
+ ).encode("utf-8")
365
+ return "sha256:" + hashlib.sha256(raw).hexdigest()
366
+
367
+
368
+ def _protocol_violation(
369
+ batch_id: str, kind: str, **evidence: int | str
370
+ ) -> ProtocolViolation:
371
+ """Build one bounded, immutable structural-violation result."""
372
+ return ProtocolViolation(
373
+ batch_id=batch_id,
374
+ kind=kind,
375
+ evidence=tuple(evidence.items()),
376
+ )
377
+
378
+
379
+ def _validate_protocol_resolution(record: dict) -> dict:
380
+ required = {"v", "t", "id", "at", "src", "payload"}
381
+ if set(record) != required:
382
+ raise JournalProtocolError(
383
+ "journal protocol resolution record shape is invalid"
384
+ )
385
+ if (
386
+ record.get("v") != LINE_VERSION
387
+ or record.get("t") != "op"
388
+ or record.get("src") != _PROTOCOL_RESOLUTION_SRC
389
+ or not isinstance(record.get("at"), str)
390
+ or not record["at"]
391
+ ):
392
+ raise JournalProtocolError(
393
+ "journal protocol resolution record identity is invalid"
394
+ )
395
+ core = {
396
+ "t": record["t"],
397
+ "at": record["at"],
398
+ "src": record["src"],
399
+ "payload": record["payload"],
400
+ }
401
+ if record.get("id") != content_id(core):
402
+ raise JournalProtocolError(
403
+ "journal protocol resolution record id does not match its content"
404
+ )
405
+ payload = record.get("payload")
406
+ if not isinstance(payload, dict) or set(payload) != {
407
+ "kind",
408
+ "violations",
409
+ "journal_high_water",
410
+ "journal_prefix_hash",
411
+ }:
412
+ raise JournalProtocolError(
413
+ "journal protocol resolution payload shape is invalid"
414
+ )
415
+ if payload.get("kind") != _PROTOCOL_RESOLUTION_KIND:
416
+ raise JournalProtocolError(
417
+ "journal protocol resolution kind is invalid"
418
+ )
419
+ high_water = payload.get("journal_high_water")
420
+ if not isinstance(high_water, dict) or set(high_water) != {"segment", "offset"}:
421
+ raise JournalProtocolError(
422
+ "journal protocol resolution high-water shape is invalid"
423
+ )
424
+ segment = high_water.get("segment")
425
+ offset = high_water.get("offset")
426
+ if (
427
+ not isinstance(segment, str)
428
+ or not segment
429
+ or type(offset) is not int
430
+ or offset < 0
431
+ ):
432
+ raise JournalProtocolError(
433
+ "journal protocol resolution high-water is invalid"
434
+ )
435
+ prefix_hash = payload.get("journal_prefix_hash")
436
+ if (
437
+ not isinstance(prefix_hash, str)
438
+ or not prefix_hash.startswith("sha256:")
439
+ or len(prefix_hash) != 71
440
+ or any(ch not in "0123456789abcdef" for ch in prefix_hash[7:])
441
+ ):
442
+ raise JournalProtocolError(
443
+ "journal protocol resolution prefix hash is invalid"
444
+ )
445
+ refs = payload.get("violations")
446
+ if not isinstance(refs, list) or not refs:
447
+ raise JournalProtocolError(
448
+ "journal protocol resolution requires exact violations"
449
+ )
450
+ normalized_refs = []
451
+ for ref in refs:
452
+ if not isinstance(ref, dict) or set(ref) != {
453
+ "batch_id",
454
+ "kind",
455
+ "fingerprint",
456
+ }:
457
+ raise JournalProtocolError(
458
+ "journal protocol resolution violation shape is invalid"
459
+ )
460
+ if any(
461
+ not isinstance(ref.get(key), str) or not ref[key]
462
+ for key in ("batch_id", "kind", "fingerprint")
463
+ ):
464
+ raise JournalProtocolError(
465
+ "journal protocol resolution violation identity is invalid"
466
+ )
467
+ normalized_refs.append(dict(ref))
468
+ if len({ref["fingerprint"] for ref in normalized_refs}) != len(normalized_refs):
469
+ raise JournalProtocolError(
470
+ "journal protocol resolution violations must be unique"
471
+ )
472
+ return {
473
+ "audit_id": record["id"],
474
+ "journal_high_water": (segment, offset),
475
+ "journal_prefix_hash": prefix_hash,
476
+ "violations": normalized_refs,
477
+ }
478
+
479
+
480
+ def _correction_action_core(record: dict) -> dict:
481
+ return {
482
+ "action": record["action"],
483
+ "id": record["id"],
484
+ "rev": record["rev"],
485
+ "at": record["at"],
486
+ "src": record["src"],
487
+ "payload": record["payload"],
488
+ }
489
+
490
+
491
+ def _validate_action_core(action: dict) -> dict:
492
+ required = {"action", "id", "rev", "at", "payload"}
493
+ if not isinstance(action, dict) or set(action) != required:
494
+ raise JournalProtocolError(
495
+ "correction action must contain action,id,rev,at,payload"
496
+ )
497
+ action_type = action.get("action")
498
+ if action_type not in {"replace", "tombstone"}:
499
+ raise JournalProtocolError("correction action must be replace or tombstone")
500
+ event_id = action.get("id")
501
+ if not isinstance(event_id, str) or not event_id:
502
+ raise JournalProtocolError("correction action id must be a non-empty string")
503
+ _validate_revision(action.get("rev"))
504
+ if not isinstance(action.get("at"), str) or not action["at"]:
505
+ raise JournalProtocolError("correction action at must be a non-empty string")
506
+ payload = action.get("payload")
507
+ if action_type == "replace":
508
+ if not isinstance(payload, dict) or not isinstance(payload.get("kind"), str):
509
+ raise JournalProtocolError(
510
+ "replacement correction payload must contain a string kind"
511
+ )
512
+ elif payload is not None:
513
+ raise JournalProtocolError("tombstone correction payload must be null")
514
+ return {
515
+ "action": action_type,
516
+ "id": event_id,
517
+ "rev": action["rev"],
518
+ "at": action["at"],
519
+ "src": "rederive",
520
+ "payload": dict(payload) if isinstance(payload, dict) else None,
521
+ }
522
+
523
+
524
+ def make_correction_batch(
525
+ *, batch_id: str, family: str, at: str, actions: list[dict]
526
+ ) -> list[dict]:
527
+ """Build begin/actions/commit records for one atomic correction batch.
528
+
529
+ Callers append the returned records in order. A crash before the final
530
+ commit marker leaves an incomplete batch that the effective selector
531
+ ignores.
532
+ """
533
+ if not isinstance(batch_id, str) or not batch_id:
534
+ raise JournalProtocolError("correction batch id must be a non-empty string")
535
+ if not isinstance(family, str) or not family:
536
+ raise JournalProtocolError("correction family must be a non-empty string")
537
+ if not isinstance(at, str) or not at:
538
+ raise JournalProtocolError("correction batch at must be a non-empty string")
539
+ if not isinstance(actions, list):
540
+ raise JournalProtocolError("correction actions must be a list")
541
+ cores = [_validate_action_core(action) for action in actions]
542
+ actions_hash = _sha256_canonical(cores)
543
+ marker = {
544
+ "v": LINE_VERSION,
545
+ "t": "correction_batch",
546
+ "id": batch_id,
547
+ "at": at,
548
+ "src": "rederive",
549
+ "family": family,
550
+ "action_count": len(cores),
551
+ "actions_hash": actions_hash,
552
+ }
553
+ records = [{**marker, "phase": "begin"}]
554
+ for seq, core in enumerate(cores):
555
+ records.append(
556
+ {
557
+ "v": LINE_VERSION,
558
+ "t": "correction",
559
+ **core,
560
+ "batch": batch_id,
561
+ "seq": seq,
562
+ }
563
+ )
564
+ records.append({**marker, "phase": "commit"})
565
+ return records
566
+
567
+
568
+ def _validate_batch_marker(record: dict) -> dict:
569
+ batch_id = record.get("id")
570
+ if not isinstance(batch_id, str) or not batch_id:
571
+ raise JournalProtocolError("correction batch id must be a non-empty string")
572
+ phase = record.get("phase")
573
+ if phase not in {"begin", "commit"}:
574
+ raise JournalProtocolError("correction batch phase must be begin or commit")
575
+ family = record.get("family")
576
+ if not isinstance(family, str) or not family:
577
+ raise JournalProtocolError("correction batch family must be a non-empty string")
578
+ at = record.get("at")
579
+ if not isinstance(at, str) or not at:
580
+ raise JournalProtocolError("correction batch at must be a non-empty string")
581
+ if record.get("src") != "rederive":
582
+ raise JournalProtocolError("correction batch src must be rederive")
583
+ count = record.get("action_count")
584
+ if type(count) is not int or count < 0:
585
+ raise JournalProtocolError(
586
+ "correction batch action_count must be a non-negative integer"
587
+ )
588
+ actions_hash = record.get("actions_hash")
589
+ if (
590
+ not isinstance(actions_hash, str)
591
+ or not actions_hash.startswith("sha256:")
592
+ or len(actions_hash) != 71
593
+ ):
594
+ raise JournalProtocolError("correction batch actions_hash is invalid")
595
+ try:
596
+ int(actions_hash[7:], 16)
597
+ except ValueError as exc:
598
+ raise JournalProtocolError("correction batch actions_hash is invalid") from exc
599
+ return {
600
+ "v": record.get("v", LINE_VERSION),
601
+ "t": "correction_batch",
602
+ "id": batch_id,
603
+ "at": at,
604
+ "src": "rederive",
605
+ "family": family,
606
+ "action_count": count,
607
+ "actions_hash": actions_hash,
608
+ }
609
+
610
+
611
+ def _validate_correction_record(record: dict) -> dict:
612
+ batch_id = record.get("batch")
613
+ if not isinstance(batch_id, str) or not batch_id:
614
+ raise JournalProtocolError("correction batch must be a non-empty string")
615
+ seq = record.get("seq")
616
+ if type(seq) is not int or seq < 0:
617
+ raise JournalProtocolError("correction seq must be a non-negative integer")
618
+ if record.get("src") != "rederive":
619
+ raise JournalProtocolError("correction src must be rederive")
620
+ core = _validate_action_core(
621
+ {
622
+ "action": record.get("action"),
623
+ "id": record.get("id"),
624
+ "rev": record.get("rev"),
625
+ "at": record.get("at"),
626
+ "payload": record.get("payload"),
627
+ }
628
+ )
629
+ return {
630
+ "v": record.get("v", LINE_VERSION),
631
+ "t": "correction",
632
+ **core,
633
+ "batch": batch_id,
634
+ "seq": seq,
635
+ }
636
+
637
+
638
+ def _candidate_from_evt(record: dict, sequence: int) -> EffectiveEvent:
639
+ event_id = record.get("id")
640
+ if not isinstance(event_id, str) or not event_id:
641
+ raise JournalProtocolError("event id must be a non-empty string")
642
+ rev = event_revision(record)
643
+ if rev != 0:
644
+ raise JournalProtocolError(
645
+ "ordinary evt revision must be 0; use a completed correction batch"
646
+ )
647
+ if not isinstance(record.get("payload"), dict):
648
+ raise JournalProtocolError("event payload must be an object")
649
+ digest = _sha256_canonical(record)
650
+ return EffectiveEvent(
651
+ event_id=event_id,
652
+ rev=rev,
653
+ status="active",
654
+ content_hash=digest,
655
+ batch_id=None,
656
+ record=record,
657
+ sequence=sequence,
658
+ )
659
+
660
+
661
+ def _candidate_from_correction(record: dict, sequence: int) -> EffectiveEvent:
662
+ core = _correction_action_core(record)
663
+ if record["action"] == "tombstone":
664
+ return EffectiveEvent(
665
+ event_id=record["id"],
666
+ rev=record["rev"],
667
+ status="tombstone",
668
+ content_hash=_sha256_canonical(core),
669
+ batch_id=record["batch"],
670
+ record=None,
671
+ sequence=sequence,
672
+ )
673
+ event = {
674
+ "v": LINE_VERSION,
675
+ "t": "evt",
676
+ "id": record["id"],
677
+ "rev": record["rev"],
678
+ "at": record["at"],
679
+ "src": record["src"],
680
+ "payload": dict(record["payload"]),
681
+ }
682
+ return EffectiveEvent(
683
+ event_id=record["id"],
684
+ rev=record["rev"],
685
+ status="active",
686
+ content_hash=_sha256_canonical(event),
687
+ batch_id=record["batch"],
688
+ record=event,
689
+ sequence=sequence,
690
+ )
691
+
692
+
693
+ def is_legacy_quota_arming_record(record: dict | None) -> bool:
694
+ """Recognize a pre-#372 qaa record whose natural id was reused."""
695
+ if not isinstance(record, dict) or event_revision(record) != 0:
696
+ return False
697
+ payload = record.get("payload") or {}
698
+ return (
699
+ payload.get("kind") == "quota_alert_arming"
700
+ and "journal_identity_version" not in payload
701
+ )
702
+
703
+
704
+ def _is_legacy_quota_arming_state(candidate: EffectiveEvent) -> bool:
705
+ return (
706
+ candidate.status == "active"
707
+ and is_legacy_quota_arming_record(candidate.record)
708
+ )
709
+
710
+
711
+ def resolve_effective_events(
712
+ records,
713
+ *,
714
+ protocol_prefix_evidence=(),
715
+ ) -> EffectiveSelection:
716
+ """Validate correction batches and select one highest revision per evt id.
717
+
718
+ Three failure classes, deliberately asymmetric (#374 §5, #402 Task A):
719
+
720
+ - Divergent same-revision EVENTS are **quarantined**: the lowest-sequence
721
+ candidate becomes the provisional winner and the group is reported on
722
+ `EffectiveSelection.conflicts`. The journal is append-only, so a divergent
723
+ line can never be un-written; raising here wedged every rebuild forever.
724
+ - The seven enumerated **structural** correction-batch violations taint their
725
+ entire batch. No action from it is a candidate; the selector continues
726
+ with distinct valid batches and reports a bounded `ProtocolViolation`.
727
+ - Invalid marker/action field shapes and every other out-of-scope
728
+ `JournalProtocolError` remain fatal. Unknown record types remain ignored.
729
+ """
730
+ candidates: list[EffectiveEvent] = []
731
+ markers: dict[str, dict[str, tuple[dict, str, str, int]]] = {}
732
+ actions: dict[str, dict[int, tuple[dict, str, int]]] = {}
733
+ resolutions: list[tuple[int, dict]] = []
734
+ tainted_batches: set[str] = set()
735
+ violations: dict[tuple[str, str, str], ProtocolViolation] = {}
736
+ violation_available_after: dict[str, int] = {}
737
+
738
+ def taint(violation: ProtocolViolation, *, available_after: int) -> None:
739
+ """Taint one batch and retain every distinct violation identity."""
740
+ tainted_batches.add(violation.batch_id)
741
+ violations[
742
+ (violation.batch_id, violation.kind, violation.fingerprint)
743
+ ] = violation
744
+ violation_available_after[violation.fingerprint] = min(
745
+ available_after,
746
+ violation_available_after.get(
747
+ violation.fingerprint,
748
+ available_after,
749
+ ),
750
+ )
751
+
752
+ for sequence, record in enumerate(records):
753
+ if not isinstance(record, dict):
754
+ continue
755
+ record_type = record.get("t")
756
+ if record_type == "evt":
757
+ candidates.append(_candidate_from_evt(record, sequence))
758
+ continue
759
+ if (
760
+ record_type == "op"
761
+ and isinstance(record.get("payload"), dict)
762
+ and record["payload"].get("kind") == _PROTOCOL_RESOLUTION_KIND
763
+ ):
764
+ resolutions.append(
765
+ (sequence, _validate_protocol_resolution(record))
766
+ )
767
+ continue
768
+ if record_type == "correction_batch":
769
+ normalized = _validate_batch_marker(record)
770
+ batch_id = normalized["id"]
771
+ phase = record["phase"]
772
+ digest = _sha256_canonical(record)
773
+ marker_identity = dict(record)
774
+ marker_identity.pop("phase", None)
775
+ identity_digest = _sha256_canonical(marker_identity)
776
+ prior = markers.setdefault(batch_id, {}).get(phase)
777
+ if prior is not None and prior[1] != digest:
778
+ taint(
779
+ _protocol_violation(
780
+ batch_id,
781
+ "marker_conflict",
782
+ phase=phase,
783
+ firstRecordHash=prior[1],
784
+ conflictingRecordHash=digest,
785
+ ),
786
+ available_after=max(prior[3], sequence),
787
+ )
788
+ if prior is None:
789
+ markers[batch_id][phase] = (
790
+ normalized,
791
+ digest,
792
+ identity_digest,
793
+ sequence,
794
+ )
795
+ continue
796
+ if record_type == "correction":
797
+ normalized = _validate_correction_record(record)
798
+ batch_id = normalized["batch"]
799
+ seq = normalized["seq"]
800
+ digest = _sha256_canonical(record)
801
+ prior = actions.setdefault(batch_id, {}).get(seq)
802
+ if prior is not None and prior[1] != digest:
803
+ taint(
804
+ _protocol_violation(
805
+ batch_id,
806
+ "action_sequence_conflict",
807
+ actionSequence=seq,
808
+ firstRecordHash=prior[1],
809
+ conflictingRecordHash=digest,
810
+ ),
811
+ available_after=max(prior[2], sequence),
812
+ )
813
+ if prior is None:
814
+ actions[batch_id][seq] = (normalized, digest, sequence)
815
+
816
+ completed: set[str] = set()
817
+ for batch_id in sorted(set(markers) | set(actions)):
818
+ batch_markers = markers.get(batch_id, {})
819
+ begin = batch_markers.get("begin")
820
+ commit = batch_markers.get("commit")
821
+ if commit is not None and begin is None:
822
+ _commit_core, commit_hash, _commit_identity, commit_sequence = commit
823
+ taint(
824
+ _protocol_violation(
825
+ batch_id,
826
+ "commit_without_begin",
827
+ commitSequence=commit_sequence,
828
+ commitRecordHash=commit_hash,
829
+ ),
830
+ available_after=commit_sequence,
831
+ )
832
+ continue
833
+ if begin is None or commit is None:
834
+ continue
835
+ begin_core, _begin_hash, begin_identity, begin_sequence = begin
836
+ commit_core, _commit_hash, commit_identity, commit_sequence = commit
837
+ if begin_core != commit_core or begin_identity != commit_identity:
838
+ taint(
839
+ _protocol_violation(
840
+ batch_id,
841
+ "marker_manifest_mismatch",
842
+ beginSequence=begin_sequence,
843
+ commitSequence=commit_sequence,
844
+ beginIdentityHash=begin_identity,
845
+ commitIdentityHash=commit_identity,
846
+ ),
847
+ available_after=max(begin_sequence, commit_sequence),
848
+ )
849
+ if begin_sequence >= commit_sequence:
850
+ taint(
851
+ _protocol_violation(
852
+ batch_id,
853
+ "record_order_violation",
854
+ beginSequence=begin_sequence,
855
+ commitSequence=commit_sequence,
856
+ recordOrderHash=_sha256_canonical(
857
+ [begin_sequence, commit_sequence]
858
+ ),
859
+ ),
860
+ available_after=max(begin_sequence, commit_sequence),
861
+ )
862
+ batch_actions = actions.get(batch_id, {})
863
+ count = begin_core["action_count"]
864
+ present_sequences = sorted(batch_actions)
865
+ complete_action_sequence = present_sequences == list(range(count))
866
+ if not complete_action_sequence:
867
+ taint(
868
+ _protocol_violation(
869
+ batch_id,
870
+ "manifest_action_sequence_mismatch",
871
+ expectedActionCount=count,
872
+ actualActionCount=len(present_sequences),
873
+ presentSequencesHash=_sha256_canonical(present_sequences),
874
+ ),
875
+ available_after=max(
876
+ [
877
+ begin_sequence,
878
+ commit_sequence,
879
+ *(
880
+ item[2]
881
+ for item in batch_actions.values()
882
+ ),
883
+ ]
884
+ ),
885
+ )
886
+ if complete_action_sequence:
887
+ action_sequences = [batch_actions[seq][2] for seq in range(count)]
888
+ if (
889
+ action_sequences != sorted(action_sequences)
890
+ or any(
891
+ not begin_sequence < sequence < commit_sequence
892
+ for sequence in action_sequences
893
+ )
894
+ ):
895
+ taint(
896
+ _protocol_violation(
897
+ batch_id,
898
+ "record_order_violation",
899
+ beginSequence=begin_sequence,
900
+ commitSequence=commit_sequence,
901
+ recordOrderHash=_sha256_canonical(
902
+ [begin_sequence, *action_sequences, commit_sequence]
903
+ ),
904
+ ),
905
+ available_after=max(
906
+ [begin_sequence, commit_sequence, *action_sequences]
907
+ ),
908
+ )
909
+ ordered_records = [
910
+ batch_actions[seq][0] for seq in range(count)
911
+ ]
912
+ cores = [
913
+ _correction_action_core(record) for record in ordered_records
914
+ ]
915
+ actual_actions_hash = _sha256_canonical(cores)
916
+ if actual_actions_hash != begin_core["actions_hash"]:
917
+ taint(
918
+ _protocol_violation(
919
+ batch_id,
920
+ "manifest_actions_hash_mismatch",
921
+ expectedActionsHash=begin_core["actions_hash"],
922
+ actualActionsHash=actual_actions_hash,
923
+ ),
924
+ available_after=max(
925
+ [begin_sequence, commit_sequence, *action_sequences]
926
+ ),
927
+ )
928
+ if batch_id in tainted_batches:
929
+ continue
930
+ completed.add(batch_id)
931
+ for seq in range(count):
932
+ normalized, _digest, sequence = batch_actions[seq]
933
+ candidates.append(_candidate_from_correction(normalized, sequence))
934
+
935
+ violation_by_fingerprint = {
936
+ violation.fingerprint: violation for violation in violations.values()
937
+ }
938
+ ordered_resolutions = sorted(resolutions, key=lambda item: item[0])
939
+ prefix_evidence = tuple(protocol_prefix_evidence)
940
+ if ordered_resolutions and len(prefix_evidence) != len(ordered_resolutions):
941
+ raise JournalProtocolError(
942
+ "journal protocol resolution requires verified raw-prefix evidence"
943
+ )
944
+ acknowledged: dict[str, AcknowledgedProtocolViolation] = {}
945
+ for (_sequence, resolution), evidence in zip(
946
+ ordered_resolutions,
947
+ prefix_evidence,
948
+ ):
949
+ if (
950
+ not isinstance(evidence, tuple)
951
+ or len(evidence) != 2
952
+ or evidence[0] != resolution["journal_high_water"]
953
+ or evidence[1] != resolution["journal_prefix_hash"]
954
+ ):
955
+ raise JournalProtocolError(
956
+ "journal protocol resolution raw-prefix binding does not match"
957
+ )
958
+ for ref in resolution["violations"]:
959
+ violation = violation_by_fingerprint.get(ref["fingerprint"])
960
+ if (
961
+ violation is None
962
+ or violation.batch_id != ref["batch_id"]
963
+ or violation.kind != ref["kind"]
964
+ ):
965
+ raise JournalProtocolError(
966
+ "journal protocol resolution references an unknown violation"
967
+ )
968
+ if _sequence <= violation_available_after[violation.fingerprint]:
969
+ raise JournalProtocolError(
970
+ "journal protocol resolution precedes the violation it resolves"
971
+ )
972
+ acknowledged.setdefault(
973
+ violation.fingerprint,
974
+ AcknowledgedProtocolViolation(
975
+ violation=violation,
976
+ audit_id=resolution["audit_id"],
977
+ journal_high_water=resolution["journal_high_water"],
978
+ journal_prefix_hash=resolution["journal_prefix_hash"],
979
+ ),
980
+ )
981
+
982
+ by_revision: dict[tuple[str, int], EffectiveEvent] = {}
983
+ # #374: divergent same-revision groups are QUARANTINED, not fatal. Keyed
984
+ # `(event_id, rev) -> {content hashes}` while grouping; filtered to the
985
+ # winning revision after selection (the only point where it is known).
986
+ divergent: dict[tuple[str, int], set] = {}
987
+ for candidate in candidates:
988
+ key = (candidate.event_id, candidate.rev)
989
+ prior = by_revision.get(key)
990
+ if prior is not None and (
991
+ prior.content_hash != candidate.content_hash
992
+ or prior.status != candidate.status
993
+ ):
994
+ if (
995
+ _is_legacy_quota_arming_state(prior)
996
+ and _is_legacy_quota_arming_state(candidate)
997
+ ):
998
+ # Legacy qaa carve-out: last-wins AND silent. Unchanged by #374
999
+ # — a pre-#372 arming record deliberately reuses its natural id
1000
+ # as a state stream, so successive lines are not a conflict.
1001
+ by_revision[key] = candidate
1002
+ continue
1003
+ divergent.setdefault(key, set()).update(
1004
+ {prior.content_hash, candidate.content_hash}
1005
+ )
1006
+ if prior is None or candidate.sequence < prior.sequence:
1007
+ by_revision[key] = candidate
1008
+
1009
+ winners: dict[str, EffectiveEvent] = {}
1010
+ for candidate in by_revision.values():
1011
+ prior = winners.get(candidate.event_id)
1012
+ if prior is None or candidate.rev > prior.rev:
1013
+ winners[candidate.event_id] = candidate
1014
+ active = [
1015
+ candidate.record
1016
+ for candidate in sorted(winners.values(), key=lambda item: item.sequence)
1017
+ if candidate.status == "active" and candidate.record is not None
1018
+ ]
1019
+ # Revision scoping: report only groups AT the winning revision. A rev-0 group
1020
+ # a completed rev-1 correction batch superseded is resolved, not outstanding.
1021
+ conflicts: list[EventConflict] = []
1022
+ for key in sorted(divergent):
1023
+ event_id, rev = key
1024
+ winner = winners.get(event_id)
1025
+ if winner is None or winner.rev != rev:
1026
+ continue
1027
+ conflicts.append(
1028
+ EventConflict(
1029
+ event_id=event_id,
1030
+ rev=rev,
1031
+ content_hashes=tuple(sorted(divergent[key])),
1032
+ selected_hash=by_revision[key].content_hash,
1033
+ )
1034
+ )
1035
+ return EffectiveSelection(
1036
+ active=active,
1037
+ by_id=winners,
1038
+ completed_batches=frozenset(completed),
1039
+ conflicts=tuple(conflicts),
1040
+ protocol_violations=tuple(
1041
+ sorted(
1042
+ (
1043
+ violation
1044
+ for violation in violations.values()
1045
+ if violation.fingerprint not in acknowledged
1046
+ ),
1047
+ key=lambda violation: (
1048
+ violation.batch_id,
1049
+ violation.kind,
1050
+ violation.fingerprint,
1051
+ ),
1052
+ )
1053
+ ),
1054
+ acknowledged_protocol_violations=tuple(
1055
+ sorted(
1056
+ acknowledged.values(),
1057
+ key=lambda item: (
1058
+ item.batch_id,
1059
+ item.kind,
1060
+ item.fingerprint,
1061
+ ),
1062
+ )
1063
+ ),
1064
+ )
1065
+
1066
+
145
1067
  # --------------------------------------------------------------------------
146
1068
  # accounts-machinery records (#341): registered op kinds folded into the
147
1069
  # `accounts` registry. These are NOT data-bearing account-stamped lines and are