cctally 1.114.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +4491 -295
- package/bin/_cctally_completion.py +1355 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +282 -9
- package/bin/_cctally_dashboard.py +8 -11
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +42 -35
- package/bin/_cctally_dashboard_sources.py +18 -9
- package/bin/_cctally_db.py +401 -9
- package/bin/_cctally_doctor.py +469 -27
- package/bin/_cctally_five_hour.py +166 -1049
- package/bin/_cctally_forecast.py +81 -9
- package/bin/_cctally_journal.py +5076 -177
- package/bin/_cctally_journal_repair.py +110 -19
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +47 -0
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +662 -0
- package/bin/_cctally_quota.py +96 -12
- package/bin/_cctally_quota_model.py +391 -83
- package/bin/_cctally_record.py +1088 -269
- package/bin/_cctally_rederive.py +854 -192
- package/bin/_cctally_refresh.py +113 -13
- package/bin/_cctally_source_analytics.py +23 -2
- package/bin/_cctally_statusline.py +7062 -505
- package/bin/_cctally_store.py +1244 -91
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +119 -60
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +569 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +177 -0
- package/bin/_lib_codex_conversation_query.py +133 -19
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +11 -3
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +4038 -0
- package/bin/_lib_doctor.py +753 -1
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +261 -10
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_pricing.py +7 -2
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +155 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +384 -52
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +393 -8
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +356 -27
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +29 -12
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +20 -2
- package/bin/_lib_merge_gate.py +0 -725
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
package/bin/_lib_journal.py
CHANGED
|
@@ -30,8 +30,11 @@ from __future__ import annotations
|
|
|
30
30
|
import datetime as dt
|
|
31
31
|
import hashlib
|
|
32
32
|
import json
|
|
33
|
+
import sys
|
|
33
34
|
from dataclasses import dataclass
|
|
34
35
|
|
|
36
|
+
import _lib_account_protocol
|
|
37
|
+
|
|
35
38
|
LINE_VERSION = 1
|
|
36
39
|
|
|
37
40
|
SEGMENT_PREFIX = "observations-"
|
|
@@ -220,6 +223,47 @@ def decode_line(raw: bytes) -> dict | None:
|
|
|
220
223
|
return obj
|
|
221
224
|
|
|
222
225
|
|
|
226
|
+
#: Strings up to this many characters are interned by `compact_retained`.
|
|
227
|
+
#: Keys, kinds, sources, account keys, ids and timestamps fall under it; a
|
|
228
|
+
#: long free-text value is rare enough that sharing it would buy nothing.
|
|
229
|
+
COMPACT_INTERN_MAX_CHARS = 64
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def compact_retained(value):
|
|
233
|
+
"""An equal copy of a decoded record whose keys and string values of at
|
|
234
|
+
most `COMPACT_INTERN_MAX_CHARS` characters are ``sys.intern``ed (#997 D15).
|
|
235
|
+
|
|
236
|
+
CPython's JSON decoder shares keys only within one ``loads`` call, so
|
|
237
|
+
every record a pass retains carries its own copy of every key and every
|
|
238
|
+
repeated short value: 381 k retained records took 0.83 GB decoded and
|
|
239
|
+
0.375 GB compacted on the maintainer's copy. Recursive through ``dict``
|
|
240
|
+
and ``list``; every other value (numbers, booleans, ``None``, long
|
|
241
|
+
strings) is shared unchanged. The result is a NEW record with new
|
|
242
|
+
containers, so equality, hashing, canonical encoding and mutation behave
|
|
243
|
+
exactly as on the plain record; only the object identity of equal strings
|
|
244
|
+
changes, and no consumer compares strings by identity.
|
|
245
|
+
|
|
246
|
+
Apply it immediately after decoding, before any consumer holds a
|
|
247
|
+
reference, so an identity check such as the rederive reader's
|
|
248
|
+
``records[index] is record`` only ever sees the compacted object.
|
|
249
|
+
"""
|
|
250
|
+
return _compact(value)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _compact(value, _intern=sys.intern):
|
|
254
|
+
kind = type(value)
|
|
255
|
+
if kind is dict:
|
|
256
|
+
return {
|
|
257
|
+
(_intern(key) if type(key) is str else key): _compact(item)
|
|
258
|
+
for key, item in value.items()
|
|
259
|
+
}
|
|
260
|
+
if kind is list:
|
|
261
|
+
return [_compact(item) for item in value]
|
|
262
|
+
if kind is str and len(value) <= COMPACT_INTERN_MAX_CHARS:
|
|
263
|
+
return _intern(value)
|
|
264
|
+
return value
|
|
265
|
+
|
|
266
|
+
|
|
223
267
|
# --------------------------------------------------------------------------
|
|
224
268
|
# identity
|
|
225
269
|
# --------------------------------------------------------------------------
|
|
@@ -379,8 +423,31 @@ def effects_payload_digest(payload: dict, refs: dict | None = None) -> str:
|
|
|
379
423
|
# fully-formed line records
|
|
380
424
|
# --------------------------------------------------------------------------
|
|
381
425
|
|
|
426
|
+
_WRITER_GENERATION = _lib_account_protocol.PROTOCOL_GENERATION
|
|
427
|
+
_WPG = _lib_account_protocol.WRITER_GENERATION_KEY
|
|
428
|
+
#: #797/#823/#840 (spec 4.6 B1): the protocol revision stamp, transport
|
|
429
|
+
#: metadata exactly like ``wpg``.
|
|
430
|
+
_WRITER_REVISION = _lib_account_protocol.PROTOCOL_REVISION
|
|
431
|
+
_WPR = _lib_account_protocol.REVISION_KEY
|
|
432
|
+
#: ``writer_revision`` default: the current revision when the record carries
|
|
433
|
+
#: the current writer generation, none for a previous release's shape.
|
|
434
|
+
_REVISION_DEFAULT = object()
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _writer_core(core: dict, writer_generation, writer_revision) -> None:
|
|
438
|
+
if writer_generation is not None:
|
|
439
|
+
core[_WPG] = writer_generation
|
|
440
|
+
if writer_revision is _REVISION_DEFAULT:
|
|
441
|
+
writer_revision = (None if writer_generation is None
|
|
442
|
+
else _WRITER_REVISION)
|
|
443
|
+
if writer_revision is not None:
|
|
444
|
+
core[_WPR] = writer_revision
|
|
445
|
+
|
|
446
|
+
|
|
382
447
|
def make_obs(at: str, src: str, provider: str, payload: dict,
|
|
383
|
-
account: str | None = None
|
|
448
|
+
account: str | None = None, *,
|
|
449
|
+
writer_generation: str | None = _WRITER_GENERATION,
|
|
450
|
+
writer_revision=_REVISION_DEFAULT) -> dict:
|
|
384
451
|
"""Build a complete ``obs`` line record (raw capture; canonicalization is
|
|
385
452
|
derivation-time, never baked into the stored line — spec §4.2).
|
|
386
453
|
|
|
@@ -388,20 +455,159 @@ def make_obs(at: str, src: str, provider: str, payload: dict,
|
|
|
388
455
|
``provider``. It is emitted ONLY when supplied, so a pre-epic / single-account
|
|
389
456
|
writer that passes nothing produces a byte-identical line (and a byte-stable
|
|
390
457
|
content id). When supplied it participates in the content id, so two obs that
|
|
391
|
-
differ only by account are distinct records.
|
|
458
|
+
differ only by account are distinct records.
|
|
459
|
+
|
|
460
|
+
``writer_generation`` (#825/#832) is the top-level ``wpg`` stamp naming the
|
|
461
|
+
writing protocol. It is part of the content-id core, so a current record and
|
|
462
|
+
a previous-release record of the same capture are distinct lines. ``None``
|
|
463
|
+
builds the previous release's exact shape (a validator re-deriving a
|
|
464
|
+
retained pre-stamp record, or a test simulating an old writer).
|
|
465
|
+
|
|
466
|
+
``writer_revision`` (#797/#823/#840 spec 4.6 B1) is the top-level ``wpr``
|
|
467
|
+
protocol revision, also part of the content-id core. It defaults to the
|
|
468
|
+
current revision for a current-generation record and to none for a
|
|
469
|
+
previous release's shape; ``None`` with the current generation builds
|
|
470
|
+
the revision-1 (#825) shape."""
|
|
392
471
|
core = {"t": "obs", "at": at, "src": src, "provider": provider, "payload": payload}
|
|
393
472
|
if account is not None:
|
|
394
473
|
core["account"] = account
|
|
474
|
+
_writer_core(core, writer_generation, writer_revision)
|
|
395
475
|
return {"v": LINE_VERSION, **core, "id": content_id(core)}
|
|
396
476
|
|
|
397
477
|
|
|
398
|
-
def make_op(at: str, src: str, payload: dict
|
|
399
|
-
|
|
478
|
+
def make_op(at: str, src: str, payload: dict, *,
|
|
479
|
+
writer_generation: str | None = _WRITER_GENERATION,
|
|
480
|
+
writer_revision=_REVISION_DEFAULT) -> dict:
|
|
481
|
+
"""Build a complete ``op`` (operator record) line — no ``provider``.
|
|
482
|
+
``writer_generation`` and ``writer_revision`` as in ``make_obs``."""
|
|
400
483
|
core = {"t": "op", "at": at, "src": src, "payload": payload}
|
|
484
|
+
_writer_core(core, writer_generation, writer_revision)
|
|
401
485
|
return {"v": LINE_VERSION, **core, "id": content_id(core)}
|
|
402
486
|
|
|
403
487
|
|
|
488
|
+
def prepare_record(record: dict) -> dict:
|
|
489
|
+
"""The single shared preparation step both append APIs apply (#825/#832
|
|
490
|
+
spec 5.1): return ``record`` carrying the current writer generation and,
|
|
491
|
+
since #797/#823/#840 (spec 4.6 B1), the current protocol revision.
|
|
492
|
+
|
|
493
|
+
A content-id record (``id`` starting ``o:``) that is not yet fully
|
|
494
|
+
stamped and whose id verifies over its current core is re-identified over
|
|
495
|
+
the stamped core, exactly as the builders compute it. Every other id —
|
|
496
|
+
semantic event ids, bootstrap ids, correction batch ids, a content id
|
|
497
|
+
that does not verify — is preserved. Idempotent: a record carrying both
|
|
498
|
+
stamps is returned unchanged. Only records about to be appended pass
|
|
499
|
+
through here, so a retained record is never restamped."""
|
|
500
|
+
if (_lib_account_protocol.is_stamped(record)
|
|
501
|
+
and _lib_account_protocol.has_current_revision(record)):
|
|
502
|
+
return record
|
|
503
|
+
stamped = _lib_account_protocol.stamp(record)
|
|
504
|
+
rid = record.get("id")
|
|
505
|
+
if (record.get("t") in ("obs", "op") and isinstance(rid, str)
|
|
506
|
+
and rid.startswith("o:")):
|
|
507
|
+
core = {k: v for k, v in record.items() if k not in ("v", "id")}
|
|
508
|
+
if content_id(core) == rid:
|
|
509
|
+
stamped_core = {k: v for k, v in stamped.items()
|
|
510
|
+
if k not in ("v", "id")}
|
|
511
|
+
stamped["id"] = content_id(stamped_core)
|
|
512
|
+
return stamped
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def unstamped_record(record: dict) -> dict:
|
|
516
|
+
"""The previous release's shape of ``record``: the inverse of
|
|
517
|
+
``prepare_record``. Drops the writer-generation stamp and, for a content-id
|
|
518
|
+
record whose id verifies over its stamped core, re-identifies it over the
|
|
519
|
+
unstamped core. Any other id is preserved. For fixtures and tests that
|
|
520
|
+
simulate a pre-stamp writer; never used on a production write path.
|
|
521
|
+
The protocol revision (``wpr``) goes with the stamp."""
|
|
522
|
+
if not isinstance(record, dict) or (
|
|
523
|
+
_WPG not in record and _WPR not in record):
|
|
524
|
+
return record
|
|
525
|
+
out = {k: v for k, v in record.items() if k not in (_WPG, _WPR)}
|
|
526
|
+
rid = record.get("id")
|
|
527
|
+
if (record.get("t") in ("obs", "op") and isinstance(rid, str)
|
|
528
|
+
and rid.startswith("o:")):
|
|
529
|
+
core = {k: v for k, v in record.items() if k not in ("v", "id")}
|
|
530
|
+
if content_id(core) == rid:
|
|
531
|
+
out["id"] = content_id(
|
|
532
|
+
{k: v for k, v in out.items() if k not in ("v", "id")})
|
|
533
|
+
return out
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def prerevision_record(record: dict) -> dict:
|
|
537
|
+
"""The revision-1 (#825 release) shape of ``record``: the current writer
|
|
538
|
+
generation without the protocol revision, re-identified like
|
|
539
|
+
``unstamped_record``. For fixtures and tests that simulate the previous
|
|
540
|
+
release; never used on a production write path."""
|
|
541
|
+
if not isinstance(record, dict) or _WPR not in record:
|
|
542
|
+
return record
|
|
543
|
+
out = {k: v for k, v in record.items() if k != _WPR}
|
|
544
|
+
rid = record.get("id")
|
|
545
|
+
if (record.get("t") in ("obs", "op") and isinstance(rid, str)
|
|
546
|
+
and rid.startswith("o:")):
|
|
547
|
+
core = {k: v for k, v in record.items() if k not in ("v", "id")}
|
|
548
|
+
if content_id(core) == rid:
|
|
549
|
+
out["id"] = content_id(
|
|
550
|
+
{k: v for k, v in out.items() if k not in ("v", "id")})
|
|
551
|
+
return out
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def legacy_writer_view():
|
|
555
|
+
"""A namespace exposing this module with every ``make_*`` builder wrapped by
|
|
556
|
+
``unstamped_record``, so a fixture builder written against the module
|
|
557
|
+
produces the previous release's exact bytes (cross-references between
|
|
558
|
+
records stay consistent because every id is the unstamped one)."""
|
|
559
|
+
import types
|
|
560
|
+
|
|
561
|
+
def _wrap(fn):
|
|
562
|
+
def build(*args, **kwargs):
|
|
563
|
+
out = fn(*args, **kwargs)
|
|
564
|
+
if isinstance(out, list):
|
|
565
|
+
return [unstamped_record(item) for item in out]
|
|
566
|
+
return unstamped_record(out)
|
|
567
|
+
build.__name__ = fn.__name__
|
|
568
|
+
build.__doc__ = fn.__doc__
|
|
569
|
+
return build
|
|
570
|
+
|
|
571
|
+
def _legacy_protocol_resolution(**kwargs):
|
|
572
|
+
# The previous release's acknowledgement kind, which the current
|
|
573
|
+
# binary never emits (#825/#832 spec 5.1).
|
|
574
|
+
current = make_protocol_resolution(**kwargs)
|
|
575
|
+
return make_op(
|
|
576
|
+
at=current["at"], src=current["src"],
|
|
577
|
+
payload={**current["payload"], "kind": _PROTOCOL_RESOLUTION_KIND},
|
|
578
|
+
writer_generation=None)
|
|
579
|
+
|
|
580
|
+
attrs = {}
|
|
581
|
+
for name, value in globals().items():
|
|
582
|
+
if name.startswith("__"):
|
|
583
|
+
continue
|
|
584
|
+
if name.startswith("make_") and callable(value):
|
|
585
|
+
value = _wrap(value)
|
|
586
|
+
attrs[name] = value
|
|
587
|
+
attrs["make_protocol_resolution"] = _legacy_protocol_resolution
|
|
588
|
+
return types.SimpleNamespace(**attrs)
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def event_content_hash(record: dict) -> str:
|
|
592
|
+
"""The semantic content hash of an evt line. The writer-generation stamp is
|
|
593
|
+
transport metadata, so it is stripped before hashing: a current emission
|
|
594
|
+
that differs from a retained pre-stamp event only by ``wpg`` is the same
|
|
595
|
+
event (a duplicate), never a same-revision conflict."""
|
|
596
|
+
return _sha256_canonical(_lib_account_protocol.strip_stamp(record))
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
#: The previous release's exact-key repair acknowledgement. The current binary
|
|
600
|
+
#: never emits it: its validator would reject the writer-generation stamp, so
|
|
601
|
+
#: current acknowledgements use the versioned kind the previous release ignores
|
|
602
|
+
#: as an unknown op (#825/#832 spec 5.1). Current readers accept every kind.
|
|
404
603
|
_PROTOCOL_RESOLUTION_KIND = "journal_protocol_resolution"
|
|
604
|
+
_PROTOCOL_RESOLUTION_V2_KIND = _lib_account_protocol.PROTOCOL_RESOLUTION_V2_KIND
|
|
605
|
+
#: #797/#823/#840 (spec 4.6 B5): the previous release's v2 validator rejects
|
|
606
|
+
#: the protocol revision stamp, so the current binary emits this successor.
|
|
607
|
+
PROTOCOL_RESOLUTION_V3_KIND = _lib_account_protocol.PROTOCOL_RESOLUTION_V3_KIND
|
|
608
|
+
PROTOCOL_RESOLUTION_KINDS = frozenset((
|
|
609
|
+
_PROTOCOL_RESOLUTION_KIND, _PROTOCOL_RESOLUTION_V2_KIND,
|
|
610
|
+
PROTOCOL_RESOLUTION_V3_KIND))
|
|
405
611
|
_PROTOCOL_RESOLUTION_SRC = "journal-repair"
|
|
406
612
|
|
|
407
613
|
|
|
@@ -441,7 +647,7 @@ def make_protocol_resolution(
|
|
|
441
647
|
at=at,
|
|
442
648
|
src=_PROTOCOL_RESOLUTION_SRC,
|
|
443
649
|
payload={
|
|
444
|
-
"kind":
|
|
650
|
+
"kind": PROTOCOL_RESOLUTION_V3_KIND,
|
|
445
651
|
"violations": refs,
|
|
446
652
|
"journal_high_water": {
|
|
447
653
|
"segment": segment,
|
|
@@ -452,6 +658,17 @@ def make_protocol_resolution(
|
|
|
452
658
|
)
|
|
453
659
|
|
|
454
660
|
|
|
661
|
+
def is_protocol_resolution(record) -> bool:
|
|
662
|
+
"""Whether ``record`` is a journal-repair acknowledgement of either
|
|
663
|
+
version."""
|
|
664
|
+
return (
|
|
665
|
+
isinstance(record, dict)
|
|
666
|
+
and record.get("t") == "op"
|
|
667
|
+
and isinstance(record.get("payload"), dict)
|
|
668
|
+
and record["payload"].get("kind") in PROTOCOL_RESOLUTION_KINDS
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
|
|
455
672
|
def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
|
|
456
673
|
"""Build a complete ``evt`` (derived) line record.
|
|
457
674
|
|
|
@@ -463,7 +680,8 @@ def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
|
|
|
463
680
|
body = dict(payload)
|
|
464
681
|
body["kind"] = kind
|
|
465
682
|
return {"v": LINE_VERSION, "t": "evt", "id": id, "rev": rev,
|
|
466
|
-
"at": at, "src": "ingest", "payload": body
|
|
683
|
+
"at": at, "src": "ingest", "payload": body,
|
|
684
|
+
_WPG: _WRITER_GENERATION, _WPR: _WRITER_REVISION}
|
|
467
685
|
|
|
468
686
|
|
|
469
687
|
# --------------------------------------------------------------------------
|
|
@@ -501,7 +719,11 @@ def _protocol_violation(
|
|
|
501
719
|
|
|
502
720
|
def _validate_protocol_resolution(record: dict) -> dict:
|
|
503
721
|
required = {"v", "t", "id", "at", "src", "payload"}
|
|
504
|
-
|
|
722
|
+
# #825/#832: a current record also carries the writer-generation stamp,
|
|
723
|
+
# which is part of its content-id core; #797/#823/#840 adds the protocol
|
|
724
|
+
# revision (spec 4.6 B5).
|
|
725
|
+
if set(record) not in (required, required | {_WPG},
|
|
726
|
+
required | {_WPG, _WPR}):
|
|
505
727
|
raise JournalProtocolError(
|
|
506
728
|
"journal protocol resolution record shape is invalid"
|
|
507
729
|
)
|
|
@@ -521,6 +743,10 @@ def _validate_protocol_resolution(record: dict) -> dict:
|
|
|
521
743
|
"src": record["src"],
|
|
522
744
|
"payload": record["payload"],
|
|
523
745
|
}
|
|
746
|
+
if _WPG in record:
|
|
747
|
+
core[_WPG] = record[_WPG]
|
|
748
|
+
if _WPR in record:
|
|
749
|
+
core[_WPR] = record[_WPR]
|
|
524
750
|
if record.get("id") != content_id(core):
|
|
525
751
|
raise JournalProtocolError(
|
|
526
752
|
"journal protocol resolution record id does not match its content"
|
|
@@ -535,7 +761,7 @@ def _validate_protocol_resolution(record: dict) -> dict:
|
|
|
535
761
|
raise JournalProtocolError(
|
|
536
762
|
"journal protocol resolution payload shape is invalid"
|
|
537
763
|
)
|
|
538
|
-
if payload.get("kind")
|
|
764
|
+
if payload.get("kind") not in PROTOCOL_RESOLUTION_KINDS:
|
|
539
765
|
raise JournalProtocolError(
|
|
540
766
|
"journal protocol resolution kind is invalid"
|
|
541
767
|
)
|
|
@@ -672,6 +898,8 @@ def make_correction_batch(
|
|
|
672
898
|
"family": family,
|
|
673
899
|
"action_count": len(cores),
|
|
674
900
|
"actions_hash": actions_hash,
|
|
901
|
+
_WPG: _WRITER_GENERATION,
|
|
902
|
+
_WPR: _WRITER_REVISION,
|
|
675
903
|
}
|
|
676
904
|
records = [{**marker, "phase": "begin"}]
|
|
677
905
|
for seq, core in enumerate(cores):
|
|
@@ -682,12 +910,35 @@ def make_correction_batch(
|
|
|
682
910
|
**core,
|
|
683
911
|
"batch": batch_id,
|
|
684
912
|
"seq": seq,
|
|
913
|
+
_WPG: _WRITER_GENERATION,
|
|
914
|
+
_WPR: _WRITER_REVISION,
|
|
685
915
|
}
|
|
686
916
|
)
|
|
687
917
|
records.append({**marker, "phase": "commit"})
|
|
688
918
|
return records
|
|
689
919
|
|
|
690
920
|
|
|
921
|
+
def revision_batch_id(batch_id: str, begins) -> str:
|
|
922
|
+
"""The batch id a retry of ``batch_id`` appends under (#797/#823/#840
|
|
923
|
+
spec 4.6 B1).
|
|
924
|
+
|
|
925
|
+
``begins`` are the retained begin markers already journalled under
|
|
926
|
+
``batch_id``. A retry reuses the id only when no begin exists or one
|
|
927
|
+
carries the current revision stamp. An interrupted batch an earlier
|
|
928
|
+
writer began without that stamp is never restamped: a revision-2 copy of
|
|
929
|
+
its lines would be a marker conflict under the original id (the fold
|
|
930
|
+
hashes markers byte-exactly), and a copy without the stamp would land
|
|
931
|
+
after the revision boundary, where admission excludes it. Its plan is
|
|
932
|
+
therefore authored afresh under a derived id, and the original begin
|
|
933
|
+
stays the inert, incomplete batch it was."""
|
|
934
|
+
begins = list(begins)
|
|
935
|
+
if not begins or any(
|
|
936
|
+
_lib_account_protocol.has_current_revision(begin)
|
|
937
|
+
for begin in begins):
|
|
938
|
+
return batch_id
|
|
939
|
+
return f"{batch_id}:wpr{_lib_account_protocol.PROTOCOL_REVISION}"
|
|
940
|
+
|
|
941
|
+
|
|
691
942
|
def _validate_batch_marker(record: dict) -> dict:
|
|
692
943
|
batch_id = record.get("id")
|
|
693
944
|
if not isinstance(batch_id, str) or not batch_id:
|
|
@@ -769,7 +1020,7 @@ def _candidate_from_evt(record: dict, sequence: int) -> EffectiveEvent:
|
|
|
769
1020
|
)
|
|
770
1021
|
if not isinstance(record.get("payload"), dict):
|
|
771
1022
|
raise JournalProtocolError("event payload must be an object")
|
|
772
|
-
digest =
|
|
1023
|
+
digest = event_content_hash(record)
|
|
773
1024
|
return EffectiveEvent(
|
|
774
1025
|
event_id=event_id,
|
|
775
1026
|
rev=rev,
|
|
@@ -919,7 +1170,7 @@ def _fold_one(fold: SelectorFold, record, sequence: int) -> None:
|
|
|
919
1170
|
if (
|
|
920
1171
|
record_type == "op"
|
|
921
1172
|
and isinstance(record.get("payload"), dict)
|
|
922
|
-
and record["payload"].get("kind")
|
|
1173
|
+
and record["payload"].get("kind") in PROTOCOL_RESOLUTION_KINDS
|
|
923
1174
|
):
|
|
924
1175
|
resolutions.append(
|
|
925
1176
|
(sequence, _validate_protocol_resolution(record))
|
|
@@ -155,12 +155,37 @@ class PrefixHashAccumulator:
|
|
|
155
155
|
|
|
156
156
|
The framing is `journal_prefix_hash`'s, verbatim: per segment, the 4-byte
|
|
157
157
|
big-endian name length, the name, the 8-byte big-endian data length, then
|
|
158
|
-
the data.
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
158
|
+
the data. The data length precedes the data, so `begin_segment` takes the
|
|
159
|
+
exact byte count the pass will feed (`size`) and frames the segment header
|
|
160
|
+
with it; every chunk is then hashed as it arrives and NO segment bytes are
|
|
161
|
+
retained (#997 D12). The buffered form held the whole current segment in a
|
|
162
|
+
`bytearray` grown by `extend` — 410 MB on the maintainer's largest segment
|
|
163
|
+
— and the freed growth steps stayed resident in the macOS allocator.
|
|
164
|
+
|
|
165
|
+
A digest at a fully fed segment end (every transition, and every boundary
|
|
166
|
+
not short of a torn tail) comes from the streamed state. A digest at any
|
|
167
|
+
other offset — an evidence point inside the current segment, a boundary
|
|
168
|
+
short of a torn trailing line, a feed that ended short of its declared
|
|
169
|
+
size — re-reads the segment through `open_segment(name)` in 1 MiB chunks,
|
|
170
|
+
which the production callers bind to the journal's one segment-read seam,
|
|
171
|
+
so an evidence re-read is visible to it.
|
|
172
|
+
|
|
173
|
+
Consumed extent (#997 D12, revision 10). Per segment the accumulator also
|
|
174
|
+
tracks the bytes actually fed (`consumed`) and a raw SHA-256 of those bytes
|
|
175
|
+
alone (the witness). An offset beyond `consumed` raises. Every re-read reads
|
|
176
|
+
through `consumed`, not only through the requested offset: one pass frames
|
|
177
|
+
the prefix digest at the offset and hashes `[0, consumed)` raw, and the
|
|
178
|
+
digest is returned only when the read was not short and that raw hash
|
|
179
|
+
equals the witness. A torn tail an appender truncated and replaced after
|
|
180
|
+
the pass read it, or a line the reader assembled from old and replacement
|
|
181
|
+
bytes across two reads, can therefore never be certified. A registered
|
|
182
|
+
boundary keeps its segment's `consumed` and witness with its saved hash
|
|
183
|
+
state. A segment fed short of its declared size is re-framed from its
|
|
184
|
+
consumed bytes through the same verified re-read at the next
|
|
185
|
+
`begin_segment`; when that re-read is unavailable or fails verification
|
|
186
|
+
the accumulator is poisoned and every later digest raises. A missing
|
|
187
|
+
opener, a short re-read or a failed verification raises
|
|
188
|
+
`PrefixEvidenceUnavailable`.
|
|
164
189
|
"""
|
|
165
190
|
|
|
166
191
|
# A transition can only ever register one boundary, and consecutive empty
|
|
@@ -176,10 +201,28 @@ class PrefixHashAccumulator:
|
|
|
176
201
|
# entry that is still live.
|
|
177
202
|
_MAX_BOUNDARIES = 16
|
|
178
203
|
|
|
179
|
-
|
|
204
|
+
#: Chunk size of an evidence re-read through the opener.
|
|
205
|
+
_REREAD_CHUNK = 1 << 20
|
|
206
|
+
|
|
207
|
+
def __init__(self, *, open_segment=None) -> None:
|
|
208
|
+
self._open_segment = open_segment
|
|
209
|
+
#: The hash of every segment before the current one, absorbed.
|
|
180
210
|
self._running = hashlib.sha256()
|
|
181
211
|
self._current_name = None
|
|
182
|
-
|
|
212
|
+
#: `_running` plus the current segment's header (framed with its
|
|
213
|
+
#: declared size) and every byte fed so far.
|
|
214
|
+
self._segment = None
|
|
215
|
+
self._declared = 0
|
|
216
|
+
#: Bytes of the current segment actually fed (its consumed extent),
|
|
217
|
+
#: and the raw SHA-256 of exactly those bytes (the witness).
|
|
218
|
+
self._fed = 0
|
|
219
|
+
self._witness = hashlib.sha256()
|
|
220
|
+
#: Set when a short segment could not be re-framed from verified
|
|
221
|
+
#: bytes: the running state no longer describes the journal.
|
|
222
|
+
self._poisoned = None
|
|
223
|
+
#: `(segment, offset) -> (state before that segment, final digest or
|
|
224
|
+
#: None, consumed, witness digest)`; a `None` final digest is re-read
|
|
225
|
+
#: through the opener, verified against the witness, when read.
|
|
183
226
|
self._boundaries: dict = {}
|
|
184
227
|
self._boundary_order: list = []
|
|
185
228
|
#: Bytes fed into an evidence digest, reported as the `protocol_evidence`
|
|
@@ -189,55 +232,92 @@ class PrefixHashAccumulator:
|
|
|
189
232
|
|
|
190
233
|
# -- feeding ----------------------------------------------------------
|
|
191
234
|
|
|
192
|
-
def begin_segment(self, name: str, boundary=None) -> None:
|
|
193
|
-
"""Start `name`,
|
|
235
|
+
def begin_segment(self, name: str, boundary=None, *, size: int) -> None:
|
|
236
|
+
"""Start `name`, which the pass will feed `size` bytes of, absorbing
|
|
237
|
+
whatever segment was open before it.
|
|
194
238
|
|
|
195
239
|
`boundary` is the `(segment, offset)` the next record's evidence would
|
|
196
|
-
name — i.e. the streaming loop's current `prior_high_water`.
|
|
197
|
-
|
|
198
|
-
immediately afterwards.
|
|
240
|
+
name — i.e. the streaming loop's current `prior_high_water`. Its hash
|
|
241
|
+
state is kept HERE because the outgoing segment's state is folded into
|
|
242
|
+
the running hash immediately afterwards.
|
|
199
243
|
"""
|
|
200
244
|
if boundary is not None:
|
|
201
245
|
self._register_boundary(boundary)
|
|
202
246
|
if self._current_name is not None:
|
|
203
|
-
|
|
204
|
-
# segment is framed WITHOUT a copy of itself. The maintainer's
|
|
205
|
-
# largest segment is 410 MB, and copying it here briefly doubled
|
|
206
|
-
# that at every transition.
|
|
207
|
-
with memoryview(self._current) as data:
|
|
208
|
-
self._absorb(self._current_name, data)
|
|
247
|
+
self._running = self._completed_segment()
|
|
209
248
|
self._current_name = name
|
|
210
|
-
self.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
self.
|
|
249
|
+
self._declared = int(size)
|
|
250
|
+
self._fed = 0
|
|
251
|
+
self._witness = hashlib.sha256()
|
|
252
|
+
self._segment = self._running.copy()
|
|
253
|
+
self._header(self._segment, name, self._declared)
|
|
254
|
+
|
|
255
|
+
def extend(self, data) -> None:
|
|
256
|
+
"""Hash raw bytes read from the segment currently open."""
|
|
257
|
+
self._segment.update(data)
|
|
258
|
+
self._witness.update(data)
|
|
259
|
+
self._fed += len(data)
|
|
215
260
|
|
|
216
261
|
# -- reading ----------------------------------------------------------
|
|
217
262
|
|
|
218
263
|
def digest_at(self, high_water) -> "str | None":
|
|
219
|
-
"""`journal_prefix_hash(high_water)`, from the streamed
|
|
264
|
+
"""`journal_prefix_hash(high_water)`, from the streamed state."""
|
|
220
265
|
if high_water is None:
|
|
221
266
|
return None
|
|
222
267
|
segment, offset = high_water
|
|
223
|
-
if
|
|
224
|
-
digest = self._running.copy()
|
|
225
|
-
with memoryview(self._current)[:offset] as data:
|
|
226
|
-
self._frame(digest, segment, data)
|
|
227
|
-
self.bytes_hashed += len(data)
|
|
228
|
-
self.digests_computed += 1
|
|
229
|
-
return "sha256:" + digest.hexdigest()
|
|
230
|
-
cached = self._boundaries.get((segment, offset))
|
|
231
|
-
if cached is None:
|
|
268
|
+
if self._poisoned is not None:
|
|
232
269
|
raise PrefixEvidenceUnavailable(
|
|
233
|
-
f"no
|
|
270
|
+
f"no verified prefix evidence for {segment}@{offset}: "
|
|
271
|
+
f"{self._poisoned}"
|
|
234
272
|
)
|
|
235
|
-
self.
|
|
273
|
+
if segment == self._current_name:
|
|
274
|
+
if self._streamed_end(offset):
|
|
275
|
+
digest = self._segment.copy()
|
|
276
|
+
else:
|
|
277
|
+
digest = self._verified_rehash(
|
|
278
|
+
self._running, segment, offset, self._fed,
|
|
279
|
+
self._witness.digest())
|
|
280
|
+
else:
|
|
281
|
+
cached = self._boundaries.get((segment, offset))
|
|
282
|
+
if cached is None:
|
|
283
|
+
raise PrefixEvidenceUnavailable(
|
|
284
|
+
f"no streamed prefix evidence for {segment}@{offset}"
|
|
285
|
+
)
|
|
286
|
+
before, final, consumed, witness = cached
|
|
287
|
+
digest = (final.copy() if final is not None
|
|
288
|
+
else self._verified_rehash(
|
|
289
|
+
before, segment, offset, consumed, witness))
|
|
290
|
+
self.bytes_hashed += offset
|
|
236
291
|
self.digests_computed += 1
|
|
237
|
-
return
|
|
292
|
+
return "sha256:" + digest.hexdigest()
|
|
238
293
|
|
|
239
294
|
# -- internals --------------------------------------------------------
|
|
240
295
|
|
|
296
|
+
def _streamed_end(self, offset) -> bool:
|
|
297
|
+
"""Whether the streamed state IS the digest at `offset`: the current
|
|
298
|
+
segment was fed exactly its declared size and `offset` is its end."""
|
|
299
|
+
return offset == self._fed == self._declared
|
|
300
|
+
|
|
301
|
+
def _completed_segment(self):
|
|
302
|
+
"""The running hash with the current segment absorbed as fed."""
|
|
303
|
+
if self._fed == self._declared or self._poisoned is not None:
|
|
304
|
+
return self._segment
|
|
305
|
+
# The feed ended short of the declared size (the segment shrank under
|
|
306
|
+
# the pass): frame the bytes actually fed, as the buffered form did,
|
|
307
|
+
# from a re-read verified against them. Bytes that cannot be verified
|
|
308
|
+
# leave no running state that describes the journal, so every later
|
|
309
|
+
# digest raises rather than certifying one.
|
|
310
|
+
try:
|
|
311
|
+
return self._verified_rehash(
|
|
312
|
+
self._running, self._current_name, self._fed, self._fed,
|
|
313
|
+
self._witness.digest())
|
|
314
|
+
except (PrefixEvidenceUnavailable, OSError) as exc:
|
|
315
|
+
self._poisoned = (
|
|
316
|
+
f"short segment {self._current_name} could not be re-framed "
|
|
317
|
+
f"from its consumed bytes ({exc})"
|
|
318
|
+
)
|
|
319
|
+
return self._segment
|
|
320
|
+
|
|
241
321
|
def _register_boundary(self, boundary) -> None:
|
|
242
322
|
segment, offset = boundary
|
|
243
323
|
if segment != self._current_name:
|
|
@@ -247,23 +327,61 @@ class PrefixHashAccumulator:
|
|
|
247
327
|
key = (segment, offset)
|
|
248
328
|
if key in self._boundaries:
|
|
249
329
|
return
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
self._boundaries[key] = ("sha256:" + digest.hexdigest(), size)
|
|
330
|
+
final = (self._segment.copy() if self._streamed_end(offset)
|
|
331
|
+
else None)
|
|
332
|
+
self._boundaries[key] = (self._running.copy(), final, self._fed,
|
|
333
|
+
self._witness.digest())
|
|
255
334
|
self._boundary_order.append(key)
|
|
256
335
|
while len(self._boundary_order) > self._MAX_BOUNDARIES:
|
|
257
336
|
self._boundaries.pop(self._boundary_order.pop(0), None)
|
|
258
337
|
|
|
259
|
-
def
|
|
260
|
-
|
|
338
|
+
def _verified_rehash(self, before, name: str, offset: int,
|
|
339
|
+
consumed: int, witness: bytes):
|
|
340
|
+
"""`before` plus `name`'s first `offset` bytes, re-read from the
|
|
341
|
+
segment through the opener in `_REREAD_CHUNK` pieces.
|
|
342
|
+
|
|
343
|
+
The read always runs through `consumed`, the bytes the pass fed, and
|
|
344
|
+
its raw SHA-256 must equal `witness`; only then are the bytes the
|
|
345
|
+
digest frames the bytes the pass read."""
|
|
346
|
+
if offset > consumed:
|
|
347
|
+
raise PrefixEvidenceUnavailable(
|
|
348
|
+
f"prefix evidence {name}@{offset} lies beyond the "
|
|
349
|
+
f"{consumed} bytes the pass consumed"
|
|
350
|
+
)
|
|
351
|
+
digest = before.copy()
|
|
352
|
+
self._header(digest, name, offset)
|
|
353
|
+
if consumed <= 0:
|
|
354
|
+
return digest
|
|
355
|
+
if self._open_segment is None:
|
|
356
|
+
raise PrefixEvidenceUnavailable(
|
|
357
|
+
f"no segment opener to re-read prefix evidence for "
|
|
358
|
+
f"{name}@{offset}"
|
|
359
|
+
)
|
|
360
|
+
raw = hashlib.sha256()
|
|
361
|
+
pos = 0
|
|
362
|
+
with self._open_segment(name) as handle:
|
|
363
|
+
while pos < consumed:
|
|
364
|
+
chunk = handle.read(min(self._REREAD_CHUNK, consumed - pos))
|
|
365
|
+
if not chunk:
|
|
366
|
+
raise PrefixEvidenceUnavailable(
|
|
367
|
+
f"short re-read of prefix evidence for {name}@{offset}"
|
|
368
|
+
)
|
|
369
|
+
raw.update(chunk)
|
|
370
|
+
if pos < offset:
|
|
371
|
+
with memoryview(chunk)[:offset - pos] as framed:
|
|
372
|
+
digest.update(framed)
|
|
373
|
+
pos += len(chunk)
|
|
374
|
+
if raw.digest() != witness:
|
|
375
|
+
raise PrefixEvidenceUnavailable(
|
|
376
|
+
f"segment {name} no longer holds the {consumed} bytes the "
|
|
377
|
+
f"pass consumed; prefix evidence {name}@{offset} is not "
|
|
378
|
+
f"certified"
|
|
379
|
+
)
|
|
380
|
+
return digest
|
|
261
381
|
|
|
262
382
|
@staticmethod
|
|
263
|
-
def
|
|
264
|
-
"""`data` is any bytes-like buffer; `hashlib` consumes it in place."""
|
|
383
|
+
def _header(digest, name: str, length: int) -> None:
|
|
265
384
|
encoded = name.encode("utf-8")
|
|
266
385
|
digest.update(len(encoded).to_bytes(4, "big"))
|
|
267
386
|
digest.update(encoded)
|
|
268
|
-
digest.update(
|
|
269
|
-
digest.update(data)
|
|
387
|
+
digest.update(int(length).to_bytes(8, "big"))
|