cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
|
@@ -6,17 +6,31 @@ This module deliberately has no filesystem, database, clock, process, or
|
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
8
|
import dataclasses
|
|
9
|
+
import hashlib
|
|
9
10
|
import json
|
|
10
11
|
import math
|
|
11
12
|
import re
|
|
12
13
|
from typing import Callable, Mapping
|
|
13
14
|
|
|
14
15
|
|
|
16
|
+
#: Candidate and tombstone documents. Every session on this machine writes
|
|
17
|
+
#: candidates, so this version may only move when peer sessions running the
|
|
18
|
+
#: previous binary can still be read.
|
|
15
19
|
SCHEMA_VERSION = 1
|
|
20
|
+
#: The control document, versioned independently (#755). It is written by
|
|
21
|
+
#: whichever session holds the persist lock and read by the next tick, so a
|
|
22
|
+
#: shape change here must not invalidate the spool documents peers are writing.
|
|
23
|
+
CONTROL_SCHEMA_VERSION = 2
|
|
24
|
+
#: Control documents this binary can read. Version 1 predates the pending
|
|
25
|
+
#: drop's deadline and retained evidence; see `_pending_drop`.
|
|
26
|
+
CONTROL_SCHEMA_VERSIONS_READ = (1, 2)
|
|
27
|
+
#: D5: a non-extendable wall clock measured from the first pending drop.
|
|
28
|
+
PENDING_DROP_DEADLINE_SECONDS = 180
|
|
16
29
|
CANDIDATE_DOCUMENT_MAX_BYTES = 4 * 1024
|
|
17
30
|
CONTROL_DOCUMENT_MAX_BYTES = 1024 * 1024
|
|
18
31
|
TOMBSTONE_DOCUMENT_MAX_BYTES = 1024
|
|
19
32
|
CANDIDATE_TOKEN_RE = re.compile(r"[0-9a-f]{64}\Z")
|
|
33
|
+
OBSERVATION_ID_RE = re.compile(r"[0-9a-f]{32}\Z")
|
|
20
34
|
AXES = ("fiveHour", "sevenDay")
|
|
21
35
|
_SIGNED_I64_MIN = -(2**63)
|
|
22
36
|
_SIGNED_I64_MAX = 2**63 - 1
|
|
@@ -81,6 +95,42 @@ class RetrySignature:
|
|
|
81
95
|
db_reset_generation: int | None
|
|
82
96
|
|
|
83
97
|
|
|
98
|
+
@dataclasses.dataclass(frozen=True)
|
|
99
|
+
class SupportingObservation:
|
|
100
|
+
"""One observation that reported the pending low.
|
|
101
|
+
|
|
102
|
+
``observation_id`` identifies the observation itself rather than the moment
|
|
103
|
+
this machine read it. It is a pure function of the reporting contributor
|
|
104
|
+
and the axis content that contributor reported, so re-reading one spool
|
|
105
|
+
record and receiving another render of the same upstream reading both
|
|
106
|
+
produce the identity already retained. Neither is a second distinct
|
|
107
|
+
confirmation, which a receipt-time comparison alone cannot express.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
observation_id: str
|
|
111
|
+
token: str
|
|
112
|
+
percent: float
|
|
113
|
+
raw_resets_at: int
|
|
114
|
+
received_at: int
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclasses.dataclass(frozen=True)
|
|
118
|
+
class RetainedLow:
|
|
119
|
+
"""The low value a pending drop will publish, and the evidence for it.
|
|
120
|
+
|
|
121
|
+
A candidate is active only while ``-5 <= now - received_at < 90`` while the
|
|
122
|
+
deadline is 180 seconds, so every contributor supporting a pending low can
|
|
123
|
+
age out of the active set before that drop's own deadline expires. The
|
|
124
|
+
record therefore carries the value and its supporting observations rather
|
|
125
|
+
than re-deriving them from a live candidate set that is empty by then.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
percent: float
|
|
129
|
+
raw_resets_at: int
|
|
130
|
+
canonical_key: int
|
|
131
|
+
observations: tuple[SupportingObservation, ...]
|
|
132
|
+
|
|
133
|
+
|
|
84
134
|
@dataclasses.dataclass(frozen=True)
|
|
85
135
|
class PendingDrop:
|
|
86
136
|
canonical_key: int
|
|
@@ -90,6 +140,12 @@ class PendingDrop:
|
|
|
90
140
|
attempts: int
|
|
91
141
|
contributors: Mapping[str, Contributor]
|
|
92
142
|
retry_signature: RetrySignature | None
|
|
143
|
+
#: First-seen plus ``PENDING_DROP_DEADLINE_SECONDS``. Persisted once and
|
|
144
|
+
#: never advanced: neither membership churn nor an upward report may move
|
|
145
|
+
#: it. Deliberately carries no default, so that every construction site
|
|
146
|
+
#: states the deadline the record will be judged against.
|
|
147
|
+
deadline_at: int
|
|
148
|
+
retained: RetainedLow | None = None
|
|
93
149
|
|
|
94
150
|
|
|
95
151
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -322,14 +378,65 @@ def _retry_signature(value: object) -> RetrySignature | None:
|
|
|
322
378
|
)
|
|
323
379
|
|
|
324
380
|
|
|
325
|
-
def
|
|
381
|
+
def _supporting_observation(value: object, *, now_epoch: int) -> SupportingObservation:
|
|
382
|
+
doc = _require_object(value, "supporting observation")
|
|
383
|
+
_require_exact_keys(
|
|
384
|
+
doc, {"observationId", "token", "percent", "rawResetsAt", "receivedAt"}
|
|
385
|
+
)
|
|
386
|
+
if not isinstance(doc["observationId"], str) or not OBSERVATION_ID_RE.fullmatch(
|
|
387
|
+
doc["observationId"]
|
|
388
|
+
):
|
|
389
|
+
raise StateValidationError("invalid observationId")
|
|
390
|
+
if not isinstance(doc["token"], str) or not CANDIDATE_TOKEN_RE.fullmatch(doc["token"]):
|
|
391
|
+
raise StateValidationError("invalid supporting observation token")
|
|
392
|
+
if not _is_int(doc["rawResetsAt"]):
|
|
393
|
+
raise StateValidationError("rawResetsAt must be an integer")
|
|
394
|
+
received = doc["receivedAt"]
|
|
395
|
+
if not _is_int(received) or not 0 <= received <= now_epoch + 5:
|
|
396
|
+
raise StateValidationError("invalid supporting receivedAt")
|
|
397
|
+
return SupportingObservation(
|
|
398
|
+
observation_id=doc["observationId"],
|
|
399
|
+
token=doc["token"],
|
|
400
|
+
percent=_percent(doc["percent"]),
|
|
401
|
+
raw_resets_at=doc["rawResetsAt"],
|
|
402
|
+
received_at=received,
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _retained_low(value: object, *, now_epoch: int) -> RetainedLow | None:
|
|
326
407
|
if value is None:
|
|
327
408
|
return None
|
|
328
|
-
doc = _require_object(value, "
|
|
329
|
-
_require_exact_keys(
|
|
330
|
-
|
|
331
|
-
|
|
409
|
+
doc = _require_object(value, "retained low")
|
|
410
|
+
_require_exact_keys(doc, {"percent", "rawResetsAt", "canonicalKey", "observations"})
|
|
411
|
+
for key in ("rawResetsAt", "canonicalKey"):
|
|
412
|
+
if not _is_int(doc[key]):
|
|
413
|
+
raise StateValidationError(f"{key} must be an integer")
|
|
414
|
+
observations = doc["observations"]
|
|
415
|
+
if not isinstance(observations, list) or len(observations) > 4096:
|
|
416
|
+
raise StateValidationError("observations must be a bounded array")
|
|
417
|
+
return RetainedLow(
|
|
418
|
+
percent=_percent(doc["percent"]),
|
|
419
|
+
raw_resets_at=doc["rawResetsAt"],
|
|
420
|
+
canonical_key=doc["canonicalKey"],
|
|
421
|
+
observations=tuple(
|
|
422
|
+
_supporting_observation(item, now_epoch=now_epoch) for item in observations
|
|
423
|
+
),
|
|
332
424
|
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _pending_drop(
|
|
428
|
+
value: object, *, now_epoch: int, control_version: int = CONTROL_SCHEMA_VERSION
|
|
429
|
+
) -> PendingDrop | None:
|
|
430
|
+
if value is None:
|
|
431
|
+
return None
|
|
432
|
+
doc = _require_object(value, "pending drop")
|
|
433
|
+
required = {
|
|
434
|
+
"canonicalKey", "reducedPercent", "firstSeenAt", "kernelStage",
|
|
435
|
+
"attempts", "contributors", "retrySignature",
|
|
436
|
+
}
|
|
437
|
+
if control_version >= 2:
|
|
438
|
+
required |= {"deadlineAt", "retained"}
|
|
439
|
+
_require_exact_keys(doc, required)
|
|
333
440
|
if not _is_int(doc["canonicalKey"]):
|
|
334
441
|
raise StateValidationError("canonicalKey must be an integer")
|
|
335
442
|
if not _is_int(doc["firstSeenAt"]) or not 0 <= doc["firstSeenAt"] <= now_epoch + 5:
|
|
@@ -355,22 +462,46 @@ def _pending_drop(value: object, *, now_epoch: int) -> PendingDrop | None:
|
|
|
355
462
|
contributors[token] = Contributor(
|
|
356
463
|
baseline_received_at=baseline, satisfied=contributor_doc["satisfied"]
|
|
357
464
|
)
|
|
465
|
+
reduced_percent = _percent(doc["reducedPercent"])
|
|
466
|
+
retry_signature = _retry_signature(doc["retrySignature"])
|
|
467
|
+
if control_version < 2:
|
|
468
|
+
# A pending drop written by the previous binary carries neither the
|
|
469
|
+
# deadline nor the retained evidence, and neither can be recovered. The
|
|
470
|
+
# evidence is not invented, and the deadline is not reconstructed from
|
|
471
|
+
# `firstSeenAt` either: that instant was stamped under the pre-#755
|
|
472
|
+
# arming rule, where a pending drop was armed from the reduced maximum
|
|
473
|
+
# rather than from a retained low, so it does not mean "when this low
|
|
474
|
+
# was first seen" under this contract. The record is therefore
|
|
475
|
+
# unreadable here and is discarded on read; the drop re-arms from live
|
|
476
|
+
# evidence at the next evaluation. The rest of the document — the
|
|
477
|
+
# projection it was read for — is kept, and its shape is still validated
|
|
478
|
+
# above, so a malformed record is refused rather than dropped quietly.
|
|
479
|
+
# A binary running the previous version discards this one's control
|
|
480
|
+
# document in the same way, for the same reason.
|
|
481
|
+
return None
|
|
482
|
+
deadline_at = doc["deadlineAt"]
|
|
483
|
+
if not _is_int(deadline_at) or deadline_at < doc["firstSeenAt"]:
|
|
484
|
+
raise StateValidationError("invalid deadlineAt")
|
|
485
|
+
retained = _retained_low(doc["retained"], now_epoch=now_epoch)
|
|
358
486
|
return PendingDrop(
|
|
359
487
|
canonical_key=doc["canonicalKey"],
|
|
360
|
-
reduced_percent=
|
|
488
|
+
reduced_percent=reduced_percent,
|
|
361
489
|
first_seen_at=doc["firstSeenAt"],
|
|
362
490
|
kernel_stage=doc["kernelStage"],
|
|
363
491
|
attempts=doc["attempts"],
|
|
364
492
|
contributors=contributors,
|
|
365
|
-
retry_signature=
|
|
493
|
+
retry_signature=retry_signature,
|
|
494
|
+
deadline_at=deadline_at,
|
|
495
|
+
retained=retained,
|
|
366
496
|
)
|
|
367
497
|
|
|
368
498
|
|
|
369
499
|
def validate_control_document(document: object, *, now_epoch: int) -> ControlState:
|
|
370
500
|
doc = _require_object(document, "control state")
|
|
371
501
|
_require_exact_keys(doc, {"schemaVersion", "dbProjection", "dbFiles", "pendingDrops"})
|
|
372
|
-
if not _is_int(doc["schemaVersion"]) or doc["schemaVersion"]
|
|
502
|
+
if not _is_int(doc["schemaVersion"]) or doc["schemaVersion"] not in CONTROL_SCHEMA_VERSIONS_READ:
|
|
373
503
|
raise StateValidationError("unsupported schemaVersion")
|
|
504
|
+
control_version = doc["schemaVersion"]
|
|
374
505
|
projection_doc = _require_object(doc["dbProjection"], "dbProjection")
|
|
375
506
|
_require_exact_keys(projection_doc, {"fiveHour", "sevenDay"})
|
|
376
507
|
files_doc = _require_object(doc["dbFiles"], "dbFiles")
|
|
@@ -387,8 +518,12 @@ def validate_control_document(document: object, *, now_epoch: int) -> ControlSta
|
|
|
387
518
|
db_files={"main": main, "wal": _fingerprint(files_doc["wal"])},
|
|
388
519
|
),
|
|
389
520
|
pending_drops={
|
|
390
|
-
"fiveHour": _pending_drop(
|
|
391
|
-
|
|
521
|
+
"fiveHour": _pending_drop(
|
|
522
|
+
pending_doc["fiveHour"], now_epoch=now_epoch, control_version=control_version
|
|
523
|
+
),
|
|
524
|
+
"sevenDay": _pending_drop(
|
|
525
|
+
pending_doc["sevenDay"], now_epoch=now_epoch, control_version=control_version
|
|
526
|
+
),
|
|
392
527
|
},
|
|
393
528
|
)
|
|
394
529
|
|
|
@@ -537,8 +672,88 @@ def _reduced_candidate(
|
|
|
537
672
|
return AxisValue(maximum, raw, canonical_key=newest_key), in_window
|
|
538
673
|
|
|
539
674
|
|
|
675
|
+
def observation_identity(token: str, axis: str, value: AxisValue) -> str:
|
|
676
|
+
"""Identify one upstream reading, independently of when it was read.
|
|
677
|
+
|
|
678
|
+
The digest covers the reporting contributor and the axis content it
|
|
679
|
+
reported. It deliberately excludes the receipt time, because the receipt
|
|
680
|
+
time records when this machine rendered the reading rather than when the
|
|
681
|
+
provider produced it: several renders of one cached upstream block carry
|
|
682
|
+
distinct receipt times and are still one observation.
|
|
683
|
+
"""
|
|
684
|
+
payload = f"{token}|{axis}|{float(value.percent)!r}|{int(value.raw_resets_at)}"
|
|
685
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _supporting_observations(
|
|
689
|
+
in_window: tuple[tuple[Candidate, AxisValue], ...], axis: str, low: float
|
|
690
|
+
) -> tuple[SupportingObservation, ...]:
|
|
691
|
+
"""The observations reporting ``low``, deduplicated by observation identity."""
|
|
692
|
+
seen: dict[str, SupportingObservation] = {}
|
|
693
|
+
for candidate, value in in_window:
|
|
694
|
+
if value.percent != low:
|
|
695
|
+
continue
|
|
696
|
+
identity = observation_identity(candidate.token, axis, value)
|
|
697
|
+
existing = seen.get(identity)
|
|
698
|
+
if existing is None or candidate.received_at > existing.received_at:
|
|
699
|
+
seen[identity] = SupportingObservation(
|
|
700
|
+
observation_id=identity,
|
|
701
|
+
token=candidate.token,
|
|
702
|
+
percent=value.percent,
|
|
703
|
+
raw_resets_at=value.raw_resets_at,
|
|
704
|
+
received_at=candidate.received_at,
|
|
705
|
+
)
|
|
706
|
+
return tuple(sorted(seen.values(), key=lambda item: item.observation_id))
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def _merged_retained(
|
|
710
|
+
previous: RetainedLow | None,
|
|
711
|
+
*,
|
|
712
|
+
canonical_key: int,
|
|
713
|
+
low: float,
|
|
714
|
+
observations: tuple[SupportingObservation, ...],
|
|
715
|
+
) -> RetainedLow:
|
|
716
|
+
"""Refresh the retained evidence from the live observations.
|
|
717
|
+
|
|
718
|
+
Observations carrying an identity already retained are the same upstream
|
|
719
|
+
reading seen again, so they replace their own entry rather than adding a
|
|
720
|
+
second confirmation. A different low replaces the record outright, which
|
|
721
|
+
is how an upward correction moves the target without restarting anything
|
|
722
|
+
else.
|
|
723
|
+
"""
|
|
724
|
+
merged: dict[str, SupportingObservation] = {}
|
|
725
|
+
if previous is not None and previous.canonical_key == canonical_key and previous.percent == low:
|
|
726
|
+
merged.update({item.observation_id: item for item in previous.observations})
|
|
727
|
+
for item in observations:
|
|
728
|
+
# `setdefault`, not assignment: an identity already retained is the same
|
|
729
|
+
# upstream reading seen again, and rewriting its receipt time would make
|
|
730
|
+
# the control document differ on every tick while nothing was confirmed.
|
|
731
|
+
merged.setdefault(item.observation_id, item)
|
|
732
|
+
ordered = tuple(sorted(merged.values(), key=lambda item: item.observation_id))
|
|
733
|
+
return RetainedLow(
|
|
734
|
+
percent=low,
|
|
735
|
+
raw_resets_at=min(item.raw_resets_at for item in ordered),
|
|
736
|
+
canonical_key=canonical_key,
|
|
737
|
+
observations=ordered,
|
|
738
|
+
)
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def _retained_axis_value(retained: RetainedLow, db_axis: AxisProjection | None) -> AxisValue:
|
|
742
|
+
"""The value a pending drop publishes, using the DB's own raw reset when it
|
|
743
|
+
describes the same physical window — the rule `_reduced_candidate` applies."""
|
|
744
|
+
if db_axis is not None and db_axis.canonical_key == retained.canonical_key:
|
|
745
|
+
raw = db_axis.raw_resets_at
|
|
746
|
+
else:
|
|
747
|
+
raw = retained.raw_resets_at
|
|
748
|
+
return AxisValue(retained.percent, raw, canonical_key=retained.canonical_key)
|
|
749
|
+
|
|
750
|
+
|
|
540
751
|
def _new_pending(
|
|
541
|
-
reduced: AxisValue,
|
|
752
|
+
reduced: AxisValue,
|
|
753
|
+
contributors: tuple[tuple[Candidate, AxisValue], ...],
|
|
754
|
+
now_epoch: int,
|
|
755
|
+
*,
|
|
756
|
+
retained: RetainedLow,
|
|
542
757
|
) -> PendingDrop:
|
|
543
758
|
return PendingDrop(
|
|
544
759
|
canonical_key=_axis_value_key(reduced),
|
|
@@ -551,6 +766,8 @@ def _new_pending(
|
|
|
551
766
|
for candidate, _ in contributors
|
|
552
767
|
},
|
|
553
768
|
retry_signature=None,
|
|
769
|
+
deadline_at=now_epoch + PENDING_DROP_DEADLINE_SECONDS,
|
|
770
|
+
retained=retained,
|
|
554
771
|
)
|
|
555
772
|
|
|
556
773
|
|
|
@@ -628,6 +845,82 @@ def _pending_kernel_attempt(
|
|
|
628
845
|
return None
|
|
629
846
|
|
|
630
847
|
|
|
848
|
+
def _has_evidence(retained: RetainedLow | None) -> bool:
|
|
849
|
+
"""Does this record carry a low with at least one supporting observation?
|
|
850
|
+
|
|
851
|
+
A retained low with no observations supports nothing, so it can neither be
|
|
852
|
+
published at expiry nor be retracted by a live contributor. Both readers of
|
|
853
|
+
a pending record consult this, because disagreeing about the evidence-free
|
|
854
|
+
case had `_reduce_axis` cancelling where `_hold_pending` would publish.
|
|
855
|
+
"""
|
|
856
|
+
return retained is not None and bool(retained.observations)
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
def _supporters_retracted(
|
|
860
|
+
pending: PendingDrop,
|
|
861
|
+
in_window: tuple[tuple[Candidate, AxisValue], ...],
|
|
862
|
+
db_axis: AxisProjection,
|
|
863
|
+
) -> bool:
|
|
864
|
+
"""Has every PRESENT retained supporter reported at or above the baseline?
|
|
865
|
+
|
|
866
|
+
Cancellation and expiry are distinct outcomes. Cancellation means a
|
|
867
|
+
contributor actively reported a value at or above the database baseline,
|
|
868
|
+
retracting the evidence it once supplied. A supporter that merely aged out
|
|
869
|
+
of the 90-second active window has reported nothing, and reading its absence
|
|
870
|
+
as retraction is the same mistake as clearing on an empty spool.
|
|
871
|
+
|
|
872
|
+
Both rules hold at once only if the decision is taken over the supporters
|
|
873
|
+
that are actually present, and only if that set is non-empty. Requiring
|
|
874
|
+
EVERY supporter to be present instead let one absent peer veto a live
|
|
875
|
+
retraction: a session that came back and said the database value was right
|
|
876
|
+
was ignored because a peer had ended, and the drop then published a low no
|
|
877
|
+
live contributor still supported.
|
|
878
|
+
"""
|
|
879
|
+
if not _has_evidence(pending.retained):
|
|
880
|
+
return False
|
|
881
|
+
supporters = {item.token for item in pending.retained.observations}
|
|
882
|
+
present = {candidate.token: value for candidate, value in in_window}
|
|
883
|
+
live = [present[token] for token in supporters if token in present]
|
|
884
|
+
if not live:
|
|
885
|
+
# Every supporter is absent, which is silence rather than retraction.
|
|
886
|
+
return False
|
|
887
|
+
return all(value.percent >= db_axis.percent for value in live)
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def _hold_pending(
|
|
891
|
+
pending: PendingDrop | None, db_axis: AxisProjection | None, now_epoch: int
|
|
892
|
+
) -> _AxisDecision:
|
|
893
|
+
"""Decide a pending drop with no eligible candidate on its axis.
|
|
894
|
+
|
|
895
|
+
An axis can lose every candidate while the spool is not empty at all,
|
|
896
|
+
because another axis still has candidates and evaluations keep running, and
|
|
897
|
+
an in-flight tombstone suppresses an axis the same way. Empty membership is
|
|
898
|
+
never unanimous confirmation, so preservation is decided here per axis and
|
|
899
|
+
per physical window rather than per spool.
|
|
900
|
+
"""
|
|
901
|
+
if pending is None:
|
|
902
|
+
return _AxisDecision("NOOP", None, None)
|
|
903
|
+
if not _has_evidence(pending.retained):
|
|
904
|
+
# Evidence-free pending state — a record built by hand, or one whose
|
|
905
|
+
# retained low carries no observation — has nothing to publish and no
|
|
906
|
+
# live contributor left to confirm it.
|
|
907
|
+
return _AxisDecision("WRITE_CONTROL", None, None)
|
|
908
|
+
if db_axis is None or db_axis.canonical_key != pending.canonical_key:
|
|
909
|
+
return _AxisDecision("NOOP", None, pending)
|
|
910
|
+
if db_axis.percent <= pending.retained.percent:
|
|
911
|
+
# The database already holds the drop, so the record has done its work.
|
|
912
|
+
return _AxisDecision("WRITE_CONTROL", None, None)
|
|
913
|
+
if now_epoch < pending.deadline_at:
|
|
914
|
+
return _AxisDecision("NOOP", None, pending)
|
|
915
|
+
publish = _retained_axis_value(pending.retained, db_axis)
|
|
916
|
+
attempted = _pending_kernel_attempt(pending, publish, db_axis)
|
|
917
|
+
if attempted is not None:
|
|
918
|
+
return _AxisDecision("PUBLISH_DB", publish, attempted)
|
|
919
|
+
# The bounded attempts are spent and no contributor remains that could
|
|
920
|
+
# change the signature, so retaining the record would never advance it.
|
|
921
|
+
return _AxisDecision("WRITE_CONTROL", None, None)
|
|
922
|
+
|
|
923
|
+
|
|
631
924
|
def _reduce_axis(
|
|
632
925
|
axis: str,
|
|
633
926
|
candidates: tuple[Candidate, ...],
|
|
@@ -640,7 +933,7 @@ def _reduce_axis(
|
|
|
640
933
|
candidates, axis=axis, tombstone=tombstone, db_axis=db_axis
|
|
641
934
|
)
|
|
642
935
|
if reduced_data is None:
|
|
643
|
-
return
|
|
936
|
+
return _hold_pending(pending, db_axis, now_epoch)
|
|
644
937
|
reduced, in_window = reduced_data
|
|
645
938
|
key = _axis_value_key(reduced)
|
|
646
939
|
|
|
@@ -649,39 +942,110 @@ def _reduce_axis(
|
|
|
649
942
|
if key > db_axis.canonical_key:
|
|
650
943
|
return _AxisDecision("PUBLISH_DB", reduced, None)
|
|
651
944
|
if key < db_axis.canonical_key:
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
945
|
+
# A stale older window says nothing about the pending newer one.
|
|
946
|
+
return _hold_pending(pending, db_axis, now_epoch)
|
|
947
|
+
# `reduced` is the MAXIMUM across active contributors, so it can equal or
|
|
948
|
+
# exceed the database value while a lower contributor is reporting a real
|
|
949
|
+
# drop. The maximum decides what is displayed; it may not erase the
|
|
950
|
+
# evidence that a drop is pending, which is why the low is read separately
|
|
951
|
+
# here and why a rise is a publication rather than a branch of its own.
|
|
952
|
+
rising = reduced.percent > db_axis.percent
|
|
953
|
+
low = min(value.percent for _, value in in_window)
|
|
954
|
+
armed = (
|
|
955
|
+
pending is not None
|
|
956
|
+
and pending.canonical_key == key
|
|
957
|
+
and _has_evidence(pending.retained)
|
|
958
|
+
)
|
|
959
|
+
|
|
960
|
+
if not armed:
|
|
961
|
+
if rising:
|
|
962
|
+
# The rise moves the baseline and no drop is pending against the old
|
|
963
|
+
# one, so there is nothing to preserve.
|
|
964
|
+
return _AxisDecision("PUBLISH_DB", reduced, None)
|
|
965
|
+
if low >= db_axis.percent:
|
|
966
|
+
return _AxisDecision(
|
|
967
|
+
"WRITE_CONTROL" if pending is not None else "NOOP", None, None
|
|
968
|
+
)
|
|
969
|
+
supporters = _supporting_observations(in_window, axis, low)
|
|
970
|
+
retained = _merged_retained(
|
|
971
|
+
None, canonical_key=key, low=low, observations=supporters
|
|
972
|
+
)
|
|
656
973
|
return _AxisDecision(
|
|
657
|
-
"WRITE_CONTROL"
|
|
658
|
-
None,
|
|
659
|
-
None,
|
|
974
|
+
"WRITE_CONTROL", None, _new_pending(reduced, in_window, now_epoch, retained=retained)
|
|
660
975
|
)
|
|
661
976
|
|
|
662
|
-
#
|
|
663
|
-
#
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
977
|
+
# A cancelled or already-recorded drop stops being pending, and a rise still
|
|
978
|
+
# publishes on the way out.
|
|
979
|
+
cancelled = _AxisDecision(
|
|
980
|
+
"PUBLISH_DB" if rising else "WRITE_CONTROL", reduced if rising else None, None
|
|
981
|
+
)
|
|
982
|
+
if _supporters_retracted(pending, in_window, db_axis):
|
|
983
|
+
return cancelled
|
|
984
|
+
if db_axis.percent <= pending.retained.percent:
|
|
985
|
+
return cancelled
|
|
670
986
|
|
|
671
987
|
reconciled = _reconcile_pending(pending, in_window)
|
|
672
|
-
if
|
|
673
|
-
|
|
988
|
+
if low < db_axis.percent:
|
|
989
|
+
supporters = _supporting_observations(in_window, axis, low)
|
|
990
|
+
retained = _merged_retained(
|
|
991
|
+
pending.retained, canonical_key=key, low=low, observations=supporters
|
|
992
|
+
)
|
|
993
|
+
else:
|
|
994
|
+
# Every present contributor is at or above the baseline while at least
|
|
995
|
+
# one supporter is merely absent. Freeze the evidence rather than
|
|
996
|
+
# rewriting it from a set that no longer contains the low.
|
|
997
|
+
retained = pending.retained
|
|
998
|
+
# An upward correction updates the reduced maximum and the retained low. It
|
|
999
|
+
# never restarts the generation: `first_seen_at`, `deadline_at` and every
|
|
1000
|
+
# other contributor's progress are carried through unchanged.
|
|
1001
|
+
updated = dataclasses.replace(
|
|
1002
|
+
reconciled, reduced_percent=reduced.percent, retained=retained
|
|
1003
|
+
)
|
|
1004
|
+
|
|
1005
|
+
if rising:
|
|
1006
|
+
# The rise moves the database baseline, and that is all it does.
|
|
1007
|
+
# Discarding the pending drop here and re-arming it against the new
|
|
1008
|
+
# baseline was an upward report advancing the single deadline D5 fixes
|
|
1009
|
+
# at the first pending drop, and it is the stall family: a peer
|
|
1010
|
+
# reporting a higher number erased another session's real reset. The
|
|
1011
|
+
# retained low keeps its meaning, because the baseline only moved UP —
|
|
1012
|
+
# a low armed below the old baseline is still a drop below the new one.
|
|
1013
|
+
return _AxisDecision("PUBLISH_DB", reduced, updated)
|
|
1014
|
+
|
|
1015
|
+
unanimous = (
|
|
1016
|
+
reduced.percent < db_axis.percent
|
|
1017
|
+
and bool(updated.contributors)
|
|
1018
|
+
and all(item.satisfied for item in updated.contributors.values())
|
|
1019
|
+
)
|
|
1020
|
+
if unanimous:
|
|
1021
|
+
attempted = _pending_kernel_attempt(updated, reduced, db_axis)
|
|
674
1022
|
if attempted is not None:
|
|
675
1023
|
return _AxisDecision("PUBLISH_DB", reduced, attempted)
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
1024
|
+
elif now_epoch >= updated.deadline_at:
|
|
1025
|
+
# D5: the first eligible evaluation after expiry publishes the retained
|
|
1026
|
+
# supported low despite unsatisfied contributors. Never the
|
|
1027
|
+
# all-contributor maximum, never the old high, and never a fresh
|
|
1028
|
+
# generation over the same drop.
|
|
1029
|
+
publish = _retained_axis_value(updated.retained, db_axis)
|
|
1030
|
+
attempted = _pending_kernel_attempt(updated, publish, db_axis)
|
|
1031
|
+
if attempted is not None:
|
|
1032
|
+
return _AxisDecision("PUBLISH_DB", publish, attempted)
|
|
1033
|
+
# The bounded attempts are spent against an unchanged signature, so this
|
|
1034
|
+
# record can no longer advance — the same state `_hold_pending`
|
|
1035
|
+
# terminates on, reached with contributors still present. Keeping it
|
|
1036
|
+
# would rewrite the control document on every tick whose membership
|
|
1037
|
+
# differs, and would let a later signature change re-publish evidence
|
|
1038
|
+
# that expired long ago. What a live contributor still reports arms a
|
|
1039
|
+
# NEW drop from current evidence at the next evaluation, which is the
|
|
1040
|
+
# difference from the unanimous path above: that one publishes the LIVE
|
|
1041
|
+
# reduced value, so re-firing it on a signature change republishes
|
|
1042
|
+
# something current.
|
|
1043
|
+
return _AxisDecision("WRITE_CONTROL", None, None)
|
|
1044
|
+
|
|
681
1045
|
return _AxisDecision(
|
|
682
|
-
"WRITE_CONTROL" if _axis_control_changed(pending,
|
|
1046
|
+
"WRITE_CONTROL" if _axis_control_changed(pending, updated) else "NOOP",
|
|
683
1047
|
None,
|
|
684
|
-
|
|
1048
|
+
updated,
|
|
685
1049
|
)
|
|
686
1050
|
|
|
687
1051
|
|