cctally 1.115.0 → 1.116.0

package/bin/_lib_statusline_candidates.py

@@ -12,6 +12,8 @@ import math
 import re
 from typing import Callable, Mapping
 
+import _lib_provider_activity
+
 
 #: Candidate and tombstone documents.  Every session on this machine writes
 #: candidates, so this version may only move when peer sessions running the
@@ -59,6 +61,11 @@ class Candidate:
     #: The immutable `AccountBinding` captured with this candidate (#825/#832
     #: spec 4.2). The reducer partitions by its account; it is never inferred.
     binding: object | None = None
+    #: The provider-activity witness (#797 spec 4.2,
+    #: `_lib_provider_activity.ActivityWitness`) captured once at this
+    #: arrival, before its intake commit. Required in every v2 candidate
+    #: document, so the intake identity hashes it; never recomputed.
+    activity: object | None = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -307,6 +314,16 @@ def _require_ascii_source(value: object) -> str:
     return value
 
 
+def validate_activity(value: object):
+    """The candidate's activity witness, or ``StateValidationError``: a
+    missing or malformed witness makes the v2 document malformed and never
+    silently reads as ``unknown`` (#797 spec 4.2)."""
+    try:
+        return _lib_provider_activity.witness_from_json(value)
+    except _lib_provider_activity.MalformedWitness as exc:
+        raise StateValidationError(str(exc)) from exc
+
+
 def _axis_value(
     value: object,
     *,
@@ -1118,7 +1135,14 @@ def _reduce_axis(
     scope=None,
     drop_allowed: bool = True,
     evidence: AxisEvidence | None = None,
+    member_facts: Mapping[str, object] | None = None,
 ) -> _AxisDecision:
+    # #797 spec 4.4 (Q3): the per-member refusal facts
+    # (`_lib_stale_proof.MemberFacts`, keyed by contributor token) are
+    # resolved before the kernel. None, or no refused member, reduces
+    # exactly as before.
+    refused = frozenset(
+        token for token, facts in (member_facts or {}).items() if facts.refused)
     reduced_data = _reduced_candidate(
         candidates, axis=axis, tombstone=tombstone, db_axis=db_axis
     )
@@ -1137,8 +1161,11 @@ def _reduce_axis(
             # ordinary first post-floor reading; one captured before it is
             # stale and is never relabelled as post-floor. The reading opens
             # no consensus and spends no attempt, under either gate.
+            # #797 spec 4.4: a refused member is excluded from the post-floor
+            # set; every other member keeps the receipt-floor test.
             post = tuple((candidate, value) for candidate, value in in_window
-                         if candidate.received_at >= int(evidence.floor_epoch))
+                         if candidate.token not in refused
+                         and candidate.received_at >= int(evidence.floor_epoch))
             if not post:
                 verdict = CLASS_STALE
             else:
@@ -1184,7 +1211,13 @@ def _reduce_axis(
     # drop.  The maximum decides what is displayed; it may not erase the
     # evidence that a drop is pending, which is why the low is read separately
     # here and why a rise is a publication rather than a branch of its own.
-    rising = reduced.percent > db_axis.percent
+    # #797 spec 4.4 (Q3): `rising` and the published value come from the
+    # members that are not refused; low evidence, supporter retraction,
+    # unanimity, pending membership, `reduced_percent`, deadlines and attempt
+    # accounting keep the unfiltered set. The eligible value applies at every
+    # rising return (plain, armed, cancelled and reconciled).
+    rise = _eligible_rise(reduced, in_window, refused, db_axis)
+    rising = rise is not None and rise.percent > db_axis.percent
     low = min(value.percent for _, value in in_window)
     armed = (
         pending is not None
@@ -1196,7 +1229,7 @@ def _reduce_axis(
         if rising:
             # The rise moves the baseline and no drop is pending against the old
             # one, so there is nothing to preserve.
-            return _AxisDecision("PUBLISH_DB", reduced, None)
+            return _AxisDecision("PUBLISH_DB", rise, None)
         if low >= db_axis.percent:
             return _AxisDecision(
                 "WRITE_CONTROL" if pending is not None else "NOOP", None, None
@@ -1212,7 +1245,7 @@ def _reduce_axis(
     # A cancelled or already-recorded drop stops being pending, and a rise still
     # publishes on the way out.
     cancelled = _AxisDecision(
-        "PUBLISH_DB" if rising else "WRITE_CONTROL", reduced if rising else None, None
+        "PUBLISH_DB" if rising else "WRITE_CONTROL", rise if rising else None, None
     )
     if _supporters_retracted(pending, in_window, db_axis):
         return cancelled
@@ -1245,7 +1278,7 @@ def _reduce_axis(
         # reporting a higher number erased another session's real reset.  The
         # retained low keeps its meaning, because the baseline only moved UP —
         # a low armed below the old baseline is still a drop below the new one.
-        return _AxisDecision("PUBLISH_DB", reduced, updated)
+        return _AxisDecision("PUBLISH_DB", rise, updated)
 
     unanimous = (
         reduced.percent < db_axis.percent
@@ -1289,6 +1322,25 @@ def _reduce_axis(
     )
 
 
+def _eligible_rise(
+    reduced: AxisValue,
+    in_window: tuple[tuple[Candidate, AxisValue], ...],
+    refused: frozenset,
+    db_axis: AxisProjection,
+) -> AxisValue | None:
+    """The same-window maximum over members that are not refused (#797 spec
+    4.4), or None when every member is refused. Without a refused member it
+    is ``reduced`` itself."""
+    if not refused or not any(c.token in refused for c, _ in in_window):
+        return reduced
+    eligible = [value.percent for candidate, value in in_window
+                if candidate.token not in refused]
+    if not eligible:
+        return None
+    return AxisValue(max(eligible), db_axis.raw_resets_at,
+                     canonical_key=reduced.canonical_key)
+
+
 def reduce_candidates(
     candidates: tuple[Candidate, ...] | list[Candidate],
     *,
@@ -1298,6 +1350,7 @@ def reduce_candidates(
     now_epoch: int,
     account_key: str | None = None,
     drop_consensus_allowed: bool = True,
+    member_facts: Mapping[str, Mapping[str, object]] | None = None,
 ) -> ReductionDecision:
     """Reduce active candidates independently for 5h and 7d axes.
 
@@ -1319,7 +1372,12 @@ def reduce_candidates(
     reading after a retiring floor); a stale reading takes the older-window
     disposition, and an unclassifiable one is named in ``unknown_axes`` (fix
     rounds K13 and K14c).
+
+    ``member_facts`` maps an axis to its per-member refusal facts (#797 spec
+    4.4, `_lib_stale_proof.MemberFacts` by contributor token). None, or an
+    axis without facts, means every member is eligible.
     """
+    facts = member_facts or {}
     active = tuple(
         candidate
         for candidate in candidates
@@ -1335,6 +1393,7 @@ def reduce_candidates(
         scope=None if account_key is None else (PROTOCOL_GENERATION, account_key, "fiveHour"),
         drop_allowed=drop_consensus_allowed,
         evidence=axis_evidence(db, "fiveHour"),
+        member_facts=facts.get("fiveHour"),
     )
     seven = _reduce_axis(
         "sevenDay",
@@ -1346,6 +1405,7 @@ def reduce_candidates(
         scope=None if account_key is None else (PROTOCOL_GENERATION, account_key, "sevenDay"),
         drop_allowed=drop_consensus_allowed,
         evidence=axis_evidence(db, "sevenDay"),
+        member_facts=facts.get("sevenDay"),
     )
     next_control = ControlState(
         db_projection=db,

package/bin/_lib_worker_launches.py

@@ -0,0 +1,315 @@
+"""The launch ledger of the detached stats rebuild workers (#997, spec D8).
+
+On 2026-10-22 an epoch-rebuild worker ran for hours and nothing recorded
+whether one worker ran throughout or one died and another replaced it: the log
+held only returned outcomes. Each worker (the epoch rebuild and the corruption
+heal) now keeps two private records in ``APP_DIR``: a run state, written only
+by the holder of the worker flock, and this launch ledger, written only under
+the worker's admission flock. This module is the pure kernel that decides what
+became of each launch; spawners, workers, ``doctor`` and ``db status`` all
+classify through it, and only spawners and workers persist the result.
+
+The central rule: **a death is settled by evidence, never by elapsed time.**
+An unsettled entry is settled when
+
+(a) the run state shows its ``launchId`` finished, which transfers that
+    outcome as ``finished:<outcome>``, or records its in-run death; or
+(b) its process is confirmed gone by identity — absent, a zombie, or a live
+    pid whose ``birth`` differs — which settles ``died-pre-start`` without a
+    start acknowledgement and ``died-in-run`` with one. The caller probes the
+    identities BEFORE it reads the files it passes here, so a re-read after the
+    probe is what this function sees.
+
+A process that is alive, or whose liveness is unknown (``ps`` failing, a live
+pid with a null ``birth``), stays unresolved however old the entry is: a slow
+child is never declared dead.
+
+Ledger shape (``schemaVersion`` 1)::
+
+    {"schemaVersion": 1,
+     "entries": [{"launchId", "at", "pid", "birth",
+                  "spawn": "reserved"|"ok"|"failed",
+                  "ack": None | {"kind": "started", "runId", "at"}
+                              | {"kind": "busy", "at"},
+                  "outcome": None | <terminal>, "settledAt"}],
+     "counts": {<terminal>: int},     # cumulative, survives pruning
+     "lastDeathAt": str | None}
+
+``pid`` and ``birth`` are null while ``spawn`` is ``reserved``; ``birth`` may
+stay null after it (``ps`` could not see the child). Unresolved entries are
+never dropped: the spawner admits no launch while ``LAUNCH_CAPACITY`` are
+unresolved (``has_launch_capacity``), which is what bounds them.
+
+``<terminal>`` is ``finished:<worker outcome>``, ``busy``, ``spawn-failed``,
+``died-pre-start`` or ``died-in-run``. Every function returns a new ledger and
+never mutates its input. Stdlib only, and importable on its own.
+"""
+from __future__ import annotations
+
+import copy
+
+SCHEMA_VERSION = 1
+#: Unresolved entries at which the spawner withholds further launches.
+LAUNCH_CAPACITY = 100
+#: Settled entries kept (the most recent); ``counts`` keeps the totals.
+SETTLED_KEEP = 10
+DEATHS = ("died-pre-start", "died-in-run")
+
+
+def empty_ledger() -> dict:
+    return {
+        "schemaVersion": SCHEMA_VERSION,
+        "entries": [],
+        "counts": {},
+        "lastDeathAt": None,
+    }
+
+
+def normalize(ledger) -> dict:
+    """A validated deep copy of ``ledger``; ``ValueError`` when malformed."""
+    if not isinstance(ledger, dict):
+        raise ValueError("launch ledger is not an object")
+    if ledger.get("schemaVersion") != SCHEMA_VERSION:
+        raise ValueError("unsupported launch ledger schemaVersion")
+    entries = ledger.get("entries", [])
+    if not isinstance(entries, list) or not all(
+            isinstance(e, dict) and isinstance(e.get("launchId"), str)
+            for e in entries):
+        raise ValueError("launch ledger entries are malformed")
+    counts = ledger.get("counts") or {}
+    if not isinstance(counts, dict) or not all(
+            isinstance(k, str) and isinstance(v, int)
+            for k, v in counts.items()):
+        raise ValueError("launch ledger counts are malformed")
+    last_death = ledger.get("lastDeathAt")
+    if last_death is not None and not isinstance(last_death, str):
+        raise ValueError("launch ledger lastDeathAt is malformed")
+    out = empty_ledger()
+    out["entries"] = [_entry(**{k: e.get(k) for k in _ENTRY_KEYS})
+                      for e in copy.deepcopy(entries)]
+    out["counts"] = dict(counts)
+    out["lastDeathAt"] = last_death
+    return out
+
+
+_ENTRY_KEYS = ("launchId", "at", "pid", "birth", "spawn", "ack", "outcome",
+               "settledAt")
+
+
+def _entry(launchId, at, pid, birth, spawn="ok", ack=None, outcome=None,
+           settledAt=None) -> dict:
+    return {
+        "launchId": launchId, "at": at, "pid": pid, "birth": birth,
+        "spawn": spawn or "ok", "ack": ack, "outcome": outcome,
+        "settledAt": settledAt,
+    }
+
+
+def find(ledger: dict, launch_id: str) -> "dict | None":
+    for entry in ledger.get("entries", ()):
+        if entry.get("launchId") == launch_id:
+            return entry
+    return None
+
+
+def unresolved(ledger: dict) -> list:
+    return [e for e in ledger.get("entries", ()) if not e.get("outcome")]
+
+
+def newest(ledger: dict) -> "dict | None":
+    """The newest launch in ledger (launch) order."""
+    entries = ledger.get("entries") or []
+    return entries[-1] if entries else None
+
+
+def _settle_in_place(ledger: dict, entry: dict, terminal: str, at) -> bool:
+    if entry.get("outcome"):
+        return False
+    entry["outcome"] = terminal
+    entry["settledAt"] = at
+    counts = ledger.setdefault("counts", {})
+    counts[terminal] = int(counts.get(terminal, 0)) + 1
+    if terminal in DEATHS:
+        ledger["lastDeathAt"] = at or entry.get("at")
+    return True
+
+
+def append_entry(ledger: dict, *, launch_id: str, at, pid, birth,
+                 spawn: str = "ok") -> dict:
+    """Record a launch. A launch id already present is left as it is (the
+    worker's own acknowledgement may have recorded it first). A failed spawn
+    is settled at once as ``spawn-failed``."""
+    out = copy.deepcopy(ledger)
+    if find(out, launch_id) is not None:
+        return out
+    entry = _entry(launch_id, at, pid, birth, spawn=spawn)
+    out.setdefault("entries", []).append(entry)
+    if spawn == "failed":
+        _settle_in_place(out, entry, "spawn-failed", at)
+    return out
+
+
+def reserve(ledger: dict, *, launch_id: str, at) -> dict:
+    """The spawner's reservation, persisted BEFORE the spawn (revision 6): a
+    spawner killed after it leaves an entry with no pid, which stays
+    unresolved with unknown liveness until the child acknowledges."""
+    out = copy.deepcopy(ledger)
+    if find(out, launch_id) is None:
+        out.setdefault("entries", []).append(
+            _entry(launch_id, at, None, None, spawn="reserved"))
+    return out
+
+
+def fill_spawn(ledger: dict, launch_id: str, *, pid, birth, at,
+               spawn: str = "ok") -> dict:
+    """Complete a reservation with the spawn's result. A failed spawn is
+    settled at once as ``spawn-failed``. An identity the worker already
+    recorded at acknowledgement is never overwritten."""
+    out = copy.deepcopy(ledger)
+    entry = find(out, launch_id)
+    if entry is None:
+        out.setdefault("entries", []).append(
+            _entry(launch_id, at, None, None, spawn="reserved"))
+        entry = out["entries"][-1]
+    if spawn == "failed":
+        if entry.get("spawn") == "reserved":
+            entry["spawn"] = "failed"
+        _settle_in_place(out, entry, "spawn-failed", at)
+        return out
+    if entry.get("pid") is None:
+        entry["pid"] = pid
+        entry["birth"] = birth
+    if entry.get("spawn") == "reserved":
+        entry["spawn"] = "ok"
+    return out
+
+
+def acknowledge(ledger: dict, launch_id: str, kind: str, run_id=None, *,
+                at, pid=None, birth=None) -> dict:
+    """A worker's acknowledgement: ``started`` (with its ``runId``) or
+    ``busy`` (which is also the entry's terminal). Fills the worker's own
+    ``pid`` and ``birth`` into a reservation the spawner never completed, and
+    appends the entry when there is none (a ledger lost to the unreadable
+    archive path)."""
+    out = copy.deepcopy(ledger)
+    entry = find(out, launch_id)
+    if entry is None:
+        entry = _entry(launch_id, at, pid, birth)
+        out.setdefault("entries", []).append(entry)
+    elif entry.get("pid") is None and pid is not None:
+        entry["pid"] = pid
+        entry["birth"] = birth
+        if entry.get("spawn") == "reserved":
+            entry["spawn"] = "ok"
+    if kind == "started":
+        entry["ack"] = {"kind": "started", "runId": run_id, "at": at}
+    elif kind == "busy":
+        entry["ack"] = {"kind": "busy", "at": at}
+        _settle_in_place(out, entry, "busy", at)
+    else:
+        raise ValueError(f"unknown acknowledgement kind {kind!r}")
+    return out
+
+
+def settle(ledger: dict, launch_id: str, terminal: str, *, at) -> dict:
+    """Give an entry its terminal outcome; the first settlement wins."""
+    out = copy.deepcopy(ledger)
+    entry = find(out, launch_id)
+    if entry is not None:
+        _settle_in_place(out, entry, terminal, at)
+    return out
+
+
+def liveness(entry: dict, identities: dict) -> str:
+    """``alive`` | ``gone`` | ``unknown`` for a ledger entry or a run (both
+    carry ``pid`` and ``birth``), from ``process_identity`` results."""
+    pid = entry.get("pid")
+    if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
+        return "unknown"
+    identity = identities.get(pid)
+    if not isinstance(identity, dict):
+        return "unknown"
+    status = identity.get("status")
+    if status == "gone":
+        return "gone"
+    if status != "alive":
+        return "unknown"
+    entry_birth, live_birth = entry.get("birth"), identity.get("birth")
+    if entry_birth and live_birth and entry_birth != live_birth:
+        return "gone"  # the pid now names a different process
+    if not entry_birth or not live_birth:
+        return "unknown"
+    return "alive"
+
+
+def classify(ledger: dict, state, identities: dict, *, now=None) -> dict:
+    """Settle what the evidence settles. Returns ``{"ledger", "settled"}``,
+    ``settled`` being the newly settled entries (copies). Never settles by
+    age; see the module docstring for rules (a) and (b)."""
+    out = copy.deepcopy(ledger)
+    run = state.get("run") if isinstance(state, dict) else None
+    run = run if isinstance(run, dict) else None
+    deaths = state.get("deaths") if isinstance(state, dict) else None
+    # A death whose ``launchId`` is not a string (a corrupt record in a
+    # state that still validates) settles nothing.
+    died_in_run = {d.get("launchId") for d in (deaths or ())
+                   if isinstance(d, dict) and isinstance(d.get("launchId"), str)
+                   and d.get("launchId")}
+    settled = []
+    for entry in out.get("entries", []):
+        if entry.get("outcome"):
+            continue
+        launch_id = entry.get("launchId")
+        terminal = None
+        if (run is not None and run.get("launchId") == launch_id
+                and run.get("status") == "finished"):
+            terminal = f"finished:{run.get('outcome')}"
+        elif launch_id in died_in_run:
+            terminal = "died-in-run"
+        elif liveness(entry, identities) == "gone":
+            ack = entry.get("ack")
+            started = isinstance(ack, dict) and ack.get("kind") == "started"
+            terminal = "died-in-run" if started else "died-pre-start"
+        if terminal is not None and _settle_in_place(out, entry, terminal,
+                                                     now):
+            settled.append(copy.deepcopy(entry))
+    return {"ledger": out, "settled": settled}
+
+
+def _protected_ids(protected_launch_ids) -> frozenset:
+    if isinstance(protected_launch_ids, str):
+        return frozenset({protected_launch_ids})
+    if not isinstance(protected_launch_ids, (list, tuple, set, frozenset)):
+        return frozenset()
+    return frozenset(i for i in protected_launch_ids if isinstance(i, str))
+
+
+def prune(ledger: dict, protected_launch_ids=None) -> dict:
+    """Bound the settled entries to the newest ``SETTLED_KEEP``, in ledger
+    order. Unresolved entries are never dropped (``has_launch_capacity``
+    bounds them); ``counts`` and ``lastDeathAt`` are kept.
+
+    ``protected_launch_ids`` (one id, or an iterable of them; a non-string
+    member protects nothing) names the entries a run state references or is
+    about to: the current ``run.launchId`` and, for a worker whose own state
+    write has not landed yet, its own launch as well. Each is exempt from the
+    bound, because the slot's entry is the record that makes the next
+    worker's death accounting exactly-once, and the settled bound then counts
+    them plus ``SETTLED_KEEP`` minus their number of others."""
+    out = copy.deepcopy(ledger)
+    entries = out.get("entries", [])
+    wanted = _protected_ids(protected_launch_ids)
+    protected = {id(e) for e in entries if e.get("launchId") in wanted}
+    done_ids = [id(e) for e in entries
+                if e.get("outcome") and id(e) not in protected]
+    settled_room = max(0, SETTLED_KEEP - sum(
+        1 for e in entries if id(e) in protected and e.get("outcome")))
+    dropped = set(done_ids[:max(0, len(done_ids) - settled_room)])
+    out["entries"] = [e for e in entries if id(e) not in dropped]
+    return out
+
+
+def has_launch_capacity(ledger: dict) -> bool:
+    """Whether the spawner may admit another launch: fewer than
+    ``LAUNCH_CAPACITY`` entries are unresolved."""
+    return len(unresolved(ledger)) < LAUNCH_CAPACITY

package/bin/cctally

@@ -404,7 +404,6 @@ diff_pricing = _lib_pricing_check.diff_pricing
 stale_allowlist_entries = _lib_pricing_check.stale_allowlist_entries
 expired_allowlist_entries = _lib_pricing_check.expired_allowlist_entries
 check_table_shapes = _lib_pricing_check.check_table_shapes
-pricing_issue_action = _lib_pricing_check.pricing_issue_action
 CoverageGap = _lib_pricing_check.CoverageGap
 DriftRow = _lib_pricing_check.DriftRow
 DriftResult = _lib_pricing_check.DriftResult
@@ -1258,13 +1257,11 @@ _delete_doomed_snapshot_rows = _cctally_record._delete_doomed_snapshot_rows
 # #834 S1 (#835) Gate A R4: the band is a connection-taking builder now,
 # because the held predicate is omitted on a store predating the column.
 _stale_replica_band_sql = _cctally_record._stale_replica_band_sql
-# #834 S1 (#835): the read-side twin of the band above — which weekly
-# values a week's credits retired, and the predicate that recognizes one.
-# `_latest_seven_day_and_window` reaches both through this namespace.
+# #834 S1 (#835): a week's credits and their floors.
+# `_latest_seven_day_and_window` reaches both through this namespace for its
+# credit-floor bound (the retirement band itself was removed by #840).
 _credit_retirement_bands = _cctally_record._credit_retirement_bands
 _latest_credit_floor_instant = _cctally_record._latest_credit_floor_instant
-_weekly_value_is_retired_replica = (
-    _cctally_record._weekly_value_is_retired_replica)
 _stale_replay_candidates = _cctally_record._stale_replay_candidates
 _count_stale_replays = _cctally_record._count_stale_replays
 _credit_preview_text = _cctally_record._credit_preview_text
@@ -1606,11 +1603,9 @@ _format_block_start = _cctally_five_hour._format_block_start
 _format_hhmm_in_tz = _cctally_five_hour._format_hhmm_in_tz
 _block_is_active = _cctally_five_hour._block_is_active
 _latest_seven_day_and_window = _cctally_five_hour._latest_seven_day_and_window
-# #834 S1 (#835) Gate A S1: the ONE read-time filter for a block's two
-# weekly axes. Both five-hour commands reach it through this namespace.
-_parsed_block_instant = _cctally_five_hour._parsed_block_instant
-_block_weekly_start_instant = _cctally_five_hour._block_weekly_start_instant
-_block_weekly_end_instant = _cctally_five_hour._block_weekly_end_instant
+# #834 S1 (#835) Gate A S1: the ONE read of a block's two weekly axes (the
+# stored values since #840). Both five-hour commands reach it through this
+# namespace.
 _credit_aware_block_weekly_axes = (
     _cctally_five_hour._credit_aware_block_weekly_axes)
 # #834 S1 (#836): the ONE five-hour milestone read, shared by the CLI