cctally 1.115.0 → 1.116.0

package/bin/_lib_quarantine.py

@@ -9,8 +9,9 @@ Stdlib only. An entry's `id` takes one of two forms:
   the `NAME` of `bin/cctally-NAME-test` and `<case>` is one scenario of that
   harness's own case inventory (`bin/cctally-NAME-test --list-cases`).
 
-`bin/cctally-test-all`, `bin/_lib_merge_gate.py` and the quick gate read the
-list through this module; the scheduled CI quarantine leg calls its CLI. An
+`bin/cctally-test-all` (through `bin/_lib_test_selection.py`) and the quick
+gate read the list through this module, and so does the nightly broad run's
+quarantine leg (bin/cctally-suite-run). An
 absent list quarantines nothing (the public mirror may carry none); a list
 that exists but cannot be read raises QuarantineError, never "nothing listed".
 

package/bin/_lib_quota_model.py

@@ -21,6 +21,7 @@ from __future__ import annotations
 import bisect
 import dataclasses
 import datetime as dt
+import decimal
 import enum
 import hashlib
 import json
@@ -1437,6 +1438,84 @@ def unattributed_units(segments, entries, *, now,
     return {"units": total, "entries": count}
 
 
+# ---------------------------------------------------------------------------
+# Correctly rounded logarithms and exponentials.
+#
+# `math.log`, `math.log1p` and `math.expm1` call the platform C library, and
+# C does not require those to be correctly rounded. glibc 2.39's `log1p` and
+# `expm1` are off by one ULP on inputs the eligibility fence reaches, which
+# moved quota fixture 12's `eligibilityFence` between Linux and macOS. These
+# compute in `decimal`, whose `ln` and `exp` are correctly rounded at the
+# context precision, from the float's exact value, and round once to a
+# double. At 80 significant digits that double rounding can differ from a
+# direct one only if the true value lies within 1e-80 of a midpoint between
+# two doubles, far below what any double input can produce. Each call uses
+# its own context, never the thread's, and the edge cases the decimal path
+# does not model are delegated to `math`, whose answers there are exact.
+# ---------------------------------------------------------------------------
+_CR_DIGITS = 80
+#: The exact sum of 1 and any double needs at most 1075 significant digits.
+_CR_EXACT = decimal.Context(prec=1200, Emin=-999999, Emax=999999,
+                            traps=[decimal.InvalidOperation])
+_CR_ONE = decimal.Decimal(1)
+
+
+def _cr_context(prec: int = _CR_DIGITS) -> decimal.Context:
+    return decimal.Context(prec=prec, rounding=decimal.ROUND_HALF_EVEN,
+                           Emin=-999999, Emax=999999,
+                           traps=[decimal.InvalidOperation,
+                                  decimal.DivisionByZero, decimal.Overflow])
+
+
+def correctly_rounded_log(x) -> float:
+    """`math.log(x)` correctly rounded on every platform.
+
+    Zero, negatives, infinities and NaN go to `math.log`, so the domain error
+    and the non-finite results are exactly the standard library's.
+    """
+    x = float(x)
+    if not math.isfinite(x) or x <= 0.0:
+        return math.log(x)
+    return float(_cr_context().ln(decimal.Decimal(x)))
+
+
+def correctly_rounded_log1p(x) -> float:
+    """`math.log1p(x)` correctly rounded on every platform.
+
+    `1 + x` is formed exactly, so a tiny `x` keeps every digit. Signed zeros,
+    `x <= -1` (the domain error), infinities and NaN go to `math.log1p`.
+    """
+    x = float(x)
+    if not math.isfinite(x) or x == 0.0 or x <= -1.0:
+        return math.log1p(x)
+    one_plus = _CR_EXACT.add(_CR_ONE, decimal.Decimal(x))
+    return float(_cr_context().ln(one_plus))
+
+
+def correctly_rounded_expm1(x) -> float:
+    """`math.expm1(x)` correctly rounded on every platform.
+
+    The working precision grows with `1/|x|`, so the subtraction of one keeps
+    80 significant digits however small `x` is. Below -64 the true value is
+    within 1.6e-28 of -1, which rounds to exactly -1. A finite result too
+    large for a double raises `OverflowError`, as `math.expm1` does. Signed
+    zeros, infinities and NaN go to `math.expm1`.
+    """
+    x = float(x)
+    if not math.isfinite(x) or x == 0.0:
+        return math.expm1(x)
+    if x < -64.0:
+        return -1.0
+    if x > 710.0:
+        return math.expm1(x)  # raises OverflowError
+    d = decimal.Decimal(x)
+    ctx = _cr_context(_CR_DIGITS + max(0, -d.adjusted()))
+    result = float(ctx.subtract(ctx.exp(d), _CR_ONE))
+    if math.isinf(result):
+        raise OverflowError("math range error")
+    return result
+
+
 # ---------------------------------------------------------------------------
 # Robust statistics, the eligibility fence, and composition support (§5, §15).
 # ---------------------------------------------------------------------------
@@ -1476,14 +1555,16 @@ def eligibility_fence(days) -> float | None:
     which carried between 0.8 and 1.2 million weighted units against a
     message claiming no local token history at all.
     """
-    xs = [math.log1p(u) for u in days if u is not None and u > 0.0]
+    xs = [correctly_rounded_log1p(u)
+          for u in days if u is not None and u > 0.0]
     if len(xs) < 2:
         return None
     centre = median(xs)
     spread = scaled_mad(xs, centre)
     if not math.isfinite(spread) or spread == 0.0:
         return None
-    return math.expm1(centre - ELIGIBILITY["fence_mads"] * spread)
+    return correctly_rounded_expm1(
+        centre - ELIGIBILITY["fence_mads"] * spread)
 
 
 def composition_centre(vectors) -> dict[str, float] | None:
@@ -1506,9 +1587,16 @@ def composition_centre(vectors) -> dict[str, float] | None:
 
 
 def tv_distance(p, centre) -> float:
-    """Total-variation distance, `0.5 * sum(|p_i - c_i|)`."""
+    """Total-variation distance, `0.5 * sum(|p_i - c_i|)`.
+
+    `math.fsum`, not `sum`: the keys come from a set, whose iteration order
+    follows PYTHONHASHSEED, and Python 3.11's float `sum()` is uncompensated,
+    so a plain sum moved the published radius by an ULP from one process to
+    the next. `fsum` is correctly rounded, so the result is the same in every
+    order and on every supported interpreter.
+    """
     keys = set(p) | set(centre)
-    return 0.5 * sum(abs(p.get(k, 0.0) - centre.get(k, 0.0)) for k in keys)
+    return 0.5 * math.fsum(abs(p.get(k, 0.0) - centre.get(k, 0.0)) for k in keys)
 
 
 def support_radius(vectors, centre) -> float | None:
@@ -1696,7 +1784,9 @@ def jackknife_fit(days) -> tuple[float, Interval] | None:
             return None
         replicates.append((total_units - row.units) / rest_meter)
     mean = sum(replicates) / n
-    se = math.sqrt((n - 1) / n * sum((v - mean) ** 2 for v in replicates))
+    # A product, not `** 2`, which is the C library's `pow`.
+    se = math.sqrt((n - 1) / n * sum((v - mean) * (v - mean)
+                                     for v in replicates))
     half = t_quantile_975(n - 1) * se
     return point, Interval(max(0.0, point - half), point + half)
 
@@ -1930,7 +2020,7 @@ def detect_change(series, *, override_split=None) -> DetectorResult:
     total = len(days)
     truncated = input_days - total
     scan_start = days[0].date if days else None
-    stats = [math.log(o.units / o.meter_delta) for o in days]
+    stats = [correctly_rounded_log(o.units / o.meter_delta) for o in days]
     min_base = int(TRUST["min_detect_baseline_days"])
     min_watch = int(DETECTOR["min_watch_days"])
     fence_mads = DETECTOR["fence_mads"]

package/bin/_lib_rebuild_progress.py

@@ -0,0 +1,146 @@
+"""Progress and resident memory for the stats index's full replays (#997).
+
+The 2026-10-22 epoch rebuild ran 4.4 hours and died without a word, because
+nothing it did between start and death reached a file. This module is the seam
+a detached worker's state file hangs off (spec D9):
+
+* ``install(sink)`` / ``uninstall()``: one process-wide sink, called as
+  ``sink(phase, counters, forced)``. Only a worker holding its flock installs
+  one; everywhere else every call below is a no-op.
+* ``phase(name, **counters)``: where the replay is now. Always written
+  (``forced``); the counters start over with the ones given.
+* ``tick(**counters)``: merges counters into the current phase's and writes
+  them at most every ``THROTTLE_SECONDS`` (``time.monotonic``), so a call
+  site can tick on every batch without turning progress into I/O.
+* ``waiting(lock)``: a context for a blocking lock wait, written as the phase
+  ``waiting:<lock>`` and replaced by the interrupted phase when it ends.
+
+A sink that raises is ignored: progress never fails a rebuild.
+
+``peak_resident_bytes`` is the process-lifetime peak the rebuild record and the
+worker state report as ``peakResidentBytes`` (spec D11). ``peakHeapBytes``, a
+``tracemalloc`` peak that is zero unless tracing, never saw the memory that
+pushed that rebuild into hours of compression.
+
+Stdlib only, and importable on its own.
+"""
+from __future__ import annotations
+
+import contextlib
+import os
+import resource
+import subprocess
+import sys
+import time
+
+#: The longest a ticking replay goes without a write (spec D9).
+THROTTLE_SECONDS = 10.0
+
+_sink = None
+_phase: "str | None" = None
+_counters: dict = {}
+_last_write: "float | None" = None
+
+
+def install(sink) -> None:
+    """Route progress to ``sink(phase, counters, forced)`` from now on."""
+    global _sink, _phase, _counters, _last_write
+    _sink = sink
+    _phase = None
+    _counters = {}
+    _last_write = None
+
+
+def uninstall() -> None:
+    """Stop routing progress; every call becomes a no-op again."""
+    global _sink, _phase, _counters, _last_write
+    _sink = None
+    _phase = None
+    _counters = {}
+    _last_write = None
+
+
+def _emit(forced: bool) -> None:
+    global _last_write
+    _last_write = time.monotonic()
+    sink = _sink
+    if sink is None:
+        return
+    try:
+        sink(_phase, dict(_counters), forced)
+    except Exception:  # progress never fails a rebuild
+        pass
+
+
+def phase(name: str, **counters) -> None:
+    """Enter phase ``name`` and write it now."""
+    global _phase, _counters
+    if _sink is None:
+        return
+    _phase = name
+    _counters = dict(counters)
+    _emit(True)
+
+
+def tick(**counters) -> None:
+    """Record ``counters`` and write them if the throttle allows."""
+    if _sink is None:
+        return
+    _counters.update(counters)
+    if (_last_write is not None
+            and time.monotonic() - _last_write < THROTTLE_SECONDS):
+        return
+    _emit(False)
+
+
+@contextlib.contextmanager
+def waiting(lock: str):
+    """Report a blocking wait on ``lock`` for as long as the body runs."""
+    global _phase, _counters
+    if _sink is None:
+        yield
+        return
+    previous, saved = _phase, dict(_counters)
+    phase(f"waiting:{lock}")
+    try:
+        yield
+    finally:
+        if _sink is not None:
+            _phase, _counters = previous, saved
+            _emit(True)
+
+
+def peak_resident_bytes() -> int:
+    """``getrusage(RUSAGE_SELF).ru_maxrss`` in bytes.
+
+    macOS reports bytes and Linux reports KiB; the unit is decided per call
+    from ``sys.platform`` so a test can exercise both branches.
+    """
+    peak = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
+    if sys.platform == "darwin":
+        return peak
+    return peak * 1024
+
+
+def current_resident_bytes() -> "int | None":
+    """This process's current resident set in bytes, or None when unknown.
+
+    Best effort, because no stdlib call reports it on macOS: Linux reads
+    ``/proc/self/statm``; elsewhere ``ps -o rss=`` (KiB) is asked under
+    ``LC_ALL=C``. Any failure answers None rather than raising, since a
+    progress reading must never fail a rebuild.
+    """
+    try:
+        with open("/proc/self/statm", "rb") as handle:
+            fields = handle.read().split()
+        return int(fields[1]) * os.sysconf("SC_PAGE_SIZE")
+    except (OSError, ValueError, IndexError):
+        pass
+    try:
+        out = subprocess.run(
+            ["ps", "-o", "rss=", "-p", str(os.getpid())],
+            capture_output=True, text=True, timeout=5,
+            env={**os.environ, "LC_ALL": "C"}, check=False)
+        return int(out.stdout.strip()) * 1024
+    except (OSError, ValueError, subprocess.SubprocessError):
+        return None

package/bin/_lib_record.py

@@ -150,6 +150,7 @@ class FiveHourSourceDecision:
 def plan_five_hour_source_local_credit(
     *, baseline_pct, pending_low_pct, pending_observation_id,
     new_pct, observation_id, drop_threshold, window_live,
+    generation=None, baseline_generation=None, pending_generation=None,
 ):
     """Classify one observation against ONE contributor's own five-hour state
     (glue call site: ``detect_reset_and_credit``'s 5h branch).
@@ -173,9 +174,30 @@ def plan_five_hour_source_local_credit(
     is deliberately no maximum gap, no maximum climb and no confirmation-value
     band, because observations arrive at an unbounded interval and every finite
     threshold on the change between two of them has a legitimate crossing.
+
+    ``generation`` (#797/#823/#840 spec 4.9, D3/D11) is the account and
+    window's current five-hour credit generation, passed only by the v2
+    detector; the historical detector passes None and keeps every rule above
+    unchanged. ``baseline_generation`` and ``pending_generation`` are the
+    generations the source's baseline and its pending descent were
+    established under. A state established under another generation is
+    OBSOLETE: a credit has been recorded, corrected or tombstoned since, so
+    its baseline may be a pre-credit high and its descent may be one another
+    source already confirmed. It can neither arm nor confirm, and it is not
+    relabelled into the new generation; this observation re-establishes the
+    baseline instead. A state recorded without a generation (the historical
+    detector, an earlier binary, or a seeded fixture) is not obsolete by
+    this rule.
     """
     new_pct = float(new_pct)
 
+    if baseline_pct is not None and generation is not None and (
+            (baseline_generation is not None
+             and baseline_generation != generation)
+            or (pending_low_pct is not None and pending_generation is not None
+                and pending_generation != generation)):
+        return FiveHourSourceDecision(FIVE_HOUR_SET_BASELINE, new_pct)
+
     if baseline_pct is None:
         # No state for this source in this window. Establish the baseline and
         # emit nothing: a first reading is not a descent, whatever any other