cctally 1.113.0 → 1.115.0

package/bin/_cctally_journal_repair.py

@@ -17,6 +17,7 @@ import _cctally_journal as _journal
 import _lib_accounts
 import _lib_journal
 import _lib_journal_router
+import _cctally_store
 from _lib_json_envelope import stamp_schema_version
 
 
@@ -68,11 +69,20 @@ def _read_prefix(high_water):
         )
     records = []
     evidence = []
+    evidence_owners: list = []
     audit_ends: dict = {}
+    audit_candidates: list = []
     malformed = 0
     prior_high_water = None
     cutover_captured = _journal._CUTOVER_UNSEEN
     hasher = _lib_journal_router.PrefixHashAccumulator()
+    # #825/#832 (spec 5.3): admission is resolved within this one pass and
+    # applied after it, before selection or normalization. An excluded record
+    # — an old process's repair acknowledgement or correction batch included —
+    # keeps its positional slot and is never selection or audit input (F7).
+    pass_boundary = _journal._PassBoundary()
+    coordinates: list = []
+    cutover_candidates: list = []
     for segment, offset, raw in _journal._iter_range_with_segments(
         None,
         high_water,
@@ -86,6 +96,7 @@ def _read_prefix(high_water):
             prior_high_water = (segment, offset + len(raw) + 1)
             continue
         record_end = (segment, offset + len(raw) + 1)
+        pass_boundary.observe(record)
         if record.get("t") == "op":
             _journal._capture_protocol_prefix_evidence(
                 record,
@@ -93,20 +104,22 @@ def _read_prefix(high_water):
                 evidence,
                 hasher=hasher,
             )
+            if len(evidence) > len(evidence_owners):
+                evidence_owners.append(len(records))
             payload = record.get("payload")
             if (
                 isinstance(payload, dict)
                 and payload.get("kind")
-                == _lib_journal._PROTOCOL_RESOLUTION_KIND
+                in _lib_journal.PROTOCOL_RESOLUTION_KINDS
             ):
-                audit_ends[record.get("id")] = record_end
-        # First cutover op wins, exactly as `find_accounts_cutover_op` scans.
-        if (
-            cutover_captured is _journal._CUTOVER_UNSEEN
-            and record.get("id") == _journal.CUTOVER_OP_ID
-        ):
-            cutover_captured = _journal._cutover_value_of(record)
+                audit_candidates.append(
+                    (len(records), record.get("id"), record_end))
+        # First ADMITTED cutover op wins, exactly as `find_accounts_cutover_op`
+        # scans; admission is known only after the pass.
+        if record.get("id") == _journal.CUTOVER_OP_ID:
+            cutover_candidates.append((record, (segment, offset)))
         records.append(_lib_journal_router.selector_slot(record))
+        coordinates.append((segment, offset))
         prior_high_water = record_end
     prefix_hash = hasher.digest_at(high_water)
     # The accumulator buffers the segment it is reading — 410 MB on the
@@ -119,6 +132,19 @@ def _read_prefix(high_water):
         raise _lib_journal.JournalProtocolError(
             f"journal prefix contains {malformed} malformed line(s)"
         )
+    boundary = pass_boundary.resolve(high_water)
+    _journal._apply_pass_admission(records, coordinates, boundary)
+    # #825/#832 fix round C (CD2-005): an excluded acknowledgement is absent
+    # for every purpose — its evidence and its audit end go with it.
+    evidence = _journal._admitted_prefix_evidence(
+        evidence, evidence_owners, records)
+    for index, audit_id, record_end in audit_candidates:
+        if records[index] is not None:
+            audit_ends[audit_id] = record_end
+    for candidate, (segment, offset) in cutover_candidates:
+        if _journal._is_admitted(candidate, segment, offset, boundary):
+            cutover_captured = _journal._cutover_value_of(candidate)
+            break
     # No suffix fallback, deliberately. `_read_only_high_water` pins the
     # canonically-last segment at its full size, so this prefix IS the whole
     # journal and an op the prefix does not contain is not in the journal at
@@ -255,6 +281,13 @@ def _repair_locks(timeout: float = 5.0):
     finally:
         _cctally_core.note_stats_maintenance_released()
         release_cache_writer_flocks(held)
+    # #825/#832 spec 4.3 revision 2: the repair's live index publication (or
+    # its no-op) posts a wake once its locks are released.
+    try:
+        import _cctally_statusline
+        _cctally_statusline.post_wake("holder:journal-repair")
+    except Exception:
+        pass
 
 
 def _iso_now() -> str:
@@ -322,6 +355,25 @@ def _post_audit_failure(requested, audit_ids, exc):
     return payload, 3
 
 
+def _post_audit_obligation_lost(requested, audit_ids, exc):
+    """Fix round K12 (K11R-3): the rebuilt index IS published (and the
+    holder wake is posted once the repair locks are released); only its
+    residual-derivation obligation was lost. The status stays `failed`,
+    with the additive `indexPublished`, and the command exits 3."""
+    payload, returncode = _post_audit_failure(requested, audit_ids, exc)
+    payload["indexPublished"] = True
+    return payload, returncode
+
+
+def _obligation_lost_message(error) -> str:
+    return (
+        "cctally: the corrected stats.db was published, but the record that "
+        "the inputs written during the rebuild still need deriving could not "
+        f"be saved ({error}). Those inputs stay in the journal and the next "
+        "ingest derives them; check that the data directory is writable."
+    )
+
+
 def _stats_has_acknowledgements(fingerprints) -> bool:
     if not _cctally_core.DB_PATH.exists():
         return False
@@ -330,9 +382,12 @@ def _stats_has_acknowledgements(fingerprints) -> bool:
             f"file:{_cctally_core.DB_PATH}?mode=ro", uri=True
         )
         try:
-            rows = conn.execute(
-                "SELECT violation_json FROM journal_protocol_violations"
-            )
+            # #825/#832 spec 6: completion evidence only from a validated
+            # current index; anything else fails closed.
+            with _cctally_store.current_stats_snapshot(conn):
+                rows = conn.execute(
+                    "SELECT violation_json FROM journal_protocol_violations"
+                ).fetchall()
             stored = {
                 item.get("fingerprint")
                 for item in (json.loads(str(row[0])) for row in rows)
@@ -341,7 +396,8 @@ def _stats_has_acknowledgements(fingerprints) -> bool:
             return set(fingerprints) <= stored
         finally:
             conn.close()
-    except (OSError, sqlite3.Error, ValueError, TypeError):
+    except (OSError, sqlite3.Error, ValueError, TypeError,
+            _cctally_store.StatsIndexNotCurrent):
         return False
 
 
@@ -368,6 +424,8 @@ def _recovery_high_water(audit_ids, audit_ends):
 
 
 def _apply(requested, initial_preview):
+    import _cctally_completion
+
     _call_pause_hook("before-lock")
     with _repair_locks():
         preview, snapshot = _preview_payload(requested)
@@ -429,10 +487,17 @@ def _apply(requested, initial_preview):
                     ),
                     high_water=audit_high_water,
                     update_quota_cache=False,
+                    complete_derivation=True,
                     before_swap=lambda: _call_crash_hook(
                         "before-rebuild-swap"
                     ),
                 )
+            except _cctally_completion.ResidualObligationNotDurable as exc:
+                return _post_audit_obligation_lost(
+                    requested,
+                    {audit["id"]},
+                    exc,
+                )
             except Exception as exc:
                 return _post_audit_failure(
                     requested,
@@ -477,10 +542,17 @@ def _apply(requested, initial_preview):
                     ),
                     high_water=recovery_high_water,
                     update_quota_cache=False,
+                    complete_derivation=True,
                     before_swap=lambda: _call_crash_hook(
                         "before-rebuild-swap"
                     ),
                 )
+            except _cctally_completion.ResidualObligationNotDurable as exc:
+                return _post_audit_obligation_lost(
+                    requested,
+                    audit_ids,
+                    exc,
+                )
             except Exception as exc:
                 return _post_audit_failure(
                     requested,
@@ -498,6 +570,18 @@ def _apply(requested, initial_preview):
 
 def cmd_db_journal_repair(args) -> int:
     """Preview structural violations without mutating the journal or indexes."""
+    if (bool(getattr(args, "yes", False))
+            and list(getattr(args, "violation", ()) or ())
+            and _journal.account_bound_cutover_enforced()):
+        # #825/#832 spec 5.5: establish the account-bound cutover before the
+        # apply's prefix-bound planning (the preview below is that plan), so
+        # the plan never races the boundary's own append.
+        try:
+            _journal.ensure_account_bound_cutover()
+        except _journal.AccountBoundCutoverDeferred:
+            pass
+        except OSError:
+            pass
     try:
         requested = list(getattr(args, "violation", ()) or ())
         payload, _snapshot = _preview_payload(requested)
@@ -585,6 +669,11 @@ def cmd_db_journal_repair(args) -> int:
         if returncode != 0:
             if bool(getattr(args, "json", False)):
                 print(json.dumps(payload, separators=(",", ":")))
+            elif payload.get("indexPublished"):
+                print(
+                    _obligation_lost_message(payload["errors"][0]),
+                    file=sys.stderr,
+                )
             else:
                 for error in payload["errors"]:
                     print(

package/bin/_cctally_milestone_history.py

@@ -817,22 +817,75 @@ def _load_codex_cycles(stats_conn, root_keys, *, include_orphaned=False,
     if not roots:
         return []
     placeholders = ",".join("?" for _ in roots)
-    orphan_clause = "" if include_orphaned else "AND orphaned_at IS NULL "
-    # `quota_window_blocks.account_key` is NOT NULL DEFAULT 'unattributed'.
-    account_clause = "" if account_key is None else "AND account_key=? "
-    account_params = () if account_key is None else (account_key,)
     # BEFORE the SQL, per #496 S5b section 4.7.
     assert_projection_readable(stats_conn)
+    # #901 (I1/G1, Amendment 1 item 2, 1b and 1c): stream every weekly block
+    # in reset order through an epoch-1017 partial expression index. Unpinned,
+    # the planner seeks a root-leading index and sorts the whole weekly
+    # history through a temp b-tree (measured for every root/account/orphan
+    # variant).
+    #
+    # The ORDER BY names the tie order explicitly: `_canonicalize_codex_
+    # cluster` keeps the first maximal member, so rows tied on one reset
+    # second must arrive in the order the pre-#901 sorter received them, and
+    # that order came from the legacy plan, which differs by root count
+    # (Amendment 1c, measured on SQLite 3.53.4). With two or more roots the
+    # legacy plan scanned `idx_quota_blocks_active`, whose order includes
+    # orphaned_at; with exactly one it seeked the UNIQUE autoindex, which has
+    # no orphaned_at (its one exception, an account-scoped live-only read, has
+    # a constant orphaned_at, so the two orders agree there). The branch
+    # counts the NORMALIZED roots above, so duplicate or empty arguments take
+    # the single-root order, exactly as they took the single-root plan.
+    #
+    # Each branch pins its own index only when that index exists; on an
+    # older index the statement stays unpinned with ordinary predicates and
+    # the same explicit order. Under the pin, the root, account and orphan
+    # predicates carry a unary `+`: SQLite treats a column that a WHERE term
+    # fixes but the scan does not seek as satisfying its ORDER BY term and
+    # then cannot match that index column, so the later ORDER BY terms fell
+    # back to a tie-run sorter ("USE TEMP B-TREE FOR LAST n TERMS OF ORDER
+    # BY", measured). Limitation: a unary `+` keeps the operand's value but
+    # removes its column affinity. That is equivalent here ONLY because these
+    # comparisons are TEXT against TEXT: the root keys are filtered to `str`
+    # above, `account_key` is a TEXT key bound as a string, and `IS NULL`
+    # involves no affinity. It is not an equivalence for numeric parameters or
+    # any other affinity-dependent comparison; a caller binding a non-string
+    # account key would need the unpinned predicate.
+    if len(roots) == 1:
+        order_index = "idx_quota_blocks_weekly_single_root_order"
+        order_by = (
+            "ORDER BY unixepoch(resets_at_utc) DESC, source_root_key ASC, "
+            "account_key ASC, logical_limit_key ASC, observed_slot ASC, "
+            "resets_at_utc ASC")
+    else:
+        order_index = "idx_quota_blocks_weekly_reset_order"
+        order_by = (
+            "ORDER BY unixepoch(resets_at_utc) DESC, source_root_key ASC, "
+            "account_key ASC, orphaned_at ASC, logical_limit_key ASC, "
+            "observed_slot ASC, resets_at_utc ASC")
+    pinned = stats_conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?",
+        (order_index,),
+    ).fetchone() is not None
+    order_hint = f"INDEXED BY {order_index} " if pinned else ""
+    plus = "+" if pinned else ""
+    orphan_clause = (
+        "" if include_orphaned else f"AND {plus}orphaned_at IS NULL ")
+    # `quota_window_blocks.account_key` is NOT NULL DEFAULT 'unattributed'.
+    account_clause = (
+        "" if account_key is None else f"AND {plus}account_key=? ")
+    account_params = () if account_key is None else (account_key,)
     rows = stats_conn.execute(
         "SELECT source_root_key, logical_limit_key, observed_slot, "
         "       window_minutes, limit_id, limit_name, account_key, "
         "       resets_at_utc, nominal_start_at_utc, current_percent "
         "FROM quota_window_blocks "
+        f"{order_hint}"
         "WHERE source='codex' AND window_minutes=10080 "
         f"{orphan_clause}"
         f"{account_clause}"
-        f"AND source_root_key IN ({placeholders}) "
-        "ORDER BY unixepoch(resets_at_utc) DESC",
+        f"AND {plus}source_root_key IN ({placeholders}) "
+        f"{order_by}",
         (*account_params, *roots),
     ).fetchall()
 
@@ -952,10 +1005,20 @@ def _codex_five_hour_rows(stats_conn, cyc, *, include_orphaned=False) -> list:
     The widening is one-directional on purpose: an unattributed CYCLE never
     picks up another account's identified 5h rows, which would violate #341's
     never-combine rule.
+
+    #901 Amendment 19 T1 (spec I1): the rows are returned whole, so they are
+    ordered here, where ``ORDER BY unixepoch(nominal_start_at_utc)`` sorted
+    them through a temp b-tree. Each physical 5h block surfaces as many
+    jittered rows, so one weekly cycle is not a bound. The key is SQLite's own
+    ``unixepoch()`` of each distinct start spelling (a table-free scalar
+    query, memoized), so it is the old key exactly, and the rows keep the
+    caller's row factory. ``list.sort`` is stable over the same scan SQLite's
+    in-memory sorter consumed, so rows tied on one start second keep their
+    order. The WHERE clause admits no NULL start.
     """
     orphan_clause = "" if include_orphaned else "AND orphaned_at IS NULL "
     assert_projection_readable(stats_conn)
-    return stats_conn.execute(
+    rows = stats_conn.execute(
         "SELECT source_root_key, logical_limit_key, observed_slot, "
         "       window_minutes, limit_id, limit_name, account_key, "
         "       resets_at_utc, nominal_start_at_utc, current_percent "
@@ -964,12 +1027,22 @@ def _codex_five_hour_rows(stats_conn, cyc, *, include_orphaned=False) -> list:
         f"{orphan_clause}"
         "AND source_root_key=? AND account_key IN (?,?) "
         "AND unixepoch(nominal_start_at_utc) < unixepoch(?) "
-        "AND unixepoch(resets_at_utc) > unixepoch(?) "
-        "ORDER BY unixepoch(nominal_start_at_utc) ASC",
+        "AND unixepoch(resets_at_utc) > unixepoch(?)",
         (cyc.root, cyc.account_key, UNATTRIBUTED,
          cyc.end.astimezone(UTC).isoformat(),
          cyc.start.astimezone(UTC).isoformat()),
     ).fetchall()
+    epochs: dict = {}
+
+    def start_epoch(row):
+        text = row[8]
+        if text not in epochs:
+            epochs[text] = stats_conn.execute(
+                "SELECT unixepoch(?)", (text,)).fetchone()[0]
+        return epochs[text]
+
+    rows.sort(key=start_epoch)
+    return rows
 
 
 def _codex_five_hour_clusters(stats_conn, cyc, *, include_orphaned=False) -> list:

package/bin/_cctally_parser.py

@@ -3787,6 +3787,28 @@ def _build_stats_corruption_heal_parser(subparsers, name, *, help_text, xref=Non
     )
     worker.set_defaults(func=c.cmd_stats_corruption_heal_internal)
 
+def _build_statusline_continue_parser(subparsers, name, *, help_text, xref=None):
+    """Build the #825/#832 detached statusline continuation worker parser."""
+    c = _cctally()
+    worker = subparsers.add_parser(
+        name,
+        help=help_text,
+        formatter_class=CLIHelpFormatter,
+        description=textwrap.dedent(
+                    """\
+                    Internal subcommand: continue deferred statusline work
+                    under the one reducer reservation it inherits, or (with
+                    --wait) observe one blocking lock's release and post one
+                    wake. Spawned by the event-driven continuation; never
+                    polls. Always returns 0.
+                    """
+                ),
+    )
+    worker.add_argument("--reservation-fd", type=int, required=True,
+                        help=argparse.SUPPRESS)
+    worker.add_argument("--wait", default=None, help=argparse.SUPPRESS)
+    worker.set_defaults(func=c.cmd_statusline_continue_internal)
+
 def _build_artifact_retention_parser(subparsers, name, *, help_text, xref=None):
     """Build the #496 S6 detached retained-artifact reclamation worker parser."""
     c = _cctally()
@@ -3827,6 +3849,24 @@ def _build_codex_replay_drain_parser(subparsers, name, *, help_text, xref=None):
     )
     rd.set_defaults(func=c.cmd_codex_replay_drain_internal)
 
+def _build_codex_prune_drain_parser(subparsers, name, *, help_text, xref=None):
+    """Build the `_codex-prune-drain` parser (#968 spec 5.7)."""
+    c = _cctally()
+    pd = subparsers.add_parser(
+        name,
+        help=help_text,
+        formatter_class=CLIHelpFormatter,
+        description=textwrap.dedent(
+                    """\
+                    Internal subcommand: detached worker that drains recorded
+                    Codex prune debt one source identity per transaction, off
+                    the hook. Always returns 0; failures are written to
+                    hook-tick.log.
+                    """
+                ),
+    )
+    pd.set_defaults(func=c.cmd_codex_prune_drain_internal)
+
 def _build_repair_symlinks_parser(subparsers, name, *, help_text, xref=None):
     """Build the `repair-symlinks` parser (registered via _REGISTRATION; #279 S6 W3).
 
@@ -3913,8 +3953,10 @@ _REGISTRATION = (
     _Reg('_codex-quota-verify', _build_codex_quota_verify_parser, argparse.SUPPRESS, None, None),
     _Reg('_stats-epoch-rebuild', _build_stats_epoch_rebuild_parser, argparse.SUPPRESS, None, None),
     _Reg('_stats-corruption-heal', _build_stats_corruption_heal_parser, argparse.SUPPRESS, None, None),
+    _Reg('_statusline-continue', _build_statusline_continue_parser, argparse.SUPPRESS, None, None),
     _Reg('_artifact-retention', _build_artifact_retention_parser, argparse.SUPPRESS, None, None),
     _Reg('_codex-replay-drain', _build_codex_replay_drain_parser, argparse.SUPPRESS, None, None),
+    _Reg('_codex-prune-drain', _build_codex_prune_drain_parser, argparse.SUPPRESS, None, None),
     _Reg('repair-symlinks', _build_repair_symlinks_parser, argparse.SUPPRESS, None, None),
 )
 

package/bin/_cctally_percent_breakdown.py

@@ -262,7 +262,8 @@ def observation_gap_marginal_cell(marginal_value: Any, *, withheld: bool) -> str
     table already says what the code means.
     """
     if marginal_value is not None:
-        return f"${float(marginal_value):.6f}"
+        c = _cctally()
+        return c._fmt_usd_accounting(float(marginal_value), 6)
     return OBSERVATION_GAP_CAUSE if withheld else "n/a"
 
 
@@ -315,7 +316,7 @@ def _render_percent_breakdown_terminal(
         rows.append([
             str(idx),
             f"{percent}%",
-            f"${cumulative:.6f}",
+            c._fmt_usd_accounting(cumulative, 6),
             c.observation_gap_marginal_cell(
                 marginal_value, withheld=(idx - 1) in withheld),
             f"{float(five_hour_value):.0f}%" if five_hour_value is not None else "n/a",

package/bin/_cctally_project.py

@@ -58,9 +58,11 @@ def _cctally():
 #: Why modelled quota is not published for a week or a whole run. A closed
 #: set. `account-not-resolved` is the run-level one (§5.3); the rest are
 #: per-week and are the reasons that week fell back to the cost share.
+#: `unvalidated-coefficient-era` is `apply_regime`'s rejection of a week
+#: outside the stored regime's coefficient era (#978 spec 4.6).
 ATTRIBUTION_CAUSES = ("account-not-resolved", "calibration-absent",
                       "regime-boundary", "unsupported-composition",
-                      "no-local-history")
+                      "no-local-history", "unvalidated-coefficient-era")
 
 #: The three bases a row or a run can carry.
 ATTRIBUTION_BASES = ("modelled", "cost-share", "withheld")
@@ -219,7 +221,8 @@ def resolve_week_attribution(regime, records, *, week_start, week_end):
             and week_end > regime.effective_until:
         return WeekAttribution("cost-share", "regime-boundary", None)
     qcg = _cctally()._load_sibling("_cctally_quota_calibration")
-    applied = qcg.apply_regime(regime, list(records))
+    applied = qcg.apply_regime(regime, list(records),
+                               interval=(week_start, week_end))
     if isinstance(applied, qcg.ApplyRejection):
         return WeekAttribution("cost-share", applied.value, None)
     return WeekAttribution("modelled", None, regime.units_per_point)
@@ -282,6 +285,9 @@ _ATTRIBUTION_CAUSE_COPY: dict = {
     "unsupported-composition":
         "the week's model mix sits outside the calibration's support",
     "no-local-history": "no local entries to model",
+    "unvalidated-coefficient-era":
+        "the week is not inside the coefficient era the calibration's token "
+        "weights are validated for",
     RESIDUAL_MISALIGNED:
         "the account, the window or the population does not align",
     RESIDUAL_OBSERVED_ABSENT: