cctally 1.88.2 → 1.89.1

package/bin/_cctally_config.py

@@ -331,9 +331,23 @@ ALLOWED_CONFIG_KEYS = (
     "budget.codex.accounts",
     "telemetry.enabled",
     "conversation.retention_days",
+    "codex.hook.ingest_budget_seconds",
 )
 
 
+#: Wall-clock budget the native Codex hook gives its ingest leg (public #5
+#: spec §4). It applies to the HOOK path only — an explicit `cctally cache-sync`
+#: still runs to completion.
+CODEX_HOOK_INGEST_BUDGET_DEFAULT_SECONDS = 5.0
+
+#: Codex kills a hook at 30 seconds. The budget is capped strictly BELOW that,
+#: not merely validated positive: a configured 60 would guarantee the timeout
+#: this whole change exists to remove, and the cap leaves room for the rest of
+#: the tick (the reconcile, the budget evaluation, interpreter startup).
+CODEX_HOOK_TIMEOUT_SECONDS = 30.0
+CODEX_HOOK_INGEST_BUDGET_MAX_SECONDS = 20.0
+
+
 _CODEX_BUDGET_LEAF_PREFIX = "budget.codex."
 
 _DEFAULT_CONVERSATION_RETENTION_DAYS = 90
@@ -374,6 +388,62 @@ def _validate_retention_days_value(raw: object) -> int:
     return value
 
 
+def _validate_codex_hook_ingest_budget_value(raw: object) -> float:
+    """Validate a ``config set`` value for ``codex.hook.ingest_budget_seconds``.
+
+    Positive is not enough. Codex kills a hook at
+    ``CODEX_HOOK_TIMEOUT_SECONDS``, so a value at or above
+    ``CODEX_HOOK_INGEST_BUDGET_MAX_SECONDS`` is rejected outright rather than
+    silently clamped: an operator who typed 60 asked for exactly the timeout
+    this budget exists to prevent, and a clamp would hide that.
+    """
+    if isinstance(raw, bool):
+        raise ValueError(
+            "codex.hook.ingest_budget_seconds must be a positive number")
+    if isinstance(raw, str):
+        try:
+            value = float(raw.strip())
+        except (TypeError, ValueError):
+            raise ValueError(
+                "codex.hook.ingest_budget_seconds must be a positive number, "
+                f"got {raw!r}")
+    elif isinstance(raw, (int, float)):
+        value = float(raw)
+    else:
+        raise ValueError(
+            "codex.hook.ingest_budget_seconds must be a positive number")
+    if not (value > 0) or value != value or value in (float("inf"),):
+        raise ValueError(
+            "codex.hook.ingest_budget_seconds must be > 0")
+    if value >= CODEX_HOOK_INGEST_BUDGET_MAX_SECONDS:
+        raise ValueError(
+            "codex.hook.ingest_budget_seconds must be < "
+            f"{CODEX_HOOK_INGEST_BUDGET_MAX_SECONDS:g} (Codex kills a hook at "
+            f"{CODEX_HOOK_TIMEOUT_SECONDS:g}s)")
+    return value
+
+
+def resolve_codex_hook_ingest_budget(config: dict) -> float:
+    """The effective hook ingest budget in seconds.
+
+    Any malformed or out-of-range persisted value degrades to the default
+    rather than raising — a hand-edited config must never be able to make the
+    hook itself fail, and the default is the safe direction here.
+    """
+    default = CODEX_HOOK_INGEST_BUDGET_DEFAULT_SECONDS
+    block = config.get("codex") if isinstance(config, dict) else None
+    if not isinstance(block, dict):
+        return default
+    hook = block.get("hook")
+    if not isinstance(hook, dict):
+        return default
+    try:
+        return _validate_codex_hook_ingest_budget_value(
+            hook.get("ingest_budget_seconds"))
+    except ValueError:
+        return default
+
+
 def resolve_retention_days(config: dict) -> int:
     """Effective conversation transcript retention in days (F8).
 
@@ -981,6 +1051,10 @@ def _config_known_value(config: dict, key: str) -> "object":
         # Effective transcript-retention window in days (default 180; 0 = keep
         # forever). Malformed persisted data surfaces the safe default (F8).
         return resolve_retention_days(config)
+    if key == "codex.hook.ingest_budget_seconds":
+        # Wall-clock budget for the native Codex hook's ingest leg (public #5).
+        # Malformed or out-of-range persisted data surfaces the default.
+        return resolve_codex_hook_ingest_budget(config)
     if key in ("update.check.enabled", "update.check.ttl_hours"):
         # Defaults mirror `_is_update_check_due` (True / 24 hours).
         # Hand-edited junk surfaces as the default — matches dashboard.bind.
@@ -1623,6 +1697,41 @@ def _cmd_config_set(args: argparse.Namespace) -> int:
                 rendered = str(normalized)
             print(f"{key}={rendered}")
         return 0
+    if key == "codex.hook.ingest_budget_seconds":
+        # Validate first; rejection short-circuits before lock acquisition.
+        try:
+            budget = _validate_codex_hook_ingest_budget_value(raw)
+        except ValueError as exc:
+            print(f"cctally: {exc}", file=sys.stderr)
+            return 2
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            existing = config.get("codex")
+            if existing is not None and not isinstance(existing, dict):
+                print(
+                    "cctally: codex config error: codex must be an object",
+                    file=sys.stderr,
+                )
+                return 2
+            block = dict(existing or {})
+            hook = block.get("hook")
+            if hook is not None and not isinstance(hook, dict):
+                print(
+                    "cctally: codex config error: codex.hook must be an object",
+                    file=sys.stderr,
+                )
+                return 2
+            hook = dict(hook or {})
+            hook["ingest_budget_seconds"] = budget
+            block["hook"] = hook
+            config["codex"] = block
+            save_config(config)
+        if getattr(args, "emit_json", False):
+            print(json.dumps(
+                {"codex": {"hook": {"ingest_budget_seconds": budget}}}, indent=2))
+        else:
+            print(f"{key}={budget:g}")
+        return 0
     if key == "conversation.retention_days":
         # Validate first; rejection short-circuits before lock acquisition (F8).
         try:
@@ -2159,6 +2268,22 @@ def _cmd_config_unset(args: argparse.Namespace) -> int:
                 save_config(config)
             # idempotent: silent on missing key
         return 0
+    if key == "codex.hook.ingest_budget_seconds":
+        # Mirror the conversation.retention_days branch: drop the leaf, then
+        # prune an empty `hook` and an empty `codex` so config.json stays tidy.
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            block = config.get("codex")
+            hook = block.get("hook") if isinstance(block, dict) else None
+            if isinstance(hook, dict) and "ingest_budget_seconds" in hook:
+                del hook["ingest_budget_seconds"]
+                if not hook:
+                    block.pop("hook", None)
+                if not block:
+                    config.pop("codex", None)
+                save_config(config)
+            # idempotent: silent on missing key
+        return 0
     if key == "conversation.retention_days":
         # Mirror the display.tz branch: drop the leaf; if the `conversation`
         # block ends up empty, drop it too. Next get resolves the 180 default.

package/bin/_cctally_core.py

@@ -349,7 +349,22 @@ STATS_WAL_SIZE_LIMIT_BYTES = 16 * 1024 * 1024  # 16777216
 # 1003 -> 1004 (#410 Task B): pair the public journal cursor with the exact
 # prefix atomically applied to the materialized index. A cursor-only hand edit
 # can no longer skip an already-durable event and make its natural key look new.
-STATS_INDEX_EPOCH = 1004
+# 1004 -> 1005 (public #5): the incremental Codex quota projection. Adds the
+# reverse map + composable per-group digest on `quota_window_blocks` and the
+# `quota_projection_ledger_state` row (change-ledger watermark, interpretation
+# version, and the two alert axes that are not functions of window dirtiness).
+# It is an epoch bump and NOT a stats migration because the registry is frozen
+# AND because an epoch-current open returns before any schema work — an
+# `add_column_if_missing` would never run on an upgraded install, so the column
+# would simply never appear. Every upgrading install therefore rebuilds stats.db
+# from the journal on first open; that is the documented resolution for an epoch
+# mismatch and a real one-time cost, not a free change.
+# 1005 -> 1006 (public #5, I2 review): the periodic verification. Adds
+# `quota_projection_ledger_state.last_full_pass_at`, the deadline a time-based
+# full pass is measured against. Same mechanical reason as 1005 — an
+# epoch-current open returns before any schema work — so it is a second bump
+# rather than an amendment to the first.
+STATS_INDEX_EPOCH = 1006
 LEGACY_STATS_HEAD = 13
 
 
@@ -1248,6 +1263,21 @@ def _apply_quota_projection_schema(conn: sqlite3.Connection) -> None:
             generation            TEXT    NOT NULL,
             orphaned_at           TEXT,
             account_key           TEXT    NOT NULL DEFAULT 'unattributed',
+            -- public #5: the REVERSE MAP. `physical_group_key` records the
+            -- physical window this block was materialized from, so the
+            -- generation sweep can be scoped to the groups a bounded pass
+            -- actually re-materialized instead of to whole roots.
+            -- `physical_group_digest` is that group's contribution to its
+            -- root's physical signature: the root value is a digest over the
+            -- root's sorted (group key, group digest) pairs, which makes it
+            -- ASSOCIATIVE. A bounded pass recomputes only the dirty groups'
+            -- digests and re-derives the root value from the stored set —
+            -- O(groups), 608 on the real store, against O(observations) at
+            -- 211K. Hanging it off the blocks is what makes it self-maintaining:
+            -- a group swept to nothing loses its blocks and drops out of the
+            -- composition with them.
+            physical_group_key    TEXT,
+            physical_group_digest TEXT,
             UNIQUE(source, source_root_key, account_key, logical_limit_key,
                    observed_slot, window_minutes, resets_at_utc)
         );
@@ -1319,6 +1349,46 @@ def _apply_quota_projection_schema(conn: sqlite3.Connection) -> None:
             PRIMARY KEY(source_root_key, account_key)
         );
 
+        -- public #5: everything the incremental projector needs to know about
+        -- its own last pass, keyed by provider source. One row, read once per
+        -- reconcile.
+        --
+        -- `watermark_seq` is the highest `quota_window_change_log.seq` this
+        -- index has consumed. It is written INSIDE the same stats transaction
+        -- as the projection it describes, so the two commit or roll back
+        -- together; a crash therefore replays a ledger range rather than
+        -- skipping one, which is safe because re-materializing a group is
+        -- idempotent. (Writing it after the commit would need a second stats
+        -- transaction, and `run_stats_ingest` is the sole stats writer.)
+        --
+        -- `interpretation_version` invalidates the mechanism itself: a
+        -- classification change alters interpreted keys with no row mutation
+        -- for the ledger to observe, so a bump queues one complete pass.
+        --
+        -- `alerts_enabled` / `next_evaluation_at_utc` are the two alert axes
+        -- that are not functions of window dirtiness — a delivery-gate
+        -- transition, and a future-clocked observation that becomes eligible
+        -- when wall time passes it with no row mutation at all.
+        --
+        -- `last_full_pass_at` is the periodic verification's deadline. Two
+        -- cases a scoped sweep structurally cannot see — a block whose physical
+        -- group is absent from the cache entirely, and a milestone on a historic
+        -- root no longer active — are otherwise repairable only by an
+        -- interpretation bump, a rebuild or a burst overflow, none of which
+        -- happen on a normal install. Every full pass stamps it, whatever
+        -- triggered it, so the deadline is satisfied by whichever caller reaches
+        -- it first and the bound on staleness is one interval rather than
+        -- forever. NULL means "never verified", which reads as due.
+        CREATE TABLE IF NOT EXISTS quota_projection_ledger_state (
+            source                 TEXT NOT NULL,
+            watermark_seq          INTEGER NOT NULL DEFAULT 0,
+            interpretation_version INTEGER NOT NULL DEFAULT 0,
+            alerts_enabled         INTEGER,
+            next_evaluation_at_utc TEXT,
+            last_full_pass_at      TEXT,
+            PRIMARY KEY(source)
+        );
+
         CREATE TABLE IF NOT EXISTS quota_alert_arming (
             id                INTEGER PRIMARY KEY AUTOINCREMENT,
             source            TEXT NOT NULL,
@@ -1361,6 +1431,20 @@ def _apply_quota_projection_schema(conn: sqlite3.Connection) -> None:
                      "quota_threshold_events", "quota_alert_arming"):
             add_column_if_missing(
                 conn, _tbl, "account_key", "TEXT NOT NULL DEFAULT 'unattributed'")
+        # public #5 backstop, and NOT redundant with the CREATE above. An
+        # epoch-MISMATCHED index resolves by rebuild and gets the fresh CREATE;
+        # a LEGACY index (`user_version <= LEGACY_STATS_HEAD`) takes the
+        # in-place cutover, where `CREATE TABLE IF NOT EXISTS` is a no-op over
+        # the table it already has — so the reverse map would never appear and
+        # every reconcile after the cutover would fail on `no such column`.
+        for _col in ("physical_group_key", "physical_group_digest"):
+            add_column_if_missing(conn, "quota_window_blocks", _col, "TEXT")
+        # Same seam, epoch 1006: a LEGACY index that already took the epoch-1005
+        # cutover carries `quota_projection_ledger_state` WITHOUT the periodic
+        # verification's deadline, and `CREATE TABLE IF NOT EXISTS` above is a
+        # no-op over the table it already has.
+        add_column_if_missing(
+            conn, "quota_projection_ledger_state", "last_full_pass_at", "TEXT")
 
 
 def open_db(*, _target_path=None) -> sqlite3.Connection:
@@ -2299,7 +2383,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
         # Schema-apply compatibility for legacy/test-mode paths that reach this
         # DDL with a pre-pair cursor row. A released epoch-1003 index does NOT
         # use this as an upgrade shortcut: the epoch mismatch rebuilds it into
-        # the complete epoch-1004 schema first.
+        # the complete current-epoch schema first.
         conn.execute(
             "UPDATE journal_cursor "
             "SET applied_segment = segment, applied_offset = offset "

package/bin/_cctally_dashboard_cache_report.py

@@ -135,6 +135,12 @@ class CacheReportDailyRow:
     net_usd: float
     anomaly_triggered: bool
     anomaly_reasons: tuple[str, ...]
+    # #443 S1. `anomaly_unevaluated` = the predicates the classifier declined
+    # to run for this row; `observed` is False only for the builder's
+    # synthetic today row. Both default so their absence reproduces pre-S1
+    # rendering exactly.
+    anomaly_unevaluated: tuple[str, ...] = ()
+    observed: bool = True
 
 
 @dataclass(frozen=True)
@@ -161,6 +167,9 @@ class CacheReportTodaySpotlight:
     anomaly_triggered: bool
     anomaly_reasons: tuple[str, ...]
     baseline_daily_row_count: int
+    # #443 S1 — see CacheReportDailyRow.
+    anomaly_unevaluated: tuple[str, ...] = ()
+    observed: bool = True
 
 
 def _cache_report_snapshot_to_dict(cr: "CacheReportSnapshot | None") -> "dict | None":
@@ -189,6 +198,8 @@ def _cache_report_snapshot_to_dict(cr: "CacheReportSnapshot | None") -> "dict |
             "anomaly_triggered": cr.today.anomaly_triggered,
             "anomaly_reasons": list(cr.today.anomaly_reasons),
             "baseline_daily_row_count": cr.today.baseline_daily_row_count,
+            "anomaly_unevaluated": list(cr.today.anomaly_unevaluated),
+            "observed": cr.today.observed,
         },
         "days": [
             {
@@ -203,6 +214,8 @@ def _cache_report_snapshot_to_dict(cr: "CacheReportSnapshot | None") -> "dict |
                 "net_usd": d.net_usd,
                 "anomaly_triggered": d.anomaly_triggered,
                 "anomaly_reasons": list(d.anomaly_reasons),
+                "anomaly_unevaluated": list(d.anomaly_unevaluated),
+                "observed": d.observed,
             }
             for d in cr.days
         ],
@@ -319,6 +332,12 @@ def _cache_report_empty(
 ):
     """The empty (no in-window entries) ``CacheReportSnapshot`` — factored so the
     warm and cold builder paths share one ``is_empty`` return (#272 §6)."""
+    crk = _cache_report_load_kernel()
+    # #443 S1 — the dataclass defaults would claim ``observed=True`` with an
+    # empty ``anomaly_unevaluated`` for a day that was definitionally never
+    # measured or classified. That is the fabricating default this session
+    # removes everywhere else, so it is stated explicitly here too even though
+    # the empty branch short-circuits before any of it renders.
     empty_today = CacheReportTodaySpotlight(
         date=today_iso,
         cache_hit_percent=0.0,
@@ -328,6 +347,8 @@ def _cache_report_empty(
         anomaly_triggered=False,
         anomaly_reasons=(),
         baseline_daily_row_count=0,
+        anomaly_unevaluated=tuple(crk.CACHE_ANOMALY_PREDICATES),
+        observed=False,
     )
     return CacheReportSnapshot(
         window_days=window_days,
@@ -555,6 +576,10 @@ def build_cache_report_snapshot(
             anomaly_triggered=False,
             anomaly_reasons=(),
             baseline_daily_row_count=baseline_daily_row_count,
+            # This row never reaches the classifier, so nothing was evaluated
+            # for it and nothing was measured behind it (#443 S1 F1/F2).
+            anomaly_unevaluated=tuple(crk.CACHE_ANOMALY_PREDICATES),
+            observed=False,
         )
     else:
         today_spotlight = CacheReportTodaySpotlight(
@@ -568,6 +593,8 @@ def build_cache_report_snapshot(
             anomaly_triggered=today_row.anomaly_triggered,
             anomaly_reasons=tuple(today_row.anomaly_reasons),
             baseline_daily_row_count=baseline_daily_row_count,
+            anomaly_unevaluated=tuple(today_row.anomaly_unevaluated),
+            observed=True,
         )
 
     # Daily rows — newest first, capped at ``window_days``.
@@ -615,6 +642,8 @@ def build_cache_report_snapshot(
                 net_usd=0.0,
                 anomaly_triggered=False,
                 anomaly_reasons=(),
+                anomaly_unevaluated=tuple(crk.CACHE_ANOMALY_PREDICATES),
+                observed=False,
             )
         )
     days_newest_first.extend(
@@ -630,6 +659,8 @@ def build_cache_report_snapshot(
             net_usd=r.net_usd,
             anomaly_triggered=r.anomaly_triggered,
             anomaly_reasons=tuple(r.anomaly_reasons),
+            anomaly_unevaluated=tuple(r.anomaly_unevaluated),
+            observed=True,
         )
         for r in raw_days_newest_first
     )

package/bin/_cctally_dashboard_conversation.py

@@ -1090,16 +1090,35 @@ def _handle_get_conversation_payload_impl(handler, path: str) -> None:
         handler._respond_json(400, {"error": "bad request"})
         return
     cq = handler._conversation_query()
-    ok, loc = handler._run_conversation_query(
-        lambda conn: cq.locate_tool_payload(
-            conn, session_id, tool_use_id, which),
-        "/api/conversation/payload")
-    if not ok:
-        return
+    mode = which
+    if which == "result":
+        # A backgrounded-MCP result lives in an attachment record, not in the
+        # placeholder tool_result a public which='result' lookup finds first.
+        # Resolve the internal carrier here; the response the client receives
+        # still carries which:"result", so the client contract is unchanged and
+        # `background_result` is never an accepted input value.
+        ok, resolved = handler._run_conversation_query(
+            lambda conn: cq.locate_result_payload(conn, session_id, tool_use_id),
+            "/api/conversation/payload")
+        if not ok:
+            return
+        mode, loc = resolved
+        if mode == "background_gone":
+            # A KNOWN background placeholder whose notification row is gone or
+            # unresolvable — 410, distinct from the 404 an unknown id returns.
+            handler._respond_json(410, {"error": "source no longer available"})
+            return
+    else:
+        ok, loc = handler._run_conversation_query(
+            lambda conn: cq.locate_tool_payload(
+                conn, session_id, tool_use_id, which),
+            "/api/conversation/payload")
+        if not ok:
+            return
     if loc is None:
         handler._respond_json(404, {"error": "not found"})
         return
-    payload = cq.read_full_payload(loc[0], loc[1], tool_use_id, which)
+    payload = cq.read_located_payload(loc, tool_use_id, mode)
     if payload is None:
         handler._respond_json(410, {"error": "source no longer available"})
         return

package/bin/_cctally_dashboard_sources.py

@@ -3188,6 +3188,50 @@ def _claude_accounts_wire(
     return cards
 
 
+def _codex_ingest_backlog_wire(
+    cache_conn: sqlite3.Connection,
+) -> "dict[str, object] | None":
+    """The hook's budgeted-ingest backlog, or ``None`` when there is none.
+
+    Public #5 spec §5. Additive and OMITTED ENTIRELY at zero, so the normal
+    payload stays byte-identical for every install whose Codex history is
+    already ingested — which is nearly all of them, nearly all the time.
+
+    It deliberately does NOT touch ``availability`` or ``freshness``. Those are
+    read by a long and explicitly non-exhaustive list of gates, and degrading a
+    shared signal for a domain-local condition is a failure this project has
+    already hit; the client renders "history still loading" from this field
+    without any shared axis moving.
+
+    Any read or shape failure degrades to ``None``. The record is a health
+    signal, not evidence, and a hand-edited one must never be able to fail the
+    envelope build.
+    """
+    try:
+        row = cache_conn.execute(
+            "SELECT value FROM cache_meta WHERE key = ? LIMIT 1",
+            ("codex_ingest_backlog",),
+        ).fetchone()
+    except sqlite3.DatabaseError:
+        return None
+    if not row or not row[0]:
+        return None
+    try:
+        record = json.loads(str(row[0]))
+        files = int(record["files"])
+        pending_bytes = int(record["bytes"])
+    except (ValueError, TypeError, KeyError):
+        return None
+    if files <= 0:
+        return None
+    since = record.get("since")
+    return {
+        "files": files,
+        "bytes": pending_bytes,
+        "since": None if since is None else str(since),
+    }
+
+
 def build_codex_source_state(
     context: DashboardReadContext,
     *,
@@ -3217,6 +3261,7 @@ def build_codex_source_state(
         context,
     )
     projection_incoherent = not coherence.coherent
+    ingest_backlog = _codex_ingest_backlog_wire(context.cache_conn)
     # The cache reader's established report surface treats the ``now`` instant
     # as inclusive.  The qualified adapter is half-open, so extend only its
     # query/result boundary by one microsecond and keep all live budget sums
@@ -3681,6 +3726,12 @@ def build_codex_source_state(
                 "projected_thresholds": context.codex_quota_projected_thresholds,
             },
             "cache_report": cache_report,
+            # public #5 §5: additive, omitted at zero (the `cycle_freshness`
+            # precedent). Never emitted as an empty object — the absence IS the
+            # "nothing owed" state, which is what keeps the normal payload
+            # byte-identical.
+            **({"ingest_backlog": ingest_backlog}
+               if ingest_backlog is not None else {}),
         },
         domain_freshness={
             "hero": (