cctally 1.65.0 → 1.67.0

package/bin/_lib_doctor.py

@@ -177,6 +177,23 @@ class DoctorState:
     # valid and default to the enabled (opt-out) posture.
     telemetry_enabled: bool = True
     telemetry_reason: str = "enabled"
+    # #279 S2 (F5a): rolling ingest parse-health records read from
+    # cache_meta keys parse_health_claude / parse_health_codex (JSON
+    # dicts; see _cctally_cache._update_parse_health_meta). None = key
+    # absent (pre-first-sync) or cache unreadable — check degrades OK.
+    parse_health_claude: Optional[dict] = None
+    parse_health_codex: Optional[dict] = None
+    # #279 S2 (F5b): PRAGMA quick_check(1) results, gathered ONLY under
+    # doctor_gather_state(deep=True) (CLI cmd_doctor) — the dashboard
+    # rebuild loop calls the gather every rebuild and quick_check on a
+    # large cache.db costs seconds. "ok" | first error line |
+    # "open failed: ..." | None = not run.
+    stats_db_quick_check: Optional[str] = None
+    cache_db_quick_check: Optional[str] = None
+    # #279 S2 (F5c): non-blocking flock probes on the two sync lock files
+    # (name -> True held / False free / None unreadable). Probe never
+    # creates files (doctor read-only contract). None = probe errored.
+    locks_held: Optional[dict] = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -1207,6 +1224,122 @@ def _check_telemetry(s: DoctorState) -> CheckResult:
     )
 
 
+_PARSE_HEALTH_RECENCY_DAYS = 7
+
+
+def _check_data_parse_health(s: DoctorState) -> CheckResult:
+    details = {"claude": s.parse_health_claude, "codex": s.parse_health_codex}
+    recent, historical_total = [], 0
+    for label, ph in (("claude", s.parse_health_claude),
+                      ("codex", s.parse_health_codex)):
+        if not isinstance(ph, dict):
+            continue
+        malformed = int(ph.get("lines_malformed", 0) or 0)
+        skipped = int(ph.get("lines_skipped", 0) or 0)
+        historical_total += malformed + skipped
+        last_anomaly = ph.get("last_anomaly_at")
+        if not last_anomaly or (malformed + skipped) == 0:
+            continue
+        try:
+            at = dt.datetime.fromisoformat(
+                str(last_anomaly).replace("Z", "+00:00"))
+            if at.tzinfo is None:
+                at = at.replace(tzinfo=dt.timezone.utc)
+        except ValueError:
+            continue
+        if (s.now_utc - at).total_seconds() <= \
+                _PARSE_HEALTH_RECENCY_DAYS * 86400.0:
+            reasons = ph.get("reasons") \
+                if isinstance(ph.get("reasons"), dict) else {}
+            top = max(reasons.items(), key=lambda kv: kv[1])[0] \
+                if reasons else None
+            recent.append((label, malformed, skipped, top))
+    if recent:
+        parts = []
+        for label, malformed, skipped, top in recent:
+            frag = f"{label}: {malformed} malformed / {skipped} drift-skipped"
+            if top:
+                frag += f" (top reason: {top})"
+            parts.append(frag)
+        return CheckResult(
+            id="data.parse_health", title="Ingest parse health",
+            severity="warn",
+            summary="; ".join(parts)
+                    + f" within {_PARSE_HEALTH_RECENCY_DAYS}d",
+            remediation=(
+                "JSONL lines are failing to parse — a Claude Code / Codex "
+                "update may have changed the session format. Check for a "
+                "cctally update or file an issue; `cctally cache-sync "
+                "--rebuild` re-baselines the counters."),
+            details=details,
+        )
+    if s.parse_health_claude is None and s.parse_health_codex is None:
+        summary = "no parse-health data yet (pre-first-sync)"
+    elif historical_total:
+        summary = (f"no recent anomalies ({historical_total} historical; "
+                   f"see details)")
+    else:
+        summary = "no parse anomalies"
+    return CheckResult(
+        id="data.parse_health", title="Ingest parse health",
+        severity="ok", summary=summary, remediation=None, details=details,
+    )
+
+
+def _check_db_integrity(s: DoctorState) -> CheckResult:
+    details = {"stats_quick_check": s.stats_db_quick_check,
+               "cache_quick_check": s.cache_db_quick_check}
+    if s.stats_db_quick_check is not None and s.stats_db_quick_check != "ok":
+        return CheckResult(
+            id="db.integrity", title="Integrity", severity="fail",
+            summary=f"stats.db quick_check: {s.stats_db_quick_check}",
+            remediation=(
+                "stats.db (the non-re-derivable DB) reports corruption. "
+                "Back up the file FIRST, then try "
+                "`sqlite3 <path> \".recover\"` into a fresh file; "
+                "`cctally db recover` does not repair corruption. Please "
+                "file a bug. Do not delete the file."),
+            details=details,
+        )
+    if s.cache_db_quick_check is not None and s.cache_db_quick_check != "ok":
+        return CheckResult(
+            id="db.integrity", title="Integrity", severity="warn",
+            summary=f"cache.db quick_check: {s.cache_db_quick_check}",
+            remediation=("cache.db is re-derivable — run "
+                         "`cctally cache-sync --rebuild`."),
+            details=details,
+        )
+    if s.stats_db_quick_check is None and s.cache_db_quick_check is None:
+        return CheckResult(
+            id="db.integrity", title="Integrity", severity="ok",
+            summary="not checked (fast gather — run `cctally doctor`)",
+            remediation=None, details=details,
+        )
+    return CheckResult(
+        id="db.integrity", title="Integrity", severity="ok",
+        summary="quick_check ok", remediation=None, details=details,
+    )
+
+
+def _check_db_lock_state(s: DoctorState) -> CheckResult:
+    locks = s.locks_held if isinstance(s.locks_held, dict) else {}
+    held = sorted(k for k, v in locks.items() if v is True)
+    details = {"locks": s.locks_held}
+    if held:
+        return CheckResult(
+            id="db.lock_state", title="Locks", severity="ok",
+            summary=(", ".join(held) + " held — an active sync or "
+                     "dashboard is running; a hold persisting across "
+                     "repeated doctor runs may indicate a wedged process"),
+            remediation=None, details=details,
+        )
+    return CheckResult(
+        id="db.lock_state", title="Locks", severity="ok",
+        summary="free" if locks else "no lock files present",
+        remediation=None, details=details,
+    )
+
+
 # Each entry is (category_id, category_title, ((check_id, evaluator_fn_name), ...)).
 # The dotted check_id is the stable JSON-contract ID (spec §5.2) AND the
 # fingerprint identity-slice key (spec §5.5). When an evaluator raises,
@@ -1232,14 +1365,17 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
     ("db", "Database", (
         ("db.stats.file", "_check_db_stats_file"),
         ("db.cache.file", "_check_db_cache_file"),
+        ("db.integrity", "_check_db_integrity"),
         ("db.version_ahead", "_check_db_version_ahead"),
         ("db.migrations.applied", "_check_db_migrations_applied"),
         ("db.migrations.pending", "_check_db_migrations_pending"),
+        ("db.lock_state", "_check_db_lock_state"),
     )),
     ("data", "Data", (
         ("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),
         ("data.cache_sync_state", "_check_data_cache_sync_state"),
         ("data.codex_cache", "_check_data_codex_cache"),
+        ("data.parse_health", "_check_data_parse_health"),
         ("data.forked_buckets", "_check_data_forked_buckets"),
         ("data.post_credit_milestones", "_check_data_post_credit_milestones"),
         ("data.conversation_sessions_rollup",
@@ -1320,7 +1456,20 @@ def run_checks(state: DoctorState) -> DoctorReport:
 
 
 def _iso_z(d: dt.datetime) -> str:
-    """Render a UTC datetime as ISO 8601 with trailing 'Z' (share-v2 convention)."""
+    """Render a UTC datetime as ISO 8601 with trailing 'Z' (share-v2 convention).
+
+    DELIBERATELY DIVERGES from the canonical ``_lib_json_envelope._iso_z``
+    (#279 S6 W4) on two axes and therefore KEEPS this local name (gate F11):
+    (1) a NAIVE datetime is treated as UTC via ``replace(tzinfo=utc)`` — the
+    canonical's ``astimezone(utc)`` would reinterpret it as host-local; and
+    (2) microseconds are preserved via ``isoformat()`` — the canonical floors
+    to whole seconds via ``%S``. Renaming this to the canonical would silently
+    degrade the doctor block for the cross-module consumers that resolve it by
+    name through the sibling loader (``_cctally_tui`` and
+    ``_cctally_dashboard_envelope``'s doctor-envelope builders, whose
+    try/except swallows a rename). Divergence pinned by
+    ``tests/test_lib_doctor.py::test_doctor_iso_z_naive_means_utc_and_keeps_microseconds``.
+    """
     if d.tzinfo is None:
         d = d.replace(tzinfo=dt.timezone.utc)
     s = d.astimezone(dt.timezone.utc).isoformat()

package/bin/_lib_five_hour.py

@@ -23,11 +23,45 @@ Spec: docs/superpowers/specs/2026-05-13-bin-cctally-split-design.md
 from __future__ import annotations
 
 import datetime as dt
+import math
 
 
 _FIVE_HOUR_JITTER_FLOOR_SECONDS = 600  # 10 minutes; tolerance band for resets_at jitter
 
 
+def five_hour_milestone_range(pct: float, max_existing: "int | None") -> range:
+    """Which integer 5h-% thresholds ``maybe_update_five_hour_block`` should
+    attempt to record for ``pct``, given the ACTIVE segment's highest already-
+    recorded threshold ``max_existing`` (#279 S4 F4).
+
+    Mirrors the milestone-detection loop's fencing exactly (glue call site:
+    ``maybe_update_five_hour_block``'s 5h-% milestone loop):
+
+      - ``current_floor = math.floor(pct + 1e-9)`` — the 1e-9 snap flushes the
+        ``0.50 * 100 == 49.99…`` ULP so the N threshold is not missed.
+      - ``current_floor < 1`` → empty range (the ``if current_floor >= 1``
+        glue guard).
+      - ``start_threshold = current_floor`` when ``max_existing is None``
+        (first observation records ONLY the current floor — no synthetic
+        1..floor-1 backfill), else ``int(max_existing) + 1``.
+      - result = ``range(start_threshold, current_floor + 1)`` — empty when
+        ``start_threshold > current_floor`` (already at/above the floor),
+        which the glue reads as ``if milestone_range:``.
+
+    Pure: the MAX(percent_threshold) probe that yields ``max_existing``, the
+    marginal-cost lookup, the per-threshold INSERT, and the alert plumbing all
+    stay in glue. (The weekly ``maybe_record_milestone`` loop shares this
+    fencing formula but keeps its own range in glue — its crossing decision
+    early-returns BEFORE the cost sync when covered, a structure this
+    range-only kernel does not encapsulate; spec §6 defers it.)
+    """
+    current_floor = math.floor(pct + 1e-9)
+    if current_floor < 1:
+        return range(0)
+    start_threshold = current_floor if max_existing is None else int(max_existing) + 1
+    return range(start_threshold, current_floor + 1)
+
+
 def _floor_to_ten_minutes(d: dt.datetime) -> dt.datetime:
     """Floor a datetime to the previous 10-minute boundary.
 
@@ -47,6 +81,33 @@ def _floor_to_ten_minutes(d: dt.datetime) -> dt.datetime:
     )
 
 
+def _round_to_ten_minutes(d: dt.datetime) -> dt.datetime:
+    """Round a datetime to the NEAREST 10-minute boundary (half up).
+
+    The *display*-side companion to ``_floor_to_ten_minutes``. Anthropic
+    ``rate_limits.5h.resets_at`` (and the derived ``block_start_at``,
+    ``block_start = reset - 5h``) carries sub-10-minute capture jitter, so
+    a reset that truly lands on ``:40`` can be recorded as ``:39``.
+    Flooring that for display shows ``:30`` — off by a full bucket — so
+    every user-facing 5h-block clock time rounds to the nearest boundary
+    instead.
+
+    Rounds the ABSOLUTE instant (epoch, tz-independent) rather than the
+    local minute, so it is correct regardless of the display zone's offset
+    (the reset is a fixed UTC instant). Returns a UTC-aware datetime;
+    callers hand it to ``format_display_dt`` for zone conversion.
+
+    NEVER use this for keys / partitioning / lookups — those require the
+    exact stored timestamp (issue #76). This is a render-only normalizer.
+    """
+    if d.tzinfo is None:
+        d = d.replace(tzinfo=dt.timezone.utc)
+    bucket = _FIVE_HOUR_JITTER_FLOOR_SECONDS  # 600s = 10 min
+    # floor(x + 0.5) is predictable half-UP (Python's round() is banker's).
+    rounded = math.floor(d.timestamp() / bucket + 0.5) * bucket
+    return dt.datetime.fromtimestamp(rounded, dt.timezone.utc)
+
+
 def _canonical_5h_window_key(
     resets_at_epoch: int,
     prior_epoch: int | None = None,

package/bin/_lib_forecast.py

@@ -0,0 +1,144 @@
+"""Forecast decision kernel for cctally.
+
+Pure-fn layer (no I/O at import time): the forecast inputs/output/budget
+dataclasses plus `_compute_forecast` — the projection + budget-headroom
+math. Values in (`ForecastInputs`), a `ForecastOutput` out. No DB reads,
+no config loads, no `_cctally()` accessor: the projection routes through
+`project_linear` (the shared, pure primitive whose real home is
+`_lib_budget`).
+
+Imported by `_cctally_forecast.py` (and `_cctally_tui.py`) and re-exported
+on the `cctally` namespace via `bin/cctally`, so the existing
+`cctally.ForecastInputs` / `mod.ForecastOutput` / `inspect.getsource(
+cctally._compute_forecast)` read paths resolve unchanged (re-export
+continuity, spec §2). Single definition of each dataclass lives here;
+everything else imports it, so class identity stays unique.
+
+Spec: docs/superpowers/specs/2026-07-09-279-s4-record-kernelization-design.md
+"""
+from __future__ import annotations
+
+import datetime as dt
+from dataclasses import dataclass
+
+from _lib_budget import project_linear
+
+
+@dataclass
+class ForecastInputs:
+    now_utc: dt.datetime
+    week_start_at: dt.datetime
+    week_end_at: dt.datetime
+    elapsed_hours: float
+    elapsed_fraction: float
+    remaining_hours: float
+    remaining_days: float
+    # Current state
+    p_now: float
+    five_hour_percent: float | None
+    spent_usd: float
+    snapshot_count: int
+    latest_snapshot_at: dt.datetime
+    # Rate inputs
+    p_24h_ago: float | None
+    t_24h_actual_hours: float | None
+    # $/1% selection
+    dollars_per_percent: float
+    dollars_per_percent_source: str  # "this_week" | "trailing_4wk_median" | "this_week_sparse"
+    # Confidence
+    confidence: str  # "high" | "low"
+    low_confidence_reasons: list[str]
+
+
+@dataclass
+class BudgetRow:
+    target_percent: int
+    pct_headroom: float | None     # None when already past target
+    dollars_per_day: float | None
+    percent_per_day: float | None
+
+
+@dataclass
+class ForecastOutput:
+    inputs: ForecastInputs
+    r_avg: float                   # pct per hour, week-avg
+    r_recent: float | None         # pct per hour, 24h recent; None if no prior sample
+    final_percent_low: float
+    final_percent_high: float
+    week_avg_projection_pct: float  # p_now + r_avg*remaining (smooth estimator)
+    projected_cap: bool
+    already_capped: bool
+    cap_at: dt.datetime | None
+    budgets: list[BudgetRow]
+
+
+def _compute_forecast(inputs: ForecastInputs, targets: list[int]) -> ForecastOutput:
+    """Implements spec §2. targets are sorted desc for stable output (100, 90, …)."""
+    # Rate methods
+    r_avg = inputs.p_now / inputs.elapsed_hours if inputs.elapsed_hours > 0 else 0.0
+    if inputs.p_24h_ago is not None and inputs.t_24h_actual_hours:
+        r_recent: float | None = max(
+            0.0, (inputs.p_now - inputs.p_24h_ago) / inputs.t_24h_actual_hours
+        )
+    else:
+        r_recent = None
+
+    # Projected final % — routed through the shared project_linear primitive
+    # (spec F1). r_recent is None ⇒ collapse to the average projection.
+    if r_recent is None:
+        final_low, final_high = project_linear(
+            inputs.p_now, inputs.remaining_hours, r_avg, r_avg
+        )
+    else:
+        a, b = project_linear(
+            inputs.p_now, inputs.remaining_hours, r_avg, r_recent
+        )
+        final_low, final_high = min(a, b), max(a, b)
+
+    # Smooth week-average projection (additive surface field). Distinct from
+    # the displayed band (which keys off final_high): this is the conservative
+    # week-average value the projected-pace alert axis fires on.
+    # p_now + r_avg*remaining (== project_linear collapsed to the single rate).
+    week_avg_projection_pct = inputs.p_now + r_avg * inputs.remaining_hours
+
+    already_capped = inputs.p_now >= 100.0
+    projected_cap = already_capped or final_high >= 100.0
+
+    cap_at: dt.datetime | None = None
+    if not already_capped and projected_cap:
+        r_pessimistic = max(r_avg, r_recent or 0.0)
+        if r_pessimistic > 0:
+            hours_to_cap = (100.0 - inputs.p_now) / r_pessimistic
+            if hours_to_cap < inputs.remaining_hours:
+                cap_at = inputs.now_utc + dt.timedelta(hours=hours_to_cap)
+
+    # Budgets
+    budgets: list[BudgetRow] = []
+    if not already_capped:
+        for t in sorted(targets, reverse=True):
+            headroom = t - inputs.p_now
+            if headroom <= 0 or inputs.remaining_days <= 0:
+                budgets.append(BudgetRow(target_percent=t, pct_headroom=None,
+                                         dollars_per_day=None, percent_per_day=None))
+                continue
+            dollars_day = (headroom * inputs.dollars_per_percent) / inputs.remaining_days
+            pct_day = headroom / inputs.remaining_days
+            budgets.append(BudgetRow(
+                target_percent=t,
+                pct_headroom=headroom,
+                dollars_per_day=dollars_day,
+                percent_per_day=pct_day,
+            ))
+
+    return ForecastOutput(
+        inputs=inputs,
+        r_avg=r_avg,
+        r_recent=r_recent,
+        final_percent_low=final_low,
+        final_percent_high=final_high,
+        week_avg_projection_pct=week_avg_projection_pct,
+        projected_cap=projected_cap,
+        already_capped=already_capped,
+        cap_at=cap_at,
+        budgets=budgets,
+    )

package/bin/_lib_json_envelope.py

@@ -0,0 +1,38 @@
+"""Pure JSON wire-format helpers: schemaVersion envelope + canonical UTC-Z serializer.
+
+stamp_schema_version() is the single chokepoint for the additive camelCase
+``schemaVersion: 1`` envelope adopted across reporting ``--json`` surfaces
+(#279 S6 W1; convention: docs/cli-contract.md). _iso_z() is the canonical
+None-safe seconds-precision UTC-Z serializer (the forecast/dashboard-envelope
+behavior; _lib_doctor._iso_z deliberately diverges — see its docstring).
+
+Pure kernel: stdlib-only, no cctally back-import (split design §5.3).
+"""
+from __future__ import annotations
+
+import datetime as dt
+
+SCHEMA_VERSION_KEY = "schemaVersion"
+
+
+def stamp_schema_version(payload: dict, *, version: int = 1,
+                         key: str = SCHEMA_VERSION_KEY) -> dict:
+    """Return a new dict with ``key: version`` first.
+
+    Always returns a shallow copy; never mutates ``payload``. If ``key`` is
+    already present the copy preserves the existing value AND key order
+    (value- and order-idempotent no-op). Formatting is the caller's concern:
+    this returns a dict, each emitter keeps its own json.dumps arguments.
+    """
+    if key in payload:
+        return dict(payload)
+    out = {key: version}
+    out.update(payload)
+    return out
+
+
+def _iso_z(d: "dt.datetime | None") -> "str | None":
+    """None-safe UTC ISO-8601 with Z suffix, seconds precision."""
+    if d is None:
+        return None
+    return d.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")