cctally 1.70.0 → 1.72.0

package/bin/_cctally_config.py

@@ -327,11 +327,76 @@ ALLOWED_CONFIG_KEYS = (
     "budget.codex.alert_thresholds",
     "budget.codex.projected_enabled",
     "telemetry.enabled",
+    "conversation.retention_days",
 )
 
 
 _CODEX_BUDGET_LEAF_PREFIX = "budget.codex."
 
+_DEFAULT_CONVERSATION_RETENTION_DAYS = 180
+
+
+def _validate_retention_days_value(raw: object) -> int:
+    """Validate a ``config set`` value for ``conversation.retention_days`` (F8).
+
+    Accept a non-negative integer; ``off`` / ``0`` canonicalize to integer 0
+    (disabled — keep transcripts forever). Reject booleans, negatives, and
+    non-numeric input by raising ``ValueError`` (the caller maps that to exit 2).
+    """
+    if isinstance(raw, bool):
+        raise ValueError(
+            "conversation.retention_days must be a non-negative integer or 'off'"
+        )
+    if isinstance(raw, str):
+        text = raw.strip().lower()
+        if text == "off":
+            return 0
+        try:
+            value = int(text, 10)
+        except (TypeError, ValueError):
+            raise ValueError(
+                "conversation.retention_days must be a non-negative integer "
+                f"or 'off', got {raw!r}"
+            )
+    elif isinstance(raw, int):
+        value = raw
+    else:
+        raise ValueError(
+            "conversation.retention_days must be a non-negative integer or 'off'"
+        )
+    if value < 0:
+        raise ValueError(
+            "conversation.retention_days must be >= 0 (use 'off' or 0 to disable)"
+        )
+    return value
+
+
+def resolve_retention_days(config: dict) -> int:
+    """Effective conversation transcript retention in days (F8).
+
+    Default is ``180``; ``0`` disables retention (keep forever). Any malformed
+    persisted value (non-int, boolean, negative, un-parseable string) degrades
+    to the safe 180 default rather than raising.
+    """
+    default = _DEFAULT_CONVERSATION_RETENTION_DAYS
+    block = config.get("conversation") if isinstance(config, dict) else None
+    if not isinstance(block, dict):
+        return default
+    stored = block.get("retention_days")
+    if stored is None:
+        return default
+    if isinstance(stored, bool):
+        return default
+    if isinstance(stored, int):
+        return stored if stored >= 0 else default
+    if isinstance(stored, str):
+        try:
+            value = int(stored.strip(), 10)
+        except (TypeError, ValueError):
+            return default
+        return value if value >= 0 else default
+    return default
+
 
 def _config_codex_leaf_value(config: dict, key: str) -> object:
     """Read one nested Codex budget leaf with its public default.
@@ -794,6 +859,10 @@ def _config_known_value(config: dict, key: str) -> "object":
             except ValueError:
                 return True
         return True
+    if key == "conversation.retention_days":
+        # Effective transcript-retention window in days (default 180; 0 = keep
+        # forever). Malformed persisted data surfaces the safe default (F8).
+        return resolve_retention_days(config)
     if key in ("update.check.enabled", "update.check.ttl_hours"):
         # Defaults mirror `_is_update_check_due` (True / 24 hours).
         # Hand-edited junk surfaces as the default — matches dashboard.bind.
@@ -1422,6 +1491,32 @@ def _cmd_config_set(args: argparse.Namespace) -> int:
                 rendered = str(normalized)
             print(f"{key}={rendered}")
         return 0
+    if key == "conversation.retention_days":
+        # Validate first; rejection short-circuits before lock acquisition (F8).
+        try:
+            normalized = _validate_retention_days_value(raw)
+        except ValueError as exc:
+            print(f"cctally: {exc}", file=sys.stderr)
+            return 2
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            existing = config.get("conversation")
+            if existing is not None and not isinstance(existing, dict):
+                print(
+                    "cctally: conversation config error: conversation must be "
+                    "an object",
+                    file=sys.stderr,
+                )
+                return 2
+            block = dict(existing or {})
+            block["retention_days"] = normalized
+            config["conversation"] = block
+            save_config(config)
+        if getattr(args, "emit_json", False):
+            print(json.dumps({"conversation": {"retention_days": normalized}}, indent=2))
+        else:
+            print(f"{key}={normalized}")
+        return 0
     if key in ("update.check.enabled", "update.check.ttl_hours"):
         # Validate first; rejection short-circuits before lock acquisition.
         if key == "update.check.enabled":
@@ -1862,6 +1957,19 @@ def _cmd_config_unset(args: argparse.Namespace) -> int:
                 save_config(config)
             # idempotent: silent on missing key
         return 0
+    if key == "conversation.retention_days":
+        # Mirror the display.tz branch: drop the leaf; if the `conversation`
+        # block ends up empty, drop it too. Next get resolves the 180 default.
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            block = config.get("conversation")
+            if isinstance(block, dict) and "retention_days" in block:
+                del block["retention_days"]
+                if not block:
+                    config.pop("conversation", None)
+                save_config(config)
+            # idempotent: silent on missing key
+        return 0
     if key in ("update.check.enabled", "update.check.ttl_hours"):
         # Mirror the dashboard.bind branch: drop the leaf, then prune
         # empty `check` and empty `update` so config.json stays tidy.

package/bin/_cctally_core.py

@@ -61,7 +61,8 @@ def _init_paths_from_env() -> None:
     """
     global APP_DIR, LEGACY_APP_DIR, LOG_DIR, DEV_MODE
     global DB_PATH, CACHE_DB_PATH
-    global CACHE_LOCK_PATH, CACHE_LOCK_CODEX_PATH, CONFIG_LOCK_PATH
+    global CACHE_LOCK_PATH, CACHE_LOCK_CODEX_PATH, CACHE_LOCK_MAINTENANCE_PATH
+    global CONFIG_LOCK_PATH
     global CONFIG_PATH, MIGRATION_ERROR_LOG_PATH, CHANGELOG_PATH
     global HOOK_TICK_LOG_DIR, HOOK_TICK_LOG_PATH, HOOK_TICK_LOG_ROTATED_PATH
     global HOOK_TICK_THROTTLE_PATH, HOOK_TICK_THROTTLE_LOCK_PATH
@@ -100,6 +101,10 @@ def _init_paths_from_env() -> None:
 
     CACHE_LOCK_PATH = APP_DIR / "cache.db.lock"
     CACHE_LOCK_CODEX_PATH = APP_DIR / "cache.db.codex.lock"
+    # #313 P3 (F7): dedicated maintenance flock serializing the transcript
+    # retention prune across processes, held ABOVE the two provider flocks so a
+    # rebuild/reingest cannot land between candidate selection and deletion.
+    CACHE_LOCK_MAINTENANCE_PATH = APP_DIR / "cache.db.maintenance.lock"
     CONFIG_LOCK_PATH = APP_DIR / "config.json.lock"
 
     CONFIG_PATH = APP_DIR / "config.json"

package/bin/_cctally_dashboard.py

@@ -1068,6 +1068,86 @@ def _load_recorded_five_hour_windows(*args, **kwargs):
     return sys.modules["cctally"]._load_recorded_five_hour_windows(*args, **kwargs)
 
 
+def _next_deadline(t0: float, interval: float, work: float) -> float:
+    """Monotonic deadline for the next sync iteration (#313 P2 / F10).
+
+    The iteration's work ended at ``t0 + work``; cool down from there for
+    ``max(interval, work)``. So the period is ``work + max(interval, work)``:
+
+    - work >= interval  → period = ``2 * work``  → CPU duty capped at 50% of one
+      core, scale-independent (a slow rebuild on a huge cache can never peg a
+      full core — the exact #313 symptom).
+    - work <  interval  → period = ``work + interval`` — byte-identical to the
+      prior loop (rebuild then a fixed ``interval`` sleep), so normal-cadence
+      behavior and its goldens are unchanged.
+
+    NOTE (spec deviation): the spec §3 pseudocode's ``deadline = t0 + max(
+    interval, work)`` cools down from ``t0`` (before the work), which for
+    work >= interval yields a ZERO cooldown (period = work, 100% duty) — it does
+    not meet the spec's own stated "period = 2*work / duty <= 50%" property or
+    §1's "<= 50% of one core" success criterion, and would leave the peg unfixed
+    (in fact worse than the old work+interval cadence). This formula cools down
+    from the work's END, which is what those properties require.
+    """
+    return (t0 + work) + max(interval, work)
+
+
+def _dashboard_sync_loop(
+    *,
+    stop,
+    interval: float,
+    run_iteration,
+    take_sync_request,
+    monotonic=time.monotonic,
+    sleep=time.sleep,
+) -> None:
+    """Run the dashboard's automatic sync loop with a work-proportional cooldown.
+
+    ``run_iteration`` performs one whole automatic iteration (rebuild plus any
+    orphan self-heal / retention maintenance the thread does), so its measured
+    duration — not just the rebuild — drives the deadline (F10). The manual
+    ``POST /api/sync`` refresh is a separate synchronous path under ``sync_lock``
+    and is unaffected; ``take_sync_request`` is the TUI force-refresh flag that
+    breaks the cooldown early.
+    """
+    while not stop.is_set():
+        t0 = monotonic()
+        run_iteration()
+        work = monotonic() - t0
+        deadline = _next_deadline(t0, interval, work)
+        while not stop.is_set() and monotonic() < deadline:
+            if take_sync_request():
+                break
+            sleep(min(0.1, max(0.0, deadline - monotonic())))
+
+
+def _dashboard_maybe_prune_retention() -> None:
+    """#313 P3 (F7): throttled transcript retention prune driven from the
+    dashboard sync thread. Runs once per iteration INSIDE the measured cooldown
+    work (Task 6), so its cost paces the deadline. Opens a dedicated cache
+    connection (no provider flock, no open transaction) for the orchestrator.
+    No-op when retention is disabled; never raises (a prune failure must not
+    crash the sync thread)."""
+    try:
+        c = sys.modules["cctally"]
+        from _cctally_config import resolve_retention_days
+        import _lib_conversation_retention as retention
+        retention_days = resolve_retention_days(c.load_config())
+        if retention_days <= 0:
+            return
+        conn = c.open_cache_db()
+        try:
+            retention._maybe_prune_conversation_retention(
+                conn,
+                now_utc=dt.datetime.now(dt.timezone.utc),
+                retention_days=retention_days,
+            )
+        finally:
+            conn.close()
+    except Exception:
+        pass
+
+
 def _make_run_sync_now(*args, **kwargs):
     return sys.modules["cctally"]._make_run_sync_now(*args, **kwargs)
 
@@ -6623,23 +6703,38 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
 
     class _DashboardSyncThread(_c_for_subclass._TuiSyncThread):
         def _run(self) -> None:
-            last_heal = _time.monotonic()
-            while not self._stop.is_set():
+            last_heal = [_time.monotonic()]
+
+            def run_iteration() -> None:
                 _run_sync_now(skip_sync=self._skip_sync)
                 # Self-heal removed-worktree orphans on a ~60s cadence (far
                 # rarer than the sync tick — a deleted worktree is not urgent).
                 # Non-blocking on the flock, so a contended tick just retries
-                # next cadence; gated off under --no-sync.
+                # next cadence; gated off under --no-sync. Runs INSIDE the
+                # measured iteration so its cost counts toward the cooldown
+                # deadline (#313 P2 / F10).
                 if (not self._skip_sync
-                        and _time.monotonic() - last_heal >= 60.0):
-                    last_heal = _time.monotonic()
+                        and _time.monotonic() - last_heal[0] >= 60.0):
+                    last_heal[0] = _time.monotonic()
                     _dashboard_self_heal_orphans(skip_sync=self._skip_sync)
-                for _ in range(int(max(1, self._interval * 10))):
-                    if self._stop.is_set():
-                        return
-                    if self._ref.take_sync_request():
-                        break
-                    _time.sleep(0.1)
+                # #313 P3 (F7): throttled transcript retention prune, inside the
+                # measured iteration so its cost is in the cooldown work. Gated
+                # off under --no-sync (frozen mode makes no cache writes).
+                if not self._skip_sync:
+                    _dashboard_maybe_prune_retention()
+
+            # Work-proportional cooldown (F10): sleep to t0 + max(interval, work)
+            # so a slow rebuild cannot peg a full core. The manual POST /api/sync
+            # refresh runs synchronously under sync_lock, independent of this
+            # cooldown, so a user force-refresh is always immediate.
+            _dashboard_sync_loop(
+                stop=self._stop,
+                interval=self._interval,
+                run_iteration=run_iteration,
+                take_sync_request=self._ref.take_sync_request,
+                monotonic=_time.monotonic,
+                sleep=_time.sleep,
+            )
 
     sync_thread = (
         None if args.no_sync