cctally 1.66.0 → 1.67.0

package/bin/_cctally_project.py

@@ -28,7 +28,7 @@ import sys
 
 from _cctally_core import (
     _command_as_of,
-    _reset_aware_floor,
+    _floored_week_max,
     eprint,
     open_db,
     parse_iso_datetime,
@@ -82,45 +82,18 @@ def _load_week_snapshots(
             "AND datetime(week_end_at) > datetime(?)",
             (until.isoformat(), since.isoformat()),
         )
-        rows = cur.fetchall()
-        # Resolve each week's clamp floor once (keyed on week_start_date) so a
-        # mid-week credit doesn't re-inflate the per-project Used %. None means
-        # "no floor" — every captured row counts.
-        floor_epoch: dict[str, int | None] = {}
-
-        def _floor_for(wsd, ws_iso, we_iso):
-            if wsd not in floor_epoch:
-                f_iso = _reset_aware_floor(conn, wsd, ws_iso, we_iso)
-                floor_epoch[wsd] = (
-                    int(parse_iso_datetime(f_iso, "project.floor").timestamp())
-                    if f_iso else None
-                )
-            return floor_epoch[wsd]
-
-        result: dict[dt.datetime, float] = {}
-        for wsd, ws_iso, we_iso, cap_iso, pct in rows:
+        rows_in = []
+        for wsd, ws_iso, we_iso, cap_iso, pct in cur.fetchall():
             if ws_iso is None or pct is None:
                 continue
-            floor = _floor_for(wsd, ws_iso, we_iso)
-            if floor is not None and cap_iso is not None:
-                try:
-                    cap_epoch = int(
-                        parse_iso_datetime(str(cap_iso), "project.cap").timestamp()
-                    )
-                except ValueError:
-                    cap_epoch = None
-                # Drop pre-floor (stale pre-credit) snapshots from the MAX.
-                if cap_epoch is not None and cap_epoch < floor:
-                    continue
-            ws = dt.datetime.fromisoformat(
-                str(ws_iso).replace("Z", "+00:00")
-            )
+            ws = dt.datetime.fromisoformat(str(ws_iso).replace("Z", "+00:00"))
             key = ws.astimezone(dt.timezone.utc)
-            pct_f = float(pct)
-            prev = result.get(key)
-            if prev is None or pct_f > prev:
-                result[key] = pct_f
-        return result
+            rows_in.append((key, wsd, ws_iso, we_iso, cap_iso, pct))
+        # Reset-aware per-week max (#290): the shared reducer resolves each
+        # week's clamp floor once (keyed on week_start_date) and drops stale
+        # pre-credit captures before taking the per-week maximum. The query
+        # above already filters NULL bounds, so canonical bounds are present.
+        return _floored_week_max(conn, rows_in)
     finally:
         conn.close()
 

package/bin/_cctally_setup.py

@@ -64,6 +64,7 @@ import pathlib
 import shutil
 import subprocess
 import sys
+import threading
 import time
 
 
@@ -1769,6 +1770,84 @@ def _setup_dry_run(args: argparse.Namespace) -> int:
     return 0
 
 
+_SETUP_PROGRESS_MIN_INTERVAL_SECONDS = 1.0
+_SETUP_PROGRESS_HEARTBEAT_POLL_SECONDS = 0.5
+
+
+def _setup_progress_enabled(*, json_mode: bool) -> bool:
+    """Whether to emit setup cold-sync progress to stderr (issue #281 S7 / R6).
+
+    ``CCTALLY_SETUP_PROGRESS='1'`` forces on, ``'0'`` forces off (both
+    directions: the golden harness force-suppresses, the timed
+    fixture-ladder run force-observes). Otherwise: on iff stderr is a TTY
+    and not ``--json`` (json is machine-readable — auto-suppress the
+    friendly notices there). Gating on ``sys.stderr`` (not stdout) is
+    correct because progress goes to stderr, so a user who pipes stdout
+    still sees it while the harness (command-substitution pipe) reads
+    non-TTY.
+    """
+    env = os.environ.get("CCTALLY_SETUP_PROGRESS")
+    if env == "1":
+        return True
+    if env == "0":
+        return False
+    return (not json_mode) and sys.stderr.isatty()
+
+
+class _SetupProgressReporter:
+    """Best-effort stderr progress for setup's cold-sync bootstrap.
+
+    Coordinates the per-file ``sync_cache(progress=)`` callback with a
+    background heartbeat so neither a single giant JSONL nor a slow OAuth
+    fetch leaves a silent gap > ~2s. All emissions throttle off one shared
+    monotonic ``last_emit`` under a lock; ``force=True`` bypasses the
+    throttle (used for the pre-sync and pre-OAuth notices). A no-op (and no
+    thread) when disabled — so it is inert in the golden harness / non-TTY
+    / ``--json``.
+    """
+
+    def __init__(self, enabled: bool) -> None:
+        self.enabled = enabled
+        self._last_emit = 0.0
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._thread: "threading.Thread | None" = None
+
+    def emit(self, msg: str, *, force: bool = False) -> None:
+        if not self.enabled:
+            return
+        with self._lock:
+            now = time.monotonic()
+            if not force and (now - self._last_emit) < _SETUP_PROGRESS_MIN_INTERVAL_SECONDS:
+                return
+            self._last_emit = now
+            eprint(msg)
+
+    def sync_callback(self, stats) -> None:
+        # `stats` is a duck-typed cache.IngestStats (reads files_processed /
+        # files_total). sync_cache invokes this per file; emit() throttles.
+        self.emit(f"  … synced {stats.files_processed}/{stats.files_total} session files")
+
+    def _heartbeat_loop(self) -> None:
+        # Fires only when no per-file line / notice emitted in the last
+        # interval, bounding intra-file and intra-fetch silence.
+        while not self._stop.wait(_SETUP_PROGRESS_HEARTBEAT_POLL_SECONDS):
+            self.emit("  … still working…")
+
+    def __enter__(self) -> "_SetupProgressReporter":
+        if self.enabled:
+            self._thread = threading.Thread(
+                target=self._heartbeat_loop, name="cctally-setup-progress", daemon=True
+            )
+            self._thread.start()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=1.0)
+
+
 def _setup_emit_text(lines: list[str]) -> None:
     for ln in lines:
         print(ln)
@@ -2075,49 +2154,72 @@ def _setup_install(args: argparse.Namespace) -> int:
         out.append("  leave (data is funneled correctly either way), but you can remove")
         out.append("  it whenever you want. We won't touch the file.")
 
-    # Bootstrap (non-fatal). sync_cache requires a connection arg — mirror
-    # the pattern from cmd_hook_tick (Task 2 fix).
+    # Bootstrap (non-fatal). Progress is stderr-only + TTY-gated so the
+    # buffered stdout transcript and --json envelope are unchanged (issue
+    # #281 S7 / R6). The reporter wraps BOTH the cold-sync ingest and the
+    # OAuth fetch under one daemon heartbeat, so neither a single giant
+    # JSONL nor a slow OAuth network call leaves a silent gap > ~2s.
+    # sync_cache is untouched — we only pass its existing progress= seam.
     bootstrap_rows: int | None = None
     bootstrap_oauth_status: str | None = None
-    try:
-        cache_conn = c.open_cache_db()
+    progress = _SetupProgressReporter(
+        _setup_progress_enabled(json_mode=getattr(args, "json", False))
+    )
+    with progress:
+        progress.emit(
+            "⏳ Syncing session history (first run can take a moment on a large history)…",
+            force=True,
+        )
         try:
-            stats = c.sync_cache(cache_conn)
-            rows = int(stats.rows_changed)
-        finally:
+            cache_conn = c.open_cache_db()
             try:
-                cache_conn.close()
-            except Exception:
-                pass
-        bootstrap_rows = rows
-        # `rows` counts both genuine INSERTs and ccusage-parity DO UPDATE
-        # replacements (see IngestStats.rows_changed). On first install
-        # this is always 0-vs-N pure inserts (cache is empty), so "N new
-        # entries" is exactly accurate. On a re-install / upgrade path
-        # with active sessions, `rows` also counts UPSERT replacements
-        # (streaming-vs-final tiebreaker swaps), so the count is more
-        # accurately "ingest activity" than "rows newly added" — but
-        # we keep "new entries" because (a) it's still a useful signal
-        # to the operator that the cache is alive, and (b) the dominant
-        # case (first install) reads literally.
-        out.append(f"✓ Synced session cache ({rows} new entries)")
-    except Exception as exc:
-        out.append(f"⚠ sync_cache during bootstrap failed: {exc}")
-        warnings += 1
-    if oauth:
-        try:
-            status, _ = c._hook_tick_oauth_refresh()
-            bootstrap_oauth_status = status
-            if status.startswith("ok"):
-                c._hook_tick_throttle_touch()
-                out.append(f"✓ Bootstrapped weekly usage ({status})")
-            else:
-                out.append(f"⚠ Bootstrap OAuth fetch: {status}")
+                stats = c.sync_cache(cache_conn, progress=progress.sync_callback)
+            finally:
+                try:
+                    cache_conn.close()
+                except Exception:
+                    pass
+            if stats.lock_contended:
+                # Another process holds the cache flock — nothing was
+                # ingested (sync_cache short-circuits with rows_changed=0).
+                # Report honestly rather than a false "Synced (0 new
+                # entries)"; leave bootstrap_rows None for the envelope.
+                out.append("⚠ Another cache sync is in progress; using existing cache.")
                 warnings += 1
+                bootstrap_rows = None
+            else:
+                rows = int(stats.rows_changed)
+                bootstrap_rows = rows
+                # `rows` counts both genuine INSERTs and ccusage-parity DO
+                # UPDATE replacements (see IngestStats.rows_changed). On
+                # first install this is always 0-vs-N pure inserts (cache is
+                # empty), so "N new entries" is exactly accurate. On a
+                # re-install / upgrade path with active sessions, `rows` also
+                # counts UPSERT replacements (streaming-vs-final tiebreaker
+                # swaps), so the count is more accurately "ingest activity"
+                # than "rows newly added" — but we keep "new entries" because
+                # (a) it's still a useful signal to the operator that the
+                # cache is alive, and (b) the dominant case (first install)
+                # reads literally.
+                out.append(f"✓ Synced session cache ({rows} new entries)")
         except Exception as exc:
-            bootstrap_oauth_status = f"err({type(exc).__name__})"
-            out.append(f"⚠ Bootstrap OAuth failed: {exc}")
+            out.append(f"⚠ sync_cache during bootstrap failed: {exc}")
             warnings += 1
+        if oauth:
+            progress.emit("⏳ Fetching current usage…", force=True)
+            try:
+                status, _ = c._hook_tick_oauth_refresh()
+                bootstrap_oauth_status = status
+                if status.startswith("ok"):
+                    c._hook_tick_throttle_touch()
+                    out.append(f"✓ Bootstrapped weekly usage ({status})")
+                else:
+                    out.append(f"⚠ Bootstrap OAuth fetch: {status}")
+                    warnings += 1
+            except Exception as exc:
+                bootstrap_oauth_status = f"err({type(exc).__name__})"
+                out.append(f"⚠ Bootstrap OAuth failed: {exc}")
+                warnings += 1
 
     out.append("")
     if warnings:
@@ -2125,13 +2227,6 @@ def _setup_install(args: argparse.Namespace) -> int:
     else:
         out.append("cctally is ready.")
     out.append("")
-    # Settings.json was modified — CC caches it at session start. The
-    # warning fires unconditionally because `_setup_install` always
-    # rewrites settings.json (legacy migration, fresh install, repair).
-    out.append("⚠ Restart Claude Code for the new hooks to take effect in any currently")
-    out.append("  open sessions. New sessions launched after this point pick them up")
-    out.append("  automatically. (settings.json is cached at session start.)")
-    out.append("")
     out.append("  Try:")
     out.append("    cctally daily              # last 30 days")
     out.append("    cctally dashboard          # live web dashboard")
@@ -2151,6 +2246,18 @@ def _setup_install(args: argparse.Namespace) -> int:
     out.append("Opt out anytime:  cctally telemetry off   (or CCTALLY_DISABLE_TELEMETRY=1)")
     out.append("How it works:     https://github.com/omrikais/cctally/blob/main/docs/telemetry.md")
 
+    # ▶ Next step LAST: the sole required action, set off from the wall of
+    # text (issue #281 S7 / R6 — it was previously buried mid-dump, right
+    # after the readiness line and followed by ~12 more lines). It's an
+    # action, not a warning, so it drops the `⚠` and never touched the
+    # `warnings` count. settings.json is cached at session start, so open
+    # sessions need a restart; `_setup_install` always rewrites settings.json
+    # (legacy migration, fresh install, repair), so this always applies.
+    out.append("")
+    out.append("▶ Next step: restart Claude Code to activate the new hooks.")
+    out.append("  Open sessions won't record until you restart (settings.json is cached at")
+    out.append("  session start). New sessions started after this pick them up automatically.")
+
     if getattr(args, "json", False):
         # JSON-safe telemetry disclosure (spec 2026-07-07 §4): a structured
         # field, never prose. Resolved READ-ONLY via `resolve_telemetry_state`

package/bin/_cctally_transcript.py

@@ -0,0 +1,191 @@
+"""cctally transcript — anonymized-by-default export + raw cross-session search
+over the conversation cache (#281 S4 R4).
+
+Thin CLI wrappers over the SAME kernels the dashboard HTTP routes use
+(``get_conversation_export`` / ``search_conversations`` / ``build_anon_plan_for_db``
+/ ``scrub_text``), so ``transcript export`` byte-matches ``GET …/export`` and the
+viewer becomes scriptable.
+
+- ``export``: anonymized Markdown by default; ``--raw`` disables the whole scrub
+  (identity + secrets) and is byte-identical to the dashboard's raw export.
+  Byte-exact emission: ``sys.stdout.buffer.write`` of the exact UTF-8 bytes (no
+  ``print``, no added trailing newline — the render already ends in exactly one),
+  or ``--output PATH`` writes the same exact bytes; nothing else on stdout.
+  Unknown session → ``transcript: …`` on stderr, exit 1.
+- ``search``: RAW output (a navigation surface, not a sharing artifact). Mirrors
+  the full HTTP filter surface; date-only values parse through the SAME
+  display-tz-aware helper the HTTP handler uses. Human table by default; ``--json``
+  emits a ``schemaVersion``-stamped, explicitly camelCased envelope.
+
+Accessor discipline: no ``_cctally_*`` sibling is imported directly; every helper
+is reached via the call-time ``_cctally()`` accessor / ``_load_sibling`` (matching
+the other command modules), except the pure ``_lib_fmt._boxed_table`` renderer.
+"""
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+
+from _cctally_core import eprint
+from _lib_fmt import _boxed_table
+
+
+def _cctally():
+    """Call-time accessor to the cctally module namespace (ns-patchable)."""
+    return sys.modules["cctally"]
+
+
+def cmd_transcript(args) -> int:
+    action = getattr(args, "transcript_action", None)
+    if action == "export":
+        return _cmd_transcript_export(args)
+    if action == "search":
+        return _cmd_transcript_search(args)
+    eprint("transcript: expected a subcommand (`export` or `search`)")
+    return 2
+
+
+# ---- export ----------------------------------------------------------------
+
+def _cmd_transcript_export(args) -> int:
+    c = _cctally()
+    session_id = args.session_id
+    scope = getattr(args, "scope", "all")
+    raw = bool(getattr(args, "raw", False))
+    output = getattr(args, "output", None)
+
+    conn = c.open_cache_db()
+    try:
+        cq = c._load_sibling("_lib_conversation_query")
+        md = cq.get_conversation_export(conn, session_id, scope)
+        if md is None:
+            eprint(f"transcript: no conversation found for session {session_id!r}")
+            return 1
+        if not raw:
+            anon = c._load_sibling("_lib_conversation_anon")
+            plan = cq.build_anon_plan_for_db(conn, home_dir=os.path.expanduser("~"))
+            md = anon.scrub_text(md, plan)
+    finally:
+        conn.close()
+
+    data = md.encode("utf-8")
+    if output:
+        pathlib.Path(output).write_bytes(data)
+    else:
+        # Byte-exact: raw bytes to stdout, no `print`, no added trailing newline.
+        sys.stdout.buffer.write(data)
+        sys.stdout.buffer.flush()
+    return 0
+
+
+# ---- search ----------------------------------------------------------------
+
+def _cmd_transcript_search(args) -> int:
+    c = _cctally()
+    query = args.query
+    kind = getattr(args, "kind", "all")
+    limit = getattr(args, "limit", 50)
+    offset = getattr(args, "offset", 0)
+    projects = getattr(args, "project", None) or None
+    models = getattr(args, "model", None) or None
+    cost_min = getattr(args, "cost_min", None)
+    cost_max = getattr(args, "cost_max", None)
+    rebuild_min = getattr(args, "rebuild_min", None)
+
+    # Date-only bounds parse through the SAME display-tz-aware helper the HTTP
+    # filter handler uses (no second parser) — reuse, don't reimplement.
+    date_from_in = getattr(args, "date_from", None)
+    date_to_in = getattr(args, "date_to", None)
+    df = dtt = None
+    if date_from_in or date_to_in:
+        tz = c.resolve_display_tz(args, c.load_config())
+        tz_name = tz.key if tz is not None else None
+        dates = c._load_sibling("_lib_dashboard_dates")
+        try:
+            df, dtt = dates.parse_filter_date_range(
+                date_from_in, date_to_in, tz_name=tz_name)
+        except ValueError as exc:
+            eprint(f"transcript: {exc}")
+            return 2
+
+    conn = c.open_cache_db()
+    try:
+        cq = c._load_sibling("_lib_conversation_query")
+        try:
+            result = cq.search_conversations(
+                conn, query, limit=limit, offset=offset, kind=kind,
+                date_from=df, date_to=dtt, projects=projects, models=models,
+                cost_min=cost_min, cost_max=cost_max, rebuild_min=rebuild_min)
+        except ValueError as exc:            # unknown kind (belt-and-suspenders)
+            eprint(f"transcript: {exc}")
+            return 2
+    finally:
+        conn.close()
+
+    if getattr(args, "json", False):
+        payload = c.stamp_schema_version(_search_to_camel(result))
+        print(_json_dumps(payload))
+        return 0
+    _render_search_table(result)
+    return 0
+
+
+def _json_dumps(payload) -> str:
+    import json
+    return json.dumps(payload, indent=2)
+
+
+def _search_to_camel(result: dict) -> dict:
+    """Explicit recursive camelCase mapping of the ``search_conversations`` result
+    (this command's own serializer — ``stamp_schema_version`` only inserts the
+    leading key). Top level then hit level, per spec §7."""
+    out = {
+        "query": result.get("query", ""),
+        "mode": result.get("mode"),
+        "hits": [_hit_to_camel(h) for h in result.get("hits", [])],
+        "total": result.get("total", 0),
+        "kind": result.get("kind"),
+        "searchDepth": result.get("search_depth"),
+    }
+    if result.get("filter_degraded"):
+        out["filterDegraded"] = True
+    return out
+
+
+def _hit_to_camel(h: dict) -> dict:
+    return {
+        "sessionId": h.get("session_id"),
+        "uuid": h.get("uuid"),
+        "projectLabel": h.get("project_label"),
+        "title": h.get("title"),
+        "ts": h.get("ts"),
+        "snippet": h.get("snippet"),
+        "matchKinds": h.get("match_kinds", []),
+        "costUsd": h.get("cost_usd", 0.0),
+    }
+
+
+def _render_search_table(result: dict) -> None:
+    hits = result.get("hits", [])
+    if not hits:
+        print("No matching transcripts.")
+        return
+    rows = []
+    for h in hits:
+        kinds = ",".join(h.get("match_kinds") or []) or "-"
+        rows.append([
+            h.get("session_id") or "",
+            h.get("ts") or "",
+            h.get("project_label") or "",
+            kinds,
+            (h.get("snippet") or "").strip(),
+        ])
+    table = _boxed_table(
+        ["Session", "When", "Project", "Kinds", "Snippet"], rows,
+        ["left", "left", "left", "left", "left"])
+    print(table)
+    total = result.get("total", len(hits))
+    depth = result.get("search_depth")
+    suffix = f"  (search depth: {depth})" if depth and depth != "full" else ""
+    print(f"\n{len(hits)} of {total} match(es){suffix}")