cctally 1.59.0 → 1.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/bin/_cctally_cache.py +323 -48
- package/bin/_cctally_core.py +8 -0
- package/bin/_cctally_dashboard.py +913 -143
- package/bin/_cctally_doctor.py +2 -0
- package/bin/_cctally_forecast.py +24 -2
- package/bin/_cctally_parser.py +28 -2
- package/bin/_cctally_tui.py +676 -16
- package/bin/_cctally_update.py +49 -9
- package/bin/_cctally_weekrefs.py +92 -2
- package/bin/_lib_doctor.py +10 -0
- package/bin/_lib_snapshot_cache.py +988 -0
- package/bin/_lib_view_models.py +117 -12
- package/bin/cctally +57 -0
- package/dashboard/static/assets/index-0jzYm75p.css +1 -0
- package/dashboard/static/assets/index-D_Ylyqsf.js +80 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-BatfACUX.css +0 -1
- package/dashboard/static/assets/index-VUPDfWul.js +0 -80
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,37 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.61.0] - 2026-07-05
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- `cache-sync --prune-orphans` cleans up cache rows left behind when a session's transcript directory is removed — for example after you delete a git worktree — so stale sessions and projects don't linger in your reports. It's a fast, targeted alternative to a full `cache-sync --rebuild`, and only removes rows it can prove are safe to drop, leaving anything ambiguous in place.
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
- The dashboard's 5-hour **Blocks** card is now a full-size half-width card the same size as the **Weekly** and **Monthly** cards, instead of a cramped third-width tile in the short bottom row where only a couple of its blocks were visible at a time. It moves up beside the other period cards — with the **Forecast** card promoted alongside it so the board stays a tidy grid — so you can see far more of the week's activity blocks at a glance, and the **Recent Alerts** card now spans the full width of the bottom row (#266).
|
|
15
|
+
- The live dashboard's warm refreshes are now much faster on large, many-reset-week usage histories. A closed subscription week's totals never change, so the **$/1% Trend** card (and its weekly-history detail view), the **Forecast**, and the **Projects** panel no longer re-cost every closed week from scratch on each refresh — each closed week is computed once and reused until the underlying usage changes, and the Projects panel additionally stops re-resolving every session's project path on each refresh (it now resolves each distinct path once instead of once per session, and reuses each closed week's per-project totals), so on the largest histories the Projects rebuild drops from seconds to a fraction of a second while staying byte-for-byte identical (#269).
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
- The live dashboard no longer pegs a CPU core after long uptime or on a large usage history. It now recognises when nothing has changed since the last refresh and reuses the previous snapshot — so an idle dashboard sits near 0% CPU while its countdowns, freshness, and health checks stay live — and when new usage does land it recomputes only the current day/week/month (and the sessions that changed) instead of re-aggregating your entire history every few seconds, so each refresh stays fast even at hundreds of thousands of sessions (#268).
|
|
19
|
+
- The dashboard now cleans up cache rows left behind by removed session directories (such as a deleted git worktree) on its own — once at startup and periodically while running — so those stale rows no longer linger until a manual rebuild, and the `[cache] N tracked file(s) no longer on disk` warning no longer repeats every few seconds.
|
|
20
|
+
- `cache-sync --rebuild` now waits for the cache lock (up to 30 seconds) and reports a non-zero exit when it still can't acquire it, instead of silently doing nothing and reporting success while a dashboard is actively syncing.
|
|
21
|
+
|
|
22
|
+
## [1.60.0] - 2026-07-03
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
- Dashboard **Recent Sessions** is denser and more useful. When every recent session uses the same model, the redundant per-row model column is dropped entirely — it previously rendered as a column of dots under a still-labelled "Model" header, which read as broken — and a single "all · model" caption in the header says it instead; a mixed set of models still shows the column. Two columns that carry real signal take its place: a **Session** title for each row (its first prompt, or an AI-generated summary when one exists) and a **Cache** hit-rate. The title is private conversation content, so it appears only when transcript viewing is enabled for how you reached the dashboard (on your own machine, or on the LAN only if you have explicitly exposed transcripts) and shows a dash otherwise. All the columns now fit at a common laptop width without sideways scrolling, and a long title shortens with the full text available on hover (#264).
|
|
26
|
+
- Dashboard **$/1% Trend** detail view reads better and gains a cost column. When there is too little history to compute a four-week median, the two formerly-empty "—" median tiles collapse into a single "needs 4 weeks" hint instead of looking broken. The weekly table gains a sortable **Cost** column (what you spent that week) and the whole view opens in a wider two-pane layout — the chart and its headline numbers on the left, the sortable weekly table on the right — so it fills the space instead of stacking tall and scrolling; sorting the table reorders only the pop-up, not the small card behind it (#264).
|
|
27
|
+
- The dashboard's usage history is split back into three independent **Daily**, **Weekly**, and **Monthly** cards, each opening its own detail view — undoing an earlier consolidation that folded them into a single "History" card and hid Weekly and Monthly behind an easy-to-miss Day/Week/Month toggle. Every card now has a discoverable header and a keyboard shortcut, and the Weekly and Monthly views open in a wider two-pane layout — the period navigator across the top, the selected period's model-and-token breakdown on the left, and a sortable table of every period on the right — so the full history fits without scrolling. The daily heatmap still deep-links a clicked day straight into the Daily view (#264).
|
|
28
|
+
- Dashboard board and hero redesigned to fix the wasted space the previous facelift left behind. The board is now a content-aware **bento grid** — cards sit in height-matched rows sized to their content and the layout goes full-width (fluid up to 2100px) instead of capping at 1600px — so a wide screen fills with data instead of stranding a tall column of half-empty full-width cards, roughly halving the page height, while still collapsing to a single column on mobile and keeping drag-to-reorder within each row. The hero is rebuilt into three zones so the two questions you actually ask each get a dominant number: weekly usage % (with 5-hour usage paired beside it and the reset countdown) and dollars spent this week (with $/1% as its sub-line), alongside a compact support column for the end-of-week forecast, the week-over-week $/1% trend, and a snapshot-freshness reading that now stays calm for a normal few-minute-old snapshot and only turns amber, then red, once the data is genuinely stale. Every card also gains a consistent ⤢ expand control in its header to open its detail view (previously some cards signalled "open" only via the share icon, and the Blocks card had no click-to-open at all), the Forecast card gains a pace bar toward its 100% cap, and the empty Alerts card shows a teaching gauge of your current usage against the configured alert thresholds (#264).
|
|
29
|
+
- Dashboard desktop board fixes that finish the bento redesign. Cards whose content is taller than their row now scroll inside their own frame instead of overflowing and rendering on top of the card below them — previously the $/1% Trend chart could spill over the Daily card beneath it, and the Weekly and Monthly cards did the same. The 5-hour **Blocks** card now shows every block for the week (scroll inside the card to reach them all) instead of only the three most recent with no way to see the rest. The **Daily** heatmap shows each day's dollar amount again on the card, not just in its detail view. And the per-card collapse arrow — which did nothing on desktop once the board became height-matched — is removed there (the ⤢ expand control opens each card's detail); it still works on smaller stacked screens (#264).
|
|
30
|
+
- Dashboard mobile fixes. The **Recent Sessions** panel title is no longer crushed to nothing on a phone; each session in the list now leads with its title (what the session was about) instead of hiding it; the at-a-glance hero is a compact card rather than three tall stacked zones; and the Daily card title no longer clips (#264).
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
- Dashboard: the empty **Recent Alerts** card no longer sits bottom-heavy with a band of empty space above it — it now shows the same "you're clear" gauge as the alerts detail view (your current usage against the configured alert thresholds), centered and sized to the card, so it reads as balanced as its Forecast and Blocks neighbours (#265).
|
|
34
|
+
- Dashboard: the **$/1% Trend** card keeps its sparkline and "older ▸ newer" legend pinned in view and scrolls the weekly table beneath them, so with only a few weeks of history the chart is no longer half-hidden below the card's internal scroll fold (#265).
|
|
35
|
+
- Dashboard: the **Blocks** card's expand (⤢) control is now disabled on a week with no activity blocks yet, instead of looking clickable but opening nothing when pressed (#265).
|
|
36
|
+
- Dashboard: the **Weekly** and **Monthly** cards now list every week and month and scroll within the card, instead of showing only the three most recent behind a scrollbar that revealed nothing more — so the whole history is reachable without opening the detail view (#265).
|
|
37
|
+
- Dashboard: the **Daily** heatmap no longer clips the top off each day-of-month number — the card row is a little taller so every day's number and its dollar amount are fully legible (#265).
|
|
38
|
+
|
|
8
39
|
## [1.59.0] - 2026-07-02
|
|
9
40
|
|
|
10
41
|
### Changed
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -106,6 +106,7 @@ import os
|
|
|
106
106
|
import pathlib
|
|
107
107
|
import sqlite3
|
|
108
108
|
import sys
|
|
109
|
+
import time
|
|
109
110
|
from dataclasses import dataclass, field
|
|
110
111
|
from typing import Any, Callable, Iterator
|
|
111
112
|
|
|
@@ -341,22 +342,40 @@ def _resolve_project_key(
|
|
|
341
342
|
is_unknown=True,
|
|
342
343
|
)
|
|
343
344
|
|
|
345
|
+
# Win 1 (#269 §14, Codex-M4 P1): raw-`project_path` fast path. The
|
|
346
|
+
# normalized cache is keyed on ``os.path.realpath(...)``, so on the base
|
|
347
|
+
# code the expensive realpath/lstat walk runs once per ENTRY (~190K on a
|
|
348
|
+
# large instance) rather than once per DISTINCT raw spelling (~10K). A
|
|
349
|
+
# namespaced raw key (``("raw", project_path)``) — a tuple that can never
|
|
350
|
+
# collide with the normalized string keys — short-circuits the common case
|
|
351
|
+
# (the same raw spelling repeated across a project's entries). This does
|
|
352
|
+
# NOT replace the normalized cache: a raw MISS still consults/populates the
|
|
353
|
+
# normalized entry, so ``mode="full-path"`` symlink-alias collapse to the
|
|
354
|
+
# first spelling seen is preserved (a second alias misses the raw fast
|
|
355
|
+
# path, hits the normalized entry, and returns the first spelling's key).
|
|
356
|
+
# Byte-identical for ALL modes.
|
|
357
|
+
raw_key = ("raw", project_path)
|
|
358
|
+
raw_hit = cache.get(raw_key)
|
|
359
|
+
if raw_hit is not None:
|
|
360
|
+
return raw_hit
|
|
361
|
+
|
|
344
362
|
if mode == "full-path":
|
|
345
363
|
normalized = os.path.realpath(os.path.expanduser(project_path))
|
|
346
364
|
key = cache.get(normalized)
|
|
347
|
-
if key is
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
cache[
|
|
365
|
+
if key is None:
|
|
366
|
+
key = ProjectKey(
|
|
367
|
+
bucket_path=normalized,
|
|
368
|
+
display_key=project_path, # raw, so user sees what they typed
|
|
369
|
+
git_root=None,
|
|
370
|
+
)
|
|
371
|
+
cache[normalized] = key
|
|
372
|
+
cache[raw_key] = key
|
|
355
373
|
return key
|
|
356
374
|
|
|
357
375
|
normalized = os.path.realpath(os.path.expanduser(project_path))
|
|
358
376
|
cached = cache.get(normalized)
|
|
359
377
|
if cached is not None:
|
|
378
|
+
cache[raw_key] = cached
|
|
360
379
|
return cached
|
|
361
380
|
|
|
362
381
|
home = os.path.expanduser("~")
|
|
@@ -371,6 +390,7 @@ def _resolve_project_key(
|
|
|
371
390
|
git_root=cur,
|
|
372
391
|
)
|
|
373
392
|
cache[normalized] = key
|
|
393
|
+
cache[raw_key] = key
|
|
374
394
|
return key
|
|
375
395
|
cur = os.path.dirname(cur)
|
|
376
396
|
|
|
@@ -381,6 +401,7 @@ def _resolve_project_key(
|
|
|
381
401
|
is_no_git=True,
|
|
382
402
|
)
|
|
383
403
|
cache[normalized] = key
|
|
404
|
+
cache[raw_key] = key
|
|
384
405
|
return key
|
|
385
406
|
|
|
386
407
|
|
|
@@ -481,6 +502,19 @@ class IngestStats:
|
|
|
481
502
|
and self.files_failed == 0)
|
|
482
503
|
|
|
483
504
|
|
|
505
|
+
@dataclass
|
|
506
|
+
class PruneResult:
|
|
507
|
+
"""Outcome of _prune_orphaned_cache_entries: how much of the derived Claude
|
|
508
|
+
surface was removed for safely-orphaned source paths, plus the orphan paths
|
|
509
|
+
left in place (residual — a gate failed, so `--rebuild` is the escape hatch)
|
|
510
|
+
and whether the flock was contended (nothing mutated)."""
|
|
511
|
+
pruned_files: int = 0
|
|
512
|
+
pruned_entries: int = 0
|
|
513
|
+
pruned_messages: int = 0
|
|
514
|
+
residual_paths: "list[str]" = field(default_factory=list)
|
|
515
|
+
contended: bool = False
|
|
516
|
+
|
|
517
|
+
|
|
484
518
|
def _progress_stderr(stats: IngestStats, *, force: bool = False) -> None:
|
|
485
519
|
"""Default stderr progress callback. Every 200 files or when forced."""
|
|
486
520
|
if not force and stats.files_processed % 200 != 0:
|
|
@@ -577,6 +611,25 @@ def _ensure_session_files_row(conn: sqlite3.Connection, source_path: str) -> Non
|
|
|
577
611
|
conn.commit()
|
|
578
612
|
|
|
579
613
|
|
|
614
|
+
# How long `cache-sync --rebuild` (and the --prune-orphans path) waits on the
|
|
615
|
+
# cache.db flock before giving up. Routine auto-syncs stay non-blocking
|
|
616
|
+
# (lock_timeout=None); an explicit rebuild is worth a bounded wait so a running
|
|
617
|
+
# dashboard's background tick doesn't turn the rebuild into a silent no-op.
|
|
618
|
+
# Read at call time in cmd_cache_sync so tests can monkeypatch it low.
|
|
619
|
+
_REBUILD_LOCK_TIMEOUT_SECONDS = 30.0
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
# Orphan-warning throttle: warn only when the detected orphan set CHANGES,
|
|
623
|
+
# so a long-lived dashboard doesn't re-spam the "[cache] N tracked file(s) no
|
|
624
|
+
# longer on disk" line every ~5s sync tick. Reset in tests.
|
|
625
|
+
_LAST_WARNED_ORPHAN_SET: "frozenset[str]" = frozenset()
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _reset_orphan_warning_throttle():
|
|
629
|
+
global _LAST_WARNED_ORPHAN_SET
|
|
630
|
+
_LAST_WARNED_ORPHAN_SET = frozenset()
|
|
631
|
+
|
|
632
|
+
|
|
580
633
|
# Flags whose presence means the cache is mid-migration / mid-reingest. A
|
|
581
634
|
# targeted (only_paths) ingest DECLINES when any is set and defers to the next
|
|
582
635
|
# full background sync — inserting through a half-migrated FTS shape or skipping
|
|
@@ -612,12 +665,153 @@ def _targeted_has_pending_global_work(conn) -> bool:
|
|
|
612
665
|
return row is not None
|
|
613
666
|
|
|
614
667
|
|
|
668
|
+
def _acquire_cache_flock(lock_fh, *, timeout):
|
|
669
|
+
"""Acquire the exclusive cache flock on ``lock_fh``.
|
|
670
|
+
|
|
671
|
+
``timeout is None`` -> a single non-blocking attempt (today's behavior
|
|
672
|
+
for routine auto-syncs): returns False immediately on contention.
|
|
673
|
+
``timeout > 0`` -> retry ``LOCK_NB`` every ~0.2s until acquired or the
|
|
674
|
+
deadline elapses. Returns True iff the lock is held on return.
|
|
675
|
+
|
|
676
|
+
A retry-with-sleep loop, NOT a SIGALRM-based blocking LOCK_EX: the
|
|
677
|
+
dashboard runs its sync on a background thread, where Python signals
|
|
678
|
+
never fire, so an alarm timeout would silently never trip.
|
|
679
|
+
"""
|
|
680
|
+
deadline = None if timeout is None else (time.monotonic() + timeout)
|
|
681
|
+
while True:
|
|
682
|
+
try:
|
|
683
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
684
|
+
return True
|
|
685
|
+
except BlockingIOError:
|
|
686
|
+
if deadline is None or time.monotonic() >= deadline:
|
|
687
|
+
return False
|
|
688
|
+
time.sleep(0.2)
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def _prune_orphaned_cache_entries(conn, *, lock_timeout=None):
|
|
692
|
+
"""Safely prune the FULL derived Claude cache surface for orphaned
|
|
693
|
+
(removed-from-disk) source paths. See the design spec
|
|
694
|
+
docs/superpowers/specs/2026-07-04-cache-orphan-prune-design.md §2.
|
|
695
|
+
|
|
696
|
+
Kept OUT of the shared sync_cache (which stays detect-only, fixture-safe):
|
|
697
|
+
only real runtimes call this — the CLI cache-sync --prune-orphans and the
|
|
698
|
+
dashboard self-heal — so synthetic /fake/… fixture paths never reach it.
|
|
699
|
+
|
|
700
|
+
Three gates decide safety per orphan path P (session_id sid):
|
|
701
|
+
A) sid is non-null and not shared by any surviving on-disk file
|
|
702
|
+
(cheap prefilter; "same session_id" is empirical, not proof).
|
|
703
|
+
B) coverage: every one of P's session_entries (msg_id, req_id) keys
|
|
704
|
+
has a conversation_messages row under P's OWN source_path (else
|
|
705
|
+
it is a uuid-less blind spot -> unprovable -> residual).
|
|
706
|
+
C) disjointness: none of P's keys appears in conversation_messages
|
|
707
|
+
under a surviving on-disk source_path (a survivor physically holds
|
|
708
|
+
the same turn -> deleting P's deduped cost row would lose it).
|
|
709
|
+
Anything failing A/B/C is left as residual (reported; for `--rebuild`).
|
|
710
|
+
|
|
711
|
+
Deletes the full derived surface for the safe set in ONE transaction
|
|
712
|
+
(conversation_file_touches -> conversation_messages [+FTS trigger] ->
|
|
713
|
+
conversation_ai_titles [+FTS trigger] -> session_entries -> session_files),
|
|
714
|
+
then recomputes the conversation_sessions rollup for exactly the pruned
|
|
715
|
+
session_ids (all their messages gone -> the stale rail rows drop). Does NOT
|
|
716
|
+
write the walk-complete marker: the next full sync_cache re-establishes it
|
|
717
|
+
on a clean walk. Acquires the cache.db.lock flock itself; a contended
|
|
718
|
+
lock_timeout returns a `contended` result without mutating.
|
|
719
|
+
"""
|
|
720
|
+
result = PruneResult()
|
|
721
|
+
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
722
|
+
_cctally_core.CACHE_LOCK_PATH.touch()
|
|
723
|
+
lock_fh = open(_cctally_core.CACHE_LOCK_PATH, "w")
|
|
724
|
+
try:
|
|
725
|
+
if not _acquire_cache_flock(lock_fh, timeout=lock_timeout):
|
|
726
|
+
result.contended = True
|
|
727
|
+
return result
|
|
728
|
+
|
|
729
|
+
tracked = conn.execute(
|
|
730
|
+
"SELECT path, size_bytes, session_id FROM session_files").fetchall()
|
|
731
|
+
on_disk = {p for (p, _sz, _sid) in tracked if os.path.isfile(p)}
|
|
732
|
+
surviving_sids = {sid for (p, _sz, sid) in tracked
|
|
733
|
+
if p in on_disk and sid is not None}
|
|
734
|
+
orphan_cands = [(p, sid) for (p, sz, sid) in tracked
|
|
735
|
+
if sz and p not in on_disk]
|
|
736
|
+
if not orphan_cands:
|
|
737
|
+
return result
|
|
738
|
+
|
|
739
|
+
safe_paths = []
|
|
740
|
+
pruned_sids = set()
|
|
741
|
+
for path, sid in orphan_cands:
|
|
742
|
+
if sid is None or sid in surviving_sids: # Gate A
|
|
743
|
+
result.residual_paths.append(path)
|
|
744
|
+
continue
|
|
745
|
+
keys = conn.execute(
|
|
746
|
+
"SELECT DISTINCT msg_id, req_id FROM session_entries "
|
|
747
|
+
"WHERE source_path=? AND msg_id IS NOT NULL AND req_id IS NOT NULL",
|
|
748
|
+
(path,)).fetchall()
|
|
749
|
+
ok = True
|
|
750
|
+
for mid, rid in keys:
|
|
751
|
+
covered = conn.execute( # Gate B
|
|
752
|
+
"SELECT 1 FROM conversation_messages "
|
|
753
|
+
"WHERE msg_id=? AND req_id=? AND source_path=? LIMIT 1",
|
|
754
|
+
(mid, rid, path)).fetchone() is not None
|
|
755
|
+
if not covered:
|
|
756
|
+
ok = False
|
|
757
|
+
break
|
|
758
|
+
shared = conn.execute( # Gate C
|
|
759
|
+
"SELECT source_path FROM conversation_messages "
|
|
760
|
+
"WHERE msg_id=? AND req_id=?", (mid, rid)).fetchall()
|
|
761
|
+
if any(sp in on_disk for (sp,) in shared):
|
|
762
|
+
ok = False
|
|
763
|
+
break
|
|
764
|
+
if not ok:
|
|
765
|
+
result.residual_paths.append(path)
|
|
766
|
+
continue
|
|
767
|
+
safe_paths.append(path)
|
|
768
|
+
pruned_sids.add(sid)
|
|
769
|
+
|
|
770
|
+
if not safe_paths:
|
|
771
|
+
return result
|
|
772
|
+
|
|
773
|
+
# No IN(...) chunking: safe_paths is bounded by the orphan count (a
|
|
774
|
+
# handful of removed files), well under SQLite's variable limit;
|
|
775
|
+
# _recompute_conversation_sessions chunks its own session-id list.
|
|
776
|
+
ph = ",".join("?" * len(safe_paths))
|
|
777
|
+
result.pruned_messages = conn.execute(
|
|
778
|
+
f"SELECT count(*) FROM conversation_messages WHERE source_path IN ({ph})",
|
|
779
|
+
safe_paths).fetchone()[0]
|
|
780
|
+
conn.execute("BEGIN")
|
|
781
|
+
try:
|
|
782
|
+
conn.execute(
|
|
783
|
+
f"DELETE FROM conversation_file_touches WHERE message_id IN "
|
|
784
|
+
f"(SELECT id FROM conversation_messages WHERE source_path IN ({ph}))",
|
|
785
|
+
safe_paths)
|
|
786
|
+
conn.execute(
|
|
787
|
+
f"DELETE FROM conversation_messages WHERE source_path IN ({ph})", safe_paths)
|
|
788
|
+
conn.execute(
|
|
789
|
+
f"DELETE FROM conversation_ai_titles WHERE source_path IN ({ph})", safe_paths)
|
|
790
|
+
result.pruned_entries = conn.execute(
|
|
791
|
+
f"DELETE FROM session_entries WHERE source_path IN ({ph})", safe_paths).rowcount
|
|
792
|
+
result.pruned_files = conn.execute(
|
|
793
|
+
f"DELETE FROM session_files WHERE path IN ({ph})", safe_paths).rowcount
|
|
794
|
+
_recompute_conversation_sessions(conn, list(pruned_sids))
|
|
795
|
+
conn.commit()
|
|
796
|
+
except BaseException:
|
|
797
|
+
conn.rollback()
|
|
798
|
+
raise
|
|
799
|
+
return result
|
|
800
|
+
finally:
|
|
801
|
+
try:
|
|
802
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
803
|
+
except OSError:
|
|
804
|
+
pass
|
|
805
|
+
lock_fh.close()
|
|
806
|
+
|
|
807
|
+
|
|
615
808
|
def sync_cache(
|
|
616
809
|
conn: sqlite3.Connection,
|
|
617
810
|
*,
|
|
618
811
|
progress: Callable[[IngestStats], None] | None = None,
|
|
619
812
|
rebuild: bool = False,
|
|
620
813
|
only_paths: "set[str] | None" = None,
|
|
814
|
+
lock_timeout: "float | None" = None,
|
|
621
815
|
) -> IngestStats:
|
|
622
816
|
"""Read-through delta ingest. Acquires an exclusive fcntl.flock; if
|
|
623
817
|
another process holds it, returns immediately with lock_contended=True
|
|
@@ -635,9 +829,7 @@ def sync_cache(
|
|
|
635
829
|
|
|
636
830
|
lock_fh = open(_cctally_core.CACHE_LOCK_PATH, "w")
|
|
637
831
|
try:
|
|
638
|
-
|
|
639
|
-
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
640
|
-
except BlockingIOError:
|
|
832
|
+
if not _acquire_cache_flock(lock_fh, timeout=lock_timeout):
|
|
641
833
|
eprint("[cache] sync already in progress; using existing cache")
|
|
642
834
|
stats.lock_contended = True
|
|
643
835
|
return stats
|
|
@@ -994,27 +1186,46 @@ def sync_cache(
|
|
|
994
1186
|
# tracked in session_files (with data already ingested) but no
|
|
995
1187
|
# longer present on disk leaves orphaned session_entries rows that
|
|
996
1188
|
# the per-file loop below never visits — it iterates only on-disk
|
|
997
|
-
# `paths`. sync_cache
|
|
998
|
-
# in-place
|
|
999
|
-
#
|
|
1000
|
-
#
|
|
1001
|
-
#
|
|
1002
|
-
#
|
|
1003
|
-
#
|
|
1004
|
-
#
|
|
1005
|
-
#
|
|
1006
|
-
#
|
|
1007
|
-
#
|
|
1008
|
-
#
|
|
1009
|
-
#
|
|
1010
|
-
#
|
|
1011
|
-
#
|
|
1012
|
-
#
|
|
1013
|
-
#
|
|
1014
|
-
#
|
|
1015
|
-
#
|
|
1016
|
-
#
|
|
1017
|
-
#
|
|
1189
|
+
# `paths`. sync_cache stays DETECT-ONLY here — it never prunes
|
|
1190
|
+
# orphans in-place. Two reasons it must not delete from this hot,
|
|
1191
|
+
# shared path: (1) the truncation hazard — under the sticky
|
|
1192
|
+
# source_path dedup a surviving file may carry the same
|
|
1193
|
+
# (msg_id, req_id) yet keep its size_bytes, so a naive per-orphan
|
|
1194
|
+
# DELETE could drop a deduped cost row the survivor still owns
|
|
1195
|
+
# without re-ingesting it; (2) fixture safety — a blanket full-reset
|
|
1196
|
+
# would wrongly fire on the legitimate "cache seeded with synthetic
|
|
1197
|
+
# source paths" fixture pattern, and only sync_cache runs against
|
|
1198
|
+
# those fixtures. So detection does two things and no more: it emits
|
|
1199
|
+
# a THROTTLED warning (once per distinct orphan set — a removed
|
|
1200
|
+
# worktree persists, so an unthrottled warn would re-spam every
|
|
1201
|
+
# dashboard tick) and it INVALIDATES the walk-complete marker. An
|
|
1202
|
+
# orphaned cache no longer faithfully mirrors disk, so it is — by the
|
|
1203
|
+
# marker's own definition — not a complete walk. We actively DELETE
|
|
1204
|
+
# any marker a PRIOR clean walk left behind (idempotently — only when
|
|
1205
|
+
# one exists, so a repeated orphaned sync doesn't churn a no-op write
|
|
1206
|
+
# txn every tick); merely withholding THIS run's end-of-loop rewrite
|
|
1207
|
+
# is not enough, since a stale marker from a previous sync would
|
|
1208
|
+
# otherwise survive and keep vouching for completeness. Setting
|
|
1209
|
+
# walk_clean=False additionally suppresses the end-of-loop rewrite so
|
|
1210
|
+
# the marker stays absent for this run. With the marker gone the
|
|
1211
|
+
# upgrade gate DEFERs the 008/009/010 recomputes (rather than
|
|
1212
|
+
# certifying aggregates that still include data from files no longer
|
|
1213
|
+
# on disk). The safe CLEANUP lives OUT of sync_cache, in
|
|
1214
|
+
# _prune_orphaned_cache_entries — it re-derives the safe orphan set
|
|
1215
|
+
# independently and removes the full derived surface under three
|
|
1216
|
+
# gates: A (session-id not shared by a survivor), B (coverage — every
|
|
1217
|
+
# orphan (msg_id, req_id) key has a conversation_messages row under
|
|
1218
|
+
# the orphan's OWN path), and C (disjointness — no key of the orphan
|
|
1219
|
+
# appears in conversation_messages under a surviving path). B + C
|
|
1220
|
+
# together close the truncation hazard soundly: C refuses to delete
|
|
1221
|
+
# any key a survivor physically holds, and B refuses the uuid-less
|
|
1222
|
+
# blind spot where coverage can't be proved (anything failing A/B/C
|
|
1223
|
+
# is left as residual, cleared by `--rebuild`). That helper is
|
|
1224
|
+
# invoked by `cache-sync --prune-orphans` and the dashboard
|
|
1225
|
+
# self-heal, never from here (so fixtures never reach a destructive
|
|
1226
|
+
# delete); `cache-sync --rebuild` remains the whole-cache re-derive.
|
|
1227
|
+
# Both cleanup paths re-establish the marker on the next clean walk.
|
|
1228
|
+
# Only paths whose row carried ingested
|
|
1018
1229
|
# bytes (size_bytes > 0) count — a size_bytes=0 row holds no
|
|
1019
1230
|
# session_entries, so its absence leaves no orphan. The DELETE +
|
|
1020
1231
|
# commit lands BEFORE the per-file read+parse loop, so no write
|
|
@@ -1027,22 +1238,43 @@ def sync_cache(
|
|
|
1027
1238
|
# for targeted: the live-tail fast path never prunes orphans (the full
|
|
1028
1239
|
# background sync owns that).
|
|
1029
1240
|
if not targeted:
|
|
1241
|
+
global _LAST_WARNED_ORPHAN_SET
|
|
1030
1242
|
on_disk_paths = {str(jp) for jp in paths}
|
|
1031
1243
|
orphaned_tracked_paths = [
|
|
1032
1244
|
p for p, (size_bytes, _, _) in existing.items()
|
|
1033
1245
|
if size_bytes and p not in on_disk_paths
|
|
1034
1246
|
]
|
|
1035
1247
|
if orphaned_tracked_paths:
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1248
|
+
# Throttle the warning: emit only when the orphan set CHANGES,
|
|
1249
|
+
# not on every ~5s dashboard tick (a removed worktree persists,
|
|
1250
|
+
# so an unthrottled warn re-spams indefinitely). The marker
|
|
1251
|
+
# invalidation below stays UNCONDITIONAL — throttling the print
|
|
1252
|
+
# must not weaken the D5a invariant.
|
|
1253
|
+
cur = frozenset(orphaned_tracked_paths)
|
|
1254
|
+
if cur != _LAST_WARNED_ORPHAN_SET:
|
|
1255
|
+
eprint(
|
|
1256
|
+
f"[cache] {len(orphaned_tracked_paths)} tracked file(s) no "
|
|
1257
|
+
f"longer on disk; invalidating walk-complete marker "
|
|
1258
|
+
f"(run `cache-sync --prune-orphans` to prune, or "
|
|
1259
|
+
f"`cache-sync --rebuild`)"
|
|
1260
|
+
)
|
|
1261
|
+
_LAST_WARNED_ORPHAN_SET = cur
|
|
1262
|
+
# Idempotent marker invalidation: only DELETE (and commit) when a
|
|
1263
|
+
# prior clean walk actually left the marker behind, so a repeated
|
|
1264
|
+
# orphaned sync doesn't churn a no-op write transaction every tick.
|
|
1265
|
+
if conn.execute(
|
|
1266
|
+
"SELECT 1 FROM cache_meta WHERE key='claude_ingest_walk_complete'"
|
|
1267
|
+
).fetchone() is not None:
|
|
1268
|
+
conn.execute(
|
|
1269
|
+
"DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'"
|
|
1270
|
+
)
|
|
1271
|
+
conn.commit()
|
|
1045
1272
|
walk_clean = False # orphaned rows -> cache doesn't mirror disk (D5a)
|
|
1273
|
+
else:
|
|
1274
|
+
# No orphans this walk: clear the throttle memory so a LATER,
|
|
1275
|
+
# distinct orphan episode (even one recreated at the same paths)
|
|
1276
|
+
# warns again rather than being silently suppressed.
|
|
1277
|
+
_LAST_WARNED_ORPHAN_SET = frozenset()
|
|
1046
1278
|
|
|
1047
1279
|
# Pre-scan for any truncation among tracked files. Under the
|
|
1048
1280
|
# ccusage-parity ON CONFLICT DO UPDATE, source_path is PINNED to
|
|
@@ -2429,6 +2661,7 @@ def sync_codex_cache(
|
|
|
2429
2661
|
*,
|
|
2430
2662
|
progress: Callable[[CodexIngestStats], None] | None = None,
|
|
2431
2663
|
rebuild: bool = False,
|
|
2664
|
+
lock_timeout: "float | None" = None,
|
|
2432
2665
|
) -> CodexIngestStats:
|
|
2433
2666
|
"""Read-through delta ingest of ~/.codex/sessions/**/*.jsonl.
|
|
2434
2667
|
|
|
@@ -2448,9 +2681,7 @@ def sync_codex_cache(
|
|
|
2448
2681
|
|
|
2449
2682
|
lock_fh = open(_cctally_core.CACHE_LOCK_CODEX_PATH, "w")
|
|
2450
2683
|
try:
|
|
2451
|
-
|
|
2452
|
-
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
2453
|
-
except BlockingIOError:
|
|
2684
|
+
if not _acquire_cache_flock(lock_fh, timeout=lock_timeout):
|
|
2454
2685
|
eprint("[codex-cache] sync already in progress; using existing cache")
|
|
2455
2686
|
stats.lock_contended = True
|
|
2456
2687
|
return stats
|
|
@@ -2991,20 +3222,63 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
2991
3222
|
source = getattr(args, "source", "all")
|
|
2992
3223
|
conn = open_cache_db()
|
|
2993
3224
|
|
|
3225
|
+
# --prune-orphans: fast, targeted cleanup of cache rows whose source
|
|
3226
|
+
# JSONL was removed from disk (e.g. a deleted git worktree), without a
|
|
3227
|
+
# full rebuild. Claude-only surface; runs the three-gate safe helper.
|
|
3228
|
+
if getattr(args, "prune_orphans", False):
|
|
3229
|
+
if source == "codex":
|
|
3230
|
+
# The prune surface is Claude-only; Codex orphans are pruned
|
|
3231
|
+
# automatically during codex sync (see sync_codex_cache). Respect
|
|
3232
|
+
# the explicit --source codex rather than silently pruning Claude.
|
|
3233
|
+
eprint(
|
|
3234
|
+
"[cache-sync] --prune-orphans applies to the Claude cache only "
|
|
3235
|
+
"(Codex orphans are pruned automatically during codex sync); "
|
|
3236
|
+
"nothing to do for --source codex."
|
|
3237
|
+
)
|
|
3238
|
+
return 0
|
|
3239
|
+
res = _prune_orphaned_cache_entries(
|
|
3240
|
+
conn, lock_timeout=_REBUILD_LOCK_TIMEOUT_SECONDS
|
|
3241
|
+
)
|
|
3242
|
+
if res.contended:
|
|
3243
|
+
eprint(
|
|
3244
|
+
"[cache-sync] prune-orphans skipped: "
|
|
3245
|
+
"another process holds the lock"
|
|
3246
|
+
)
|
|
3247
|
+
return 1
|
|
3248
|
+
eprint(
|
|
3249
|
+
f"[cache-sync] pruned {res.pruned_files} orphaned file(s), "
|
|
3250
|
+
f"{res.pruned_entries} cost row(s), {res.pruned_messages} message(s)"
|
|
3251
|
+
)
|
|
3252
|
+
if res.residual_paths:
|
|
3253
|
+
eprint(
|
|
3254
|
+
f"[cache-sync] {len(res.residual_paths)} orphan(s) left in place "
|
|
3255
|
+
f"(shared session or missing conversation evidence); "
|
|
3256
|
+
f"run `cache-sync --rebuild` to clear them"
|
|
3257
|
+
)
|
|
3258
|
+
return 0
|
|
3259
|
+
|
|
2994
3260
|
# Note: when --rebuild is set we delegate the DELETE to sync_cache /
|
|
2995
3261
|
# sync_codex_cache, which execute it AFTER acquiring the flock. A
|
|
2996
3262
|
# pre-sync DELETE here would wipe the cache even when the subsequent
|
|
2997
3263
|
# sync loses the lock race and bails — leaving the user with empty
|
|
2998
|
-
# state. See sync_cache() / sync_codex_cache() docstrings.
|
|
3264
|
+
# state. See sync_cache() / sync_codex_cache() docstrings. A rebuild
|
|
3265
|
+
# is worth a bounded wait on the flock (vs the non-blocking auto-sync)
|
|
3266
|
+
# so a running dashboard's background tick doesn't silently no-op it;
|
|
3267
|
+
# if it still can't acquire, we report + exit non-zero rather than lie.
|
|
3268
|
+
lt = _REBUILD_LOCK_TIMEOUT_SECONDS if args.rebuild else None
|
|
3269
|
+
contended = False
|
|
2999
3270
|
|
|
3000
3271
|
if source in ("claude", "all"):
|
|
3001
|
-
stats = sync_cache(
|
|
3272
|
+
stats = sync_cache(
|
|
3273
|
+
conn, progress=_progress_stderr, rebuild=args.rebuild, lock_timeout=lt
|
|
3274
|
+
)
|
|
3002
3275
|
_progress_stderr(stats, force=True)
|
|
3003
3276
|
if stats.lock_contended and args.rebuild:
|
|
3004
3277
|
eprint(
|
|
3005
3278
|
"[cache-sync] rebuild skipped (claude): "
|
|
3006
3279
|
"another process holds the lock"
|
|
3007
3280
|
)
|
|
3281
|
+
contended = True
|
|
3008
3282
|
elif not stats.lock_contended:
|
|
3009
3283
|
eprint(
|
|
3010
3284
|
f"[cache-sync] claude done: {stats.files_processed} processed, "
|
|
@@ -3015,7 +3289,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
3015
3289
|
|
|
3016
3290
|
if source in ("codex", "all"):
|
|
3017
3291
|
stats = sync_codex_cache(
|
|
3018
|
-
conn, progress=_progress_codex_stderr, rebuild=args.rebuild
|
|
3292
|
+
conn, progress=_progress_codex_stderr, rebuild=args.rebuild, lock_timeout=lt
|
|
3019
3293
|
)
|
|
3020
3294
|
_progress_codex_stderr(stats, force=True)
|
|
3021
3295
|
if stats.lock_contended and args.rebuild:
|
|
@@ -3023,6 +3297,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
3023
3297
|
"[cache-sync] rebuild skipped (codex): "
|
|
3024
3298
|
"another process holds the lock"
|
|
3025
3299
|
)
|
|
3300
|
+
contended = True
|
|
3026
3301
|
elif not stats.lock_contended:
|
|
3027
3302
|
eprint(
|
|
3028
3303
|
f"[cache-sync] codex done: {stats.files_processed} processed, "
|
|
@@ -3031,4 +3306,4 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
3031
3306
|
f"{stats.rows_changed} rows changed"
|
|
3032
3307
|
)
|
|
3033
3308
|
|
|
3034
|
-
return 0
|
|
3309
|
+
return 1 if contended else 0
|
package/bin/_cctally_core.py
CHANGED
|
@@ -160,6 +160,14 @@ def _is_dev_checkout() -> bool:
|
|
|
160
160
|
return (_repo_root() / ".git").exists()
|
|
161
161
|
|
|
162
162
|
|
|
163
|
+
def is_preview_channel() -> bool:
|
|
164
|
+
"""True when running under the maintainer-local preview channel
|
|
165
|
+
(the `cctally-preview` wrapper sets CCTALLY_CHANNEL=preview). Single
|
|
166
|
+
source of truth for every preview-marker surface (dashboard port +
|
|
167
|
+
envelope, TUI header, --version, doctor) so the gate can't drift."""
|
|
168
|
+
return os.environ.get("CCTALLY_CHANNEL") == "preview"
|
|
169
|
+
|
|
170
|
+
|
|
163
171
|
def _real_prod_data_dir() -> pathlib.Path:
|
|
164
172
|
"""The REAL user's prod data dir (~/.local/share/cctally), resolved from
|
|
165
173
|
the password database rather than $HOME so it is immune to a faked HOME.
|