cctally 1.60.0 → 1.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/bin/_cctally_cache.py +323 -48
- package/bin/_cctally_core.py +8 -0
- package/bin/_cctally_dashboard.py +866 -141
- package/bin/_cctally_doctor.py +2 -0
- package/bin/_cctally_forecast.py +24 -2
- package/bin/_cctally_parser.py +28 -2
- package/bin/_cctally_tui.py +643 -16
- package/bin/_cctally_update.py +49 -9
- package/bin/_cctally_weekrefs.py +92 -2
- package/bin/_lib_doctor.py +10 -0
- package/bin/_lib_snapshot_cache.py +988 -0
- package/bin/_lib_view_models.py +107 -12
- package/bin/cctally +57 -0
- package/dashboard/static/assets/index-0jzYm75p.css +1 -0
- package/dashboard/static/assets/{index-DQH6UPc_.js → index-D_Ylyqsf.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-uGwZSssU.css +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,20 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.61.0] - 2026-07-05
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- `cache-sync --prune-orphans` cleans up cache rows left behind when a session's transcript directory is removed — for example after you delete a git worktree — so stale sessions and projects don't linger in your reports. It's a fast, targeted alternative to a full `cache-sync --rebuild`, and only removes rows it can prove are safe to drop, leaving anything ambiguous in place.
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
- The dashboard's 5-hour **Blocks** card is now a full-size half-width card the same size as the **Weekly** and **Monthly** cards, instead of a cramped third-width tile in the short bottom row where only a couple of its blocks were visible at a time. It moves up beside the other period cards — with the **Forecast** card promoted alongside it so the board stays a tidy grid — so you can see far more of the week's activity blocks at a glance, and the **Recent Alerts** card now spans the full width of the bottom row (#266).
|
|
15
|
+
- The live dashboard's warm refreshes are now much faster on large, many-reset-week usage histories. A closed subscription week's totals never change, so the **$/1% Trend** card (and its weekly-history detail view), the **Forecast**, and the **Projects** panel no longer re-cost every closed week from scratch on each refresh — each closed week is computed once and reused until the underlying usage changes, and the Projects panel additionally stops re-resolving every session's project path on each refresh (it now resolves each distinct path once instead of once per session, and reuses each closed week's per-project totals), so on the largest histories the Projects rebuild drops from seconds to a fraction of a second while staying byte-for-byte identical (#269).
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
- The live dashboard no longer pegs a CPU core after long uptime or on a large usage history. It now recognises when nothing has changed since the last refresh and reuses the previous snapshot — so an idle dashboard sits near 0% CPU while its countdowns, freshness, and health checks stay live — and when new usage does land it recomputes only the current day/week/month (and the sessions that changed) instead of re-aggregating your entire history every few seconds, so each refresh stays fast even at hundreds of thousands of sessions (#268).
|
|
19
|
+
- The dashboard now cleans up cache rows left behind by removed session directories (such as a deleted git worktree) on its own — once at startup and periodically while running — so those stale rows no longer linger until a manual rebuild, and the `[cache] N tracked file(s) no longer on disk` warning no longer repeats every few seconds.
|
|
20
|
+
- `cache-sync --rebuild` now waits for the cache lock (up to 30 seconds) and reports a non-zero exit when it still can't acquire it, instead of silently doing nothing and reporting success while a dashboard is actively syncing.
|
|
21
|
+
|
|
8
22
|
## [1.60.0] - 2026-07-03
|
|
9
23
|
|
|
10
24
|
### Changed
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -106,6 +106,7 @@ import os
|
|
|
106
106
|
import pathlib
|
|
107
107
|
import sqlite3
|
|
108
108
|
import sys
|
|
109
|
+
import time
|
|
109
110
|
from dataclasses import dataclass, field
|
|
110
111
|
from typing import Any, Callable, Iterator
|
|
111
112
|
|
|
@@ -341,22 +342,40 @@ def _resolve_project_key(
|
|
|
341
342
|
is_unknown=True,
|
|
342
343
|
)
|
|
343
344
|
|
|
345
|
+
# Win 1 (#269 §14, Codex-M4 P1): raw-`project_path` fast path. The
|
|
346
|
+
# normalized cache is keyed on ``os.path.realpath(...)``, so on the base
|
|
347
|
+
# code the expensive realpath/lstat walk runs once per ENTRY (~190K on a
|
|
348
|
+
# large instance) rather than once per DISTINCT raw spelling (~10K). A
|
|
349
|
+
# namespaced raw key (``("raw", project_path)``) — a tuple that can never
|
|
350
|
+
# collide with the normalized string keys — short-circuits the common case
|
|
351
|
+
# (the same raw spelling repeated across a project's entries). This does
|
|
352
|
+
# NOT replace the normalized cache: a raw MISS still consults/populates the
|
|
353
|
+
# normalized entry, so ``mode="full-path"`` symlink-alias collapse to the
|
|
354
|
+
# first spelling seen is preserved (a second alias misses the raw fast
|
|
355
|
+
# path, hits the normalized entry, and returns the first spelling's key).
|
|
356
|
+
# Byte-identical for ALL modes.
|
|
357
|
+
raw_key = ("raw", project_path)
|
|
358
|
+
raw_hit = cache.get(raw_key)
|
|
359
|
+
if raw_hit is not None:
|
|
360
|
+
return raw_hit
|
|
361
|
+
|
|
344
362
|
if mode == "full-path":
|
|
345
363
|
normalized = os.path.realpath(os.path.expanduser(project_path))
|
|
346
364
|
key = cache.get(normalized)
|
|
347
|
-
if key is
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
cache[
|
|
365
|
+
if key is None:
|
|
366
|
+
key = ProjectKey(
|
|
367
|
+
bucket_path=normalized,
|
|
368
|
+
display_key=project_path, # raw, so user sees what they typed
|
|
369
|
+
git_root=None,
|
|
370
|
+
)
|
|
371
|
+
cache[normalized] = key
|
|
372
|
+
cache[raw_key] = key
|
|
355
373
|
return key
|
|
356
374
|
|
|
357
375
|
normalized = os.path.realpath(os.path.expanduser(project_path))
|
|
358
376
|
cached = cache.get(normalized)
|
|
359
377
|
if cached is not None:
|
|
378
|
+
cache[raw_key] = cached
|
|
360
379
|
return cached
|
|
361
380
|
|
|
362
381
|
home = os.path.expanduser("~")
|
|
@@ -371,6 +390,7 @@ def _resolve_project_key(
|
|
|
371
390
|
git_root=cur,
|
|
372
391
|
)
|
|
373
392
|
cache[normalized] = key
|
|
393
|
+
cache[raw_key] = key
|
|
374
394
|
return key
|
|
375
395
|
cur = os.path.dirname(cur)
|
|
376
396
|
|
|
@@ -381,6 +401,7 @@ def _resolve_project_key(
|
|
|
381
401
|
is_no_git=True,
|
|
382
402
|
)
|
|
383
403
|
cache[normalized] = key
|
|
404
|
+
cache[raw_key] = key
|
|
384
405
|
return key
|
|
385
406
|
|
|
386
407
|
|
|
@@ -481,6 +502,19 @@ class IngestStats:
|
|
|
481
502
|
and self.files_failed == 0)
|
|
482
503
|
|
|
483
504
|
|
|
505
|
+
@dataclass
|
|
506
|
+
class PruneResult:
|
|
507
|
+
"""Outcome of _prune_orphaned_cache_entries: how much of the derived Claude
|
|
508
|
+
surface was removed for safely-orphaned source paths, plus the orphan paths
|
|
509
|
+
left in place (residual — a gate failed, so `--rebuild` is the escape hatch)
|
|
510
|
+
and whether the flock was contended (nothing mutated)."""
|
|
511
|
+
pruned_files: int = 0
|
|
512
|
+
pruned_entries: int = 0
|
|
513
|
+
pruned_messages: int = 0
|
|
514
|
+
residual_paths: "list[str]" = field(default_factory=list)
|
|
515
|
+
contended: bool = False
|
|
516
|
+
|
|
517
|
+
|
|
484
518
|
def _progress_stderr(stats: IngestStats, *, force: bool = False) -> None:
|
|
485
519
|
"""Default stderr progress callback. Every 200 files or when forced."""
|
|
486
520
|
if not force and stats.files_processed % 200 != 0:
|
|
@@ -577,6 +611,25 @@ def _ensure_session_files_row(conn: sqlite3.Connection, source_path: str) -> Non
|
|
|
577
611
|
conn.commit()
|
|
578
612
|
|
|
579
613
|
|
|
614
|
+
# How long `cache-sync --rebuild` (and the --prune-orphans path) waits on the
|
|
615
|
+
# cache.db flock before giving up. Routine auto-syncs stay non-blocking
|
|
616
|
+
# (lock_timeout=None); an explicit rebuild is worth a bounded wait so a running
|
|
617
|
+
# dashboard's background tick doesn't turn the rebuild into a silent no-op.
|
|
618
|
+
# Read at call time in cmd_cache_sync so tests can monkeypatch it low.
|
|
619
|
+
_REBUILD_LOCK_TIMEOUT_SECONDS = 30.0
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
# Orphan-warning throttle: warn only when the detected orphan set CHANGES,
|
|
623
|
+
# so a long-lived dashboard doesn't re-spam the "[cache] N tracked file(s) no
|
|
624
|
+
# longer on disk" line every ~5s sync tick. Reset in tests.
|
|
625
|
+
_LAST_WARNED_ORPHAN_SET: "frozenset[str]" = frozenset()
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _reset_orphan_warning_throttle():
|
|
629
|
+
global _LAST_WARNED_ORPHAN_SET
|
|
630
|
+
_LAST_WARNED_ORPHAN_SET = frozenset()
|
|
631
|
+
|
|
632
|
+
|
|
580
633
|
# Flags whose presence means the cache is mid-migration / mid-reingest. A
|
|
581
634
|
# targeted (only_paths) ingest DECLINES when any is set and defers to the next
|
|
582
635
|
# full background sync — inserting through a half-migrated FTS shape or skipping
|
|
@@ -612,12 +665,153 @@ def _targeted_has_pending_global_work(conn) -> bool:
|
|
|
612
665
|
return row is not None
|
|
613
666
|
|
|
614
667
|
|
|
668
|
+
def _acquire_cache_flock(lock_fh, *, timeout):
|
|
669
|
+
"""Acquire the exclusive cache flock on ``lock_fh``.
|
|
670
|
+
|
|
671
|
+
``timeout is None`` -> a single non-blocking attempt (today's behavior
|
|
672
|
+
for routine auto-syncs): returns False immediately on contention.
|
|
673
|
+
``timeout > 0`` -> retry ``LOCK_NB`` every ~0.2s until acquired or the
|
|
674
|
+
deadline elapses. Returns True iff the lock is held on return.
|
|
675
|
+
|
|
676
|
+
A retry-with-sleep loop, NOT a SIGALRM-based blocking LOCK_EX: the
|
|
677
|
+
dashboard runs its sync on a background thread, where Python signals
|
|
678
|
+
never fire, so an alarm timeout would silently never trip.
|
|
679
|
+
"""
|
|
680
|
+
deadline = None if timeout is None else (time.monotonic() + timeout)
|
|
681
|
+
while True:
|
|
682
|
+
try:
|
|
683
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
684
|
+
return True
|
|
685
|
+
except BlockingIOError:
|
|
686
|
+
if deadline is None or time.monotonic() >= deadline:
|
|
687
|
+
return False
|
|
688
|
+
time.sleep(0.2)
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def _prune_orphaned_cache_entries(conn, *, lock_timeout=None):
|
|
692
|
+
"""Safely prune the FULL derived Claude cache surface for orphaned
|
|
693
|
+
(removed-from-disk) source paths. See the design spec
|
|
694
|
+
docs/superpowers/specs/2026-07-04-cache-orphan-prune-design.md §2.
|
|
695
|
+
|
|
696
|
+
Kept OUT of the shared sync_cache (which stays detect-only, fixture-safe):
|
|
697
|
+
only real runtimes call this — the CLI cache-sync --prune-orphans and the
|
|
698
|
+
dashboard self-heal — so synthetic /fake/… fixture paths never reach it.
|
|
699
|
+
|
|
700
|
+
Three gates decide safety per orphan path P (session_id sid):
|
|
701
|
+
A) sid is non-null and not shared by any surviving on-disk file
|
|
702
|
+
(cheap prefilter; "same session_id" is empirical, not proof).
|
|
703
|
+
B) coverage: every one of P's session_entries (msg_id, req_id) keys
|
|
704
|
+
has a conversation_messages row under P's OWN source_path (else
|
|
705
|
+
it is a uuid-less blind spot -> unprovable -> residual).
|
|
706
|
+
C) disjointness: none of P's keys appears in conversation_messages
|
|
707
|
+
under a surviving on-disk source_path (a survivor physically holds
|
|
708
|
+
the same turn -> deleting P's deduped cost row would lose it).
|
|
709
|
+
Anything failing A/B/C is left as residual (reported; for `--rebuild`).
|
|
710
|
+
|
|
711
|
+
Deletes the full derived surface for the safe set in ONE transaction
|
|
712
|
+
(conversation_file_touches -> conversation_messages [+FTS trigger] ->
|
|
713
|
+
conversation_ai_titles [+FTS trigger] -> session_entries -> session_files),
|
|
714
|
+
then recomputes the conversation_sessions rollup for exactly the pruned
|
|
715
|
+
session_ids (all their messages gone -> the stale rail rows drop). Does NOT
|
|
716
|
+
write the walk-complete marker: the next full sync_cache re-establishes it
|
|
717
|
+
on a clean walk. Acquires the cache.db.lock flock itself; a contended
|
|
718
|
+
lock_timeout returns a `contended` result without mutating.
|
|
719
|
+
"""
|
|
720
|
+
result = PruneResult()
|
|
721
|
+
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
722
|
+
_cctally_core.CACHE_LOCK_PATH.touch()
|
|
723
|
+
lock_fh = open(_cctally_core.CACHE_LOCK_PATH, "w")
|
|
724
|
+
try:
|
|
725
|
+
if not _acquire_cache_flock(lock_fh, timeout=lock_timeout):
|
|
726
|
+
result.contended = True
|
|
727
|
+
return result
|
|
728
|
+
|
|
729
|
+
tracked = conn.execute(
|
|
730
|
+
"SELECT path, size_bytes, session_id FROM session_files").fetchall()
|
|
731
|
+
on_disk = {p for (p, _sz, _sid) in tracked if os.path.isfile(p)}
|
|
732
|
+
surviving_sids = {sid for (p, _sz, sid) in tracked
|
|
733
|
+
if p in on_disk and sid is not None}
|
|
734
|
+
orphan_cands = [(p, sid) for (p, sz, sid) in tracked
|
|
735
|
+
if sz and p not in on_disk]
|
|
736
|
+
if not orphan_cands:
|
|
737
|
+
return result
|
|
738
|
+
|
|
739
|
+
safe_paths = []
|
|
740
|
+
pruned_sids = set()
|
|
741
|
+
for path, sid in orphan_cands:
|
|
742
|
+
if sid is None or sid in surviving_sids: # Gate A
|
|
743
|
+
result.residual_paths.append(path)
|
|
744
|
+
continue
|
|
745
|
+
keys = conn.execute(
|
|
746
|
+
"SELECT DISTINCT msg_id, req_id FROM session_entries "
|
|
747
|
+
"WHERE source_path=? AND msg_id IS NOT NULL AND req_id IS NOT NULL",
|
|
748
|
+
(path,)).fetchall()
|
|
749
|
+
ok = True
|
|
750
|
+
for mid, rid in keys:
|
|
751
|
+
covered = conn.execute( # Gate B
|
|
752
|
+
"SELECT 1 FROM conversation_messages "
|
|
753
|
+
"WHERE msg_id=? AND req_id=? AND source_path=? LIMIT 1",
|
|
754
|
+
(mid, rid, path)).fetchone() is not None
|
|
755
|
+
if not covered:
|
|
756
|
+
ok = False
|
|
757
|
+
break
|
|
758
|
+
shared = conn.execute( # Gate C
|
|
759
|
+
"SELECT source_path FROM conversation_messages "
|
|
760
|
+
"WHERE msg_id=? AND req_id=?", (mid, rid)).fetchall()
|
|
761
|
+
if any(sp in on_disk for (sp,) in shared):
|
|
762
|
+
ok = False
|
|
763
|
+
break
|
|
764
|
+
if not ok:
|
|
765
|
+
result.residual_paths.append(path)
|
|
766
|
+
continue
|
|
767
|
+
safe_paths.append(path)
|
|
768
|
+
pruned_sids.add(sid)
|
|
769
|
+
|
|
770
|
+
if not safe_paths:
|
|
771
|
+
return result
|
|
772
|
+
|
|
773
|
+
# No IN(...) chunking: safe_paths is bounded by the orphan count (a
|
|
774
|
+
# handful of removed files), well under SQLite's variable limit;
|
|
775
|
+
# _recompute_conversation_sessions chunks its own session-id list.
|
|
776
|
+
ph = ",".join("?" * len(safe_paths))
|
|
777
|
+
result.pruned_messages = conn.execute(
|
|
778
|
+
f"SELECT count(*) FROM conversation_messages WHERE source_path IN ({ph})",
|
|
779
|
+
safe_paths).fetchone()[0]
|
|
780
|
+
conn.execute("BEGIN")
|
|
781
|
+
try:
|
|
782
|
+
conn.execute(
|
|
783
|
+
f"DELETE FROM conversation_file_touches WHERE message_id IN "
|
|
784
|
+
f"(SELECT id FROM conversation_messages WHERE source_path IN ({ph}))",
|
|
785
|
+
safe_paths)
|
|
786
|
+
conn.execute(
|
|
787
|
+
f"DELETE FROM conversation_messages WHERE source_path IN ({ph})", safe_paths)
|
|
788
|
+
conn.execute(
|
|
789
|
+
f"DELETE FROM conversation_ai_titles WHERE source_path IN ({ph})", safe_paths)
|
|
790
|
+
result.pruned_entries = conn.execute(
|
|
791
|
+
f"DELETE FROM session_entries WHERE source_path IN ({ph})", safe_paths).rowcount
|
|
792
|
+
result.pruned_files = conn.execute(
|
|
793
|
+
f"DELETE FROM session_files WHERE path IN ({ph})", safe_paths).rowcount
|
|
794
|
+
_recompute_conversation_sessions(conn, list(pruned_sids))
|
|
795
|
+
conn.commit()
|
|
796
|
+
except BaseException:
|
|
797
|
+
conn.rollback()
|
|
798
|
+
raise
|
|
799
|
+
return result
|
|
800
|
+
finally:
|
|
801
|
+
try:
|
|
802
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
803
|
+
except OSError:
|
|
804
|
+
pass
|
|
805
|
+
lock_fh.close()
|
|
806
|
+
|
|
807
|
+
|
|
615
808
|
def sync_cache(
|
|
616
809
|
conn: sqlite3.Connection,
|
|
617
810
|
*,
|
|
618
811
|
progress: Callable[[IngestStats], None] | None = None,
|
|
619
812
|
rebuild: bool = False,
|
|
620
813
|
only_paths: "set[str] | None" = None,
|
|
814
|
+
lock_timeout: "float | None" = None,
|
|
621
815
|
) -> IngestStats:
|
|
622
816
|
"""Read-through delta ingest. Acquires an exclusive fcntl.flock; if
|
|
623
817
|
another process holds it, returns immediately with lock_contended=True
|
|
@@ -635,9 +829,7 @@ def sync_cache(
|
|
|
635
829
|
|
|
636
830
|
lock_fh = open(_cctally_core.CACHE_LOCK_PATH, "w")
|
|
637
831
|
try:
|
|
638
|
-
|
|
639
|
-
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
640
|
-
except BlockingIOError:
|
|
832
|
+
if not _acquire_cache_flock(lock_fh, timeout=lock_timeout):
|
|
641
833
|
eprint("[cache] sync already in progress; using existing cache")
|
|
642
834
|
stats.lock_contended = True
|
|
643
835
|
return stats
|
|
@@ -994,27 +1186,46 @@ def sync_cache(
|
|
|
994
1186
|
# tracked in session_files (with data already ingested) but no
|
|
995
1187
|
# longer present on disk leaves orphaned session_entries rows that
|
|
996
1188
|
# the per-file loop below never visits — it iterates only on-disk
|
|
997
|
-
# `paths`. sync_cache
|
|
998
|
-
# in-place
|
|
999
|
-
#
|
|
1000
|
-
#
|
|
1001
|
-
#
|
|
1002
|
-
#
|
|
1003
|
-
#
|
|
1004
|
-
#
|
|
1005
|
-
#
|
|
1006
|
-
#
|
|
1007
|
-
#
|
|
1008
|
-
#
|
|
1009
|
-
#
|
|
1010
|
-
#
|
|
1011
|
-
#
|
|
1012
|
-
#
|
|
1013
|
-
#
|
|
1014
|
-
#
|
|
1015
|
-
#
|
|
1016
|
-
#
|
|
1017
|
-
#
|
|
1189
|
+
# `paths`. sync_cache stays DETECT-ONLY here — it never prunes
|
|
1190
|
+
# orphans in-place. Two reasons it must not delete from this hot,
|
|
1191
|
+
# shared path: (1) the truncation hazard — under the sticky
|
|
1192
|
+
# source_path dedup a surviving file may carry the same
|
|
1193
|
+
# (msg_id, req_id) yet keep its size_bytes, so a naive per-orphan
|
|
1194
|
+
# DELETE could drop a deduped cost row the survivor still owns
|
|
1195
|
+
# without re-ingesting it; (2) fixture safety — a blanket full-reset
|
|
1196
|
+
# would wrongly fire on the legitimate "cache seeded with synthetic
|
|
1197
|
+
# source paths" fixture pattern, and only sync_cache runs against
|
|
1198
|
+
# those fixtures. So detection does two things and no more: it emits
|
|
1199
|
+
# a THROTTLED warning (once per distinct orphan set — a removed
|
|
1200
|
+
# worktree persists, so an unthrottled warn would re-spam every
|
|
1201
|
+
# dashboard tick) and it INVALIDATES the walk-complete marker. An
|
|
1202
|
+
# orphaned cache no longer faithfully mirrors disk, so it is — by the
|
|
1203
|
+
# marker's own definition — not a complete walk. We actively DELETE
|
|
1204
|
+
# any marker a PRIOR clean walk left behind (idempotently — only when
|
|
1205
|
+
# one exists, so a repeated orphaned sync doesn't churn a no-op write
|
|
1206
|
+
# txn every tick); merely withholding THIS run's end-of-loop rewrite
|
|
1207
|
+
# is not enough, since a stale marker from a previous sync would
|
|
1208
|
+
# otherwise survive and keep vouching for completeness. Setting
|
|
1209
|
+
# walk_clean=False additionally suppresses the end-of-loop rewrite so
|
|
1210
|
+
# the marker stays absent for this run. With the marker gone the
|
|
1211
|
+
# upgrade gate DEFERs the 008/009/010 recomputes (rather than
|
|
1212
|
+
# certifying aggregates that still include data from files no longer
|
|
1213
|
+
# on disk). The safe CLEANUP lives OUT of sync_cache, in
|
|
1214
|
+
# _prune_orphaned_cache_entries — it re-derives the safe orphan set
|
|
1215
|
+
# independently and removes the full derived surface under three
|
|
1216
|
+
# gates: A (session-id not shared by a survivor), B (coverage — every
|
|
1217
|
+
# orphan (msg_id, req_id) key has a conversation_messages row under
|
|
1218
|
+
# the orphan's OWN path), and C (disjointness — no key of the orphan
|
|
1219
|
+
# appears in conversation_messages under a surviving path). B + C
|
|
1220
|
+
# together close the truncation hazard soundly: C refuses to delete
|
|
1221
|
+
# any key a survivor physically holds, and B refuses the uuid-less
|
|
1222
|
+
# blind spot where coverage can't be proved (anything failing A/B/C
|
|
1223
|
+
# is left as residual, cleared by `--rebuild`). That helper is
|
|
1224
|
+
# invoked by `cache-sync --prune-orphans` and the dashboard
|
|
1225
|
+
# self-heal, never from here (so fixtures never reach a destructive
|
|
1226
|
+
# delete); `cache-sync --rebuild` remains the whole-cache re-derive.
|
|
1227
|
+
# Both cleanup paths re-establish the marker on the next clean walk.
|
|
1228
|
+
# Only paths whose row carried ingested
|
|
1018
1229
|
# bytes (size_bytes > 0) count — a size_bytes=0 row holds no
|
|
1019
1230
|
# session_entries, so its absence leaves no orphan. The DELETE +
|
|
1020
1231
|
# commit lands BEFORE the per-file read+parse loop, so no write
|
|
@@ -1027,22 +1238,43 @@ def sync_cache(
|
|
|
1027
1238
|
# for targeted: the live-tail fast path never prunes orphans (the full
|
|
1028
1239
|
# background sync owns that).
|
|
1029
1240
|
if not targeted:
|
|
1241
|
+
global _LAST_WARNED_ORPHAN_SET
|
|
1030
1242
|
on_disk_paths = {str(jp) for jp in paths}
|
|
1031
1243
|
orphaned_tracked_paths = [
|
|
1032
1244
|
p for p, (size_bytes, _, _) in existing.items()
|
|
1033
1245
|
if size_bytes and p not in on_disk_paths
|
|
1034
1246
|
]
|
|
1035
1247
|
if orphaned_tracked_paths:
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1248
|
+
# Throttle the warning: emit only when the orphan set CHANGES,
|
|
1249
|
+
# not on every ~5s dashboard tick (a removed worktree persists,
|
|
1250
|
+
# so an unthrottled warn re-spams indefinitely). The marker
|
|
1251
|
+
# invalidation below stays UNCONDITIONAL — throttling the print
|
|
1252
|
+
# must not weaken the D5a invariant.
|
|
1253
|
+
cur = frozenset(orphaned_tracked_paths)
|
|
1254
|
+
if cur != _LAST_WARNED_ORPHAN_SET:
|
|
1255
|
+
eprint(
|
|
1256
|
+
f"[cache] {len(orphaned_tracked_paths)} tracked file(s) no "
|
|
1257
|
+
f"longer on disk; invalidating walk-complete marker "
|
|
1258
|
+
f"(run `cache-sync --prune-orphans` to prune, or "
|
|
1259
|
+
f"`cache-sync --rebuild`)"
|
|
1260
|
+
)
|
|
1261
|
+
_LAST_WARNED_ORPHAN_SET = cur
|
|
1262
|
+
# Idempotent marker invalidation: only DELETE (and commit) when a
|
|
1263
|
+
# prior clean walk actually left the marker behind, so a repeated
|
|
1264
|
+
# orphaned sync doesn't churn a no-op write transaction every tick.
|
|
1265
|
+
if conn.execute(
|
|
1266
|
+
"SELECT 1 FROM cache_meta WHERE key='claude_ingest_walk_complete'"
|
|
1267
|
+
).fetchone() is not None:
|
|
1268
|
+
conn.execute(
|
|
1269
|
+
"DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'"
|
|
1270
|
+
)
|
|
1271
|
+
conn.commit()
|
|
1045
1272
|
walk_clean = False # orphaned rows -> cache doesn't mirror disk (D5a)
|
|
1273
|
+
else:
|
|
1274
|
+
# No orphans this walk: clear the throttle memory so a LATER,
|
|
1275
|
+
# distinct orphan episode (even one recreated at the same paths)
|
|
1276
|
+
# warns again rather than being silently suppressed.
|
|
1277
|
+
_LAST_WARNED_ORPHAN_SET = frozenset()
|
|
1046
1278
|
|
|
1047
1279
|
# Pre-scan for any truncation among tracked files. Under the
|
|
1048
1280
|
# ccusage-parity ON CONFLICT DO UPDATE, source_path is PINNED to
|
|
@@ -2429,6 +2661,7 @@ def sync_codex_cache(
|
|
|
2429
2661
|
*,
|
|
2430
2662
|
progress: Callable[[CodexIngestStats], None] | None = None,
|
|
2431
2663
|
rebuild: bool = False,
|
|
2664
|
+
lock_timeout: "float | None" = None,
|
|
2432
2665
|
) -> CodexIngestStats:
|
|
2433
2666
|
"""Read-through delta ingest of ~/.codex/sessions/**/*.jsonl.
|
|
2434
2667
|
|
|
@@ -2448,9 +2681,7 @@ def sync_codex_cache(
|
|
|
2448
2681
|
|
|
2449
2682
|
lock_fh = open(_cctally_core.CACHE_LOCK_CODEX_PATH, "w")
|
|
2450
2683
|
try:
|
|
2451
|
-
|
|
2452
|
-
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
2453
|
-
except BlockingIOError:
|
|
2684
|
+
if not _acquire_cache_flock(lock_fh, timeout=lock_timeout):
|
|
2454
2685
|
eprint("[codex-cache] sync already in progress; using existing cache")
|
|
2455
2686
|
stats.lock_contended = True
|
|
2456
2687
|
return stats
|
|
@@ -2991,20 +3222,63 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
2991
3222
|
source = getattr(args, "source", "all")
|
|
2992
3223
|
conn = open_cache_db()
|
|
2993
3224
|
|
|
3225
|
+
# --prune-orphans: fast, targeted cleanup of cache rows whose source
|
|
3226
|
+
# JSONL was removed from disk (e.g. a deleted git worktree), without a
|
|
3227
|
+
# full rebuild. Claude-only surface; runs the three-gate safe helper.
|
|
3228
|
+
if getattr(args, "prune_orphans", False):
|
|
3229
|
+
if source == "codex":
|
|
3230
|
+
# The prune surface is Claude-only; Codex orphans are pruned
|
|
3231
|
+
# automatically during codex sync (see sync_codex_cache). Respect
|
|
3232
|
+
# the explicit --source codex rather than silently pruning Claude.
|
|
3233
|
+
eprint(
|
|
3234
|
+
"[cache-sync] --prune-orphans applies to the Claude cache only "
|
|
3235
|
+
"(Codex orphans are pruned automatically during codex sync); "
|
|
3236
|
+
"nothing to do for --source codex."
|
|
3237
|
+
)
|
|
3238
|
+
return 0
|
|
3239
|
+
res = _prune_orphaned_cache_entries(
|
|
3240
|
+
conn, lock_timeout=_REBUILD_LOCK_TIMEOUT_SECONDS
|
|
3241
|
+
)
|
|
3242
|
+
if res.contended:
|
|
3243
|
+
eprint(
|
|
3244
|
+
"[cache-sync] prune-orphans skipped: "
|
|
3245
|
+
"another process holds the lock"
|
|
3246
|
+
)
|
|
3247
|
+
return 1
|
|
3248
|
+
eprint(
|
|
3249
|
+
f"[cache-sync] pruned {res.pruned_files} orphaned file(s), "
|
|
3250
|
+
f"{res.pruned_entries} cost row(s), {res.pruned_messages} message(s)"
|
|
3251
|
+
)
|
|
3252
|
+
if res.residual_paths:
|
|
3253
|
+
eprint(
|
|
3254
|
+
f"[cache-sync] {len(res.residual_paths)} orphan(s) left in place "
|
|
3255
|
+
f"(shared session or missing conversation evidence); "
|
|
3256
|
+
f"run `cache-sync --rebuild` to clear them"
|
|
3257
|
+
)
|
|
3258
|
+
return 0
|
|
3259
|
+
|
|
2994
3260
|
# Note: when --rebuild is set we delegate the DELETE to sync_cache /
|
|
2995
3261
|
# sync_codex_cache, which execute it AFTER acquiring the flock. A
|
|
2996
3262
|
# pre-sync DELETE here would wipe the cache even when the subsequent
|
|
2997
3263
|
# sync loses the lock race and bails — leaving the user with empty
|
|
2998
|
-
# state. See sync_cache() / sync_codex_cache() docstrings.
|
|
3264
|
+
# state. See sync_cache() / sync_codex_cache() docstrings. A rebuild
|
|
3265
|
+
# is worth a bounded wait on the flock (vs the non-blocking auto-sync)
|
|
3266
|
+
# so a running dashboard's background tick doesn't silently no-op it;
|
|
3267
|
+
# if it still can't acquire, we report + exit non-zero rather than lie.
|
|
3268
|
+
lt = _REBUILD_LOCK_TIMEOUT_SECONDS if args.rebuild else None
|
|
3269
|
+
contended = False
|
|
2999
3270
|
|
|
3000
3271
|
if source in ("claude", "all"):
|
|
3001
|
-
stats = sync_cache(
|
|
3272
|
+
stats = sync_cache(
|
|
3273
|
+
conn, progress=_progress_stderr, rebuild=args.rebuild, lock_timeout=lt
|
|
3274
|
+
)
|
|
3002
3275
|
_progress_stderr(stats, force=True)
|
|
3003
3276
|
if stats.lock_contended and args.rebuild:
|
|
3004
3277
|
eprint(
|
|
3005
3278
|
"[cache-sync] rebuild skipped (claude): "
|
|
3006
3279
|
"another process holds the lock"
|
|
3007
3280
|
)
|
|
3281
|
+
contended = True
|
|
3008
3282
|
elif not stats.lock_contended:
|
|
3009
3283
|
eprint(
|
|
3010
3284
|
f"[cache-sync] claude done: {stats.files_processed} processed, "
|
|
@@ -3015,7 +3289,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
3015
3289
|
|
|
3016
3290
|
if source in ("codex", "all"):
|
|
3017
3291
|
stats = sync_codex_cache(
|
|
3018
|
-
conn, progress=_progress_codex_stderr, rebuild=args.rebuild
|
|
3292
|
+
conn, progress=_progress_codex_stderr, rebuild=args.rebuild, lock_timeout=lt
|
|
3019
3293
|
)
|
|
3020
3294
|
_progress_codex_stderr(stats, force=True)
|
|
3021
3295
|
if stats.lock_contended and args.rebuild:
|
|
@@ -3023,6 +3297,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
3023
3297
|
"[cache-sync] rebuild skipped (codex): "
|
|
3024
3298
|
"another process holds the lock"
|
|
3025
3299
|
)
|
|
3300
|
+
contended = True
|
|
3026
3301
|
elif not stats.lock_contended:
|
|
3027
3302
|
eprint(
|
|
3028
3303
|
f"[cache-sync] codex done: {stats.files_processed} processed, "
|
|
@@ -3031,4 +3306,4 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
3031
3306
|
f"{stats.rows_changed} rows changed"
|
|
3032
3307
|
)
|
|
3033
3308
|
|
|
3034
|
-
return 0
|
|
3309
|
+
return 1 if contended else 0
|
package/bin/_cctally_core.py
CHANGED
|
@@ -160,6 +160,14 @@ def _is_dev_checkout() -> bool:
|
|
|
160
160
|
return (_repo_root() / ".git").exists()
|
|
161
161
|
|
|
162
162
|
|
|
163
|
+
def is_preview_channel() -> bool:
|
|
164
|
+
"""True when running under the maintainer-local preview channel
|
|
165
|
+
(the `cctally-preview` wrapper sets CCTALLY_CHANNEL=preview). Single
|
|
166
|
+
source of truth for every preview-marker surface (dashboard port +
|
|
167
|
+
envelope, TUI header, --version, doctor) so the gate can't drift."""
|
|
168
|
+
return os.environ.get("CCTALLY_CHANNEL") == "preview"
|
|
169
|
+
|
|
170
|
+
|
|
163
171
|
def _real_prod_data_dir() -> pathlib.Path:
|
|
164
172
|
"""The REAL user's prod data dir (~/.local/share/cctally), resolved from
|
|
165
173
|
the password database rather than $HOME so it is immune to a faked HOME.
|