cctally 1.72.0 → 1.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/bin/_cctally_cache.py +271 -55
- package/bin/_cctally_core.py +20 -40
- package/bin/_cctally_dashboard.py +35 -1
- package/bin/_cctally_dashboard_conversation.py +731 -94
- package/bin/_cctally_dashboard_share.py +38 -3
- package/bin/_cctally_dashboard_sources.py +812 -74
- package/bin/_cctally_db.py +104 -1
- package/bin/_cctally_doctor.py +86 -4
- package/bin/_cctally_parser.py +17 -4
- package/bin/_cctally_record.py +162 -33
- package/bin/_cctally_refresh.py +58 -36
- package/bin/_cctally_source_analytics.py +288 -3
- package/bin/_cctally_statusline.py +811 -82
- package/bin/_cctally_transcript.py +223 -7
- package/bin/_cctally_tui.py +17 -14
- package/bin/_lib_aggregators.py +48 -34
- package/bin/_lib_codex_conversation_export.py +124 -0
- package/bin/_lib_codex_conversation_query.py +500 -28
- package/bin/_lib_codex_conversation_watch.py +323 -0
- package/bin/_lib_conversation_dispatch.py +269 -7
- package/bin/_lib_conversation_query.py +88 -0
- package/bin/_lib_dashboard_sources.py +20 -7
- package/bin/_lib_doctor.py +106 -0
- package/bin/_lib_jsonl.py +11 -0
- package/bin/_lib_pricing.py +7 -4
- package/bin/_lib_pricing_check.py +7 -1
- package/bin/_lib_quota.py +12 -11
- package/bin/_lib_statusline_candidates.py +734 -0
- package/bin/_lib_view_models.py +50 -14
- package/bin/cctally +31 -2
- package/dashboard/static/assets/index-DeQjEMO6.js +80 -0
- package/dashboard/static/assets/index-ZiPO8veo.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +4 -1
- package/dashboard/static/assets/index-BWadAHxC.js +0 -80
- package/dashboard/static/assets/index-D2nwo_ln.css +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,28 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.74.0] - 2026-07-19
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- Dashboard Daily, Weekly, and Monthly details now use the same bounded history density across Claude, Codex, and All sources—30 days, 12 weeks, and 8 months—with compact date labels and contained mobile tables; Codex Blocks remains explicitly unavailable when native 5-hour activity does not exist. (#319)
|
|
12
|
+
- Dashboard Trend, Projects, Cache Report, Forecast, and Alerts details now keep their canonical composed structure for Codex and All sources, show provider-native values or explicit unavailable reasons in every slot, and route provider-qualified project rows through the source-bound detail foundation. (#319)
|
|
13
|
+
- Dashboard source-qualified Session and Project details now preserve Codex accounting totals when project metadata is incomplete; the combined source retains server-issued Claude Block identities and opens the canonical Block modal; source details share the standard focus, Escape, return-focus, and scroll-lock lifecycle; and desktop Share previews retain the captured provider. (#319)
|
|
14
|
+
- Codex accounting no longer double-counts parent token events copied into a forked/subagent rollout before its first model context. Those records remain available to the physical conversation reader but are excluded from accounting; cache migration 027 safely replays retained Codex rollouts to remove existing `unknown` duplicates. Consequently, `cctally pricing-check` and `doctor` no longer report those ingest artifacts as a missing embedded price, while real unknown model IDs remain actionable. The complete scoped OpenAI model set and tracked rates were reverified against the live pricing sources, and the embedded pricing snapshot date is now 2026-07-19.
|
|
15
|
+
- Dashboard Codex Weekly rows now follow observed native seven-day quota-cycle re-anchors instead of calendar weeks, collapsing reset jitter and clipping an older cycle when OpenAI grants an early reset. Codex quota forecasts now use elapsed native-cycle pace rather than activity-only bursts, avoiding misleading projections pinned at 100%. (#312)
|
|
16
|
+
- Dashboard Codex source reads now retain accounting, quota, budget, session, and forensics panels when retained historical rows still lack project attribution. The cache replay path repairs this metadata when possible; until then the source is fresh-but-partial and disables Projects only, with a rebuild remedy. (#312)
|
|
17
|
+
- Dashboard Codex hero cost and token counters now cover the active native seven-day reset cycle, rather than an unrelated calendar slice. A missing or conflicting active seven-day boundary leaves quota and budget visible while clearly marking the hero unavailable; Codex panels regain bounded native chrome and source warnings degrade only the affected domain. (#312)
|
|
18
|
+
|
|
19
|
+
## [1.73.0] - 2026-07-18
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
- Codex conversations (#294 S7): the S6 kernel layer is now reachable. The dashboard's twelve conversation routes qualify in place — an entity id beginning `v1.` opts into the provider-neutral envelope (`ok`/`normalization_pending` → `200` JSON, `not_found` → `404`, payload `gone` → `410`, export → `200 text/markdown`, Codex media → `404 capability_unsupported`), and the three collection routes (`/api/conversations`, `/api/conversations/facets`, `/api/conversation/search`) take a strict `?source={claude,codex}`. Absent qualification, the Claude routes stay byte-identical, and the transcript privacy gate remains the first act of every handler. (#294)
|
|
23
|
+
- `cctally transcript export` now accepts a Claude `sessionId` **or** a `v1.` conversation key (Codex conversations are addressable only by their `v1.` key), anonymizes qualified exports by default via a provider-aware plan (`--raw` escapes), and adds a Codex-only `--speed {auto,standard,fast}` (rejected on any non-Codex ref); the CLI export byte-matches the dashboard `/export` download in both anonymize and raw modes. (#294)
|
|
24
|
+
- `cctally transcript search --source {claude,codex}` searches either provider. The Codex output is its own `Key / When / Project / Kinds / Snippet` table and a stamped-first camelCase JSON envelope paginated by an opaque base64url `--cursor`; the default-Claude table and JSON stay byte-frozen. (#294)
|
|
25
|
+
- Codex conversation live-tail: `GET /api/conversation/<v1key>/events` preflights as plain JSON before any SSE bytes and then streams `conversationKey`-framed frames, and a budgeted directory-frontier discovery joins child threads spawned mid-watch. Targeted Codex ingest keeps each tick's cost proportional to the watched conversation. (#294)
|
|
26
|
+
|
|
27
|
+
### Fixed
|
|
28
|
+
- Statusline usage persistence now arbitrates concurrent regular-pool Claude sessions through privacy-safe per-session candidates instead of accepting the first lock winner. A fresher 5h or 7d observation is selected independent of render arrival order; lower same-window values retain a full-cycle consensus path for credits/resets, including an extra revalidated tick for reset-to-zero. This replaces the internal 25-second persistence throttle; the existing `statusLine.refreshInterval: 30` remains the idle-time trigger. OAuth and manual credits share a fail-closed selected-state recovery protocol, the periodic dashboard remains OAuth-free and publishes accepted observations through its normal SSE path, and `cctally doctor` reports passive pipeline evidence when repair is needed. (#318)
|
|
29
|
+
|
|
8
30
|
## [1.72.0] - 2026-07-17
|
|
9
31
|
|
|
10
32
|
### Added
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -692,6 +692,78 @@ def _discover_codex_files_with_roots() -> list[CodexDiscoveredFile]:
|
|
|
692
692
|
return discovered
|
|
693
693
|
|
|
694
694
|
|
|
695
|
+
def _qualify_codex_targets(only_paths: "set[str]") -> list[CodexDiscoveredFile]:
|
|
696
|
+
"""Resolve each requested path through the ordered configured roots exactly
|
|
697
|
+
as full discovery would (spec §5.1) — producing the same per-file facts a
|
|
698
|
+
``CodexDiscoveredFile`` carries (configured spelling, physical path, provider
|
|
699
|
+
root, walk root, source-root key) with first-match containment + physical
|
|
700
|
+
dedup. A path resolving under no configured root, not a ``*.jsonl`` file, or
|
|
701
|
+
an alias of an already-resolved physical file is DROPPED (clean, not
|
|
702
|
+
ingested). Targeted mode's analogue of ``_discover_codex_files_with_roots``:
|
|
703
|
+
it does NOT walk any tree — it qualifies only the caller's exact paths."""
|
|
704
|
+
roots = _codex_provider_roots()
|
|
705
|
+
resolved: list[CodexDiscoveredFile] = []
|
|
706
|
+
seen_physical: set[pathlib.Path] = set()
|
|
707
|
+
# Deterministic order so first-match physical dedup is stable across a set.
|
|
708
|
+
for p in sorted(only_paths):
|
|
709
|
+
candidate = pathlib.Path(p)
|
|
710
|
+
for root in roots:
|
|
711
|
+
try:
|
|
712
|
+
inside = candidate.is_relative_to(root.walk_root)
|
|
713
|
+
except (ValueError, TypeError):
|
|
714
|
+
inside = False
|
|
715
|
+
if not inside:
|
|
716
|
+
continue
|
|
717
|
+
# Under this root's walk boundary: it is qualified here (first match)
|
|
718
|
+
# or dropped — a different-spelled alias never re-qualifies under a
|
|
719
|
+
# later root, matching full discovery's yielded-source-path identity.
|
|
720
|
+
if candidate.suffix != ".jsonl" or not candidate.is_file():
|
|
721
|
+
break # vanished / non-rollout under this root → drop (clean)
|
|
722
|
+
physical = _canonical_codex_path(candidate)
|
|
723
|
+
if physical in seen_physical:
|
|
724
|
+
break # first-match physical dedup → drop the alias
|
|
725
|
+
seen_physical.add(physical)
|
|
726
|
+
resolved.append(CodexDiscoveredFile(
|
|
727
|
+
source_path=candidate,
|
|
728
|
+
physical_path=physical,
|
|
729
|
+
provider_root=root.provider_root,
|
|
730
|
+
walk_root=root.walk_root,
|
|
731
|
+
source_root_key=root.source_root_key,
|
|
732
|
+
))
|
|
733
|
+
break
|
|
734
|
+
return resolved
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _load_codex_session_files_rows(
|
|
738
|
+
conn: sqlite3.Connection, paths: "list[str]"
|
|
739
|
+
) -> dict:
|
|
740
|
+
"""Cursor rows from ``codex_session_files`` for ONLY the given paths (spec
|
|
741
|
+
§5.1 — the targeted preload must never load every row like the full-sync
|
|
742
|
+
path). Same 12-tuple value shape as ``sync_codex_cache``'s full ``existing``
|
|
743
|
+
map, so the per-file delta logic is byte-identical between the two modes."""
|
|
744
|
+
out: dict = {}
|
|
745
|
+
if not paths:
|
|
746
|
+
return out
|
|
747
|
+
cols = (
|
|
748
|
+
"path, size_bytes, mtime_ns, last_byte_offset, "
|
|
749
|
+
"last_session_id, last_model, last_total_tokens, source_root_key, "
|
|
750
|
+
"last_native_thread_id, last_root_thread_id, last_parent_thread_id, "
|
|
751
|
+
"last_conversation_key, last_turn_id"
|
|
752
|
+
)
|
|
753
|
+
for i in range(0, len(paths), 400):
|
|
754
|
+
chunk = paths[i:i + 400]
|
|
755
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
756
|
+
for row in conn.execute(
|
|
757
|
+
f"SELECT {cols} FROM codex_session_files WHERE path IN ({placeholders})",
|
|
758
|
+
chunk,
|
|
759
|
+
):
|
|
760
|
+
out[row[0]] = (
|
|
761
|
+
row[1], row[2], row[3], row[4], row[5], row[6], row[7],
|
|
762
|
+
row[8], row[9], row[10], row[11], row[12],
|
|
763
|
+
)
|
|
764
|
+
return out
|
|
765
|
+
|
|
766
|
+
|
|
695
767
|
def _delete_codex_file_derived_rows(
|
|
696
768
|
conn: sqlite3.Connection,
|
|
697
769
|
path_str: str,
|
|
@@ -741,8 +813,30 @@ def _delete_codex_file_derived_rows(
|
|
|
741
813
|
)
|
|
742
814
|
|
|
743
815
|
|
|
744
|
-
def _clear_codex_derived_rows(conn: sqlite3.Connection) ->
|
|
745
|
-
"""Clear every re-derivable Codex row family
|
|
816
|
+
def _clear_codex_derived_rows(conn: sqlite3.Connection) -> bool:
|
|
817
|
+
"""Clear every re-derivable Codex row family and report whether state changed.
|
|
818
|
+
|
|
819
|
+
The FTS5 ``delete-all`` command can increment SQLite's cumulative change
|
|
820
|
+
counter even when the semantic Codex surface was already empty. Callers
|
|
821
|
+
use this return value, rather than ``Connection.total_changes``, when
|
|
822
|
+
deciding whether to advance the physical-mutation sequence.
|
|
823
|
+
"""
|
|
824
|
+
state_changed = any(
|
|
825
|
+
conn.execute(query).fetchone() is not None
|
|
826
|
+
for query in (
|
|
827
|
+
"SELECT 1 FROM codex_session_entries LIMIT 1",
|
|
828
|
+
"SELECT 1 FROM quota_window_snapshots WHERE source = 'codex' LIMIT 1",
|
|
829
|
+
"SELECT 1 FROM codex_conversation_threads LIMIT 1",
|
|
830
|
+
"SELECT 1 FROM codex_conversation_events LIMIT 1",
|
|
831
|
+
"SELECT 1 FROM codex_session_files LIMIT 1",
|
|
832
|
+
"SELECT 1 FROM codex_source_roots LIMIT 1",
|
|
833
|
+
"SELECT 1 FROM codex_conversation_messages LIMIT 1",
|
|
834
|
+
"SELECT 1 FROM codex_conversation_file_touches LIMIT 1",
|
|
835
|
+
"SELECT 1 FROM codex_conversation_rollups LIMIT 1",
|
|
836
|
+
"SELECT 1 FROM cache_meta "
|
|
837
|
+
"WHERE key='codex_quota_projection_certificate' LIMIT 1",
|
|
838
|
+
)
|
|
839
|
+
)
|
|
746
840
|
conn.execute("DELETE FROM codex_session_entries")
|
|
747
841
|
conn.execute("DELETE FROM quota_window_snapshots WHERE source = 'codex'")
|
|
748
842
|
conn.execute("DELETE FROM codex_conversation_threads")
|
|
@@ -760,6 +854,7 @@ def _clear_codex_derived_rows(conn: sqlite3.Connection) -> None:
|
|
|
760
854
|
conn.execute(
|
|
761
855
|
"DELETE FROM cache_meta WHERE key='codex_quota_projection_certificate'"
|
|
762
856
|
)
|
|
857
|
+
return state_changed
|
|
763
858
|
|
|
764
859
|
|
|
765
860
|
def _bump_codex_physical_mutation_seq(conn: sqlite3.Connection) -> None:
|
|
@@ -1075,8 +1170,14 @@ def _write_codex_file_batch(
|
|
|
1075
1170
|
normalized_rows: list,
|
|
1076
1171
|
normalized_touches: list,
|
|
1077
1172
|
active_root_keys: set[str],
|
|
1173
|
+
prune_roots: bool = True,
|
|
1078
1174
|
) -> int:
|
|
1079
|
-
"""Write one fully-buffered Codex file atomically and return entry changes.
|
|
1175
|
+
"""Write one fully-buffered Codex file atomically and return entry changes.
|
|
1176
|
+
|
|
1177
|
+
``prune_roots`` gates the whole-tree ``_prune_inactive_codex_source_roots``
|
|
1178
|
+
call: a targeted (only_paths) ingest passes ``False`` so it never deletes a
|
|
1179
|
+
``codex_source_roots`` row for a root it wasn't asked about (spec §5.1
|
|
1180
|
+
whole-tree bypass — ``active_root_keys`` then covers only the targets)."""
|
|
1080
1181
|
now_iso = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
1081
1182
|
# #294 S6: rollups are recomputed-affected-or-deleted (§3.2), so capture the
|
|
1082
1183
|
# conversation keys this file's normalized rows touch BEFORE a reset delete
|
|
@@ -1174,7 +1275,8 @@ def _write_codex_file_batch(
|
|
|
1174
1275
|
last_conversation_key, last_turn_id,
|
|
1175
1276
|
),
|
|
1176
1277
|
)
|
|
1177
|
-
|
|
1278
|
+
if prune_roots:
|
|
1279
|
+
_prune_inactive_codex_source_roots(conn, active_root_keys)
|
|
1178
1280
|
# A file batch owns accounting, quota, thread, event, root, and cursor
|
|
1179
1281
|
# facts as one physical unit. Keep the version bump in that same commit so
|
|
1180
1282
|
# a rolled-back batch never appears newer to the dashboard signature.
|
|
@@ -3719,6 +3821,25 @@ class CodexIngestStats:
|
|
|
3719
3821
|
lines_malformed: int = 0
|
|
3720
3822
|
token_events_skipped: int = 0
|
|
3721
3823
|
skip_reasons: dict = field(default_factory=dict)
|
|
3824
|
+
# #294 S7 targeted (only_paths) live-tail fast-path fields. Default-clean so
|
|
3825
|
+
# every existing only_paths=None caller reads targeted_clean=True and is
|
|
3826
|
+
# otherwise unaffected — the exact Claude ``IngestStats`` semantics (§5.1).
|
|
3827
|
+
# ``deferred_reason`` carries the whole-call preflight decline
|
|
3828
|
+
# (shrink/requalification — Codex's ENTIRE pending-global condition; there is
|
|
3829
|
+
# NO ``cache_meta`` decline-marker tuple, pinned §5.1). ``files_failed``
|
|
3830
|
+
# counts per-file declines (post-preflight late shrink/requalification, I/O,
|
|
3831
|
+
# normalization, DB exception).
|
|
3832
|
+
files_failed: int = 0
|
|
3833
|
+
deferred_reason: "str | None" = None
|
|
3834
|
+
|
|
3835
|
+
@property
|
|
3836
|
+
def targeted_clean(self) -> bool:
|
|
3837
|
+
"""True ⇔ a targeted ingest fully applied: not contended, not deferred,
|
|
3838
|
+
and no per-file failure (§5.1). The watch loop emits + advances ``seen``
|
|
3839
|
+
only when this is True — byte-for-byte the Claude ``IngestStats`` rule."""
|
|
3840
|
+
return (not self.lock_contended
|
|
3841
|
+
and self.deferred_reason is None
|
|
3842
|
+
and self.files_failed == 0)
|
|
3722
3843
|
|
|
3723
3844
|
|
|
3724
3845
|
def _progress_codex_stderr(stats: CodexIngestStats, *, force: bool = False) -> None:
|
|
@@ -3736,8 +3857,10 @@ def sync_codex_cache(
|
|
|
3736
3857
|
*,
|
|
3737
3858
|
progress: Callable[[CodexIngestStats], None] | None = None,
|
|
3738
3859
|
rebuild: bool = False,
|
|
3860
|
+
only_paths: "set[str] | None" = None,
|
|
3739
3861
|
lock_timeout: "float | None" = None,
|
|
3740
3862
|
_on_first_file_rollback: Callable[[], None] | None = None,
|
|
3863
|
+
_on_file_committed: Callable[[str], None] | None = None,
|
|
3741
3864
|
) -> CodexIngestStats:
|
|
3742
3865
|
"""Read-through delta ingest of ~/.codex/sessions/**/*.jsonl.
|
|
3743
3866
|
|
|
@@ -3777,6 +3900,16 @@ def sync_codex_cache(
|
|
|
3777
3900
|
stats.lock_contended = True
|
|
3778
3901
|
return stats
|
|
3779
3902
|
|
|
3903
|
+
# #294 S7 targeted (only_paths) live-tail fast path (§5.1). Mutually
|
|
3904
|
+
# exclusive with rebuild (matches the Claude rule). Targeted mode
|
|
3905
|
+
# qualifies ONLY the caller's paths, scopes the cursor preload to them,
|
|
3906
|
+
# and bypasses every whole-tree operation (orphan prune, root prune,
|
|
3907
|
+
# global quota reconcile) — see the guards threaded through below.
|
|
3908
|
+
targeted = only_paths is not None
|
|
3909
|
+
if targeted and rebuild:
|
|
3910
|
+
raise ValueError(
|
|
3911
|
+
"sync_codex_cache: only_paths is incompatible with rebuild")
|
|
3912
|
+
|
|
3780
3913
|
# F4 (#313): the reconcile trigger gate is "did the Codex physical
|
|
3781
3914
|
# mutation sequence advance during this sync", NOT rows_changed —
|
|
3782
3915
|
# rows_changed counts only inserted accounting rows and misses
|
|
@@ -3793,17 +3926,19 @@ def sync_codex_cache(
|
|
|
3793
3926
|
# Clear INSIDE the lock — see sync_cache() for the full
|
|
3794
3927
|
# rationale. Done before the existing SELECT so delta
|
|
3795
3928
|
# detection sees an empty baseline.
|
|
3796
|
-
|
|
3797
|
-
_clear_codex_derived_rows(conn)
|
|
3798
|
-
if conn.total_changes != before_clear:
|
|
3929
|
+
if _clear_codex_derived_rows(conn):
|
|
3799
3930
|
_bump_codex_physical_mutation_seq(conn)
|
|
3800
3931
|
conn.commit()
|
|
3801
3932
|
eprint("[cache-sync] rebuild: cleared Codex cached entries")
|
|
3802
3933
|
|
|
3803
3934
|
# Pure read (glob + is_file only); safe to run before the SELECT and
|
|
3804
|
-
# the per-file loop, where no cache.db write lock may be held.
|
|
3935
|
+
# the per-file loop, where no cache.db write lock may be held. Targeted
|
|
3936
|
+
# mode qualifies ONLY the requested paths — never a tree walk (§5.1).
|
|
3805
3937
|
with _perf.phase("discover") as _p_disc:
|
|
3806
|
-
|
|
3938
|
+
if targeted:
|
|
3939
|
+
files = _qualify_codex_targets(only_paths)
|
|
3940
|
+
else:
|
|
3941
|
+
files = _discover_codex_files_with_roots()
|
|
3807
3942
|
stats.files_total = len(files)
|
|
3808
3943
|
_p_disc.set_count(len(files))
|
|
3809
3944
|
|
|
@@ -3820,7 +3955,7 @@ def sync_codex_cache(
|
|
|
3820
3955
|
# ingest (same invariant as the --rebuild clear above). Concurrent
|
|
3821
3956
|
# processes with different $CODEX_HOME would prune each other; the
|
|
3822
3957
|
# flock serializes them and that is a pathological configuration.
|
|
3823
|
-
if not rebuild: # --rebuild already cleared
|
|
3958
|
+
if not rebuild and not targeted: # --rebuild already cleared; targeted bypasses
|
|
3824
3959
|
current_file_identities = {
|
|
3825
3960
|
(str(item.source_path), item.source_root_key) for item in files
|
|
3826
3961
|
}
|
|
@@ -3875,19 +4010,53 @@ def sync_codex_cache(
|
|
|
3875
4010
|
# delta detection consults size alone (Codex rollout JSONLs are
|
|
3876
4011
|
# append-only, so a size change is a sufficient signal and mtime
|
|
3877
4012
|
# is prone to clock-skew false-positives).
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
4013
|
+
if targeted:
|
|
4014
|
+
# §5.1: the cursor preload queries codex_session_files for the
|
|
4015
|
+
# REQUESTED paths only (the full-sync path loads every row; targeted
|
|
4016
|
+
# must not, so its cost stays proportional to its targets).
|
|
4017
|
+
existing = _load_codex_session_files_rows(
|
|
4018
|
+
conn, [str(item.source_path) for item in files])
|
|
4019
|
+
else:
|
|
4020
|
+
existing = {
|
|
4021
|
+
row[0]: (
|
|
4022
|
+
row[1], row[2], row[3], row[4], row[5], row[6], row[7],
|
|
4023
|
+
row[8], row[9], row[10], row[11], row[12],
|
|
4024
|
+
)
|
|
4025
|
+
for row in conn.execute(
|
|
4026
|
+
"SELECT path, size_bytes, mtime_ns, last_byte_offset, "
|
|
4027
|
+
"last_session_id, last_model, last_total_tokens, source_root_key, "
|
|
4028
|
+
"last_native_thread_id, last_root_thread_id, last_parent_thread_id, "
|
|
4029
|
+
"last_conversation_key, last_turn_id "
|
|
4030
|
+
"FROM codex_session_files"
|
|
4031
|
+
)
|
|
4032
|
+
}
|
|
4033
|
+
|
|
4034
|
+
# §5.1 whole-call read-only shrink/requalification preflight. Because
|
|
4035
|
+
# ``targeted_clean`` is CALL-wide while the Codex path commits per file,
|
|
4036
|
+
# a shrink/requalification on ANY resolved target declines the whole call
|
|
4037
|
+
# with ZERO mutations (no partial commit where a healthy file's cursor
|
|
4038
|
+
# advances but no event can be emitted). A shrink landing AFTER this
|
|
4039
|
+
# snapshot is caught per-file at its write turn (below). This is Codex's
|
|
4040
|
+
# ENTIRE pending-global condition — there is NO cache_meta decline-marker
|
|
4041
|
+
# tuple (pinned §5.1). A target that vanished between qualification and
|
|
4042
|
+
# here is skipped, not declined — the per-file loop clean-drops it.
|
|
4043
|
+
if targeted:
|
|
4044
|
+
for discovered in files:
|
|
4045
|
+
prev = existing.get(str(discovered.source_path))
|
|
4046
|
+
if prev is None:
|
|
4047
|
+
continue # brand-new file: nothing to shrink/requalify
|
|
4048
|
+
prev_size = prev[0]
|
|
4049
|
+
prev_root_key = prev[6]
|
|
4050
|
+
if prev_root_key != discovered.source_root_key:
|
|
4051
|
+
stats.deferred_reason = "requalification"
|
|
4052
|
+
return stats
|
|
4053
|
+
try:
|
|
4054
|
+
cur_size = discovered.source_path.stat().st_size
|
|
4055
|
+
except OSError:
|
|
4056
|
+
continue # vanished post-qualify → clean-drop in the loop
|
|
4057
|
+
if cur_size < prev_size:
|
|
4058
|
+
stats.deferred_reason = "truncation"
|
|
4059
|
+
return stats
|
|
3891
4060
|
|
|
3892
4061
|
# #279 S2 F4: ONE coarse `walk` phase bracketing the per-file loop
|
|
3893
4062
|
# (count = files_processed, never per-row — §2 rule). Manual CM so
|
|
@@ -3901,6 +4070,8 @@ def sync_codex_cache(
|
|
|
3901
4070
|
st = jp.stat()
|
|
3902
4071
|
except OSError as exc:
|
|
3903
4072
|
eprint(f"[codex-cache] stat failed for {jp}: {exc}")
|
|
4073
|
+
if targeted:
|
|
4074
|
+
stats.files_failed += 1 # §5.1 I/O decline → call dirty
|
|
3904
4075
|
continue
|
|
3905
4076
|
|
|
3906
4077
|
size = st.st_size
|
|
@@ -3929,6 +4100,17 @@ def sync_codex_cache(
|
|
|
3929
4100
|
int(prev_ttot) if prev_ttot is not None else None
|
|
3930
4101
|
)
|
|
3931
4102
|
requalified = prev_root_key != discovered.source_root_key
|
|
4103
|
+
if targeted and (requalified or size < prev_size):
|
|
4104
|
+
# §5.1 preflight-snapshot scoped: a shrink or requalification
|
|
4105
|
+
# landing AFTER the preflight is declined HERE, per file —
|
|
4106
|
+
# earlier per-file commits in this call stand, the call still
|
|
4107
|
+
# reports dirty (files_failed → not targeted_clean), so the
|
|
4108
|
+
# watch advances no cursor and emits nothing, and recovery
|
|
4109
|
+
# rides the next full sync's per-file reset. Targeted mode
|
|
4110
|
+
# NEVER runs the full-file offset-0 reset re-ingest (below)
|
|
4111
|
+
# — that whole-cache-affecting escalation is the full sync's.
|
|
4112
|
+
stats.files_failed += 1
|
|
4113
|
+
continue
|
|
3932
4114
|
if not requalified and size == prev_size:
|
|
3933
4115
|
stats.files_skipped_unchanged += 1
|
|
3934
4116
|
continue
|
|
@@ -4070,6 +4252,8 @@ def sync_codex_cache(
|
|
|
4070
4252
|
stats.skip_reasons[_r] = stats.skip_reasons.get(_r, 0) + _n
|
|
4071
4253
|
except OSError as exc:
|
|
4072
4254
|
eprint(f"[codex-cache] could not read {jp}: {exc}")
|
|
4255
|
+
if targeted:
|
|
4256
|
+
stats.files_failed += 1 # §5.1 I/O decline → call dirty
|
|
4073
4257
|
continue
|
|
4074
4258
|
|
|
4075
4259
|
# Pull terminal session_id/model from the iterator's tracker.
|
|
@@ -4119,11 +4303,22 @@ def sync_codex_cache(
|
|
|
4119
4303
|
# seeded from the persisted resume seed. Buffered before the first DML
|
|
4120
4304
|
# (like every other family), so the single retry re-runs the same
|
|
4121
4305
|
# in-memory batch. The terminal sticky turn persists as last_turn_id.
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4306
|
+
try:
|
|
4307
|
+
norm_result = _lib_codex_conversation.normalize_codex_events(
|
|
4308
|
+
events_list,
|
|
4309
|
+
initial=_lib_codex_conversation.CodexStickyState(
|
|
4310
|
+
turn_id=initial_turn_id, model=initial_model),
|
|
4311
|
+
)
|
|
4312
|
+
except Exception as exc: # noqa: BLE001
|
|
4313
|
+
# §5.1: a normalization exception during a targeted ingest is a
|
|
4314
|
+
# per-file decline → the call is dirty (files_failed) but earlier
|
|
4315
|
+
# commits stand. A full sync keeps its historical crash-loud
|
|
4316
|
+
# behavior (there the whole walk is authoritative).
|
|
4317
|
+
if not targeted:
|
|
4318
|
+
raise
|
|
4319
|
+
eprint(f"[codex-cache] normalization failed for {jp}: {exc}")
|
|
4320
|
+
stats.files_failed += 1
|
|
4321
|
+
continue
|
|
4127
4322
|
new_last_turn_id = norm_result.terminal.turn_id
|
|
4128
4323
|
|
|
4129
4324
|
# Every derived row above was buffered before the first DML. A
|
|
@@ -4155,6 +4350,9 @@ def sync_codex_cache(
|
|
|
4155
4350
|
normalized_rows=norm_result.rows,
|
|
4156
4351
|
normalized_touches=norm_result.touches,
|
|
4157
4352
|
active_root_keys={item.source_root_key for item in files},
|
|
4353
|
+
# §5.1 whole-tree bypass: targeted mode never prunes
|
|
4354
|
+
# codex_source_roots for roots outside its target set.
|
|
4355
|
+
prune_roots=not targeted,
|
|
4158
4356
|
)
|
|
4159
4357
|
except sqlite3.DatabaseError as exc:
|
|
4160
4358
|
conn.rollback()
|
|
@@ -4174,8 +4372,16 @@ def sync_codex_cache(
|
|
|
4174
4372
|
committed = True
|
|
4175
4373
|
break
|
|
4176
4374
|
if not committed:
|
|
4375
|
+
if targeted:
|
|
4376
|
+
stats.files_failed += 1 # §5.1 DB-exception decline → dirty
|
|
4177
4377
|
continue
|
|
4178
4378
|
|
|
4379
|
+
# Private test seam (§5.1 post-preflight late-shrink race): fires
|
|
4380
|
+
# after each file's successful commit, so a race test can shrink a
|
|
4381
|
+
# not-yet-written target and assert the earlier commit stands.
|
|
4382
|
+
if _on_file_committed is not None:
|
|
4383
|
+
_on_file_committed(path_str)
|
|
4384
|
+
|
|
4179
4385
|
if progress is not None:
|
|
4180
4386
|
progress(stats)
|
|
4181
4387
|
|
|
@@ -4230,36 +4436,46 @@ def sync_codex_cache(
|
|
|
4230
4436
|
# projection, so the skip decision on that branch is DEFERRED to the
|
|
4231
4437
|
# post-flock stats-side signature check below — making the gate's
|
|
4232
4438
|
# skip-condition identical to reconcile's own short-circuit-condition.
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4439
|
+
#
|
|
4440
|
+
# §5.1: a targeted (only_paths) ingest NEVER invokes the global quota
|
|
4441
|
+
# reconciler (its observation load reconciles all roots at seconds-scale
|
|
4442
|
+
# cost). Quota projection is deferred to the next full sync, which the
|
|
4443
|
+
# ordinary hook cadence supplies. Skip the whole decision block so
|
|
4444
|
+
# project_after_unlock / deferred_cert_* / did_from_zero_replay keep
|
|
4445
|
+
# their no-op defaults — the post-flock reconcile paths below then all
|
|
4446
|
+
# short-circuit for a targeted call.
|
|
4447
|
+
if not targeted:
|
|
4448
|
+
cur_seq = codex_physical_mutation_seq(conn)
|
|
4449
|
+
if cur_seq != seq_before:
|
|
4450
|
+
project_after_unlock = True
|
|
4240
4451
|
else:
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
certificate is not None
|
|
4244
|
-
and certificate[0] == cur_seq
|
|
4245
|
-
and active_roots <= set(certificate[1])
|
|
4246
|
-
)
|
|
4247
|
-
if certificate_current:
|
|
4248
|
-
# The CACHE certificate is current, but that alone does NOT
|
|
4249
|
-
# prove stats.db still holds the projection (F1). Defer the
|
|
4250
|
-
# stats-side signature match until AFTER the Codex flock
|
|
4251
|
-
# releases below — only then may the reconcile be skipped.
|
|
4452
|
+
active_roots = _cache_root_keys(conn)
|
|
4453
|
+
if not active_roots:
|
|
4252
4454
|
project_after_unlock = False
|
|
4253
|
-
deferred_cert_roots = set(active_roots)
|
|
4254
|
-
assert certificate is not None
|
|
4255
|
-
deferred_cert_sigs = dict(certificate[1])
|
|
4256
4455
|
else:
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4456
|
+
certificate = load_codex_quota_projection_certificate(conn)
|
|
4457
|
+
certificate_current = (
|
|
4458
|
+
certificate is not None
|
|
4459
|
+
and certificate[0] == cur_seq
|
|
4460
|
+
and active_roots <= set(certificate[1])
|
|
4461
|
+
)
|
|
4462
|
+
if certificate_current:
|
|
4463
|
+
# The CACHE certificate is current, but that alone does
|
|
4464
|
+
# NOT prove stats.db still holds the projection (F1).
|
|
4465
|
+
# Defer the stats-side signature match until AFTER the
|
|
4466
|
+
# Codex flock releases below — only then may the
|
|
4467
|
+
# reconcile be skipped.
|
|
4468
|
+
project_after_unlock = False
|
|
4469
|
+
deferred_cert_roots = set(active_roots)
|
|
4470
|
+
assert certificate is not None
|
|
4471
|
+
deferred_cert_sigs = dict(certificate[1])
|
|
4472
|
+
else:
|
|
4473
|
+
project_after_unlock = True
|
|
4474
|
+
# #313 P3 (F9): a Codex rebuild or a truncation/requalification
|
|
4475
|
+
# re-ingest replays from offset 0 and restores >retention-day
|
|
4476
|
+
# codex_conversation_events. Force an unthrottled prune after the
|
|
4477
|
+
# flock releases (below), so restored old rows don't persist for 24h.
|
|
4478
|
+
did_from_zero_replay = rebuild or stats.files_reset_truncated > 0
|
|
4263
4479
|
finally:
|
|
4264
4480
|
try:
|
|
4265
4481
|
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
package/bin/_cctally_core.py
CHANGED
|
@@ -67,6 +67,9 @@ def _init_paths_from_env() -> None:
|
|
|
67
67
|
global HOOK_TICK_LOG_DIR, HOOK_TICK_LOG_PATH, HOOK_TICK_LOG_ROTATED_PATH
|
|
68
68
|
global HOOK_TICK_THROTTLE_PATH, HOOK_TICK_THROTTLE_LOCK_PATH
|
|
69
69
|
global STATUSLINE_OBSERVE_MARKER_PATH, STATUSLINE_PERSIST_LOCK_PATH
|
|
70
|
+
global STATUSLINE_CANDIDATE_DIR, STATUSLINE_SELECTED_PATH
|
|
71
|
+
global STATUSLINE_TRANSPORT_MARKER_PATH
|
|
72
|
+
global STATUSLINE_AUTHORITATIVE_7D_PATH, STATUSLINE_AUTHORITATIVE_5H_PATH
|
|
70
73
|
global OAUTH_BACKOFF_MARKER_PATH, OAUTH_BACKOFF_COUNT_PATH
|
|
71
74
|
global UPDATE_STATE_PATH, UPDATE_SUPPRESS_PATH
|
|
72
75
|
global UPDATE_LOCK_PATH, UPDATE_LOG_PATH, UPDATE_LOG_ROTATED_PATH
|
|
@@ -126,25 +129,17 @@ def _init_paths_from_env() -> None:
|
|
|
126
129
|
HOOK_TICK_THROTTLE_PATH = APP_DIR / "hook-tick.last-fetch"
|
|
127
130
|
HOOK_TICK_THROTTLE_LOCK_PATH = APP_DIR / "hook-tick.last-fetch.lock"
|
|
128
131
|
|
|
129
|
-
# Statusline
|
|
130
|
-
#
|
|
131
|
-
#
|
|
132
|
-
#
|
|
133
|
-
# like HOOK_TICK_THROTTLE_PATH) — touched after every statusline
|
|
134
|
-
# persist that fed valid 7d input through cmd_record_usage, INCLUDING
|
|
135
|
-
# a dedup no-op. Represents "the statusline is alive and feeding".
|
|
136
|
-
# The statusline persist throttle AND the OAuth backfill gate both
|
|
137
|
-
# key off this, NOT snapshot age (cmd_record_usage dedups unchanged
|
|
138
|
-
# percentages without refreshing captured_at, so snapshot age keeps
|
|
139
|
-
# growing while the statusline is actively feeding).
|
|
140
|
-
# - STATUSLINE_PERSIST_LOCK_PATH: cross-process flock serializing the
|
|
141
|
-
# detached persist feeder so a multi-session render herd yields at
|
|
142
|
-
# most one snapshot per throttle window.
|
|
143
|
-
# - OAUTH_BACKOFF_MARKER_PATH: shared 429 cooldown. Stores an ABSOLUTE
|
|
144
|
-
# epoch deadline in the file CONTENT (never the mtime — future-dating
|
|
145
|
-
# an mtime would corrupt HOOK_TICK_THROTTLE_PATH's age reading).
|
|
132
|
+
# Statusline candidate arbitration (#318). The spool and derived selected
|
|
133
|
+
# state are entirely APP_DIR-derived so dev/data-dir redirection remains
|
|
134
|
+
# complete. The observation marker now means selected/authoritative
|
|
135
|
+
# freshness; transport liveness has its own marker.
|
|
146
136
|
STATUSLINE_OBSERVE_MARKER_PATH = APP_DIR / "statusline-observe.last"
|
|
147
137
|
STATUSLINE_PERSIST_LOCK_PATH = APP_DIR / "statusline-persist.lock"
|
|
138
|
+
STATUSLINE_CANDIDATE_DIR = APP_DIR / "statusline-candidates"
|
|
139
|
+
STATUSLINE_SELECTED_PATH = APP_DIR / "statusline-selected.json"
|
|
140
|
+
STATUSLINE_TRANSPORT_MARKER_PATH = APP_DIR / "statusline-transport.last"
|
|
141
|
+
STATUSLINE_AUTHORITATIVE_7D_PATH = APP_DIR / "statusline-authoritative-7d.json"
|
|
142
|
+
STATUSLINE_AUTHORITATIVE_5H_PATH = APP_DIR / "statusline-authoritative-5h.json"
|
|
148
143
|
OAUTH_BACKOFF_MARKER_PATH = APP_DIR / "oauth-backoff.until"
|
|
149
144
|
# Consecutive-429 counter (text int) driving the headerless exponential
|
|
150
145
|
# backoff (base * 2**count). Separate from the deadline marker so the
|
|
@@ -246,38 +241,23 @@ def _real_prod_data_dir() -> pathlib.Path:
|
|
|
246
241
|
# Internal (no config UI — YAGNI, spec §Out of scope); test injection only.
|
|
247
242
|
# Spec 2026-07-17-usage-statusline-fallback-design.
|
|
248
243
|
#
|
|
249
|
-
# STATUSLINE_PERSIST_THROTTLE_SECONDS: min seconds between statusline
|
|
250
|
-
# persist attempts (keyed off STATUSLINE_OBSERVE_MARKER_PATH liveness).
|
|
251
|
-
# Retuned 60.0 -> 25.0 for #311: the setup-managed statusLine.refreshInterval
|
|
252
|
-
# timer fires the statusline on a 30s cadence, and the throttle MUST be
|
|
253
|
-
# strictly LESS than that interval. When interval <= throttle, phase jitter
|
|
254
|
-
# produces beat-frequency skips (a tick at 59.9s of marker age is throttled;
|
|
255
|
-
# the next persist waits ~120s), so a 60/60 pairing oscillates 60<->120s.
|
|
256
|
-
# With interval 30 > throttle 25, every tick whose predecessor's record
|
|
257
|
-
# completed promptly (within interval - throttle = 5s) passes the gate.
|
|
258
|
-
# CADENCE QUALIFIER: the marker is touched at persist COMPLETION, so a tick
|
|
259
|
-
# observes marker age = 30 - d where d is the previous record's duration;
|
|
260
|
-
# if d > 5s (the record kernel can run cmd_sync_week JSONL scans) that tick
|
|
261
|
-
# throttles and the cycle degrades to skip-one-tick (~60s), self-correcting
|
|
262
|
-
# on the next tick. Exact-30s is NOT an invariant (no attempt-start marker
|
|
263
|
-
# exists — one would let a hung/failed record claim liveness and suppress
|
|
264
|
-
# the OAuth backfill). Still strictly better than the 60/60 beat's 60-120s.
|
|
265
244
|
# OAUTH_BACKFILL_STALE_SECONDS: the OAuth poll only backfills once the
|
|
266
|
-
#
|
|
267
|
-
#
|
|
268
|
-
# statusline is the primary writer and OAuth only covers its absence
|
|
269
|
-
# (300 > 25 still holds).
|
|
245
|
+
# selected-freshness marker is at least this stale. The 30-second statusline
|
|
246
|
+
# transport tick does not run OAuth.
|
|
270
247
|
# OAUTH_BACKOFF_BASE_SECONDS / OAUTH_BACKOFF_CAP_SECONDS: the headerless
|
|
271
248
|
# exponential 429 backoff (base * 2**consecutive_429, capped).
|
|
272
|
-
|
|
249
|
+
STATUSLINE_CANDIDATE_TTL_SECONDS = 90
|
|
250
|
+
STATUSLINE_CANDIDATE_FUTURE_SKEW_SECONDS = 5
|
|
251
|
+
STATUSLINE_CANDIDATE_DOCUMENT_MAX_BYTES = 4 * 1024
|
|
252
|
+
STATUSLINE_SELECTED_DOCUMENT_MAX_BYTES = 1024 * 1024
|
|
253
|
+
STATUSLINE_TOMBSTONE_DOCUMENT_MAX_BYTES = 1024
|
|
273
254
|
# STATUSLINE_REFRESH_INTERVAL_DEFAULT (#311): the value `cctally setup`
|
|
274
255
|
# writes into Claude Code's settings.json `statusLine.refreshInterval` when a
|
|
275
256
|
# recognized cctally statusLine block lacks one. Claude Code re-runs the
|
|
276
257
|
# statusline command on this fixed timer "in addition to the event-driven
|
|
277
258
|
# updates", which keeps the usage-persistence feeder ticking while a parent
|
|
278
259
|
# session waits on a long subagent (event-driven updates go quiet then). MUST
|
|
279
|
-
#
|
|
280
|
-
# above. Add-when-absent only; a user-set value is never mutated.
|
|
260
|
+
# Add-when-absent only; a user-set value is never mutated.
|
|
281
261
|
STATUSLINE_REFRESH_INTERVAL_DEFAULT = 30
|
|
282
262
|
OAUTH_BACKFILL_STALE_SECONDS = 300.0
|
|
283
263
|
OAUTH_BACKOFF_BASE_SECONDS = 60.0
|
|
@@ -763,6 +763,35 @@ def _build_codex_source_detail(snapshot, *, resource: str, key: str) -> dict[str
|
|
|
763
763
|
raise SourceResourceNotFound()
|
|
764
764
|
|
|
765
765
|
|
|
766
|
+
def _codex_partial_source_detail(snapshot, row: Mapping, *, resource: str,
|
|
767
|
+
key: str) -> dict[str, Any]:
|
|
768
|
+
"""Keep a published cache-only row usable when project metadata is partial."""
|
|
769
|
+
detail = _source_safe_native_detail(
|
|
770
|
+
row, source="codex", resource=resource, key=key,
|
|
771
|
+
)
|
|
772
|
+
detail.update({
|
|
773
|
+
"metadata_availability": "partial",
|
|
774
|
+
"metadata_reason": (
|
|
775
|
+
"Codex project metadata is incomplete; available accounting totals "
|
|
776
|
+
"are preserved."
|
|
777
|
+
),
|
|
778
|
+
})
|
|
779
|
+
if resource == "session":
|
|
780
|
+
detail.setdefault("models", [])
|
|
781
|
+
detail["model_breakdowns"] = []
|
|
782
|
+
elif resource == "project":
|
|
783
|
+
generated_at = getattr(snapshot, "generated_at", None)
|
|
784
|
+
detail.setdefault("range_start", detail.get("first_seen"))
|
|
785
|
+
detail.setdefault(
|
|
786
|
+
"range_end",
|
|
787
|
+
generated_at.astimezone(dt.timezone.utc).isoformat()
|
|
788
|
+
if isinstance(generated_at, dt.datetime) else detail.get("last_seen"),
|
|
789
|
+
)
|
|
790
|
+
detail["models"] = []
|
|
791
|
+
detail["sessions"] = []
|
|
792
|
+
return detail
|
|
793
|
+
|
|
794
|
+
|
|
766
795
|
def _source_safe_claude_session_detail(detail, *, key: str) -> dict[str, Any]:
|
|
767
796
|
"""Adapt the existing Claude session detail builder to S4's safe route."""
|
|
768
797
|
payload = _session_detail_to_envelope(detail)
|
|
@@ -958,7 +987,12 @@ def build_source_detail(*, snapshot, source: str, resource: str,
|
|
|
958
987
|
row = source_detail_lookup(snapshot.source_bundle, source, resource, key)
|
|
959
988
|
if source == "claude":
|
|
960
989
|
return _build_claude_source_detail(snapshot, resource=resource, key=key)
|
|
961
|
-
|
|
990
|
+
try:
|
|
991
|
+
return _build_codex_source_detail(snapshot, resource=resource, key=key)
|
|
992
|
+
except sys.modules["_cctally_source_analytics"].QualifiedMetadataUnavailable:
|
|
993
|
+
return _codex_partial_source_detail(
|
|
994
|
+
snapshot, row, resource=resource, key=key,
|
|
995
|
+
)
|
|
962
996
|
|
|
963
997
|
_ensure_sibling_loaded("_cctally_dashboard_share")
|
|
964
998
|
from _cctally_dashboard_share import (
|