cctally 1.72.0 → 1.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/bin/_cctally_cache.py +245 -50
- package/bin/_cctally_core.py +20 -40
- package/bin/_cctally_dashboard_conversation.py +731 -94
- package/bin/_cctally_doctor.py +59 -0
- package/bin/_cctally_parser.py +17 -4
- package/bin/_cctally_record.py +162 -33
- package/bin/_cctally_refresh.py +58 -36
- package/bin/_cctally_statusline.py +811 -82
- package/bin/_cctally_transcript.py +223 -7
- package/bin/_lib_codex_conversation_export.py +124 -0
- package/bin/_lib_codex_conversation_query.py +500 -28
- package/bin/_lib_codex_conversation_watch.py +323 -0
- package/bin/_lib_conversation_dispatch.py +269 -7
- package/bin/_lib_conversation_query.py +88 -0
- package/bin/_lib_doctor.py +62 -0
- package/bin/_lib_statusline_candidates.py +734 -0
- package/bin/cctally +31 -2
- package/package.json +4 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,17 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.73.0] - 2026-07-18
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- Codex conversations (#294 S7): the S6 kernel layer is now reachable. The dashboard's twelve conversation routes qualify in place — an entity id beginning `v1.` opts into the provider-neutral envelope (`ok`/`normalization_pending` → `200` JSON, `not_found` → `404`, payload `gone` → `410`, export → `200 text/markdown`, Codex media → `404 capability_unsupported`), and the three collection routes (`/api/conversations`, `/api/conversations/facets`, `/api/conversation/search`) take a strict `?source={claude,codex}`. Absent qualification, the Claude routes stay byte-identical, and the transcript privacy gate remains the first act of every handler. (#294)
|
|
12
|
+
- `cctally transcript export` now accepts a Claude `sessionId` **or** a `v1.` conversation key (Codex conversations are addressable only by their `v1.` key), anonymizes qualified exports by default via a provider-aware plan (`--raw` escapes), and adds a Codex-only `--speed {auto,standard,fast}` (rejected on any non-Codex ref); the CLI export byte-matches the dashboard `/export` download in both anonymize and raw modes. (#294)
|
|
13
|
+
- `cctally transcript search --source {claude,codex}` searches either provider. The Codex output is its own `Key / When / Project / Kinds / Snippet` table and a stamped-first camelCase JSON envelope paginated by an opaque base64url `--cursor`; the default-Claude table and JSON stay byte-frozen. (#294)
|
|
14
|
+
- Codex conversation live-tail: `GET /api/conversation/<v1key>/events` preflights as plain JSON before any SSE bytes and then streams `conversationKey`-framed frames, and a budgeted directory-frontier discovery joins child threads spawned mid-watch. Targeted Codex ingest keeps each tick's cost proportional to the watched conversation. (#294)
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
- Statusline usage persistence now arbitrates concurrent regular-pool Claude sessions through privacy-safe per-session candidates instead of accepting the first lock winner. A fresher 5h or 7d observation is selected independent of render arrival order; lower same-window values retain a full-cycle consensus path for credits/resets, including an extra revalidated tick for reset-to-zero. This replaces the internal 25-second persistence throttle; the existing `statusLine.refreshInterval: 30` remains the idle-time trigger. OAuth and manual credits share a fail-closed selected-state recovery protocol, the periodic dashboard remains OAuth-free and publishes accepted observations through its normal SSE path, and `cctally doctor` reports passive pipeline evidence when repair is needed. (#318)
|
|
18
|
+
|
|
8
19
|
## [1.72.0] - 2026-07-17
|
|
9
20
|
|
|
10
21
|
### Added
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -692,6 +692,78 @@ def _discover_codex_files_with_roots() -> list[CodexDiscoveredFile]:
|
|
|
692
692
|
return discovered
|
|
693
693
|
|
|
694
694
|
|
|
695
|
+
def _qualify_codex_targets(only_paths: "set[str]") -> list[CodexDiscoveredFile]:
|
|
696
|
+
"""Resolve each requested path through the ordered configured roots exactly
|
|
697
|
+
as full discovery would (spec §5.1) — producing the same per-file facts a
|
|
698
|
+
``CodexDiscoveredFile`` carries (configured spelling, physical path, provider
|
|
699
|
+
root, walk root, source-root key) with first-match containment + physical
|
|
700
|
+
dedup. A path resolving under no configured root, not a ``*.jsonl`` file, or
|
|
701
|
+
an alias of an already-resolved physical file is DROPPED (clean, not
|
|
702
|
+
ingested). Targeted mode's analogue of ``_discover_codex_files_with_roots``:
|
|
703
|
+
it does NOT walk any tree — it qualifies only the caller's exact paths."""
|
|
704
|
+
roots = _codex_provider_roots()
|
|
705
|
+
resolved: list[CodexDiscoveredFile] = []
|
|
706
|
+
seen_physical: set[pathlib.Path] = set()
|
|
707
|
+
# Deterministic order so first-match physical dedup is stable across a set.
|
|
708
|
+
for p in sorted(only_paths):
|
|
709
|
+
candidate = pathlib.Path(p)
|
|
710
|
+
for root in roots:
|
|
711
|
+
try:
|
|
712
|
+
inside = candidate.is_relative_to(root.walk_root)
|
|
713
|
+
except (ValueError, TypeError):
|
|
714
|
+
inside = False
|
|
715
|
+
if not inside:
|
|
716
|
+
continue
|
|
717
|
+
# Under this root's walk boundary: it is qualified here (first match)
|
|
718
|
+
# or dropped — a different-spelled alias never re-qualifies under a
|
|
719
|
+
# later root, matching full discovery's yielded-source-path identity.
|
|
720
|
+
if candidate.suffix != ".jsonl" or not candidate.is_file():
|
|
721
|
+
break # vanished / non-rollout under this root → drop (clean)
|
|
722
|
+
physical = _canonical_codex_path(candidate)
|
|
723
|
+
if physical in seen_physical:
|
|
724
|
+
break # first-match physical dedup → drop the alias
|
|
725
|
+
seen_physical.add(physical)
|
|
726
|
+
resolved.append(CodexDiscoveredFile(
|
|
727
|
+
source_path=candidate,
|
|
728
|
+
physical_path=physical,
|
|
729
|
+
provider_root=root.provider_root,
|
|
730
|
+
walk_root=root.walk_root,
|
|
731
|
+
source_root_key=root.source_root_key,
|
|
732
|
+
))
|
|
733
|
+
break
|
|
734
|
+
return resolved
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _load_codex_session_files_rows(
|
|
738
|
+
conn: sqlite3.Connection, paths: "list[str]"
|
|
739
|
+
) -> dict:
|
|
740
|
+
"""Cursor rows from ``codex_session_files`` for ONLY the given paths (spec
|
|
741
|
+
§5.1 — the targeted preload must never load every row like the full-sync
|
|
742
|
+
path). Same 12-tuple value shape as ``sync_codex_cache``'s full ``existing``
|
|
743
|
+
map, so the per-file delta logic is byte-identical between the two modes."""
|
|
744
|
+
out: dict = {}
|
|
745
|
+
if not paths:
|
|
746
|
+
return out
|
|
747
|
+
cols = (
|
|
748
|
+
"path, size_bytes, mtime_ns, last_byte_offset, "
|
|
749
|
+
"last_session_id, last_model, last_total_tokens, source_root_key, "
|
|
750
|
+
"last_native_thread_id, last_root_thread_id, last_parent_thread_id, "
|
|
751
|
+
"last_conversation_key, last_turn_id"
|
|
752
|
+
)
|
|
753
|
+
for i in range(0, len(paths), 400):
|
|
754
|
+
chunk = paths[i:i + 400]
|
|
755
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
756
|
+
for row in conn.execute(
|
|
757
|
+
f"SELECT {cols} FROM codex_session_files WHERE path IN ({placeholders})",
|
|
758
|
+
chunk,
|
|
759
|
+
):
|
|
760
|
+
out[row[0]] = (
|
|
761
|
+
row[1], row[2], row[3], row[4], row[5], row[6], row[7],
|
|
762
|
+
row[8], row[9], row[10], row[11], row[12],
|
|
763
|
+
)
|
|
764
|
+
return out
|
|
765
|
+
|
|
766
|
+
|
|
695
767
|
def _delete_codex_file_derived_rows(
|
|
696
768
|
conn: sqlite3.Connection,
|
|
697
769
|
path_str: str,
|
|
@@ -1075,8 +1147,14 @@ def _write_codex_file_batch(
|
|
|
1075
1147
|
normalized_rows: list,
|
|
1076
1148
|
normalized_touches: list,
|
|
1077
1149
|
active_root_keys: set[str],
|
|
1150
|
+
prune_roots: bool = True,
|
|
1078
1151
|
) -> int:
|
|
1079
|
-
"""Write one fully-buffered Codex file atomically and return entry changes.
|
|
1152
|
+
"""Write one fully-buffered Codex file atomically and return entry changes.
|
|
1153
|
+
|
|
1154
|
+
``prune_roots`` gates the whole-tree ``_prune_inactive_codex_source_roots``
|
|
1155
|
+
call: a targeted (only_paths) ingest passes ``False`` so it never deletes a
|
|
1156
|
+
``codex_source_roots`` row for a root it wasn't asked about (spec §5.1
|
|
1157
|
+
whole-tree bypass — ``active_root_keys`` then covers only the targets)."""
|
|
1080
1158
|
now_iso = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
1081
1159
|
# #294 S6: rollups are recomputed-affected-or-deleted (§3.2), so capture the
|
|
1082
1160
|
# conversation keys this file's normalized rows touch BEFORE a reset delete
|
|
@@ -1174,7 +1252,8 @@ def _write_codex_file_batch(
|
|
|
1174
1252
|
last_conversation_key, last_turn_id,
|
|
1175
1253
|
),
|
|
1176
1254
|
)
|
|
1177
|
-
|
|
1255
|
+
if prune_roots:
|
|
1256
|
+
_prune_inactive_codex_source_roots(conn, active_root_keys)
|
|
1178
1257
|
# A file batch owns accounting, quota, thread, event, root, and cursor
|
|
1179
1258
|
# facts as one physical unit. Keep the version bump in that same commit so
|
|
1180
1259
|
# a rolled-back batch never appears newer to the dashboard signature.
|
|
@@ -3719,6 +3798,25 @@ class CodexIngestStats:
|
|
|
3719
3798
|
lines_malformed: int = 0
|
|
3720
3799
|
token_events_skipped: int = 0
|
|
3721
3800
|
skip_reasons: dict = field(default_factory=dict)
|
|
3801
|
+
# #294 S7 targeted (only_paths) live-tail fast-path fields. Default-clean so
|
|
3802
|
+
# every existing only_paths=None caller reads targeted_clean=True and is
|
|
3803
|
+
# otherwise unaffected — the exact Claude ``IngestStats`` semantics (§5.1).
|
|
3804
|
+
# ``deferred_reason`` carries the whole-call preflight decline
|
|
3805
|
+
# (shrink/requalification — Codex's ENTIRE pending-global condition; there is
|
|
3806
|
+
# NO ``cache_meta`` decline-marker tuple, pinned §5.1). ``files_failed``
|
|
3807
|
+
# counts per-file declines (post-preflight late shrink/requalification, I/O,
|
|
3808
|
+
# normalization, DB exception).
|
|
3809
|
+
files_failed: int = 0
|
|
3810
|
+
deferred_reason: "str | None" = None
|
|
3811
|
+
|
|
3812
|
+
@property
|
|
3813
|
+
def targeted_clean(self) -> bool:
|
|
3814
|
+
"""True ⇔ a targeted ingest fully applied: not contended, not deferred,
|
|
3815
|
+
and no per-file failure (§5.1). The watch loop emits + advances ``seen``
|
|
3816
|
+
only when this is True — byte-for-byte the Claude ``IngestStats`` rule."""
|
|
3817
|
+
return (not self.lock_contended
|
|
3818
|
+
and self.deferred_reason is None
|
|
3819
|
+
and self.files_failed == 0)
|
|
3722
3820
|
|
|
3723
3821
|
|
|
3724
3822
|
def _progress_codex_stderr(stats: CodexIngestStats, *, force: bool = False) -> None:
|
|
@@ -3736,8 +3834,10 @@ def sync_codex_cache(
|
|
|
3736
3834
|
*,
|
|
3737
3835
|
progress: Callable[[CodexIngestStats], None] | None = None,
|
|
3738
3836
|
rebuild: bool = False,
|
|
3837
|
+
only_paths: "set[str] | None" = None,
|
|
3739
3838
|
lock_timeout: "float | None" = None,
|
|
3740
3839
|
_on_first_file_rollback: Callable[[], None] | None = None,
|
|
3840
|
+
_on_file_committed: Callable[[str], None] | None = None,
|
|
3741
3841
|
) -> CodexIngestStats:
|
|
3742
3842
|
"""Read-through delta ingest of ~/.codex/sessions/**/*.jsonl.
|
|
3743
3843
|
|
|
@@ -3777,6 +3877,16 @@ def sync_codex_cache(
|
|
|
3777
3877
|
stats.lock_contended = True
|
|
3778
3878
|
return stats
|
|
3779
3879
|
|
|
3880
|
+
# #294 S7 targeted (only_paths) live-tail fast path (§5.1). Mutually
|
|
3881
|
+
# exclusive with rebuild (matches the Claude rule). Targeted mode
|
|
3882
|
+
# qualifies ONLY the caller's paths, scopes the cursor preload to them,
|
|
3883
|
+
# and bypasses every whole-tree operation (orphan prune, root prune,
|
|
3884
|
+
# global quota reconcile) — see the guards threaded through below.
|
|
3885
|
+
targeted = only_paths is not None
|
|
3886
|
+
if targeted and rebuild:
|
|
3887
|
+
raise ValueError(
|
|
3888
|
+
"sync_codex_cache: only_paths is incompatible with rebuild")
|
|
3889
|
+
|
|
3780
3890
|
# F4 (#313): the reconcile trigger gate is "did the Codex physical
|
|
3781
3891
|
# mutation sequence advance during this sync", NOT rows_changed —
|
|
3782
3892
|
# rows_changed counts only inserted accounting rows and misses
|
|
@@ -3801,9 +3911,13 @@ def sync_codex_cache(
|
|
|
3801
3911
|
eprint("[cache-sync] rebuild: cleared Codex cached entries")
|
|
3802
3912
|
|
|
3803
3913
|
# Pure read (glob + is_file only); safe to run before the SELECT and
|
|
3804
|
-
# the per-file loop, where no cache.db write lock may be held.
|
|
3914
|
+
# the per-file loop, where no cache.db write lock may be held. Targeted
|
|
3915
|
+
# mode qualifies ONLY the requested paths — never a tree walk (§5.1).
|
|
3805
3916
|
with _perf.phase("discover") as _p_disc:
|
|
3806
|
-
|
|
3917
|
+
if targeted:
|
|
3918
|
+
files = _qualify_codex_targets(only_paths)
|
|
3919
|
+
else:
|
|
3920
|
+
files = _discover_codex_files_with_roots()
|
|
3807
3921
|
stats.files_total = len(files)
|
|
3808
3922
|
_p_disc.set_count(len(files))
|
|
3809
3923
|
|
|
@@ -3820,7 +3934,7 @@ def sync_codex_cache(
|
|
|
3820
3934
|
# ingest (same invariant as the --rebuild clear above). Concurrent
|
|
3821
3935
|
# processes with different $CODEX_HOME would prune each other; the
|
|
3822
3936
|
# flock serializes them and that is a pathological configuration.
|
|
3823
|
-
if not rebuild: # --rebuild already cleared
|
|
3937
|
+
if not rebuild and not targeted: # --rebuild already cleared; targeted bypasses
|
|
3824
3938
|
current_file_identities = {
|
|
3825
3939
|
(str(item.source_path), item.source_root_key) for item in files
|
|
3826
3940
|
}
|
|
@@ -3875,19 +3989,53 @@ def sync_codex_cache(
|
|
|
3875
3989
|
# delta detection consults size alone (Codex rollout JSONLs are
|
|
3876
3990
|
# append-only, so a size change is a sufficient signal and mtime
|
|
3877
3991
|
# is prone to clock-skew false-positives).
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3992
|
+
if targeted:
|
|
3993
|
+
# §5.1: the cursor preload queries codex_session_files for the
|
|
3994
|
+
# REQUESTED paths only (the full-sync path loads every row; targeted
|
|
3995
|
+
# must not, so its cost stays proportional to its targets).
|
|
3996
|
+
existing = _load_codex_session_files_rows(
|
|
3997
|
+
conn, [str(item.source_path) for item in files])
|
|
3998
|
+
else:
|
|
3999
|
+
existing = {
|
|
4000
|
+
row[0]: (
|
|
4001
|
+
row[1], row[2], row[3], row[4], row[5], row[6], row[7],
|
|
4002
|
+
row[8], row[9], row[10], row[11], row[12],
|
|
4003
|
+
)
|
|
4004
|
+
for row in conn.execute(
|
|
4005
|
+
"SELECT path, size_bytes, mtime_ns, last_byte_offset, "
|
|
4006
|
+
"last_session_id, last_model, last_total_tokens, source_root_key, "
|
|
4007
|
+
"last_native_thread_id, last_root_thread_id, last_parent_thread_id, "
|
|
4008
|
+
"last_conversation_key, last_turn_id "
|
|
4009
|
+
"FROM codex_session_files"
|
|
4010
|
+
)
|
|
4011
|
+
}
|
|
4012
|
+
|
|
4013
|
+
# §5.1 whole-call read-only shrink/requalification preflight. Because
|
|
4014
|
+
# ``targeted_clean`` is CALL-wide while the Codex path commits per file,
|
|
4015
|
+
# a shrink/requalification on ANY resolved target declines the whole call
|
|
4016
|
+
# with ZERO mutations (no partial commit where a healthy file's cursor
|
|
4017
|
+
# advances but no event can be emitted). A shrink landing AFTER this
|
|
4018
|
+
# snapshot is caught per-file at its write turn (below). This is Codex's
|
|
4019
|
+
# ENTIRE pending-global condition — there is NO cache_meta decline-marker
|
|
4020
|
+
# tuple (pinned §5.1). A target that vanished between qualification and
|
|
4021
|
+
# here is skipped, not declined — the per-file loop clean-drops it.
|
|
4022
|
+
if targeted:
|
|
4023
|
+
for discovered in files:
|
|
4024
|
+
prev = existing.get(str(discovered.source_path))
|
|
4025
|
+
if prev is None:
|
|
4026
|
+
continue # brand-new file: nothing to shrink/requalify
|
|
4027
|
+
prev_size = prev[0]
|
|
4028
|
+
prev_root_key = prev[6]
|
|
4029
|
+
if prev_root_key != discovered.source_root_key:
|
|
4030
|
+
stats.deferred_reason = "requalification"
|
|
4031
|
+
return stats
|
|
4032
|
+
try:
|
|
4033
|
+
cur_size = discovered.source_path.stat().st_size
|
|
4034
|
+
except OSError:
|
|
4035
|
+
continue # vanished post-qualify → clean-drop in the loop
|
|
4036
|
+
if cur_size < prev_size:
|
|
4037
|
+
stats.deferred_reason = "truncation"
|
|
4038
|
+
return stats
|
|
3891
4039
|
|
|
3892
4040
|
# #279 S2 F4: ONE coarse `walk` phase bracketing the per-file loop
|
|
3893
4041
|
# (count = files_processed, never per-row — §2 rule). Manual CM so
|
|
@@ -3901,6 +4049,8 @@ def sync_codex_cache(
|
|
|
3901
4049
|
st = jp.stat()
|
|
3902
4050
|
except OSError as exc:
|
|
3903
4051
|
eprint(f"[codex-cache] stat failed for {jp}: {exc}")
|
|
4052
|
+
if targeted:
|
|
4053
|
+
stats.files_failed += 1 # §5.1 I/O decline → call dirty
|
|
3904
4054
|
continue
|
|
3905
4055
|
|
|
3906
4056
|
size = st.st_size
|
|
@@ -3929,6 +4079,17 @@ def sync_codex_cache(
|
|
|
3929
4079
|
int(prev_ttot) if prev_ttot is not None else None
|
|
3930
4080
|
)
|
|
3931
4081
|
requalified = prev_root_key != discovered.source_root_key
|
|
4082
|
+
if targeted and (requalified or size < prev_size):
|
|
4083
|
+
# §5.1 preflight-snapshot scoped: a shrink or requalification
|
|
4084
|
+
# landing AFTER the preflight is declined HERE, per file —
|
|
4085
|
+
# earlier per-file commits in this call stand, the call still
|
|
4086
|
+
# reports dirty (files_failed → not targeted_clean), so the
|
|
4087
|
+
# watch advances no cursor and emits nothing, and recovery
|
|
4088
|
+
# rides the next full sync's per-file reset. Targeted mode
|
|
4089
|
+
# NEVER runs the full-file offset-0 reset re-ingest (below)
|
|
4090
|
+
# — that whole-cache-affecting escalation is the full sync's.
|
|
4091
|
+
stats.files_failed += 1
|
|
4092
|
+
continue
|
|
3932
4093
|
if not requalified and size == prev_size:
|
|
3933
4094
|
stats.files_skipped_unchanged += 1
|
|
3934
4095
|
continue
|
|
@@ -4070,6 +4231,8 @@ def sync_codex_cache(
|
|
|
4070
4231
|
stats.skip_reasons[_r] = stats.skip_reasons.get(_r, 0) + _n
|
|
4071
4232
|
except OSError as exc:
|
|
4072
4233
|
eprint(f"[codex-cache] could not read {jp}: {exc}")
|
|
4234
|
+
if targeted:
|
|
4235
|
+
stats.files_failed += 1 # §5.1 I/O decline → call dirty
|
|
4073
4236
|
continue
|
|
4074
4237
|
|
|
4075
4238
|
# Pull terminal session_id/model from the iterator's tracker.
|
|
@@ -4119,11 +4282,22 @@ def sync_codex_cache(
|
|
|
4119
4282
|
# seeded from the persisted resume seed. Buffered before the first DML
|
|
4120
4283
|
# (like every other family), so the single retry re-runs the same
|
|
4121
4284
|
# in-memory batch. The terminal sticky turn persists as last_turn_id.
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4285
|
+
try:
|
|
4286
|
+
norm_result = _lib_codex_conversation.normalize_codex_events(
|
|
4287
|
+
events_list,
|
|
4288
|
+
initial=_lib_codex_conversation.CodexStickyState(
|
|
4289
|
+
turn_id=initial_turn_id, model=initial_model),
|
|
4290
|
+
)
|
|
4291
|
+
except Exception as exc: # noqa: BLE001
|
|
4292
|
+
# §5.1: a normalization exception during a targeted ingest is a
|
|
4293
|
+
# per-file decline → the call is dirty (files_failed) but earlier
|
|
4294
|
+
# commits stand. A full sync keeps its historical crash-loud
|
|
4295
|
+
# behavior (there the whole walk is authoritative).
|
|
4296
|
+
if not targeted:
|
|
4297
|
+
raise
|
|
4298
|
+
eprint(f"[codex-cache] normalization failed for {jp}: {exc}")
|
|
4299
|
+
stats.files_failed += 1
|
|
4300
|
+
continue
|
|
4127
4301
|
new_last_turn_id = norm_result.terminal.turn_id
|
|
4128
4302
|
|
|
4129
4303
|
# Every derived row above was buffered before the first DML. A
|
|
@@ -4155,6 +4329,9 @@ def sync_codex_cache(
|
|
|
4155
4329
|
normalized_rows=norm_result.rows,
|
|
4156
4330
|
normalized_touches=norm_result.touches,
|
|
4157
4331
|
active_root_keys={item.source_root_key for item in files},
|
|
4332
|
+
# §5.1 whole-tree bypass: targeted mode never prunes
|
|
4333
|
+
# codex_source_roots for roots outside its target set.
|
|
4334
|
+
prune_roots=not targeted,
|
|
4158
4335
|
)
|
|
4159
4336
|
except sqlite3.DatabaseError as exc:
|
|
4160
4337
|
conn.rollback()
|
|
@@ -4174,8 +4351,16 @@ def sync_codex_cache(
|
|
|
4174
4351
|
committed = True
|
|
4175
4352
|
break
|
|
4176
4353
|
if not committed:
|
|
4354
|
+
if targeted:
|
|
4355
|
+
stats.files_failed += 1 # §5.1 DB-exception decline → dirty
|
|
4177
4356
|
continue
|
|
4178
4357
|
|
|
4358
|
+
# Private test seam (§5.1 post-preflight late-shrink race): fires
|
|
4359
|
+
# after each file's successful commit, so a race test can shrink a
|
|
4360
|
+
# not-yet-written target and assert the earlier commit stands.
|
|
4361
|
+
if _on_file_committed is not None:
|
|
4362
|
+
_on_file_committed(path_str)
|
|
4363
|
+
|
|
4179
4364
|
if progress is not None:
|
|
4180
4365
|
progress(stats)
|
|
4181
4366
|
|
|
@@ -4230,36 +4415,46 @@ def sync_codex_cache(
|
|
|
4230
4415
|
# projection, so the skip decision on that branch is DEFERRED to the
|
|
4231
4416
|
# post-flock stats-side signature check below — making the gate's
|
|
4232
4417
|
# skip-condition identical to reconcile's own short-circuit-condition.
|
|
4233
|
-
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4418
|
+
#
|
|
4419
|
+
# §5.1: a targeted (only_paths) ingest NEVER invokes the global quota
|
|
4420
|
+
# reconciler (its observation load reconciles all roots at seconds-scale
|
|
4421
|
+
# cost). Quota projection is deferred to the next full sync, which the
|
|
4422
|
+
# ordinary hook cadence supplies. Skip the whole decision block so
|
|
4423
|
+
# project_after_unlock / deferred_cert_* / did_from_zero_replay keep
|
|
4424
|
+
# their no-op defaults — the post-flock reconcile paths below then all
|
|
4425
|
+
# short-circuit for a targeted call.
|
|
4426
|
+
if not targeted:
|
|
4427
|
+
cur_seq = codex_physical_mutation_seq(conn)
|
|
4428
|
+
if cur_seq != seq_before:
|
|
4429
|
+
project_after_unlock = True
|
|
4240
4430
|
else:
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
certificate is not None
|
|
4244
|
-
and certificate[0] == cur_seq
|
|
4245
|
-
and active_roots <= set(certificate[1])
|
|
4246
|
-
)
|
|
4247
|
-
if certificate_current:
|
|
4248
|
-
# The CACHE certificate is current, but that alone does NOT
|
|
4249
|
-
# prove stats.db still holds the projection (F1). Defer the
|
|
4250
|
-
# stats-side signature match until AFTER the Codex flock
|
|
4251
|
-
# releases below — only then may the reconcile be skipped.
|
|
4431
|
+
active_roots = _cache_root_keys(conn)
|
|
4432
|
+
if not active_roots:
|
|
4252
4433
|
project_after_unlock = False
|
|
4253
|
-
deferred_cert_roots = set(active_roots)
|
|
4254
|
-
assert certificate is not None
|
|
4255
|
-
deferred_cert_sigs = dict(certificate[1])
|
|
4256
4434
|
else:
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4435
|
+
certificate = load_codex_quota_projection_certificate(conn)
|
|
4436
|
+
certificate_current = (
|
|
4437
|
+
certificate is not None
|
|
4438
|
+
and certificate[0] == cur_seq
|
|
4439
|
+
and active_roots <= set(certificate[1])
|
|
4440
|
+
)
|
|
4441
|
+
if certificate_current:
|
|
4442
|
+
# The CACHE certificate is current, but that alone does
|
|
4443
|
+
# NOT prove stats.db still holds the projection (F1).
|
|
4444
|
+
# Defer the stats-side signature match until AFTER the
|
|
4445
|
+
# Codex flock releases below — only then may the
|
|
4446
|
+
# reconcile be skipped.
|
|
4447
|
+
project_after_unlock = False
|
|
4448
|
+
deferred_cert_roots = set(active_roots)
|
|
4449
|
+
assert certificate is not None
|
|
4450
|
+
deferred_cert_sigs = dict(certificate[1])
|
|
4451
|
+
else:
|
|
4452
|
+
project_after_unlock = True
|
|
4453
|
+
# #313 P3 (F9): a Codex rebuild or a truncation/requalification
|
|
4454
|
+
# re-ingest replays from offset 0 and restores >retention-day
|
|
4455
|
+
# codex_conversation_events. Force an unthrottled prune after the
|
|
4456
|
+
# flock releases (below), so restored old rows don't persist for 24h.
|
|
4457
|
+
did_from_zero_replay = rebuild or stats.files_reset_truncated > 0
|
|
4263
4458
|
finally:
|
|
4264
4459
|
try:
|
|
4265
4460
|
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
package/bin/_cctally_core.py
CHANGED
|
@@ -67,6 +67,9 @@ def _init_paths_from_env() -> None:
|
|
|
67
67
|
global HOOK_TICK_LOG_DIR, HOOK_TICK_LOG_PATH, HOOK_TICK_LOG_ROTATED_PATH
|
|
68
68
|
global HOOK_TICK_THROTTLE_PATH, HOOK_TICK_THROTTLE_LOCK_PATH
|
|
69
69
|
global STATUSLINE_OBSERVE_MARKER_PATH, STATUSLINE_PERSIST_LOCK_PATH
|
|
70
|
+
global STATUSLINE_CANDIDATE_DIR, STATUSLINE_SELECTED_PATH
|
|
71
|
+
global STATUSLINE_TRANSPORT_MARKER_PATH
|
|
72
|
+
global STATUSLINE_AUTHORITATIVE_7D_PATH, STATUSLINE_AUTHORITATIVE_5H_PATH
|
|
70
73
|
global OAUTH_BACKOFF_MARKER_PATH, OAUTH_BACKOFF_COUNT_PATH
|
|
71
74
|
global UPDATE_STATE_PATH, UPDATE_SUPPRESS_PATH
|
|
72
75
|
global UPDATE_LOCK_PATH, UPDATE_LOG_PATH, UPDATE_LOG_ROTATED_PATH
|
|
@@ -126,25 +129,17 @@ def _init_paths_from_env() -> None:
|
|
|
126
129
|
HOOK_TICK_THROTTLE_PATH = APP_DIR / "hook-tick.last-fetch"
|
|
127
130
|
HOOK_TICK_THROTTLE_LOCK_PATH = APP_DIR / "hook-tick.last-fetch.lock"
|
|
128
131
|
|
|
129
|
-
# Statusline
|
|
130
|
-
#
|
|
131
|
-
#
|
|
132
|
-
#
|
|
133
|
-
# like HOOK_TICK_THROTTLE_PATH) — touched after every statusline
|
|
134
|
-
# persist that fed valid 7d input through cmd_record_usage, INCLUDING
|
|
135
|
-
# a dedup no-op. Represents "the statusline is alive and feeding".
|
|
136
|
-
# The statusline persist throttle AND the OAuth backfill gate both
|
|
137
|
-
# key off this, NOT snapshot age (cmd_record_usage dedups unchanged
|
|
138
|
-
# percentages without refreshing captured_at, so snapshot age keeps
|
|
139
|
-
# growing while the statusline is actively feeding).
|
|
140
|
-
# - STATUSLINE_PERSIST_LOCK_PATH: cross-process flock serializing the
|
|
141
|
-
# detached persist feeder so a multi-session render herd yields at
|
|
142
|
-
# most one snapshot per throttle window.
|
|
143
|
-
# - OAUTH_BACKOFF_MARKER_PATH: shared 429 cooldown. Stores an ABSOLUTE
|
|
144
|
-
# epoch deadline in the file CONTENT (never the mtime — future-dating
|
|
145
|
-
# an mtime would corrupt HOOK_TICK_THROTTLE_PATH's age reading).
|
|
132
|
+
# Statusline candidate arbitration (#318). The spool and derived selected
|
|
133
|
+
# state are entirely APP_DIR-derived so dev/data-dir redirection remains
|
|
134
|
+
# complete. The observation marker now means selected/authoritative
|
|
135
|
+
# freshness; transport liveness has its own marker.
|
|
146
136
|
STATUSLINE_OBSERVE_MARKER_PATH = APP_DIR / "statusline-observe.last"
|
|
147
137
|
STATUSLINE_PERSIST_LOCK_PATH = APP_DIR / "statusline-persist.lock"
|
|
138
|
+
STATUSLINE_CANDIDATE_DIR = APP_DIR / "statusline-candidates"
|
|
139
|
+
STATUSLINE_SELECTED_PATH = APP_DIR / "statusline-selected.json"
|
|
140
|
+
STATUSLINE_TRANSPORT_MARKER_PATH = APP_DIR / "statusline-transport.last"
|
|
141
|
+
STATUSLINE_AUTHORITATIVE_7D_PATH = APP_DIR / "statusline-authoritative-7d.json"
|
|
142
|
+
STATUSLINE_AUTHORITATIVE_5H_PATH = APP_DIR / "statusline-authoritative-5h.json"
|
|
148
143
|
OAUTH_BACKOFF_MARKER_PATH = APP_DIR / "oauth-backoff.until"
|
|
149
144
|
# Consecutive-429 counter (text int) driving the headerless exponential
|
|
150
145
|
# backoff (base * 2**count). Separate from the deadline marker so the
|
|
@@ -246,38 +241,23 @@ def _real_prod_data_dir() -> pathlib.Path:
|
|
|
246
241
|
# Internal (no config UI — YAGNI, spec §Out of scope); test injection only.
|
|
247
242
|
# Spec 2026-07-17-usage-statusline-fallback-design.
|
|
248
243
|
#
|
|
249
|
-
# STATUSLINE_PERSIST_THROTTLE_SECONDS: min seconds between statusline
|
|
250
|
-
# persist attempts (keyed off STATUSLINE_OBSERVE_MARKER_PATH liveness).
|
|
251
|
-
# Retuned 60.0 -> 25.0 for #311: the setup-managed statusLine.refreshInterval
|
|
252
|
-
# timer fires the statusline on a 30s cadence, and the throttle MUST be
|
|
253
|
-
# strictly LESS than that interval. When interval <= throttle, phase jitter
|
|
254
|
-
# produces beat-frequency skips (a tick at 59.9s of marker age is throttled;
|
|
255
|
-
# the next persist waits ~120s), so a 60/60 pairing oscillates 60<->120s.
|
|
256
|
-
# With interval 30 > throttle 25, every tick whose predecessor's record
|
|
257
|
-
# completed promptly (within interval - throttle = 5s) passes the gate.
|
|
258
|
-
# CADENCE QUALIFIER: the marker is touched at persist COMPLETION, so a tick
|
|
259
|
-
# observes marker age = 30 - d where d is the previous record's duration;
|
|
260
|
-
# if d > 5s (the record kernel can run cmd_sync_week JSONL scans) that tick
|
|
261
|
-
# throttles and the cycle degrades to skip-one-tick (~60s), self-correcting
|
|
262
|
-
# on the next tick. Exact-30s is NOT an invariant (no attempt-start marker
|
|
263
|
-
# exists — one would let a hung/failed record claim liveness and suppress
|
|
264
|
-
# the OAuth backfill). Still strictly better than the 60/60 beat's 60-120s.
|
|
265
244
|
# OAUTH_BACKFILL_STALE_SECONDS: the OAuth poll only backfills once the
|
|
266
|
-
#
|
|
267
|
-
#
|
|
268
|
-
# statusline is the primary writer and OAuth only covers its absence
|
|
269
|
-
# (300 > 25 still holds).
|
|
245
|
+
# selected-freshness marker is at least this stale. The 30-second statusline
|
|
246
|
+
# transport tick does not run OAuth.
|
|
270
247
|
# OAUTH_BACKOFF_BASE_SECONDS / OAUTH_BACKOFF_CAP_SECONDS: the headerless
|
|
271
248
|
# exponential 429 backoff (base * 2**consecutive_429, capped).
|
|
272
|
-
|
|
249
|
+
STATUSLINE_CANDIDATE_TTL_SECONDS = 90
|
|
250
|
+
STATUSLINE_CANDIDATE_FUTURE_SKEW_SECONDS = 5
|
|
251
|
+
STATUSLINE_CANDIDATE_DOCUMENT_MAX_BYTES = 4 * 1024
|
|
252
|
+
STATUSLINE_SELECTED_DOCUMENT_MAX_BYTES = 1024 * 1024
|
|
253
|
+
STATUSLINE_TOMBSTONE_DOCUMENT_MAX_BYTES = 1024
|
|
273
254
|
# STATUSLINE_REFRESH_INTERVAL_DEFAULT (#311): the value `cctally setup`
|
|
274
255
|
# writes into Claude Code's settings.json `statusLine.refreshInterval` when a
|
|
275
256
|
# recognized cctally statusLine block lacks one. Claude Code re-runs the
|
|
276
257
|
# statusline command on this fixed timer "in addition to the event-driven
|
|
277
258
|
# updates", which keeps the usage-persistence feeder ticking while a parent
|
|
278
259
|
# session waits on a long subagent (event-driven updates go quiet then). MUST
|
|
279
|
-
#
|
|
280
|
-
# above. Add-when-absent only; a user-set value is never mutated.
|
|
260
|
+
# Add-when-absent only; a user-set value is never mutated.
|
|
281
261
|
STATUSLINE_REFRESH_INTERVAL_DEFAULT = 30
|
|
282
262
|
OAUTH_BACKFILL_STALE_SECONDS = 300.0
|
|
283
263
|
OAUTH_BACKOFF_BASE_SECONDS = 60.0
|