cctally 1.87.2 → 1.88.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/bin/_cctally_cache.py +134 -2
- package/bin/_cctally_dashboard_envelope.py +3 -2
- package/bin/_cctally_dashboard_sources.py +306 -180
- package/bin/_cctally_db.py +122 -0
- package/bin/_cctally_doctor.py +25 -1
- package/bin/_cctally_transcript.py +7 -2
- package/bin/_cctally_tui.py +3 -2
- package/bin/_lib_codex_conversation.py +17 -0
- package/bin/_lib_codex_conversation_query.py +16 -2
- package/bin/_lib_conversation_retention.py +25 -1
- package/bin/_lib_dashboard_sources.py +8 -1
- package/bin/_lib_doctor.py +57 -0
- package/bin/_lib_jsonl.py +35 -2
- package/package.json +1 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -3925,6 +3925,54 @@ def _conv_001_adopt_schema_version_marker(conn: sqlite3.Connection) -> None:
|
|
|
3925
3925
|
conn.commit()
|
|
3926
3926
|
|
|
3927
3927
|
|
|
3928
|
+
@conversations_migration("002_codex_thread_source_inference_replay")
|
|
3929
|
+
def _conv_002_codex_thread_source_inference_replay(
|
|
3930
|
+
conn: sqlite3.Connection,
|
|
3931
|
+
) -> None:
|
|
3932
|
+
"""Arm the conversations half of the byte-zero Codex replay.
|
|
3933
|
+
|
|
3934
|
+
Spec:
|
|
3935
|
+
``docs/superpowers/specs/2026-07-30-codex-thread-source-inference-design.md``
|
|
3936
|
+
§4.3.
|
|
3937
|
+
|
|
3938
|
+
Writes a marker ONLY; it clears no table. ``sync_codex_conversations``
|
|
3939
|
+
consumes it, and DEFERS while the cache-side marker is still pending —
|
|
3940
|
+
``_recompute_codex_rollups`` resolves project attribution from the cache-side
|
|
3941
|
+
thread row, and a missing one stamps a materialized ``"(unassigned)"`` the
|
|
3942
|
+
read path then prefers permanently.
|
|
3943
|
+
|
|
3944
|
+
The key is DISTINCT from ``conversation_rebuild_codex_pending`` on purpose.
|
|
3945
|
+
``_ensure_codex_conversation_contract`` consumes that one by replaying
|
|
3946
|
+
normalization over already-retained events — which preserves their NULL
|
|
3947
|
+
conversation keys — and then deletes it, so a ``dashboard --no-sync`` or
|
|
3948
|
+
qualified CLI read between this migration and the next real sync would
|
|
3949
|
+
silently discard the repair.
|
|
3950
|
+
|
|
3951
|
+
Takes the Codex conversations provider flock first, the way cache migration
|
|
3952
|
+
``028_split_conversation_store`` does, and DEFERS on contention. The
|
|
3953
|
+
conversations dispatcher runs inside ``_conversations_open_guarded``, which
|
|
3954
|
+
holds ``CONVERSATIONS_LOCK_MAINTENANCE_PATH`` only SHARED, while
|
|
3955
|
+
``sync_codex_conversations`` serializes on ``CONVERSATIONS_LOCK_CODEX_PATH``
|
|
3956
|
+
— so without this lock the marker can be armed in the middle of a walk that
|
|
3957
|
+
already read it as absent, and that walk's finalize would clear a replay it
|
|
3958
|
+
never performed. Deferring leaves the migration pending, so it arms cleanly
|
|
3959
|
+
at the next open.
|
|
3960
|
+
|
|
3961
|
+
Idempotent: re-running rewrites the same marker. NO self-stamp — the
|
|
3962
|
+
dispatcher central-stamps on a clean return (#140).
|
|
3963
|
+
"""
|
|
3964
|
+
import _cctally_cache
|
|
3965
|
+
|
|
3966
|
+
held = _acquire_conversations_db_codex_provider_flock(
|
|
3967
|
+
conn, migration="conversations 002 thread_source replay")
|
|
3968
|
+
try:
|
|
3969
|
+
_set_cache_meta(
|
|
3970
|
+
conn, _cctally_cache.CODEX_CONVERSATION_REPLAY_FROM_ZERO_KEY, "1")
|
|
3971
|
+
conn.commit()
|
|
3972
|
+
finally:
|
|
3973
|
+
_release_cache_db_writer_flocks(held)
|
|
3974
|
+
|
|
3975
|
+
|
|
3928
3976
|
# #177 S6: the consolidated multi-column external-content FTS5 table that
|
|
3929
3977
|
# replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
|
|
3930
3978
|
# pair. The three column names MUST match the conversation_messages columns BY
|
|
@@ -4683,6 +4731,44 @@ def _acquire_cache_db_codex_provider_flock(
|
|
|
4683
4731
|
return held
|
|
4684
4732
|
|
|
4685
4733
|
|
|
4734
|
+
def _acquire_conversations_db_codex_provider_flock(
|
|
4735
|
+
conn: sqlite3.Connection,
|
|
4736
|
+
*,
|
|
4737
|
+
migration: str,
|
|
4738
|
+
) -> list[int]:
|
|
4739
|
+
"""Take the ``<conversations.db>.codex.lock`` sibling, or DEFER.
|
|
4740
|
+
|
|
4741
|
+
The conversations dispatcher runs under a SHARED maintenance flock, so —
|
|
4742
|
+
unlike the cache dispatcher — it does not exclude the provider sync that
|
|
4743
|
+
owns the marker lifecycle. A conversations handler that writes a marker
|
|
4744
|
+
``sync_codex_conversations`` consumes must therefore hold the same
|
|
4745
|
+
provider lock that sync holds, or it can arm mid-walk and have its marker
|
|
4746
|
+
swallowed by a walk that already read it as absent.
|
|
4747
|
+
|
|
4748
|
+
Derived from the connection (the lock-path helper is store-agnostic: main DB
|
|
4749
|
+
file + ``.codex.lock``), so a migration test never contends on the caller's
|
|
4750
|
+
real conversations lock.
|
|
4751
|
+
"""
|
|
4752
|
+
provider_path = _cache_db_codex_lock_path_for_conn(conn)
|
|
4753
|
+
if provider_path is None:
|
|
4754
|
+
return []
|
|
4755
|
+
|
|
4756
|
+
from _lib_cache_writer_lock import acquire_ordered_flocks
|
|
4757
|
+
|
|
4758
|
+
try:
|
|
4759
|
+
held = acquire_ordered_flocks([(provider_path, fcntl.LOCK_EX)])
|
|
4760
|
+
except OSError as exc:
|
|
4761
|
+
raise MigrationGateNotMet(
|
|
4762
|
+
f"conversations.db Codex lock unavailable; deferring {migration}"
|
|
4763
|
+
) from exc
|
|
4764
|
+
if held is None:
|
|
4765
|
+
raise MigrationGateNotMet(
|
|
4766
|
+
f"conversations.db Codex lock held by a concurrent Codex "
|
|
4767
|
+
f"conversation sync; deferring {migration}"
|
|
4768
|
+
)
|
|
4769
|
+
return held
|
|
4770
|
+
|
|
4771
|
+
|
|
4686
4772
|
def _release_cache_db_writer_flocks(held: list[int]) -> None:
|
|
4687
4773
|
from _lib_cache_writer_lock import release_cache_writer_flocks
|
|
4688
4774
|
|
|
@@ -6135,6 +6221,42 @@ def _034_codex_window_spend_adoption(conn: sqlite3.Connection) -> None:
|
|
|
6135
6221
|
_release_cache_db_writer_flocks(held)
|
|
6136
6222
|
|
|
6137
6223
|
|
|
6224
|
+
@cache_migration("035_codex_thread_source_inference_replay")
|
|
6225
|
+
def _035_codex_thread_source_inference_replay(conn: sqlite3.Connection) -> None:
|
|
6226
|
+
"""Arm a byte-zero Codex replay so rollouts whose ``session_meta`` omits
|
|
6227
|
+
``thread_source`` gain a conversation identity.
|
|
6228
|
+
|
|
6229
|
+
Spec:
|
|
6230
|
+
``docs/superpowers/specs/2026-07-30-codex-thread-source-inference-design.md``
|
|
6231
|
+
§4.3.
|
|
6232
|
+
|
|
6233
|
+
The repair must re-read the rollout bytes: the retained events carry NULL
|
|
6234
|
+
conversation keys and the in-place normalization replay preserves them.
|
|
6235
|
+
|
|
6236
|
+
Writes a marker ONLY. ``sync_codex_cache`` consumes it and ORs it into its
|
|
6237
|
+
own ``rebuild``, which is what makes the rebuild path capture
|
|
6238
|
+
``rebuild_known_identities`` BEFORE the clear. Clearing here instead would
|
|
6239
|
+
delete ``codex_session_files`` out of band, leaving the next ordinary sync
|
|
6240
|
+
with an empty snapshot — every re-read rollout would fall through to the
|
|
6241
|
+
live-``auth.json`` branch and pre-mechanism Codex spend would be
|
|
6242
|
+
re-attributed to whoever is authenticated now (#416 spec D1). Migrations 026
|
|
6243
|
+
and 027 predate that snapshot and are already stamped, so this would be the
|
|
6244
|
+
first migration to hit it live.
|
|
6245
|
+
|
|
6246
|
+
No provider flock: unlike handlers 024-027 and 034 this writes one
|
|
6247
|
+
``cache_meta`` row and touches no Codex-derived table, so a concurrent
|
|
6248
|
+
``sync_codex_cache`` has nothing to interleave with — and if one is mid-walk,
|
|
6249
|
+
arming the marker simply defers the replay to the following sync.
|
|
6250
|
+
|
|
6251
|
+
Idempotent: re-running rewrites the same marker. NO self-stamp — the
|
|
6252
|
+
dispatcher central-stamps on a clean return (#140).
|
|
6253
|
+
"""
|
|
6254
|
+
import _cctally_cache
|
|
6255
|
+
|
|
6256
|
+
_set_cache_meta(conn, _cctally_cache.CODEX_REPLAY_FROM_ZERO_KEY, "1")
|
|
6257
|
+
conn.commit()
|
|
6258
|
+
|
|
6259
|
+
|
|
6138
6260
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
6139
6261
|
|
|
6140
6262
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -1125,12 +1125,25 @@ def _doctor_gather_state_impl(
|
|
|
1125
1125
|
# #416 review B4: the durable record that a torn Codex `auth.json` halted
|
|
1126
1126
|
# ingest. Same cache_meta read, same degrade-to-None-on-anything contract.
|
|
1127
1127
|
codex_torn_deferred = None
|
|
1128
|
+
# The byte-zero Codex replay stall signal. The marker itself is a bare "1";
|
|
1129
|
+
# the sibling `blocked` record is the JSON one, so it is read through the
|
|
1130
|
+
# same loop while the marker gets a plain existence probe. Key names come
|
|
1131
|
+
# from the kernel constants, never inline literals.
|
|
1132
|
+
codex_replay_pending = None
|
|
1133
|
+
codex_replay_blocked = None
|
|
1134
|
+
try:
|
|
1135
|
+
import _lib_codex_conversation as _codex_kern
|
|
1136
|
+
_blocked_key = _codex_kern.CODEX_REPLAY_BLOCKED_KEY
|
|
1137
|
+
_pending_key = _codex_kern.CODEX_REPLAY_FROM_ZERO_KEY
|
|
1138
|
+
except Exception:
|
|
1139
|
+
_blocked_key = "codex_replay_from_zero_blocked"
|
|
1140
|
+
_pending_key = "codex_replay_from_zero_pending"
|
|
1128
1141
|
try:
|
|
1129
1142
|
if _cache_probe_allowed and _cctally_core.CACHE_DB_PATH.exists():
|
|
1130
1143
|
conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
|
|
1131
1144
|
try:
|
|
1132
1145
|
for _key in ("parse_health_claude", "parse_health_codex",
|
|
1133
|
-
"codex_torn_auth_deferred"):
|
|
1146
|
+
"codex_torn_auth_deferred", _blocked_key):
|
|
1134
1147
|
try:
|
|
1135
1148
|
row = conn.execute(
|
|
1136
1149
|
"SELECT value FROM cache_meta WHERE key = ?",
|
|
@@ -1143,10 +1156,19 @@ def _doctor_gather_state_impl(
|
|
|
1143
1156
|
parse_health_claude = _parsed
|
|
1144
1157
|
elif _key == "parse_health_codex":
|
|
1145
1158
|
parse_health_codex = _parsed
|
|
1159
|
+
elif _key == _blocked_key:
|
|
1160
|
+
codex_replay_blocked = _parsed
|
|
1146
1161
|
else:
|
|
1147
1162
|
codex_torn_deferred = _parsed
|
|
1148
1163
|
except (sqlite3.OperationalError, ValueError):
|
|
1149
1164
|
pass
|
|
1165
|
+
try:
|
|
1166
|
+
codex_replay_pending = conn.execute(
|
|
1167
|
+
"SELECT 1 FROM cache_meta WHERE key = ?",
|
|
1168
|
+
(_pending_key,),
|
|
1169
|
+
).fetchone() is not None
|
|
1170
|
+
except sqlite3.OperationalError:
|
|
1171
|
+
pass
|
|
1150
1172
|
finally:
|
|
1151
1173
|
conn.close()
|
|
1152
1174
|
except Exception:
|
|
@@ -1722,6 +1744,8 @@ def _doctor_gather_state_impl(
|
|
|
1722
1744
|
parse_health_claude=parse_health_claude,
|
|
1723
1745
|
parse_health_codex=parse_health_codex,
|
|
1724
1746
|
codex_torn_deferred=codex_torn_deferred,
|
|
1747
|
+
codex_replay_pending=codex_replay_pending,
|
|
1748
|
+
codex_replay_blocked=codex_replay_blocked,
|
|
1725
1749
|
stats_db_quick_check=stats_db_quick_check,
|
|
1726
1750
|
cache_db_quick_check=cache_db_quick_check,
|
|
1727
1751
|
conversations_db_quick_check=conversations_db_quick_check,
|
|
@@ -49,9 +49,13 @@ def cmd_transcript(args) -> int:
|
|
|
49
49
|
# ---- export ----------------------------------------------------------------
|
|
50
50
|
|
|
51
51
|
_SPEED_ONLY_CODEX_MSG = "transcript: --speed applies only to Codex conversations"
|
|
52
|
+
# Deliberately generic about WHAT is pending. Three distinct conditions reach
|
|
53
|
+
# this branch — a cache predating migration 025, a store with the contract
|
|
54
|
+
# rebuild marker armed, and a store with a byte-zero Codex replay marker armed —
|
|
55
|
+
# and naming the migration told users to wait for work applied long ago.
|
|
52
56
|
_PENDING_EXPORT_MSG = (
|
|
53
57
|
"transcript: Codex conversation is not yet normalized "
|
|
54
|
-
"(
|
|
58
|
+
"(the pending work runs on the next cache open) — retry shortly")
|
|
55
59
|
|
|
56
60
|
|
|
57
61
|
def _emit_export(md: str, output) -> None:
|
|
@@ -213,9 +217,10 @@ def _cmd_transcript_search_claude(args) -> int:
|
|
|
213
217
|
|
|
214
218
|
# ---- search (Codex) --------------------------------------------------------
|
|
215
219
|
|
|
220
|
+
# Same generic wording as `_PENDING_EXPORT_MSG`, for the same reason.
|
|
216
221
|
_CODEX_SEARCH_PENDING_MSG = (
|
|
217
222
|
"transcript: Codex conversations are not yet normalized "
|
|
218
|
-
"(
|
|
223
|
+
"(the pending work runs on the next cache open); no results yet")
|
|
219
224
|
|
|
220
225
|
|
|
221
226
|
def _cmd_transcript_search_codex(args) -> int:
|
package/bin/_cctally_tui.py
CHANGED
|
@@ -271,6 +271,7 @@ from _cctally_dashboard_sources import (
|
|
|
271
271
|
resolve_dashboard_source_semantics,
|
|
272
272
|
)
|
|
273
273
|
from _lib_dashboard_sources import (
|
|
274
|
+
SOURCE_SCHEMA_VERSION,
|
|
274
275
|
CapabilityRecord,
|
|
275
276
|
SourceDashboardBundle,
|
|
276
277
|
SourceDashboardState,
|
|
@@ -2803,7 +2804,7 @@ def _tui_build_source_bundle(
|
|
|
2803
2804
|
codex = refresh_codex_source_clock(codex, now_utc=now_utc)
|
|
2804
2805
|
combined = compose_all_state(claude, codex)
|
|
2805
2806
|
bundle = SourceDashboardBundle(
|
|
2806
|
-
source_schema_version=
|
|
2807
|
+
source_schema_version=SOURCE_SCHEMA_VERSION,
|
|
2807
2808
|
default_source="claude",
|
|
2808
2809
|
source_order=("claude", "codex", "all"),
|
|
2809
2810
|
sources={"claude": claude, "codex": codex, "all": combined},
|
|
@@ -2877,7 +2878,7 @@ def _tui_hydrating_source_bundle() -> SourceDashboardBundle:
|
|
|
2877
2878
|
domain_freshness={"hero": "stale", "quota": "stale", "sessions": "stale"},
|
|
2878
2879
|
)
|
|
2879
2880
|
return SourceDashboardBundle(
|
|
2880
|
-
source_schema_version=
|
|
2881
|
+
source_schema_version=SOURCE_SCHEMA_VERSION,
|
|
2881
2882
|
default_source="claude",
|
|
2882
2883
|
source_order=("claude", "codex", "all"),
|
|
2883
2884
|
sources={"claude": claude, "codex": codex, "all": compose_all_state(claude, codex)},
|
|
@@ -74,6 +74,23 @@ CODEX_TITLE_MAX = _TITLE_MAX
|
|
|
74
74
|
# migration (the store is wholly re-derivable).
|
|
75
75
|
CODEX_CONVERSATION_CONTRACT_VERSION = "5"
|
|
76
76
|
|
|
77
|
+
# Byte-zero Codex replay markers (spec
|
|
78
|
+
# docs/superpowers/specs/2026-07-30-codex-thread-source-inference-design.md
|
|
79
|
+
# §4.3). One `cache_meta` key per store, armed by cache migration 035 /
|
|
80
|
+
# conversations migration 002 and consumed inside the sync function that owns
|
|
81
|
+
# the correct replay semantics. They live in this kernel because three layers
|
|
82
|
+
# read them — the cache glue that consumes them, the read-side authority probe,
|
|
83
|
+
# and the doctor I/O shell — and a SQL string literal in any one of them would
|
|
84
|
+
# silently outlive a rename of the others.
|
|
85
|
+
CODEX_REPLAY_FROM_ZERO_KEY = "codex_replay_from_zero_pending"
|
|
86
|
+
CODEX_CONVERSATION_REPLAY_FROM_ZERO_KEY = (
|
|
87
|
+
"codex_conversation_replay_from_zero_pending"
|
|
88
|
+
)
|
|
89
|
+
# Written when a whole-tree Codex sync completed and still could NOT consume
|
|
90
|
+
# `CODEX_REPLAY_FROM_ZERO_KEY` — the replay is stalled, not merely pending, and
|
|
91
|
+
# Codex transcript ingest is deferred behind it. Read by `doctor`.
|
|
92
|
+
CODEX_REPLAY_BLOCKED_KEY = "codex_replay_from_zero_blocked"
|
|
93
|
+
|
|
77
94
|
# Structural wrapper prefixes skipped during title selection (§4.3), pinned from
|
|
78
95
|
# the corpus (title-wrapper-window). Prefix-structural, never content heuristics.
|
|
79
96
|
CODEX_TITLE_SKIP_PREFIXES: tuple[str, ...] = (
|
|
@@ -35,6 +35,13 @@ from _lib_pricing import _calculate_codex_entry_cost
|
|
|
35
35
|
# authoritative; a held-lock deferral leaves it pending.
|
|
36
36
|
CODEX_NORMALIZATION_MIGRATION = "025_codex_conversation_normalization"
|
|
37
37
|
|
|
38
|
+
# The provider-local rebuild marker migration 028's byte-zero replay arms. Its
|
|
39
|
+
# sibling — the thread_source-inference replay marker — is
|
|
40
|
+
# ``kern.CODEX_CONVERSATION_REPLAY_FROM_ZERO_KEY``; both are named, never
|
|
41
|
+
# inlined as SQL literals, so a rename cannot leave this probe testing a key
|
|
42
|
+
# nothing writes any more.
|
|
43
|
+
CODEX_CONTRACT_REBUILD_MARKER = "conversation_rebuild_codex_pending"
|
|
44
|
+
|
|
38
45
|
# Domain separations for the opaque item-key encoding (§5.2). The source-path
|
|
39
46
|
# fingerprint is a domain-separated hash, NEVER a raw path (privacy-safe).
|
|
40
47
|
CODEX_ITEM_KEY_DOMAIN = b"cctally-codex-item-key-v1\0"
|
|
@@ -93,6 +100,12 @@ def codex_normalization_authoritative(conn: sqlite3.Connection) -> bool:
|
|
|
93
100
|
Split stores use their provider-local rebuild marker: current schema alone
|
|
94
101
|
is not authority while migration 028's byte-zero replay is pending. Legacy
|
|
95
102
|
monolithic/bare connections retain the migration-025 stamp contract.
|
|
103
|
+
|
|
104
|
+
EITHER pending marker withholds authority. The thread_source-inference
|
|
105
|
+
replay (conversations migration 002) is armed by its own key precisely
|
|
106
|
+
because the contract replay must not consume it, so a probe that tested only
|
|
107
|
+
the contract marker would report a not-yet-repaired store as authoritative
|
|
108
|
+
to every ``--no-sync`` read.
|
|
96
109
|
"""
|
|
97
110
|
try:
|
|
98
111
|
split = conn.execute(
|
|
@@ -101,8 +114,9 @@ def codex_normalization_authoritative(conn: sqlite3.Connection) -> bool:
|
|
|
101
114
|
).fetchone() is not None
|
|
102
115
|
if split:
|
|
103
116
|
pending = conn.execute(
|
|
104
|
-
"SELECT 1 FROM cache_meta "
|
|
105
|
-
|
|
117
|
+
"SELECT 1 FROM cache_meta WHERE key IN (?,?) LIMIT 1",
|
|
118
|
+
(CODEX_CONTRACT_REBUILD_MARKER,
|
|
119
|
+
kern.CODEX_CONVERSATION_REPLAY_FROM_ZERO_KEY),
|
|
106
120
|
).fetchone() is not None
|
|
107
121
|
version = conn.execute(
|
|
108
122
|
"SELECT value FROM cache_meta "
|
|
@@ -255,7 +255,11 @@ def _maybe_prune_conversation_retention(
|
|
|
255
255
|
skipped (retention disabled, throttled within 24h, or a lock contended).
|
|
256
256
|
|
|
257
257
|
Concurrency (F7): a dedicated non-blocking MAINTENANCE flock serializes prune
|
|
258
|
-
attempts across processes (a second dashboard skips cleanly).
|
|
258
|
+
attempts across processes (a second dashboard skips cleanly). It is claimed
|
|
259
|
+
EXCLUSIVE and then downgraded to SHARED for the pass proper, so a long prune
|
|
260
|
+
cannot starve the fail-closed panel readers that sample it
|
|
261
|
+
``LOCK_SH | LOCK_NB``; a rival ``LOCK_EX | LOCK_NB`` claim still fails
|
|
262
|
+
against the held SHARED, so the serialization is unchanged. Under it, the
|
|
259
263
|
two provider flocks are taken in a FIXED order (Claude then Codex),
|
|
260
264
|
non-blocking, so a rebuild/reingest mid-flight makes the prune skip this
|
|
261
265
|
cycle rather than race between candidate selection and deletion. The prune of
|
|
@@ -297,6 +301,26 @@ def _maybe_prune_conversation_retention(
|
|
|
297
301
|
fcntl.flock(codex_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
298
302
|
except (BlockingIOError, OSError):
|
|
299
303
|
return None # a Codex sync is mid-flight; retry next cycle
|
|
304
|
+
# Downgrade the maintenance flock to SHARED for the pass proper.
|
|
305
|
+
# The EXCLUSIVE acquire above is what wins the race; holding it
|
|
306
|
+
# exclusive for the whole pass additionally locked out the
|
|
307
|
+
# fail-CLOSED panel readers, which take this flock
|
|
308
|
+
# `LOCK_SH | LOCK_NB` and blank their column on any contention
|
|
309
|
+
# (`read_session_titles_bounded` -> every Claude session title in
|
|
310
|
+
# the Recent Sessions card, for the minutes a large prune runs).
|
|
311
|
+
# A prune is a writer, not a family replacement: concurrent
|
|
312
|
+
# writes are already excluded by the two provider flocks, and
|
|
313
|
+
# the replacement paths readers guard against take this flock
|
|
314
|
+
# EXCLUSIVE, which a held SHARED still blocks. Rival prunes and
|
|
315
|
+
# `db vacuum` claim `LOCK_EX | LOCK_NB`, which also still fails.
|
|
316
|
+
# MUST stay after both provider flocks: a flock conversion is
|
|
317
|
+
# not atomic, so a rival can slip into the downgrade window —
|
|
318
|
+
# the provider flocks it then fails to claim are what make that
|
|
319
|
+
# harmless.
|
|
320
|
+
try:
|
|
321
|
+
fcntl.flock(maint_fh, fcntl.LOCK_SH)
|
|
322
|
+
except OSError:
|
|
323
|
+
pass # keep the exclusive hold; correctness is unchanged
|
|
300
324
|
cutoff = now_utc - dt.timedelta(days=int(retention_days))
|
|
301
325
|
conn.execute("BEGIN IMMEDIATE")
|
|
302
326
|
try:
|
|
@@ -22,7 +22,14 @@ CapabilityStatus = Literal[
|
|
|
22
22
|
"supported", "derived", "unavailable", "deferred", "not_applicable",
|
|
23
23
|
]
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
# #429 §3.2 — bumped to 2 because `active[].captured_at` changed MEANING (the
|
|
26
|
+
# newest physical observation, not the interpreted baseline). The in-place
|
|
27
|
+
# update flow `execvp`s the server while the already-loaded client reconnects
|
|
28
|
+
# over its existing EventSource without reloading its JS
|
|
29
|
+
# (`UpdateRunningModal.tsx`, `store/sse.ts`), so an old client demonstrably
|
|
30
|
+
# does meet a new server; `docs/cli-contract.md` calls changing a value's
|
|
31
|
+
# meaning breaking. Version-aware client reaction is deliberately deferred.
|
|
32
|
+
SOURCE_SCHEMA_VERSION = 2
|
|
26
33
|
DEFAULT_SOURCE = "claude"
|
|
27
34
|
SOURCE_ORDER = ("claude", "codex", "all")
|
|
28
35
|
SOURCE_FRESHNESS_DOMAINS = ("hero", "quota", "sessions")
|
package/bin/_lib_doctor.py
CHANGED
|
@@ -189,6 +189,14 @@ class DoctorState:
|
|
|
189
189
|
# by the next whole-tree sync that defers none. None = key absent (the
|
|
190
190
|
# normal state) or cache unreadable — the check degrades OK.
|
|
191
191
|
codex_torn_deferred: Optional[dict] = None
|
|
192
|
+
# The byte-zero Codex replay's stall signal. `codex_replay_pending` mirrors
|
|
193
|
+
# the `codex_replay_from_zero_pending` cache_meta marker; `codex_replay_blocked`
|
|
194
|
+
# is `{"at": iso, "files_failed": N, "files_deferred_torn": N}`, written by a
|
|
195
|
+
# whole-tree Codex sync that RAN and still could not consume it, and deleted
|
|
196
|
+
# by the next clean whole-tree sync. Both None = key absent (the normal
|
|
197
|
+
# state) or cache unreadable — the check degrades OK.
|
|
198
|
+
codex_replay_pending: Optional[bool] = None
|
|
199
|
+
codex_replay_blocked: Optional[dict] = None
|
|
192
200
|
# #279 S2 (F5b): PRAGMA quick_check(1) results, gathered ONLY under
|
|
193
201
|
# doctor_gather_state(deep=True) (CLI cmd_doctor) — the dashboard
|
|
194
202
|
# rebuild loop calls the gather every rebuild and quick_check on a
|
|
@@ -1107,6 +1115,50 @@ def _check_data_codex_cache(s: DoctorState) -> CheckResult:
|
|
|
1107
1115
|
)
|
|
1108
1116
|
|
|
1109
1117
|
|
|
1118
|
+
def _check_data_codex_replay(s: DoctorState) -> CheckResult:
|
|
1119
|
+
"""WARN while a byte-zero Codex replay is STALLED rather than merely pending.
|
|
1120
|
+
|
|
1121
|
+
`sync_codex_conversations` defers on the cache-side replay marker, so as long
|
|
1122
|
+
as that marker stands no Codex transcript is ingested at all. The deferral is
|
|
1123
|
+
protective — running ahead of the replayed thread rows stamps a materialized
|
|
1124
|
+
`"(unassigned)"` project the read path then prefers permanently — so the fix
|
|
1125
|
+
is never to drop it. But a whole-tree sync that runs and still cannot consume
|
|
1126
|
+
the marker (a persistently torn `auth.json`, a repeated per-file DB error)
|
|
1127
|
+
holds that deferral open indefinitely, and `cache-sync` still exits 0. This
|
|
1128
|
+
leg is the only standing signal that Codex transcript ingest has stopped.
|
|
1129
|
+
|
|
1130
|
+
A marker that is merely pending is NOT reported: it is the ordinary state
|
|
1131
|
+
between the migration and the next sync, and it clears on its own. The WARN
|
|
1132
|
+
needs the durable `blocked` record, which only a completed-but-unsuccessful
|
|
1133
|
+
whole-tree walk writes.
|
|
1134
|
+
"""
|
|
1135
|
+
blocked = s.codex_replay_blocked or {}
|
|
1136
|
+
at = blocked.get("at")
|
|
1137
|
+
details = {
|
|
1138
|
+
"pending": bool(s.codex_replay_pending),
|
|
1139
|
+
"blocked_at": at,
|
|
1140
|
+
"files_failed": blocked.get("files_failed"),
|
|
1141
|
+
"files_deferred_torn": blocked.get("files_deferred_torn"),
|
|
1142
|
+
}
|
|
1143
|
+
if s.codex_replay_pending and isinstance(at, str) and at:
|
|
1144
|
+
return CheckResult(
|
|
1145
|
+
id="data.codex_replay", title="Codex transcript replay",
|
|
1146
|
+
severity="warn",
|
|
1147
|
+
summary=(f"stalled since {at} — Codex transcript ingest is "
|
|
1148
|
+
"deferred until it completes"),
|
|
1149
|
+
remediation=("Check the Codex auth.json (re-run `codex login` if it "
|
|
1150
|
+
"is truncated), then `cctally cache-sync --source codex`"),
|
|
1151
|
+
details=details,
|
|
1152
|
+
)
|
|
1153
|
+
return CheckResult(
|
|
1154
|
+
id="data.codex_replay", title="Codex transcript replay",
|
|
1155
|
+
severity="ok",
|
|
1156
|
+
summary="pending (clears on the next Codex sync)"
|
|
1157
|
+
if s.codex_replay_pending else "none pending",
|
|
1158
|
+
remediation=None, details=details,
|
|
1159
|
+
)
|
|
1160
|
+
|
|
1161
|
+
|
|
1110
1162
|
def _check_data_codex_project_metadata(s: DoctorState) -> CheckResult:
|
|
1111
1163
|
"""Report the identity-safe all-history Codex metadata partition."""
|
|
1112
1164
|
if s.codex_project_metadata_error is not None:
|
|
@@ -2680,7 +2732,12 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
2680
2732
|
("data.statusline_pipeline", "_check_statusline_pipeline"),
|
|
2681
2733
|
("data.cache_sync_state", "_check_data_cache_sync_state"),
|
|
2682
2734
|
("data.codex_cache", "_check_data_codex_cache"),
|
|
2735
|
+
# `data.codex_project_metadata` must stay IMMEDIATELY after
|
|
2736
|
+
# `data.codex_cache` (pinned by
|
|
2737
|
+
# tests/test_doctor_codex_project_metadata.py), so the replay leg goes
|
|
2738
|
+
# after the pair rather than between them.
|
|
2683
2739
|
("data.codex_project_metadata", "_check_data_codex_project_metadata"),
|
|
2740
|
+
("data.codex_replay", "_check_data_codex_replay"),
|
|
2684
2741
|
("data.codex_quota", "_check_data_codex_quota"),
|
|
2685
2742
|
("data.parse_health", "_check_data_parse_health"),
|
|
2686
2743
|
("data.forked_buckets", "_check_data_forked_buckets"),
|
package/bin/_lib_jsonl.py
CHANGED
|
@@ -549,15 +549,48 @@ def _canonical_container(value: object) -> str | None:
|
|
|
549
549
|
return _codex_canonical_json(value)
|
|
550
550
|
|
|
551
551
|
|
|
552
|
+
# Codex is mid-rollout on `thread_source`: Desktop 0.146.0-alpha.3.1 emits it,
|
|
553
|
+
# the CLI 0.146.0 behind the MCP server does not. Abandoning the identity when
|
|
554
|
+
# it is absent costs the rollout its thread row AND every normalized message,
|
|
555
|
+
# so infer the category the provider would most likely emit instead. Inferring
|
|
556
|
+
# rather than minting a null parent keeps one identity across the release that
|
|
557
|
+
# starts emitting the field.
|
|
558
|
+
_CODEX_DEFAULT_THREAD_SOURCE = "user"
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def _inferred_codex_thread_source(payload: dict[str, Any]) -> str:
|
|
562
|
+
"""The thread-origin category for one ``session_meta`` payload.
|
|
563
|
+
|
|
564
|
+
Per-record and stateless: a later ``session_meta`` never inherits an earlier
|
|
565
|
+
one's category, so a file's key sequence is a function of its own bytes.
|
|
566
|
+
The string form of ``source`` is deliberately never used — ``source:
|
|
567
|
+
"vscode"`` co-occurs with ``thread_source: "user"``, so the client name and
|
|
568
|
+
the origin category are orthogonal vocabularies.
|
|
569
|
+
"""
|
|
570
|
+
explicit = _codex_string(payload.get("thread_source"))
|
|
571
|
+
if explicit is not None:
|
|
572
|
+
return explicit
|
|
573
|
+
source = payload.get("source")
|
|
574
|
+
if isinstance(source, dict) and len(source) == 1:
|
|
575
|
+
# A single-key object is the shape Codex uses for `{"subagent": {...}}`.
|
|
576
|
+
# Guard every malformed variant: the identity encoder REJECTS an empty
|
|
577
|
+
# parent key, so an unguarded rule turns bad metadata into an ingest
|
|
578
|
+
# exception rather than a degraded-but-working ingest.
|
|
579
|
+
only_key = _codex_string(next(iter(source)))
|
|
580
|
+
if only_key is not None:
|
|
581
|
+
return only_key
|
|
582
|
+
return _CODEX_DEFAULT_THREAD_SOURCE
|
|
583
|
+
|
|
584
|
+
|
|
552
585
|
def _thread_metadata_from_session_meta(
|
|
553
586
|
payload: dict[str, Any], path_str: str, source_root_key: str | None,
|
|
554
587
|
) -> CodexThreadMetadata:
|
|
555
588
|
accounting_id = _codex_string(payload.get("id"))
|
|
556
589
|
native_thread_id = _codex_string(payload.get("session_id")) or accounting_id
|
|
557
|
-
root_thread_id =
|
|
590
|
+
root_thread_id = _inferred_codex_thread_source(payload)
|
|
558
591
|
parent_thread_id = _codex_string(payload.get("forked_from_id"))
|
|
559
592
|
conversation_key = None
|
|
560
|
-
if native_thread_id is not None
|
|
593
|
+
if native_thread_id is not None:
|
|
561
594
|
conversation_key = canonical_identity_from_root_key(
|
|
562
595
|
"codex", "conversation", source_root_key, native_thread_id, root_thread_id
|
|
563
596
|
)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cctally",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.88.1",
|
|
4
4
|
"description": "Claude Code usage tracker and local dashboard for Pro/Max subscription limits - weekly cost-per-percent trend, quota forecasts, threshold alerts. ccusage-compatible.",
|
|
5
5
|
"homepage": "https://github.com/omrikais/cctally",
|
|
6
6
|
"repository": {
|