cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
|
@@ -21,6 +21,12 @@ from _cctally_core import (
|
|
|
21
21
|
)
|
|
22
22
|
from _cctally_cache import _codex_provider_roots
|
|
23
23
|
import _lib_accounts
|
|
24
|
+
from _lib_codex_metadata import (
|
|
25
|
+
DecodedCodexProjectMetadata,
|
|
26
|
+
codex_metadata_is_malformed,
|
|
27
|
+
decode_codex_project_metadata,
|
|
28
|
+
resolve_codex_threads_table,
|
|
29
|
+
)
|
|
24
30
|
from _lib_quota import build_blocks
|
|
25
31
|
from _lib_pricing import _calculate_codex_entry_cost
|
|
26
32
|
from _cctally_quota import load_codex_quota_observations
|
|
@@ -46,7 +52,22 @@ UTC = dt.timezone.utc
|
|
|
46
52
|
|
|
47
53
|
|
|
48
54
|
class QualifiedMetadataUnavailable(RuntimeError):
|
|
49
|
-
"""S1-qualified metadata is absent or cannot be truthfully read.
|
|
55
|
+
"""S1-qualified metadata is absent or cannot be truthfully read.
|
|
56
|
+
|
|
57
|
+
#834 S2 (#829): ``transient`` says which of those two it is. A failure of
|
|
58
|
+
the READ — a SQLite error, a contended ingest lock, a cache that would not
|
|
59
|
+
open — can succeed on the next refresh and must NOT tell the reader to
|
|
60
|
+
rebuild their Codex cache. A row that is deterministically unqualifiable
|
|
61
|
+
will still be unqualifiable next time, and rebuilding IS its remedy.
|
|
62
|
+
|
|
63
|
+
The default is ``True`` because an unclassified failure is safer treated
|
|
64
|
+
as transient: naming a rebuild that would not help is worse than saying a
|
|
65
|
+
build will retry.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
def __init__(self, *args: object, transient: bool = True) -> None:
|
|
69
|
+
super().__init__(*args)
|
|
70
|
+
self.transient = bool(transient)
|
|
50
71
|
|
|
51
72
|
|
|
52
73
|
class SourceUsageError(ValueError):
|
|
@@ -61,10 +82,21 @@ class CodexProjectMetadataHealth:
|
|
|
61
82
|
qualified_rows: int
|
|
62
83
|
missing_conversation_key_rows: int
|
|
63
84
|
missing_thread_join_rows: int
|
|
85
|
+
#: #845 §4.3. Entries whose project metadata is present but undecodable,
|
|
86
|
+
#: counted over THIS population and net of whatever this population's
|
|
87
|
+
#: aggregate already counted for another reason.
|
|
88
|
+
undecodable_metadata_rows: int = 0
|
|
89
|
+
#: Rows with a refused timestamp or token operand, net of key/join reasons.
|
|
90
|
+
malformed_accounting_rows: int = 0
|
|
64
91
|
|
|
65
92
|
@property
|
|
66
93
|
def incomplete_rows(self) -> int:
|
|
67
|
-
return
|
|
94
|
+
return (
|
|
95
|
+
self.missing_conversation_key_rows
|
|
96
|
+
+ self.missing_thread_join_rows
|
|
97
|
+
+ self.undecodable_metadata_rows
|
|
98
|
+
+ self.malformed_accounting_rows
|
|
99
|
+
)
|
|
68
100
|
|
|
69
101
|
|
|
70
102
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -86,6 +118,11 @@ class RootedCodexAccountingEntry:
|
|
|
86
118
|
# reserved ``unattributed`` sentinel). Carried so the per-account hero cards
|
|
87
119
|
# can scope spend without a second query shape.
|
|
88
120
|
account_key: str = _lib_accounts.UNATTRIBUTED
|
|
121
|
+
# #845 §4.4: the per-conversation identity the accounting-identity map is
|
|
122
|
+
# keyed by. A path-keyed map cannot carry a per-conversation decision,
|
|
123
|
+
# because one rollout path may host a healthy and a malformed conversation
|
|
124
|
+
# at once and one decision would be wrong for one of them.
|
|
125
|
+
conversation_key: str = ""
|
|
89
126
|
|
|
90
127
|
|
|
91
128
|
_OPAQUE_PROJECT_KEY_RE = re.compile(r"^project:[0-9a-f]{24}$")
|
|
@@ -97,7 +134,9 @@ _QUALIFIED_CODEX_ENTRIES_SQL = """
|
|
|
97
134
|
entries.conversation_key, entries.model, entries.account_key,
|
|
98
135
|
entries.input_tokens, entries.cached_input_tokens,
|
|
99
136
|
entries.output_tokens, entries.reasoning_output_tokens,
|
|
100
|
-
entries.total_tokens,
|
|
137
|
+
entries.total_tokens,
|
|
138
|
+
CAST(threads.cwd AS BLOB) AS cwd_blob,
|
|
139
|
+
CAST(threads.git_json AS BLOB) AS git_json_blob,
|
|
101
140
|
threads.conversation_key AS joined_conversation_key,
|
|
102
141
|
threads.source_root_key AS joined_source_root_key,
|
|
103
142
|
entries.id AS cache_entry_id
|
|
@@ -128,33 +167,166 @@ def _qualified_codex_path_entries_sql(identity_count: int) -> str:
|
|
|
128
167
|
)
|
|
129
168
|
|
|
130
169
|
|
|
131
|
-
|
|
132
|
-
|
|
170
|
+
#: The thread leg of the decode inventory (§4.3 item 1). ``{threads}`` is the
|
|
171
|
+
#: raw table reference ``resolve_codex_threads_table`` supplies at call time, so
|
|
172
|
+
#: a conversations connection reads the attached cache rather than failing, and
|
|
173
|
+
#: an account-scoped TEMP view over the unqualified name cannot hide the rows.
|
|
174
|
+
#: Measured on the production store: 2,646 rows, 2.2 ms.
|
|
175
|
+
_CODEX_THREAD_INVENTORY_SQL = """
|
|
176
|
+
SELECT conversation_key, source_root_key, native_thread_id, last_seen_utc,
|
|
177
|
+
CAST(cwd AS BLOB) AS cwd_blob,
|
|
178
|
+
CAST(git_json AS BLOB) AS git_json_blob
|
|
179
|
+
FROM {threads}
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
#: The alias leg (§4.3 item 2). Issued only when this cache generation carries
|
|
183
|
+
#: ``last_native_thread_id``. Measured: 2,986 rows, millisecond-scale.
|
|
184
|
+
_CODEX_FILE_ALIAS_INVENTORY_SQL = """
|
|
185
|
+
SELECT files.source_root_key, files.path, files.last_native_thread_id
|
|
133
186
|
FROM codex_session_files AS files
|
|
134
|
-
JOIN codex_conversation_threads AS inherited
|
|
135
|
-
ON inherited.source_root_key = files.source_root_key
|
|
136
|
-
AND inherited.native_thread_id = files.last_native_thread_id
|
|
137
187
|
WHERE files.last_native_thread_id IS NOT NULL
|
|
138
188
|
AND files.last_native_thread_id != ''
|
|
139
|
-
ORDER BY inherited.last_seen_utc DESC, inherited.conversation_key DESC
|
|
140
189
|
"""
|
|
141
190
|
|
|
142
191
|
|
|
143
|
-
def
|
|
144
|
-
"""Constrain
|
|
192
|
+
def _codex_alias_inventory_sql(identity_count: int) -> str:
|
|
193
|
+
"""Constrain the alias leg to dirty physical paths."""
|
|
145
194
|
if identity_count < 1:
|
|
146
195
|
raise ValueError("identity_count must be positive")
|
|
147
196
|
predicates = " OR ".join(
|
|
148
197
|
"(files.source_root_key = ? AND files.path = ?)"
|
|
149
198
|
for _ in range(identity_count)
|
|
150
199
|
)
|
|
151
|
-
return
|
|
200
|
+
return _CODEX_FILE_ALIAS_INVENTORY_SQL.replace(
|
|
152
201
|
" WHERE files.last_native_thread_id IS NOT NULL",
|
|
153
202
|
f" WHERE ({predicates})\n"
|
|
154
203
|
" AND files.last_native_thread_id IS NOT NULL",
|
|
155
204
|
)
|
|
156
205
|
|
|
157
206
|
|
|
207
|
+
def _alias_winner_sort_key(last_seen_utc: object, conversation_key: object) -> tuple:
|
|
208
|
+
"""Reproduce ``ORDER BY last_seen_utc DESC, conversation_key DESC``.
|
|
209
|
+
|
|
210
|
+
The retired statement ordered its joined rows that way and the qualifier
|
|
211
|
+
kept the FIRST row per ``(source_root_key, path)``, so the winner is the
|
|
212
|
+
maximum under this key. SQLite sorts NULL FIRST ascending, hence LAST
|
|
213
|
+
descending, so a NULL ``last_seen_utc`` loses to every stored value; the
|
|
214
|
+
leading flag is what reproduces that, and ``max`` is what reproduces
|
|
215
|
+
"first row wins".
|
|
216
|
+
"""
|
|
217
|
+
has_last_seen = 0 if last_seen_utc is None else 1
|
|
218
|
+
return (
|
|
219
|
+
has_last_seen,
|
|
220
|
+
"" if last_seen_utc is None else str(last_seen_utc),
|
|
221
|
+
str(conversation_key or ""),
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@dataclasses.dataclass(frozen=True)
|
|
226
|
+
class CodexThreadMetadataInventory:
|
|
227
|
+
"""One linear pass over the thread and alias tables (§4.3).
|
|
228
|
+
|
|
229
|
+
This replaces the SQL join in the retired
|
|
230
|
+
``_INHERITED_CODEX_PROJECT_METADATA_SQL``, whose plan on the current schema
|
|
231
|
+
is ``SCAN files`` then ``SEARCH inherited USING INDEX
|
|
232
|
+
idx_codex_threads_source_root``: on a one-root machine every file row
|
|
233
|
+
rescans all 2,646 threads and the statement costs 1,362 ms. No index is
|
|
234
|
+
added, because a new index needs a real cache migration, which M4 excludes.
|
|
235
|
+
|
|
236
|
+
``undecodable_direct`` and ``undecodable_inherited`` are what make the
|
|
237
|
+
counting of §4.3 zero-cost on a clean store: both are empty, so no
|
|
238
|
+
per-identity query runs at all.
|
|
239
|
+
"""
|
|
240
|
+
|
|
241
|
+
direct_by_conversation: "dict[tuple[str, str], DecodedCodexProjectMetadata]"
|
|
242
|
+
inherited_by_path: "dict[tuple[str, str], DecodedCodexProjectMetadata]"
|
|
243
|
+
inherited_winner_by_path: "dict[tuple[str, str], tuple[str, str]]"
|
|
244
|
+
undecodable_direct: "frozenset[tuple[str, str]]"
|
|
245
|
+
undecodable_inherited: "frozenset[tuple[str, str]]"
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def load_codex_thread_metadata_inventory(
|
|
249
|
+
conn, *, paths: "Iterable[tuple[str, str]] | None" = None,
|
|
250
|
+
) -> CodexThreadMetadataInventory:
|
|
251
|
+
"""Read the thread and alias tables once each and decode every row.
|
|
252
|
+
|
|
253
|
+
``paths`` bounds only the ALIAS leg, to the ``(source_root_key, path)``
|
|
254
|
+
identities a delta-resumed read actually asked for. The thread leg stays
|
|
255
|
+
whole-table because the alias winner for any path can be any thread sharing
|
|
256
|
+
its ``last_native_thread_id``.
|
|
257
|
+
"""
|
|
258
|
+
threads_table = resolve_codex_threads_table(conn)
|
|
259
|
+
direct_by_conversation: "dict[tuple[str, str], DecodedCodexProjectMetadata]" = {}
|
|
260
|
+
candidates_by_native: "dict[tuple[str, str], tuple]" = {}
|
|
261
|
+
undecodable_direct: "set[tuple[str, str]]" = set()
|
|
262
|
+
for row in conn.execute(
|
|
263
|
+
_CODEX_THREAD_INVENTORY_SQL.format(threads=threads_table)
|
|
264
|
+
):
|
|
265
|
+
(
|
|
266
|
+
conversation_key, root_key, native_thread_id, last_seen_utc,
|
|
267
|
+
cwd_blob, git_json_blob,
|
|
268
|
+
) = row
|
|
269
|
+
identity = (str(root_key or ""), str(conversation_key or ""))
|
|
270
|
+
decoded = decode_codex_project_metadata(cwd_blob, git_json_blob)
|
|
271
|
+
if all(identity):
|
|
272
|
+
direct_by_conversation[identity] = decoded
|
|
273
|
+
if decoded.undecodable_fields:
|
|
274
|
+
undecodable_direct.add(identity)
|
|
275
|
+
native_identity = (identity[0], str(native_thread_id or ""))
|
|
276
|
+
if not all(native_identity):
|
|
277
|
+
continue
|
|
278
|
+
sort_key = _alias_winner_sort_key(last_seen_utc, conversation_key)
|
|
279
|
+
current = candidates_by_native.get(native_identity)
|
|
280
|
+
if current is None or sort_key > current[0]:
|
|
281
|
+
candidates_by_native[native_identity] = (
|
|
282
|
+
sort_key, str(conversation_key or ""), decoded,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
inherited_by_path: "dict[tuple[str, str], DecodedCodexProjectMetadata]" = {}
|
|
286
|
+
inherited_winner_by_path: "dict[tuple[str, str], tuple[str, str]]" = {}
|
|
287
|
+
undecodable_inherited: "set[tuple[str, str]]" = set()
|
|
288
|
+
if _supports_native_file_aliases(conn):
|
|
289
|
+
identities = (
|
|
290
|
+
None if paths is None
|
|
291
|
+
else tuple(sorted({
|
|
292
|
+
(str(root), str(path)) for root, path in paths
|
|
293
|
+
if str(root) and str(path)
|
|
294
|
+
}))
|
|
295
|
+
)
|
|
296
|
+
if identities is not None and not identities:
|
|
297
|
+
alias_rows: tuple = ()
|
|
298
|
+
else:
|
|
299
|
+
alias_rows = tuple(conn.execute(
|
|
300
|
+
_CODEX_FILE_ALIAS_INVENTORY_SQL if identities is None
|
|
301
|
+
else _codex_alias_inventory_sql(len(identities)),
|
|
302
|
+
tuple(
|
|
303
|
+
value for identity in (identities or ()) for value in identity
|
|
304
|
+
),
|
|
305
|
+
))
|
|
306
|
+
for root_key, path, native_thread_id in alias_rows:
|
|
307
|
+
identity = (str(root_key or ""), str(path or ""))
|
|
308
|
+
if not all(identity):
|
|
309
|
+
continue
|
|
310
|
+
winner = candidates_by_native.get(
|
|
311
|
+
(identity[0], str(native_thread_id or "")))
|
|
312
|
+
if winner is None:
|
|
313
|
+
continue
|
|
314
|
+
_sort_key, winning_conversation, decoded = winner
|
|
315
|
+
inherited_by_path[identity] = decoded
|
|
316
|
+
inherited_winner_by_path[identity] = (
|
|
317
|
+
identity[0], winning_conversation)
|
|
318
|
+
if decoded.undecodable_fields:
|
|
319
|
+
undecodable_inherited.add(identity)
|
|
320
|
+
|
|
321
|
+
return CodexThreadMetadataInventory(
|
|
322
|
+
direct_by_conversation=direct_by_conversation,
|
|
323
|
+
inherited_by_path=inherited_by_path,
|
|
324
|
+
inherited_winner_by_path=inherited_winner_by_path,
|
|
325
|
+
undecodable_direct=frozenset(undecodable_direct),
|
|
326
|
+
undecodable_inherited=frozenset(undecodable_inherited),
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
|
|
158
330
|
_CODEX_ACCOUNTING_ENTRIES_SQL = """
|
|
159
331
|
SELECT timestamp_utc, source_root_key, conversation_key, model,
|
|
160
332
|
input_tokens, cached_input_tokens, output_tokens,
|
|
@@ -178,7 +350,7 @@ _CODEX_ACCOUNTING_ENTRIES_SQL = """
|
|
|
178
350
|
_ROOTED_CODEX_ACCOUNTING_ENTRIES_SQL = """
|
|
179
351
|
SELECT timestamp_utc, session_id, source_path, source_root_key, model,
|
|
180
352
|
input_tokens, cached_input_tokens, output_tokens,
|
|
181
|
-
reasoning_output_tokens, total_tokens, account_key
|
|
353
|
+
reasoning_output_tokens, total_tokens, account_key, conversation_key
|
|
182
354
|
FROM codex_session_entries INDEXED BY idx_codex_entries_ts_root_conversation
|
|
183
355
|
WHERE timestamp_utc >= ?
|
|
184
356
|
AND timestamp_utc < ?
|
|
@@ -206,7 +378,26 @@ _CODEX_PROJECT_METADATA_HEALTH_SQL = """
|
|
|
206
378
|
WHERE files.path = entries.source_path
|
|
207
379
|
AND files.source_root_key = entries.source_root_key
|
|
208
380
|
)
|
|
209
|
-
THEN 1 ELSE 0 END), 0) AS missing_thread_join_rows
|
|
381
|
+
THEN 1 ELSE 0 END), 0) AS missing_thread_join_rows,
|
|
382
|
+
COALESCE(SUM(CASE
|
|
383
|
+
WHEN entries.conversation_key IS NOT NULL
|
|
384
|
+
AND entries.conversation_key != ''
|
|
385
|
+
AND NOT (
|
|
386
|
+
threads.conversation_key IS NULL
|
|
387
|
+
AND NOT EXISTS (
|
|
388
|
+
SELECT 1 FROM codex_session_files AS files
|
|
389
|
+
JOIN codex_conversation_threads AS inherited
|
|
390
|
+
ON inherited.source_root_key = entries.source_root_key
|
|
391
|
+
AND inherited.native_thread_id = files.last_native_thread_id
|
|
392
|
+
WHERE files.path = entries.source_path
|
|
393
|
+
AND files.source_root_key = entries.source_root_key
|
|
394
|
+
)
|
|
395
|
+
)
|
|
396
|
+
AND codex_accounting_row_usable(
|
|
397
|
+
entries.timestamp_utc, entries.input_tokens,
|
|
398
|
+
entries.cached_input_tokens, entries.output_tokens,
|
|
399
|
+
entries.reasoning_output_tokens, entries.total_tokens) = 0
|
|
400
|
+
THEN 1 ELSE 0 END), 0) AS malformed_accounting_rows
|
|
210
401
|
FROM codex_session_entries AS entries
|
|
211
402
|
LEFT JOIN codex_conversation_threads AS threads
|
|
212
403
|
ON threads.conversation_key = entries.conversation_key
|
|
@@ -217,6 +408,37 @@ _CODEX_PROJECT_METADATA_HEALTH_SQL = """
|
|
|
217
408
|
|
|
218
409
|
|
|
219
410
|
_CODEX_PROJECT_METADATA_HEALTH_LEGACY_SQL = """
|
|
411
|
+
SELECT
|
|
412
|
+
COUNT(*) AS total_rows,
|
|
413
|
+
COALESCE(SUM(CASE
|
|
414
|
+
WHEN entries.conversation_key IS NULL OR entries.conversation_key = ''
|
|
415
|
+
THEN 1 ELSE 0 END), 0) AS missing_conversation_key_rows,
|
|
416
|
+
COALESCE(SUM(CASE
|
|
417
|
+
WHEN entries.conversation_key IS NOT NULL
|
|
418
|
+
AND entries.conversation_key != ''
|
|
419
|
+
AND threads.conversation_key IS NULL
|
|
420
|
+
THEN 1 ELSE 0 END), 0) AS missing_thread_join_rows,
|
|
421
|
+
COALESCE(SUM(CASE
|
|
422
|
+
WHEN entries.conversation_key IS NOT NULL
|
|
423
|
+
AND entries.conversation_key != ''
|
|
424
|
+
AND threads.conversation_key IS NOT NULL
|
|
425
|
+
AND codex_accounting_row_usable(
|
|
426
|
+
entries.timestamp_utc, entries.input_tokens,
|
|
427
|
+
entries.cached_input_tokens, entries.output_tokens,
|
|
428
|
+
entries.reasoning_output_tokens, entries.total_tokens) = 0
|
|
429
|
+
THEN 1 ELSE 0 END), 0) AS malformed_accounting_rows
|
|
430
|
+
FROM codex_session_entries AS entries
|
|
431
|
+
LEFT JOIN codex_conversation_threads AS threads
|
|
432
|
+
ON threads.conversation_key = entries.conversation_key
|
|
433
|
+
AND threads.source_root_key = entries.source_root_key
|
|
434
|
+
WHERE (? IS NULL OR entries.timestamp_utc >= ?)
|
|
435
|
+
AND (? IS NULL OR entries.timestamp_utc < ?)
|
|
436
|
+
"""
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
# A pre-native-alias cache may also predate the accounting operand columns.
|
|
440
|
+
# Doctor must still report its metadata partition instead of failing the read.
|
|
441
|
+
_CODEX_PROJECT_METADATA_HEALTH_PRE_OPERAND_SQL = """
|
|
220
442
|
SELECT
|
|
221
443
|
COUNT(*) AS total_rows,
|
|
222
444
|
COALESCE(SUM(CASE
|
|
@@ -236,12 +458,373 @@ _CODEX_PROJECT_METADATA_HEALTH_LEGACY_SQL = """
|
|
|
236
458
|
"""
|
|
237
459
|
|
|
238
460
|
|
|
239
|
-
|
|
240
|
-
|
|
461
|
+
# === #834 S2 (#828) — the bounded detail-horizon metadata probe ============
|
|
462
|
+
#
|
|
463
|
+
# `_CODEX_PROJECT_METADATA_HEALTH_SQL` above counts key, join and accounting
|
|
464
|
+
# operand failures. The detail probe answers the larger read horizon.
|
|
465
|
+
#
|
|
466
|
+
# The horizon is the other half. The generation's accounting window is roughly
|
|
467
|
+
# thirty calendar days while both detail routes read a YEAR, so a malformed row
|
|
468
|
+
# aged thirty-one to three hundred and sixty-five days set no flag at all and
|
|
469
|
+
# the page rendered in full with that row's cost silently absent.
|
|
470
|
+
#
|
|
471
|
+
# ONE LIMIT, stated rather than hidden. This probe and the qualified
|
|
472
|
+
# reader share the same TEXT range bounds, so a timestamp malformed enough to
|
|
473
|
+
# sort outside the window is invisible to BOTH — such a row also contributes to
|
|
474
|
+
# no figure and reaches no route, so the two agree about it by construction.
|
|
475
|
+
# The operand predicate below calls the reader's own timestamp and integer
|
|
476
|
+
# parsers, so SQLite's looser datetime and signed-digit grammars cannot turn a
|
|
477
|
+
# refused row into a healthy probe result.
|
|
478
|
+
|
|
479
|
+
#: The five integer operands the qualified reader parses. `total_tokens` is
|
|
480
|
+
#: included in the same translated conversion as the other four operands.
|
|
481
|
+
_CODEX_TOKEN_OPERANDS = (
|
|
482
|
+
"input_tokens", "cached_input_tokens", "output_tokens",
|
|
483
|
+
"reasoning_output_tokens", "total_tokens",
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _codex_accounting_row_usable(*values: object) -> int:
|
|
488
|
+
"""Use the reader's exact parsers for the six stored accounting values."""
|
|
489
|
+
try:
|
|
490
|
+
_parse_timestamp(values[0])
|
|
491
|
+
for value in values[1:]:
|
|
492
|
+
int(value)
|
|
493
|
+
except (QualifiedMetadataUnavailable, TypeError, ValueError, OverflowError):
|
|
494
|
+
return 0
|
|
495
|
+
return 1
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
_CODEX_ACCOUNTING_ROW_USABLE_SQL = (
|
|
499
|
+
"codex_accounting_row_usable(entries.timestamp_utc, "
|
|
500
|
+
+ ", ".join(f"entries.{column}" for column in _CODEX_TOKEN_OPERANDS)
|
|
501
|
+
+ ")"
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _register_codex_accounting_probe(cache_conn: sqlite3.Connection) -> None:
|
|
506
|
+
cache_conn.create_function(
|
|
507
|
+
"codex_accounting_row_usable", 6, _codex_accounting_row_usable,
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
_CODEX_DETAIL_PROBE_JOIN_SQL = """
|
|
511
|
+
entries.conversation_key IS NOT NULL
|
|
512
|
+
AND entries.conversation_key != ''
|
|
513
|
+
AND threads.conversation_key IS NULL
|
|
514
|
+
AND NOT EXISTS (
|
|
515
|
+
SELECT 1 FROM codex_session_files AS files
|
|
516
|
+
JOIN codex_conversation_threads AS inherited
|
|
517
|
+
ON inherited.source_root_key = entries.source_root_key
|
|
518
|
+
AND inherited.native_thread_id = files.last_native_thread_id
|
|
519
|
+
WHERE files.path = entries.source_path
|
|
520
|
+
AND files.source_root_key = entries.source_root_key
|
|
521
|
+
)"""
|
|
522
|
+
|
|
523
|
+
_CODEX_DETAIL_PROBE_JOIN_LEGACY_SQL = """
|
|
524
|
+
entries.conversation_key IS NOT NULL
|
|
525
|
+
AND entries.conversation_key != ''
|
|
526
|
+
AND threads.conversation_key IS NULL"""
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _codex_detail_metadata_probe_sql(cache_conn: sqlite3.Connection) -> str:
|
|
530
|
+
"""Compose the probe over whichever thread-alias shape this cache carries."""
|
|
531
|
+
_register_codex_accounting_probe(cache_conn)
|
|
532
|
+
join_predicate = (
|
|
533
|
+
_CODEX_DETAIL_PROBE_JOIN_SQL if _supports_native_file_aliases(cache_conn)
|
|
534
|
+
else _CODEX_DETAIL_PROBE_JOIN_LEGACY_SQL
|
|
535
|
+
)
|
|
536
|
+
# Direct non-null bounds, never the nullable `(? IS NULL OR ...)` form the
|
|
537
|
+
# unbounded doctor helper needs: that form is opaque to the query planner,
|
|
538
|
+
# so it cannot use the timestamp-leading index and reads every row.
|
|
539
|
+
return f"""
|
|
540
|
+
SELECT
|
|
541
|
+
COUNT(*) AS total_rows,
|
|
542
|
+
COALESCE(SUM(CASE WHEN
|
|
543
|
+
entries.conversation_key IS NULL OR entries.conversation_key = ''
|
|
544
|
+
OR ({join_predicate}
|
|
545
|
+
)
|
|
546
|
+
OR {_CODEX_ACCOUNTING_ROW_USABLE_SQL} = 0
|
|
547
|
+
THEN 1 ELSE 0 END), 0) AS unqualifiable_rows
|
|
548
|
+
FROM codex_session_entries AS entries
|
|
549
|
+
INDEXED BY idx_codex_entries_ts_root_conversation
|
|
550
|
+
LEFT JOIN codex_conversation_threads AS threads
|
|
551
|
+
ON threads.conversation_key = entries.conversation_key
|
|
552
|
+
AND threads.source_root_key = entries.source_root_key
|
|
553
|
+
WHERE entries.timestamp_utc >= ?
|
|
554
|
+
AND entries.timestamp_utc < ?
|
|
555
|
+
"""
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
#: How many affected ids one overlap statement carries. The subtraction runs
|
|
559
|
+
#: over the ids the per-identity queries already returned, so the chunk bounds
|
|
560
|
+
#: the SQL text rather than the work.
|
|
561
|
+
_OVERLAP_ID_CHUNK = 400
|
|
562
|
+
|
|
563
|
+
#: One query per undecodable identity, never a disjunction over all of them: a
|
|
564
|
+
#: multi-identity `OR` predicate makes SQLite abandon the index and scan the
|
|
565
|
+
#: table, which is why `load_codex_quota_observations` shards too.
|
|
566
|
+
_UNDECODABLE_DIRECT_ENTRIES_SQL = """
|
|
567
|
+
SELECT entries.id, entries.source_path, entries.conversation_key
|
|
568
|
+
FROM codex_session_entries AS entries
|
|
569
|
+
INDEXED BY idx_codex_entries_conversation
|
|
570
|
+
WHERE entries.source_root_key = ?
|
|
571
|
+
AND entries.conversation_key = ?
|
|
572
|
+
"""
|
|
573
|
+
|
|
574
|
+
_UNDECODABLE_INHERITED_ENTRIES_SQL = """
|
|
575
|
+
SELECT entries.id, entries.source_path, entries.conversation_key
|
|
576
|
+
FROM codex_session_entries AS entries
|
|
577
|
+
INDEXED BY idx_codex_entries_root_path
|
|
578
|
+
WHERE entries.source_root_key = ?
|
|
579
|
+
AND entries.source_path = ?
|
|
580
|
+
"""
|
|
581
|
+
|
|
582
|
+
_BOUNDED_ENTRIES_PREDICATE = """
|
|
583
|
+
AND entries.timestamp_utc >= ?
|
|
584
|
+
AND entries.timestamp_utc < ?
|
|
585
|
+
"""
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def _undecodable_entries_sql(base: str, *, bounded: bool) -> str:
|
|
589
|
+
return base + (_BOUNDED_ENTRIES_PREDICATE if bounded else "")
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _codex_population_reason_sql(
|
|
593
|
+
cache_conn: sqlite3.Connection, *, include_probe_reasons: bool,
|
|
594
|
+
) -> str:
|
|
595
|
+
"""This population's OWN reason predicate, reproduced not re-implemented.
|
|
596
|
+
|
|
597
|
+
The aggregates count a missing conversation key, a missing thread join,
|
|
598
|
+
and a refused accounting operand. A row with undecodable metadata and a
|
|
599
|
+
bad timestamp is already counted by the operand branch, so it is
|
|
600
|
+
subtracted from the undecodable count for each population.
|
|
601
|
+
|
|
602
|
+
The reason semantics — the shared accounting parser, the alias
|
|
603
|
+
``NOT EXISTS`` — are never re-implemented in Python: the predicate text
|
|
604
|
+
here is the same text the aggregates evaluate.
|
|
605
|
+
"""
|
|
606
|
+
join_predicate = (
|
|
607
|
+
_CODEX_DETAIL_PROBE_JOIN_SQL if _supports_native_file_aliases(cache_conn)
|
|
608
|
+
else _CODEX_DETAIL_PROBE_JOIN_LEGACY_SQL
|
|
609
|
+
)
|
|
610
|
+
reasons = [
|
|
611
|
+
"(entries.conversation_key IS NULL OR entries.conversation_key = '')",
|
|
612
|
+
f"({join_predicate})",
|
|
613
|
+
]
|
|
614
|
+
if include_probe_reasons:
|
|
615
|
+
_register_codex_accounting_probe(cache_conn)
|
|
616
|
+
reasons.append(f"{_CODEX_ACCOUNTING_ROW_USABLE_SQL} = 0")
|
|
617
|
+
return "\n OR ".join(reasons)
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _codex_overlap_sql(
|
|
621
|
+
cache_conn: sqlite3.Connection, *, include_probe_reasons: bool,
|
|
622
|
+
bounded: bool, chunk_size: int,
|
|
623
|
+
) -> str:
|
|
624
|
+
"""The population's complete caller shape, over one chunk of ids.
|
|
625
|
+
|
|
626
|
+
The callers' reason predicates are not standalone expressions: they
|
|
627
|
+
reference the ``threads`` join alias and the inherited-alias ``NOT
|
|
628
|
+
EXISTS``, so this statement reproduces the aggregate's own ``LEFT JOIN``
|
|
629
|
+
and its own time bounds rather than evaluating the predicate in isolation.
|
|
630
|
+
"""
|
|
631
|
+
placeholders = ",".join("?" for _ in range(chunk_size))
|
|
632
|
+
bounds = _BOUNDED_ENTRIES_PREDICATE if bounded else ""
|
|
633
|
+
return f"""
|
|
634
|
+
SELECT entries.id
|
|
635
|
+
FROM codex_session_entries AS entries
|
|
636
|
+
LEFT JOIN codex_conversation_threads AS threads
|
|
637
|
+
ON threads.conversation_key = entries.conversation_key
|
|
638
|
+
AND threads.source_root_key = entries.source_root_key
|
|
639
|
+
WHERE entries.id IN ({placeholders}){bounds}
|
|
640
|
+
AND ({_codex_population_reason_sql(
|
|
641
|
+
cache_conn, include_probe_reasons=include_probe_reasons)})
|
|
642
|
+
"""
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def count_undecodable_codex_metadata_rows(
|
|
646
|
+
cache_conn: sqlite3.Connection,
|
|
647
|
+
*,
|
|
648
|
+
bound_start: "str | None",
|
|
649
|
+
bound_end: "str | None",
|
|
650
|
+
include_probe_reasons: bool,
|
|
651
|
+
inventory: "CodexThreadMetadataInventory | None" = None,
|
|
652
|
+
) -> int:
|
|
653
|
+
"""Entries this population must count for undecodable project metadata.
|
|
654
|
+
|
|
655
|
+
Zero-cost on a clean store: both undecodable sets are empty, so no
|
|
656
|
+
per-identity query runs at all and this returns before touching the
|
|
657
|
+
entries table.
|
|
658
|
+
|
|
659
|
+
The affected ids are unioned across the direct and inherited identities,
|
|
660
|
+
and a second bounded query per population returns the subset that
|
|
661
|
+
population's aggregate already counted for another reason; the decode count
|
|
662
|
+
is the union minus that subset, so no entry is counted twice.
|
|
663
|
+
"""
|
|
664
|
+
if inventory is None:
|
|
665
|
+
inventory = load_codex_thread_metadata_inventory(cache_conn)
|
|
666
|
+
if not inventory.undecodable_direct and not inventory.undecodable_inherited:
|
|
667
|
+
return 0
|
|
668
|
+
bounded = bound_start is not None and bound_end is not None
|
|
669
|
+
bounds: tuple = (bound_start, bound_end) if bounded else ()
|
|
670
|
+
|
|
671
|
+
affected: set[int] = set()
|
|
672
|
+
seeks = (
|
|
673
|
+
(_UNDECODABLE_DIRECT_ENTRIES_SQL, inventory.undecodable_direct),
|
|
674
|
+
(_UNDECODABLE_INHERITED_ENTRIES_SQL, inventory.undecodable_inherited),
|
|
675
|
+
)
|
|
676
|
+
for base_sql, identities in seeks:
|
|
677
|
+
if not identities:
|
|
678
|
+
continue
|
|
679
|
+
sql = _undecodable_entries_sql(base_sql, bounded=bounded)
|
|
680
|
+
for root_key, second in sorted(identities):
|
|
681
|
+
for entry_id, source_path, conversation_key in cache_conn.execute(
|
|
682
|
+
sql, (root_key, second, *bounds),
|
|
683
|
+
):
|
|
684
|
+
direct = inventory.direct_by_conversation.get(
|
|
685
|
+
(str(root_key), str(conversation_key or "")))
|
|
686
|
+
inherited = inventory.inherited_by_path.get(
|
|
687
|
+
(str(root_key), str(source_path or "")))
|
|
688
|
+
if codex_metadata_is_malformed(direct, inherited):
|
|
689
|
+
affected.add(int(entry_id))
|
|
690
|
+
if not affected:
|
|
691
|
+
return 0
|
|
692
|
+
|
|
693
|
+
already_counted: set[int] = set()
|
|
694
|
+
ordered = sorted(affected)
|
|
695
|
+
for offset in range(0, len(ordered), _OVERLAP_ID_CHUNK):
|
|
696
|
+
chunk = ordered[offset:offset + _OVERLAP_ID_CHUNK]
|
|
697
|
+
sql = _codex_overlap_sql(
|
|
698
|
+
cache_conn, include_probe_reasons=include_probe_reasons,
|
|
699
|
+
bounded=bounded, chunk_size=len(chunk),
|
|
700
|
+
)
|
|
701
|
+
already_counted.update(
|
|
702
|
+
int(row[0]) for row in cache_conn.execute(sql, (*chunk, *bounds))
|
|
703
|
+
)
|
|
704
|
+
return len(affected - already_counted)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def explain_undecodable_codex_overlap(
|
|
708
|
+
cache_conn: sqlite3.Connection,
|
|
709
|
+
*,
|
|
710
|
+
include_probe_reasons: bool,
|
|
711
|
+
bounded: bool,
|
|
712
|
+
chunk_size: int = 1,
|
|
713
|
+
) -> "tuple[str, ...]":
|
|
714
|
+
"""The overlap statement's query plan, so a test can prove it seeks."""
|
|
715
|
+
sql = _codex_overlap_sql(
|
|
716
|
+
cache_conn, include_probe_reasons=include_probe_reasons,
|
|
717
|
+
bounded=bounded, chunk_size=chunk_size,
|
|
718
|
+
)
|
|
719
|
+
params = tuple(range(chunk_size)) + ((("", "")) if bounded else ())
|
|
720
|
+
rows = cache_conn.execute("EXPLAIN QUERY PLAN " + sql, params).fetchall()
|
|
721
|
+
return tuple(str(row[-1]) for row in rows)
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def explain_undecodable_codex_identity_seek(
|
|
725
|
+
cache_conn: sqlite3.Connection, *, inherited: bool, bounded: bool,
|
|
726
|
+
) -> "tuple[str, ...]":
|
|
727
|
+
"""One per-identity counting query's plan, so a test can prove it seeks."""
|
|
728
|
+
sql = _undecodable_entries_sql(
|
|
729
|
+
_UNDECODABLE_INHERITED_ENTRIES_SQL if inherited
|
|
730
|
+
else _UNDECODABLE_DIRECT_ENTRIES_SQL,
|
|
731
|
+
bounded=bounded,
|
|
732
|
+
)
|
|
733
|
+
params = ("", "") + (("", "") if bounded else ())
|
|
734
|
+
rows = cache_conn.execute("EXPLAIN QUERY PLAN " + sql, params).fetchall()
|
|
735
|
+
return tuple(str(row[-1]) for row in rows)
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _probe_bounds(start: dt.datetime, end: dt.datetime) -> tuple[str, str]:
|
|
739
|
+
bound_start = _metadata_health_bound(start, name="start")
|
|
740
|
+
bound_end = _metadata_health_bound(end, name="end")
|
|
741
|
+
if bound_start is None or bound_end is None:
|
|
742
|
+
raise ValueError("the bounded probe requires both bounds")
|
|
743
|
+
if start > end:
|
|
744
|
+
raise ValueError("start must not be after end")
|
|
745
|
+
return bound_start, bound_end
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def probe_codex_detail_metadata_health(
|
|
749
|
+
*,
|
|
750
|
+
cache_conn: sqlite3.Connection,
|
|
751
|
+
start: dt.datetime,
|
|
752
|
+
end: dt.datetime,
|
|
753
|
+
inventory: "CodexThreadMetadataInventory | None" = None,
|
|
754
|
+
) -> CodexProjectMetadataHealth:
|
|
755
|
+
"""Count rows in ``[start, end)`` the qualified reader would refuse.
|
|
756
|
+
|
|
757
|
+
Bounds are REQUIRED and both timezone-aware, unlike
|
|
758
|
+
``load_codex_project_metadata_health``, which keeps its nullable predicate
|
|
759
|
+
for ``doctor``'s all-history read. The two are separate entry points on
|
|
760
|
+
purpose: they answer questions over different populations, and folding one
|
|
761
|
+
into the other would let a year-old malformed row redefine thirty-day
|
|
762
|
+
aggregate availability.
|
|
763
|
+
|
|
764
|
+
The returned partition reports the count under
|
|
765
|
+
``missing_conversation_key_rows`` because this probe is a single verdict
|
|
766
|
+
rather than a per-cause breakdown; ``incomplete_rows`` is the figure every
|
|
767
|
+
caller reads.
|
|
768
|
+
"""
|
|
769
|
+
bound_start, bound_end = _probe_bounds(start, end)
|
|
770
|
+
_register_codex_accounting_probe(cache_conn)
|
|
771
|
+
row = cache_conn.execute(
|
|
772
|
+
_codex_detail_metadata_probe_sql(cache_conn), (bound_start, bound_end),
|
|
773
|
+
).fetchone()
|
|
774
|
+
if row is None or len(row) != 2:
|
|
775
|
+
raise RuntimeError("Codex detail metadata probe returned no partition")
|
|
241
776
|
try:
|
|
242
|
-
|
|
243
|
-
except
|
|
244
|
-
|
|
777
|
+
total_rows, unqualifiable_rows = (int(value) for value in row)
|
|
778
|
+
except (TypeError, ValueError) as exc:
|
|
779
|
+
raise RuntimeError("Codex detail metadata probe returned invalid counts") from exc
|
|
780
|
+
if total_rows < 0 or unqualifiable_rows < 0 or unqualifiable_rows > total_rows:
|
|
781
|
+
raise RuntimeError("Codex detail metadata probe partition is invalid")
|
|
782
|
+
undecodable_rows = count_undecodable_codex_metadata_rows(
|
|
783
|
+
cache_conn,
|
|
784
|
+
bound_start=bound_start,
|
|
785
|
+
bound_end=bound_end,
|
|
786
|
+
include_probe_reasons=True,
|
|
787
|
+
inventory=inventory,
|
|
788
|
+
)
|
|
789
|
+
if unqualifiable_rows + undecodable_rows > total_rows:
|
|
790
|
+
raise RuntimeError("Codex detail metadata probe partition is invalid")
|
|
791
|
+
return CodexProjectMetadataHealth(
|
|
792
|
+
total_rows=total_rows,
|
|
793
|
+
qualified_rows=total_rows - unqualifiable_rows - undecodable_rows,
|
|
794
|
+
missing_conversation_key_rows=unqualifiable_rows,
|
|
795
|
+
missing_thread_join_rows=0,
|
|
796
|
+
undecodable_metadata_rows=undecodable_rows,
|
|
797
|
+
)
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def explain_codex_detail_metadata_probe(
|
|
801
|
+
*,
|
|
802
|
+
cache_conn: sqlite3.Connection,
|
|
803
|
+
start: dt.datetime,
|
|
804
|
+
end: dt.datetime,
|
|
805
|
+
) -> tuple[str, ...]:
|
|
806
|
+
"""Return the probe's query plan, so a test can prove it seeks."""
|
|
807
|
+
bound_start, bound_end = _probe_bounds(start, end)
|
|
808
|
+
rows = cache_conn.execute(
|
|
809
|
+
"EXPLAIN QUERY PLAN " + _codex_detail_metadata_probe_sql(cache_conn),
|
|
810
|
+
(bound_start, bound_end),
|
|
811
|
+
).fetchall()
|
|
812
|
+
return tuple(str(row[-1]) for row in rows)
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _supports_native_file_aliases(cache_conn: sqlite3.Connection) -> bool:
|
|
816
|
+
"""Return whether this cache generation can link child files to a root task.
|
|
817
|
+
|
|
818
|
+
#846 §4.6, the second taxonomy hole. This used to answer ``False`` both
|
|
819
|
+
when the PRAGMA succeeded and the column was absent, and when the PRAGMA
|
|
820
|
+
itself failed. The two are different facts: the first is an old schema and
|
|
821
|
+
the second is a failed read, and answering "old schema" for a failed read
|
|
822
|
+
silently dropped the inherited alias population from every caller. A
|
|
823
|
+
``sqlite3.Error`` now propagates, and the combined verdict of §4.6 turns it
|
|
824
|
+
into a transient generation rather than a quietly narrower one.
|
|
825
|
+
"""
|
|
826
|
+
columns = cache_conn.execute(
|
|
827
|
+
"PRAGMA table_info(codex_session_files)").fetchall()
|
|
245
828
|
return any(len(row) > 1 and row[1] == "last_native_thread_id" for row in columns)
|
|
246
829
|
|
|
247
830
|
|
|
@@ -250,12 +833,17 @@ def _cctally():
|
|
|
250
833
|
|
|
251
834
|
|
|
252
835
|
def _parse_timestamp(value: object) -> dt.datetime:
|
|
836
|
+
# #834 S2 (#829): deterministic. A stored timestamp that will not parse, or
|
|
837
|
+
# that carries no offset, is a property of the ROW; the next refresh reads
|
|
838
|
+
# the same bytes and refuses again.
|
|
253
839
|
try:
|
|
254
840
|
timestamp = dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
255
841
|
except (TypeError, ValueError) as exc:
|
|
256
|
-
raise QualifiedMetadataUnavailable(
|
|
842
|
+
raise QualifiedMetadataUnavailable(
|
|
843
|
+
"Codex accounting metadata is unavailable", transient=False) from exc
|
|
257
844
|
if timestamp.tzinfo is None or timestamp.utcoffset() is None:
|
|
258
|
-
raise QualifiedMetadataUnavailable(
|
|
845
|
+
raise QualifiedMetadataUnavailable(
|
|
846
|
+
"Codex accounting metadata is unavailable", transient=False)
|
|
259
847
|
return timestamp.astimezone(UTC)
|
|
260
848
|
|
|
261
849
|
|
|
@@ -275,6 +863,7 @@ def load_codex_project_metadata_health(
|
|
|
275
863
|
cache_conn: sqlite3.Connection,
|
|
276
864
|
start: dt.datetime | None = None,
|
|
277
865
|
end: dt.datetime | None = None,
|
|
866
|
+
inventory: "CodexThreadMetadataInventory | None" = None,
|
|
278
867
|
) -> CodexProjectMetadataHealth:
|
|
279
868
|
"""Classify retained Codex accounting in one root-qualified SQL read.
|
|
280
869
|
|
|
@@ -289,19 +878,48 @@ def load_codex_project_metadata_health(
|
|
|
289
878
|
if start is not None and end is not None and start > end:
|
|
290
879
|
raise ValueError("start must not be after end")
|
|
291
880
|
|
|
881
|
+
entry_columns = {
|
|
882
|
+
str(row[1]) for row in cache_conn.execute(
|
|
883
|
+
"PRAGMA table_info(codex_session_entries)")
|
|
884
|
+
}
|
|
885
|
+
has_operands = set(_CODEX_TOKEN_OPERANDS).issubset(entry_columns)
|
|
886
|
+
if has_operands:
|
|
887
|
+
_register_codex_accounting_probe(cache_conn)
|
|
888
|
+
native_aliases = _supports_native_file_aliases(cache_conn)
|
|
889
|
+
if has_operands:
|
|
890
|
+
sql = (_CODEX_PROJECT_METADATA_HEALTH_SQL if native_aliases
|
|
891
|
+
else _CODEX_PROJECT_METADATA_HEALTH_LEGACY_SQL)
|
|
892
|
+
else:
|
|
893
|
+
sql = _CODEX_PROJECT_METADATA_HEALTH_PRE_OPERAND_SQL
|
|
292
894
|
row = cache_conn.execute(
|
|
293
|
-
|
|
294
|
-
else _CODEX_PROJECT_METADATA_HEALTH_LEGACY_SQL),
|
|
895
|
+
sql,
|
|
295
896
|
(bound_start, bound_start, bound_end, bound_end),
|
|
296
897
|
).fetchone()
|
|
297
|
-
if row is None or len(row) != 3:
|
|
898
|
+
if row is None or len(row) != (4 if has_operands else 3):
|
|
298
899
|
raise RuntimeError("Codex project metadata health query returned no partition")
|
|
299
900
|
try:
|
|
300
|
-
|
|
901
|
+
counts = tuple(int(value) for value in row)
|
|
902
|
+
total_rows, missing_key_rows, missing_join_rows = counts[:3]
|
|
903
|
+
malformed_rows = counts[3] if has_operands else 0
|
|
301
904
|
except (TypeError, ValueError) as exc:
|
|
302
905
|
raise RuntimeError("Codex project metadata health query returned invalid counts") from exc
|
|
303
|
-
|
|
304
|
-
|
|
906
|
+
# #845 §4.3. The same partition over the same population, with a third
|
|
907
|
+
# reason. Doctor's all-history call omits both timestamp predicates, which
|
|
908
|
+
# is exactly the nullable-bound shape this function already answers for.
|
|
909
|
+
undecodable_rows = count_undecodable_codex_metadata_rows(
|
|
910
|
+
cache_conn,
|
|
911
|
+
bound_start=bound_start,
|
|
912
|
+
bound_end=bound_end,
|
|
913
|
+
include_probe_reasons=has_operands,
|
|
914
|
+
inventory=inventory,
|
|
915
|
+
)
|
|
916
|
+
qualified_rows = (
|
|
917
|
+
total_rows - missing_key_rows - missing_join_rows
|
|
918
|
+
- malformed_rows - undecodable_rows)
|
|
919
|
+
values = (
|
|
920
|
+
total_rows, qualified_rows, missing_key_rows, missing_join_rows,
|
|
921
|
+
undecodable_rows, malformed_rows,
|
|
922
|
+
)
|
|
305
923
|
if any(value < 0 for value in values) or total_rows != sum(values[1:]):
|
|
306
924
|
raise RuntimeError("Codex project metadata health partition is invalid")
|
|
307
925
|
return CodexProjectMetadataHealth(
|
|
@@ -309,6 +927,8 @@ def load_codex_project_metadata_health(
|
|
|
309
927
|
qualified_rows=qualified_rows,
|
|
310
928
|
missing_conversation_key_rows=missing_key_rows,
|
|
311
929
|
missing_thread_join_rows=missing_join_rows,
|
|
930
|
+
undecodable_metadata_rows=undecodable_rows,
|
|
931
|
+
malformed_accounting_rows=malformed_rows,
|
|
312
932
|
)
|
|
313
933
|
|
|
314
934
|
|
|
@@ -373,11 +993,17 @@ def load_cached_rooted_codex_accounting_entries(
|
|
|
373
993
|
|
|
374
994
|
result: list[RootedCodexAccountingEntry] = []
|
|
375
995
|
for row in rows:
|
|
996
|
+
# A corrupt operand cannot contribute cost or tokens. The generation's
|
|
997
|
+
# bounded probes count this physical row once; keep every other row.
|
|
998
|
+
if not _codex_accounting_row_usable(
|
|
999
|
+
row[0], row[5], row[6], row[7], row[8], row[9],
|
|
1000
|
+
):
|
|
1001
|
+
continue
|
|
376
1002
|
try:
|
|
377
1003
|
(
|
|
378
1004
|
timestamp_raw, session_id_raw, source_path_raw, source_root_key_raw,
|
|
379
1005
|
model_raw, input_raw, cached_input_raw, output_raw, reasoning_raw,
|
|
380
|
-
total_raw, account_key_raw,
|
|
1006
|
+
total_raw, account_key_raw, conversation_key_raw,
|
|
381
1007
|
) = row
|
|
382
1008
|
source_path = source_path_raw if isinstance(source_path_raw, str) else ""
|
|
383
1009
|
source_root_key = source_root_key_raw if isinstance(source_root_key_raw, str) else ""
|
|
@@ -416,6 +1042,10 @@ def load_cached_rooted_codex_accounting_entries(
|
|
|
416
1042
|
total_tokens=total_tokens,
|
|
417
1043
|
cost_usd=cost_usd,
|
|
418
1044
|
account_key=entry_account_key,
|
|
1045
|
+
conversation_key=(
|
|
1046
|
+
conversation_key_raw
|
|
1047
|
+
if isinstance(conversation_key_raw, str) else ""
|
|
1048
|
+
),
|
|
419
1049
|
))
|
|
420
1050
|
return tuple(result)
|
|
421
1051
|
|
|
@@ -468,6 +1098,63 @@ def _git_resolved_key(value: object) -> str | None:
|
|
|
468
1098
|
return "git:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
469
1099
|
|
|
470
1100
|
|
|
1101
|
+
class _CodexProjectResolution:
|
|
1102
|
+
"""The per-entry project precedence, stated ONCE (#815).
|
|
1103
|
+
|
|
1104
|
+
Per entry the precedence is: thread ``cwd``, then inherited ``cwd``, then
|
|
1105
|
+
thread ``git_json``, then inherited ``git_json``, then ``(unassigned)``.
|
|
1106
|
+
The first two outrank BOTH git members because ``cwd = row["cwd"] or
|
|
1107
|
+
inherited["cwd"]`` is evaluated before either ``git_json`` is consulted —
|
|
1108
|
+
so a thread carrying no ``cwd`` and a direct ``git_json`` resolving to
|
|
1109
|
+
project A, read through a path whose inherited ``cwd`` resolves to project
|
|
1110
|
+
B, belongs to B.
|
|
1111
|
+
|
|
1112
|
+
Extracted so that the scoped project read resolves keys through exactly the
|
|
1113
|
+
same code the unscoped qualifier uses. Two implementations of this
|
|
1114
|
+
precedence is the shape in which one of them silently stops matching the
|
|
1115
|
+
other, and here that would mean a project detail quietly losing rows.
|
|
1116
|
+
|
|
1117
|
+
The caches are the ones the qualifier already kept: there are 337 distinct
|
|
1118
|
+
working directories on the production store against 216,165 accounting
|
|
1119
|
+
rows, so the per-``cwd`` cache does almost all the work.
|
|
1120
|
+
"""
|
|
1121
|
+
|
|
1122
|
+
__slots__ = ("_group", "_resolver_cache", "_by_cwd", "_by_git_json")
|
|
1123
|
+
|
|
1124
|
+
def __init__(self, group: str = "git-root") -> None:
|
|
1125
|
+
self._group = group
|
|
1126
|
+
self._resolver_cache: dict[object, object] = {}
|
|
1127
|
+
self._by_cwd: dict[str, object] = {}
|
|
1128
|
+
self._by_git_json: dict[str, str | None] = {}
|
|
1129
|
+
|
|
1130
|
+
def resolve(self, cwd: object, git_json: object) -> tuple[str, str]:
|
|
1131
|
+
"""Return ``(resolved_key, project_label)`` for one already-merged pair."""
|
|
1132
|
+
if isinstance(cwd, str) and cwd:
|
|
1133
|
+
project = self._by_cwd.get(cwd)
|
|
1134
|
+
if project is None:
|
|
1135
|
+
project = _cctally()._resolve_project_key(
|
|
1136
|
+
cwd, self._group, self._resolver_cache)
|
|
1137
|
+
self._by_cwd[cwd] = project
|
|
1138
|
+
cwd_label = _project_label(cwd)
|
|
1139
|
+
return project.bucket_path, (
|
|
1140
|
+
cwd_label if cwd_label in {"(home)", "(root)"}
|
|
1141
|
+
else _project_label(project.display_key)
|
|
1142
|
+
)
|
|
1143
|
+
if isinstance(git_json, str) and git_json not in self._by_git_json:
|
|
1144
|
+
self._by_git_json[git_json] = _git_resolved_key(git_json)
|
|
1145
|
+
git_key = (
|
|
1146
|
+
self._by_git_json.get(git_json) if isinstance(git_json, str) else None
|
|
1147
|
+
)
|
|
1148
|
+
if git_key is None:
|
|
1149
|
+
return "(unassigned)", "(unassigned)"
|
|
1150
|
+
return git_key, "Git project"
|
|
1151
|
+
|
|
1152
|
+
def project_key(self, root_key: str, cwd: object, git_json: object) -> str:
|
|
1153
|
+
"""The opaque key one metadata pair resolves to, under one root."""
|
|
1154
|
+
resolved, _label = self.resolve(cwd, git_json)
|
|
1155
|
+
return opaque_project_key("codex", root_key, resolved)
|
|
1156
|
+
|
|
1157
|
+
|
|
471
1158
|
def _require_joined_metadata(row: sqlite3.Row) -> tuple[str, str]:
|
|
472
1159
|
root_key = row["source_root_key"]
|
|
473
1160
|
conversation_key = row["conversation_key"]
|
|
@@ -477,7 +1164,11 @@ def _require_joined_metadata(row: sqlite3.Row) -> tuple[str, str]:
|
|
|
477
1164
|
or row["joined_conversation_key"] != conversation_key
|
|
478
1165
|
or row["joined_source_root_key"] != root_key
|
|
479
1166
|
):
|
|
480
|
-
|
|
1167
|
+
# #834 S2 (#829): deterministic. A missing key or an absent same-root
|
|
1168
|
+
# thread join is a property of the ROW, and rebuilding the Codex cache
|
|
1169
|
+
# is its remedy.
|
|
1170
|
+
raise QualifiedMetadataUnavailable(
|
|
1171
|
+
"Codex qualified project metadata is unavailable", transient=False)
|
|
481
1172
|
return root_key, conversation_key
|
|
482
1173
|
|
|
483
1174
|
|
|
@@ -490,6 +1181,7 @@ def load_qualified_codex_entries(
|
|
|
490
1181
|
group: str = "git-root",
|
|
491
1182
|
cache_conn: sqlite3.Connection | None = None,
|
|
492
1183
|
source_identities: Iterable[tuple[str, str]] | None = None,
|
|
1184
|
+
inventory: "CodexThreadMetadataInventory | None" = None,
|
|
493
1185
|
) -> tuple[QualifiedCodexEntry, ...]:
|
|
494
1186
|
"""Load exactly one bounded, root-qualified Codex accounting read.
|
|
495
1187
|
|
|
@@ -546,19 +1238,15 @@ def load_qualified_codex_entries(
|
|
|
546
1238
|
*(value for identity in (identities or ()) for value in identity),
|
|
547
1239
|
)
|
|
548
1240
|
rows = tuple(conn.execute(sql, params))
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
for inherited in conn.execute(inherited_sql, inherited_params):
|
|
559
|
-
identity = (str(inherited["source_root_key"] or ""), str(inherited["path"] or ""))
|
|
560
|
-
if all(identity):
|
|
561
|
-
inherited_metadata.setdefault(identity, inherited)
|
|
1241
|
+
# §4.3: one inventory read per build. The caller supplies the build's
|
|
1242
|
+
# own inventory when it has one; its alias leg is unbounded, which is a
|
|
1243
|
+
# SUPERSET of the `paths=identities` leg this read would have loaded,
|
|
1244
|
+
# and every lookup below is by identity, so the wider map answers each
|
|
1245
|
+
# row identically.
|
|
1246
|
+
inherited_metadata = (
|
|
1247
|
+
load_codex_thread_metadata_inventory(conn, paths=identities)
|
|
1248
|
+
if inventory is None else inventory
|
|
1249
|
+
).inherited_by_path
|
|
562
1250
|
except sqlite3.Error as exc:
|
|
563
1251
|
raise QualifiedMetadataUnavailable("Codex qualified project metadata is unavailable") from exc
|
|
564
1252
|
finally:
|
|
@@ -567,9 +1255,23 @@ def load_qualified_codex_entries(
|
|
|
567
1255
|
else:
|
|
568
1256
|
conn.row_factory = previous_row_factory
|
|
569
1257
|
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
1258
|
+
return _qualify_codex_rows(
|
|
1259
|
+
rows, inherited_metadata, speed=speed, group=group)
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def _qualify_codex_rows(
|
|
1263
|
+
rows, inherited_metadata, *, speed: str, group: str = "git-root",
|
|
1264
|
+
projects: "_CodexProjectResolution | None" = None,
|
|
1265
|
+
) -> tuple[QualifiedCodexEntry, ...]:
|
|
1266
|
+
"""Turn accounting rows plus their metadata into qualified entries.
|
|
1267
|
+
|
|
1268
|
+
The one implementation of the per-entry qualification, shared by the
|
|
1269
|
+
unscoped year-long read and the project-scoped read (#815), so a scoped
|
|
1270
|
+
request can never assign a row differently from the unscoped one.
|
|
1271
|
+
"""
|
|
1272
|
+
c = _cctally()
|
|
1273
|
+
if projects is None:
|
|
1274
|
+
projects = _CodexProjectResolution(group)
|
|
573
1275
|
result: list[QualifiedCodexEntry] = []
|
|
574
1276
|
for row in rows:
|
|
575
1277
|
root_key = row["source_root_key"]
|
|
@@ -584,39 +1286,42 @@ def load_qualified_codex_entries(
|
|
|
584
1286
|
or not isinstance(conversation_key, str) or not conversation_key
|
|
585
1287
|
):
|
|
586
1288
|
raise
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
if git_key is None:
|
|
605
|
-
resolved_key = "(unassigned)"
|
|
606
|
-
project_label = "(unassigned)"
|
|
607
|
-
else:
|
|
608
|
-
resolved_key = git_key
|
|
609
|
-
project_label = "Git project"
|
|
1289
|
+
# #845 §4.1/§4.2. The two thread columns arrive as BLOBs and are
|
|
1290
|
+
# decoded field by field, so one undecodable byte no longer fails the
|
|
1291
|
+
# statement that selected it. An entry whose resolver path REACHES an
|
|
1292
|
+
# undecodable field cannot be attributed, and this reader has no
|
|
1293
|
+
# partial-result carrier, so it refuses the whole read — the same
|
|
1294
|
+
# deterministic refusal `_parse_timestamp` and the cost operands
|
|
1295
|
+
# already use. It does not skip the row.
|
|
1296
|
+
direct = decode_codex_project_metadata(
|
|
1297
|
+
row["cwd_blob"], row["git_json_blob"])
|
|
1298
|
+
if codex_metadata_is_malformed(direct, inherited):
|
|
1299
|
+
raise QualifiedMetadataUnavailable(
|
|
1300
|
+
"Codex qualified project metadata is unavailable",
|
|
1301
|
+
transient=False)
|
|
1302
|
+
cwd = direct.cwd or (inherited.cwd if inherited is not None else None)
|
|
1303
|
+
git_json = direct.git_json or (
|
|
1304
|
+
inherited.git_json if inherited is not None else None)
|
|
1305
|
+
resolved_key, project_label = projects.resolve(cwd, git_json)
|
|
610
1306
|
try:
|
|
1307
|
+
timestamp = _parse_timestamp(row["timestamp_utc"])
|
|
1308
|
+
input_tokens = int(row["input_tokens"])
|
|
1309
|
+
cached_input_tokens = int(row["cached_input_tokens"])
|
|
1310
|
+
output_tokens = int(row["output_tokens"])
|
|
1311
|
+
reasoning_output_tokens = int(row["reasoning_output_tokens"])
|
|
1312
|
+
total_tokens = int(row["total_tokens"])
|
|
611
1313
|
cost_usd = c._calculate_codex_entry_cost(
|
|
612
|
-
str(row["model"]),
|
|
613
|
-
|
|
614
|
-
|
|
1314
|
+
str(row["model"]), input_tokens,
|
|
1315
|
+
cached_input_tokens, output_tokens,
|
|
1316
|
+
reasoning_output_tokens, speed=speed,
|
|
615
1317
|
)
|
|
616
1318
|
except (TypeError, ValueError, OverflowError) as exc:
|
|
617
|
-
|
|
1319
|
+
# #834 S2 (#829): deterministic. A stored token operand `int()`
|
|
1320
|
+
# refuses is a property of the ROW.
|
|
1321
|
+
raise QualifiedMetadataUnavailable(
|
|
1322
|
+
"Codex qualified accounting is unavailable", transient=False) from exc
|
|
618
1323
|
result.append(QualifiedCodexEntry(
|
|
619
|
-
timestamp=
|
|
1324
|
+
timestamp=timestamp,
|
|
620
1325
|
session_id=str(row["session_id"] or ""),
|
|
621
1326
|
source_path=str(row["source_path"] or ""),
|
|
622
1327
|
source_root_key=root_key,
|
|
@@ -624,11 +1329,11 @@ def load_qualified_codex_entries(
|
|
|
624
1329
|
project_key=opaque_project_key("codex", root_key, resolved_key),
|
|
625
1330
|
project_label=project_label,
|
|
626
1331
|
model=str(row["model"]),
|
|
627
|
-
input_tokens=
|
|
628
|
-
cached_input_tokens=
|
|
629
|
-
output_tokens=
|
|
630
|
-
reasoning_output_tokens=
|
|
631
|
-
total_tokens=
|
|
1332
|
+
input_tokens=input_tokens,
|
|
1333
|
+
cached_input_tokens=cached_input_tokens,
|
|
1334
|
+
output_tokens=output_tokens,
|
|
1335
|
+
reasoning_output_tokens=reasoning_output_tokens,
|
|
1336
|
+
total_tokens=total_tokens,
|
|
632
1337
|
cost_usd=cost_usd,
|
|
633
1338
|
# NULL ≡ unattributed — the cache-read rule (#416 §5.2). Carried,
|
|
634
1339
|
# never grouped; see the field's comment on `QualifiedCodexEntry`.
|
|
@@ -638,6 +1343,258 @@ def load_qualified_codex_entries(
|
|
|
638
1343
|
return tuple(result)
|
|
639
1344
|
|
|
640
1345
|
|
|
1346
|
+
def _require_sql_replace(sql: str, marker: str, replacement: str) -> str:
|
|
1347
|
+
"""Rewrite ``marker`` in ``sql``, refusing when it is not there.
|
|
1348
|
+
|
|
1349
|
+
A silent ``str.replace`` no-op is the failure mode this guards. Both scoped
|
|
1350
|
+
constants below are derived from ``_QUALIFIED_CODEX_ENTRIES_SQL`` by
|
|
1351
|
+
rewriting three markers in it. Reflow that constant — re-indent it, move its
|
|
1352
|
+
``ORDER BY``, rename its index hint — and every replacement quietly matches
|
|
1353
|
+
nothing: the derived statement keeps the year-wide `WHERE`, loses its
|
|
1354
|
+
identity predicate, and is then executed with FOUR bound parameters against
|
|
1355
|
+
TWO placeholders. That is an unscoped read plus a binding error, arriving at
|
|
1356
|
+
import time with no diagnostic naming the constant that moved.
|
|
1357
|
+
"""
|
|
1358
|
+
if marker not in sql:
|
|
1359
|
+
raise AssertionError(
|
|
1360
|
+
"_QUALIFIED_CODEX_ENTRIES_SQL no longer contains the marker "
|
|
1361
|
+
f"{marker!r}; the scoped Codex accounting statements are derived "
|
|
1362
|
+
"from it by rewriting that text, so a reflow of that constant must "
|
|
1363
|
+
"be mirrored here rather than left to no-op into an unscoped read"
|
|
1364
|
+
)
|
|
1365
|
+
return sql.replace(marker, replacement)
|
|
1366
|
+
|
|
1367
|
+
|
|
1368
|
+
#: The accounting read, re-shaped to seek ONE cache identity. Both forms keep
|
|
1369
|
+
#: the qualifier's own SELECT list and its `LEFT JOIN` verbatim, so a scoped
|
|
1370
|
+
#: row arrives in exactly the shape `_qualify_codex_rows` expects.
|
|
1371
|
+
#:
|
|
1372
|
+
#: ONE QUERY PER IDENTITY, never a disjunction over all of them. A
|
|
1373
|
+
#: multi-identity `OR` predicate makes SQLite abandon the index and scan the
|
|
1374
|
+
#: table — the reason `_cctally_quota.load_codex_quota_observations` issues one
|
|
1375
|
+
#: shard per physical group, and the reason
|
|
1376
|
+
#: `tests/test_codex_entries_root_path_index.py` exists at all. A scan hiding
|
|
1377
|
+
#: behind a correct row count is invisible to a materialisation counter, so
|
|
1378
|
+
#: both shapes below carry an `EXPLAIN QUERY PLAN` regression.
|
|
1379
|
+
#:
|
|
1380
|
+
#: THE `ORDER BY` COMES OFF. The unscoped statement orders its rows because it
|
|
1381
|
+
#: is the whole population; each shard here is one of many whose union
|
|
1382
|
+
#: `load_codex_project_scoped_entries` re-sorts in Python anyway, so a SQL sort
|
|
1383
|
+
#: is superseded the moment it returns. It is not free: the forced `INDEXED BY`
|
|
1384
|
+
#: cannot supply that order, so SQLite builds a temporary B-tree per shard to
|
|
1385
|
+
#: produce an ordering nothing reads.
|
|
1386
|
+
_CONVERSATION_SCOPED_CODEX_ENTRIES_SQL = _require_sql_replace(
|
|
1387
|
+
_require_sql_replace(
|
|
1388
|
+
_require_sql_replace(
|
|
1389
|
+
_QUALIFIED_CODEX_ENTRIES_SQL,
|
|
1390
|
+
"INDEXED BY idx_codex_entries_ts_root_conversation",
|
|
1391
|
+
"INDEXED BY idx_codex_entries_conversation",
|
|
1392
|
+
),
|
|
1393
|
+
" WHERE entries.timestamp_utc >= ?",
|
|
1394
|
+
" WHERE entries.conversation_key = ?\n"
|
|
1395
|
+
" AND entries.source_root_key = ?\n"
|
|
1396
|
+
" AND entries.timestamp_utc >= ?",
|
|
1397
|
+
),
|
|
1398
|
+
" ORDER BY entries.timestamp_utc ASC, entries.source_root_key ASC,\n"
|
|
1399
|
+
" entries.conversation_key ASC, entries.id ASC\n",
|
|
1400
|
+
"",
|
|
1401
|
+
)
|
|
1402
|
+
|
|
1403
|
+
_ROOT_PATH_SCOPED_CODEX_ENTRIES_SQL = _require_sql_replace(
|
|
1404
|
+
_require_sql_replace(
|
|
1405
|
+
_require_sql_replace(
|
|
1406
|
+
_QUALIFIED_CODEX_ENTRIES_SQL,
|
|
1407
|
+
"INDEXED BY idx_codex_entries_ts_root_conversation",
|
|
1408
|
+
"INDEXED BY idx_codex_entries_root_path",
|
|
1409
|
+
),
|
|
1410
|
+
" WHERE entries.timestamp_utc >= ?",
|
|
1411
|
+
" WHERE entries.source_root_key = ?\n"
|
|
1412
|
+
" AND entries.source_path = ?\n"
|
|
1413
|
+
" AND entries.timestamp_utc >= ?",
|
|
1414
|
+
),
|
|
1415
|
+
" ORDER BY entries.timestamp_utc ASC, entries.source_root_key ASC,\n"
|
|
1416
|
+
" entries.conversation_key ASC, entries.id ASC\n",
|
|
1417
|
+
"",
|
|
1418
|
+
)
|
|
1419
|
+
|
|
1420
|
+
|
|
1421
|
+
def resolve_codex_project_scope(
|
|
1422
|
+
conn, matches_project_key, *, group: str = "git-root",
|
|
1423
|
+
) -> "tuple[tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], dict]":
|
|
1424
|
+
"""Stage one: which cache identities can carry the requested project (#815)?
|
|
1425
|
+
|
|
1426
|
+
``matches_project_key`` is a predicate over the OPAQUE project key rather
|
|
1427
|
+
than a value, because the dashboard identifies a project by
|
|
1428
|
+
``dashboard_resource_key``, which is a non-reversible digest OVER that
|
|
1429
|
+
opaque key. Neither identifier can be turned back into the other, so the
|
|
1430
|
+
only way to decide membership is to derive a candidate's opaque key and let
|
|
1431
|
+
the caller say whether it is the one requested.
|
|
1432
|
+
|
|
1433
|
+
``opaque_project_key`` is not reversible, so the requested key cannot
|
|
1434
|
+
become a SQL predicate. It has to be re-derived over the metadata tables
|
|
1435
|
+
and compared — roughly 5,637 rows against 216,165 accounting rows on the
|
|
1436
|
+
production store, with at most 337 git-root resolutions. Spec §4.6 accepts
|
|
1437
|
+
that metadata read as disclosed resolution evidence.
|
|
1438
|
+
|
|
1439
|
+
Returns ``(conversations, paths, inherited_metadata)``: the
|
|
1440
|
+
``(source_root_key, conversation_key)`` identities to seek, the
|
|
1441
|
+
``(source_root_key, source_path)`` identities to seek, and the inherited
|
|
1442
|
+
map stage two needs to qualify whatever it finds.
|
|
1443
|
+
|
|
1444
|
+
WHY TWO SHARDS AND WHY THEY ARE COMPLETE. An entry's key is
|
|
1445
|
+
``combined(thread, inherited)`` under the precedence in
|
|
1446
|
+
``_CodexProjectResolution``. That value differs from the thread's own key
|
|
1447
|
+
only when the thread carries no ``cwd`` AND the inherited row supplies one,
|
|
1448
|
+
or when neither carries a ``cwd`` and only the inherited row carries a
|
|
1449
|
+
``git_json`` — and in both of those cases the result is exactly the
|
|
1450
|
+
inherited row's OWN key. So every eligible entry is reachable either from a
|
|
1451
|
+
thread whose own metadata resolves to the request, or from an aliased path
|
|
1452
|
+
whose own metadata does. That is what keeps the case the review
|
|
1453
|
+
constructed: a thread with no ``cwd`` and a direct ``git_json`` resolving
|
|
1454
|
+
to project A, read through a path whose inherited ``cwd`` resolves to
|
|
1455
|
+
project B, belongs to B and is found through the path shard.
|
|
1456
|
+
|
|
1457
|
+
Both shards deliberately OVER-collect — an entry of a listed conversation
|
|
1458
|
+
read through a path that reassigns it is selected here and dropped later.
|
|
1459
|
+
Stage two re-qualifies every row and gates on the resolved key, so
|
|
1460
|
+
over-collection costs rows read and never correctness. Under-collection
|
|
1461
|
+
would silently lose a project's spend, which is why the shards are derived
|
|
1462
|
+
from the precedence rather than from a guess about which is "the" axis.
|
|
1463
|
+
"""
|
|
1464
|
+
projects = _CodexProjectResolution(group)
|
|
1465
|
+
previous_row_factory = conn.row_factory
|
|
1466
|
+
try:
|
|
1467
|
+
# Set here rather than assumed, so this resolver is usable on a plain
|
|
1468
|
+
# caller-owned connection as well as inside the scoped read.
|
|
1469
|
+
conn.row_factory = sqlite3.Row
|
|
1470
|
+
inventory = load_codex_thread_metadata_inventory(conn)
|
|
1471
|
+
finally:
|
|
1472
|
+
conn.row_factory = previous_row_factory
|
|
1473
|
+
conversations: list[tuple[str, str]] = []
|
|
1474
|
+
for (root_key, conversation_key), direct in (
|
|
1475
|
+
inventory.direct_by_conversation.items()
|
|
1476
|
+
):
|
|
1477
|
+
# #845 §4.1. A thread whose own resolver path reaches an undecodable
|
|
1478
|
+
# field resolves to nothing, so it can carry no request. Handing its
|
|
1479
|
+
# decoded `None` fields to the resolver would answer `(unassigned)`
|
|
1480
|
+
# and select the conversation for an unrelated request.
|
|
1481
|
+
if codex_metadata_is_malformed(direct, None):
|
|
1482
|
+
continue
|
|
1483
|
+
if matches_project_key(
|
|
1484
|
+
projects.project_key(root_key, direct.cwd, direct.git_json)
|
|
1485
|
+
):
|
|
1486
|
+
conversations.append((root_key, conversation_key))
|
|
1487
|
+
paths: list[tuple[str, str]] = []
|
|
1488
|
+
for (root_key, path), inherited in inventory.inherited_by_path.items():
|
|
1489
|
+
if codex_metadata_is_malformed(None, inherited):
|
|
1490
|
+
continue
|
|
1491
|
+
if matches_project_key(projects.project_key(
|
|
1492
|
+
root_key, inherited.cwd, inherited.git_json,
|
|
1493
|
+
)):
|
|
1494
|
+
paths.append((root_key, path))
|
|
1495
|
+
return (
|
|
1496
|
+
tuple(sorted(set(conversations))), tuple(sorted(set(paths))),
|
|
1497
|
+
inventory.inherited_by_path,
|
|
1498
|
+
)
|
|
1499
|
+
|
|
1500
|
+
|
|
1501
|
+
def load_codex_project_scoped_entries(
|
|
1502
|
+
start: dt.datetime,
|
|
1503
|
+
end: dt.datetime,
|
|
1504
|
+
*,
|
|
1505
|
+
speed: str,
|
|
1506
|
+
matches_project_key,
|
|
1507
|
+
cache_conn,
|
|
1508
|
+
group: str = "git-root",
|
|
1509
|
+
) -> tuple[QualifiedCodexEntry, ...]:
|
|
1510
|
+
"""One project's qualified accounting, without loading the year (#815).
|
|
1511
|
+
|
|
1512
|
+
BOTH STAGES RUN UNDER ONE EXPLICIT READ TRANSACTION. Python's ``sqlite3``
|
|
1513
|
+
driver does not begin a transaction for a ``SELECT`` — this repository
|
|
1514
|
+
records that at ``bin/_cctally_cache.py:7607`` — and the detail route's
|
|
1515
|
+
context opener merely opens a connection. A cache writer can therefore
|
|
1516
|
+
commit a new conversation belonging to this project between the two
|
|
1517
|
+
stages: the single joined statement the unscoped read issues would see
|
|
1518
|
+
both the thread and its entries, while a staged selector whose identity set
|
|
1519
|
+
came from the earlier statement would miss the entries. Silently
|
|
1520
|
+
under-reporting during ingest is worse than being slow.
|
|
1521
|
+
|
|
1522
|
+
THE TRANSACTION IS HELD ACROSS FILESYSTEM CALLS, which is a residual rather
|
|
1523
|
+
than an oversight (spec §9). Stage one resolves each distinct ``cwd``
|
|
1524
|
+
through ``_resolve_project_key``, which calls ``os.path.realpath`` and
|
|
1525
|
+
walks parents with ``os.path.exists`` — up to 337 distinct working
|
|
1526
|
+
directories on the production store. In WAL mode a held read transaction
|
|
1527
|
+
blocks checkpointing, and this repository already carries a cache.db
|
|
1528
|
+
WAL-bloat incident. The measured whole-route cost is 68.6 ms, so the window
|
|
1529
|
+
is small, but on a slow or network-backed path a single ``realpath`` can
|
|
1530
|
+
block for much longer. The per-``cwd`` resolver cache is what keeps the
|
|
1531
|
+
call count near the number of distinct directories rather than the number
|
|
1532
|
+
of threads.
|
|
1533
|
+
"""
|
|
1534
|
+
if start.tzinfo is None or start.utcoffset() is None:
|
|
1535
|
+
raise ValueError("start must be timezone-aware")
|
|
1536
|
+
if end.tzinfo is None or end.utcoffset() is None:
|
|
1537
|
+
raise ValueError("end must be timezone-aware")
|
|
1538
|
+
if end <= start:
|
|
1539
|
+
raise ValueError("end must be after start")
|
|
1540
|
+
|
|
1541
|
+
conn = cache_conn
|
|
1542
|
+
previous_row_factory = conn.row_factory
|
|
1543
|
+
started_transaction = False
|
|
1544
|
+
try:
|
|
1545
|
+
if not conn.in_transaction:
|
|
1546
|
+
conn.execute("BEGIN")
|
|
1547
|
+
started_transaction = True
|
|
1548
|
+
conn.row_factory = sqlite3.Row
|
|
1549
|
+
conversations, paths, inherited_metadata = resolve_codex_project_scope(
|
|
1550
|
+
conn, matches_project_key, group=group)
|
|
1551
|
+
if not conversations and not paths:
|
|
1552
|
+
return ()
|
|
1553
|
+
start_iso = start.astimezone(UTC).isoformat()
|
|
1554
|
+
end_iso = end.astimezone(UTC).isoformat()
|
|
1555
|
+
selected: "dict[int, sqlite3.Row]" = {}
|
|
1556
|
+
for root_key, conversation_key in conversations:
|
|
1557
|
+
for row in conn.execute(
|
|
1558
|
+
_CONVERSATION_SCOPED_CODEX_ENTRIES_SQL,
|
|
1559
|
+
(conversation_key, root_key, start_iso, end_iso),
|
|
1560
|
+
):
|
|
1561
|
+
selected[int(row["cache_entry_id"])] = row
|
|
1562
|
+
for root_key, path in paths:
|
|
1563
|
+
for row in conn.execute(
|
|
1564
|
+
_ROOT_PATH_SCOPED_CODEX_ENTRIES_SQL,
|
|
1565
|
+
(root_key, path, start_iso, end_iso),
|
|
1566
|
+
):
|
|
1567
|
+
selected[int(row["cache_entry_id"])] = row
|
|
1568
|
+
# The union is re-ordered the way the unscoped statement orders it, so
|
|
1569
|
+
# every downstream "first row wins" or "last row wins" rule sees the
|
|
1570
|
+
# same sequence it would have seen.
|
|
1571
|
+
rows = [
|
|
1572
|
+
selected[identity] for identity in sorted(
|
|
1573
|
+
selected,
|
|
1574
|
+
key=lambda identity: (
|
|
1575
|
+
str(selected[identity]["timestamp_utc"] or ""),
|
|
1576
|
+
str(selected[identity]["source_root_key"] or ""),
|
|
1577
|
+
str(selected[identity]["conversation_key"] or ""),
|
|
1578
|
+
identity,
|
|
1579
|
+
),
|
|
1580
|
+
)
|
|
1581
|
+
]
|
|
1582
|
+
except sqlite3.Error as exc:
|
|
1583
|
+
raise QualifiedMetadataUnavailable(
|
|
1584
|
+
"Codex qualified project metadata is unavailable") from exc
|
|
1585
|
+
finally:
|
|
1586
|
+
conn.row_factory = previous_row_factory
|
|
1587
|
+
if started_transaction:
|
|
1588
|
+
conn.rollback()
|
|
1589
|
+
qualified = _qualify_codex_rows(
|
|
1590
|
+
rows, inherited_metadata, speed=speed, group=group)
|
|
1591
|
+
# THE FINAL GATE. Over-collection above is deliberate, so the scoped read
|
|
1592
|
+
# can never admit a row the unscoped read would have assigned elsewhere.
|
|
1593
|
+
return tuple(
|
|
1594
|
+
entry for entry in qualified
|
|
1595
|
+
if matches_project_key(entry.project_key))
|
|
1596
|
+
|
|
1597
|
+
|
|
641
1598
|
def load_codex_accounting_entries(
|
|
642
1599
|
start: dt.datetime, end: dt.datetime, *, speed: str, sync: bool = True,
|
|
643
1600
|
force_direct: bool = False,
|