cctally 1.83.0 → 1.84.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +2 -4
- package/bin/_cctally_account.py +144 -11
- package/bin/_cctally_alerts.py +3 -1
- package/bin/_cctally_cache.py +1396 -33
- package/bin/_cctally_dashboard.py +46 -2
- package/bin/_cctally_dashboard_conversation.py +16 -7
- package/bin/_cctally_dashboard_envelope.py +12 -0
- package/bin/_cctally_dashboard_share.py +58 -5
- package/bin/_cctally_dashboard_sources.py +641 -38
- package/bin/_cctally_db.py +173 -7
- package/bin/_cctally_doctor.py +112 -27
- package/bin/_cctally_journal.py +761 -74
- package/bin/_cctally_milestone_history.py +73 -16
- package/bin/_cctally_quota.py +313 -15
- package/bin/_cctally_source_analytics.py +4 -1
- package/bin/_cctally_tui.py +7 -3
- package/bin/_lib_doctor.py +67 -2
- package/bin/_lib_journal.py +48 -0
- package/bin/_lib_jsonl.py +119 -0
- package/bin/_lib_quota.py +222 -13
- package/bin/_lib_rederive.py +12 -0
- package/bin/_lib_source_analytics.py +13 -0
- package/bin/_lib_source_identity.py +24 -0
- package/dashboard/static/assets/index-DlVVJeS4.js +92 -0
- package/dashboard/static/assets/index-OYBkyglj.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
- package/dashboard/static/assets/index-3bgCMVHb.js +0 -92
- package/dashboard/static/assets/index-D27EIHEI.css +0 -1
package/bin/_cctally_journal.py
CHANGED
|
@@ -532,7 +532,7 @@ def journal_high_water() -> tuple[str, int] | None:
|
|
|
532
532
|
#
|
|
533
533
|
# 1. HW snapshot (leaf lock, µs). journal_high_water()
|
|
534
534
|
# 2. read+decode cursor -> HW, counting malformed. _read_range()
|
|
535
|
-
# 3. cache leg (Codex quota)
|
|
535
|
+
# 3. cache leg (Codex quota + attribution) before stats. CACHE_APPLIER seam
|
|
536
536
|
# 4. ONE `BEGIN IMMEDIATE`:
|
|
537
537
|
# a. replay journal evt lines (apply-only, NO alerts). _apply_evt()
|
|
538
538
|
# b. per-record sequential PIPELINE over obs/op. PIPELINE hooks
|
|
@@ -548,14 +548,19 @@ def journal_high_water() -> tuple[str, int] | None:
|
|
|
548
548
|
# milestones, resets/credits, cost snapshots, budgets); the
|
|
549
549
|
# built-in `_pipeline_op_weekly_credit_floor` op fold ships
|
|
550
550
|
# here (spec §5.3 "fold op").
|
|
551
|
-
#
|
|
552
|
-
#
|
|
553
|
-
#
|
|
554
|
-
#
|
|
555
|
-
#
|
|
556
|
-
#
|
|
557
|
-
#
|
|
558
|
-
#
|
|
551
|
+
# CACHE_APPLIER the composite Codex cache leg (Task 7; #416 widened it from
|
|
552
|
+
# quota-only to quota + `codex_file_account` attribution ops,
|
|
553
|
+
# wired to `_cache_applier` below; `QUOTA_APPLIER` remains as
|
|
554
|
+
# its back-compat alias). Contract: (decoded) -> stop_index |
|
|
555
|
+
# None. `decoded` is the ordered list of (record, segment,
|
|
556
|
+
# offset); a non-None int is a prefix-stop boundary (busy
|
|
557
|
+
# global or Codex cache writer flock, or an incomplete cache
|
|
558
|
+
# write): the cycle processes decoded[:stop] and advances the
|
|
559
|
+
# cursor to decoded[stop]'s offset (spec §5.2 step 3). ONE
|
|
560
|
+
# applier, ONE `BEGIN IMMEDIATE`, ONE stop across BOTH
|
|
561
|
+
# families — see the #416 review-F1 note at `_cache_applier`
|
|
562
|
+
# for why a second independent applier is unsafe here.
|
|
563
|
+
# Always-on: a Claude-only batch scans + returns None.
|
|
559
564
|
# codex_apply per-cycle `(ctx) -> None` closure (Task 7, a `run_stats_
|
|
560
565
|
# ingest` arg, not a module global) run in step 4b'' on
|
|
561
566
|
# ctx.conn — the seam every Codex on-demand stats.db writer
|
|
@@ -944,36 +949,63 @@ def _write_cursor(conn: sqlite3.Connection, segment: str, offset: int) -> None:
|
|
|
944
949
|
)
|
|
945
950
|
|
|
946
951
|
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
+
_SEGMENT_READ_CHUNK = 256 * 1024
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def _iter_segment_lines(seg_path, lo: int, hi: int):
|
|
956
|
+
"""Stream `(basename, absolute-offset, raw-line-without-newline)` for every
|
|
957
|
+
complete line in `[lo, hi)`, holding at most one chunk plus one partial line
|
|
958
|
+
in memory. `hi` is a line boundary (a HW snapshot size or an immutable prior
|
|
959
|
+
segment's full size), so no partial trailing line appears."""
|
|
960
|
+
name = seg_path.name
|
|
952
961
|
with open(seg_path, "rb") as fh:
|
|
953
962
|
fh.seek(lo)
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
963
|
+
pos = lo
|
|
964
|
+
buf = b""
|
|
965
|
+
buf_at = lo
|
|
966
|
+
while pos < hi:
|
|
967
|
+
data = fh.read(min(_SEGMENT_READ_CHUNK, hi - pos))
|
|
968
|
+
if not data:
|
|
969
|
+
break
|
|
970
|
+
pos += len(data)
|
|
971
|
+
buf = buf + data if buf else data
|
|
972
|
+
start = 0
|
|
973
|
+
while True:
|
|
974
|
+
nl = buf.find(b"\n", start)
|
|
975
|
+
if nl == -1:
|
|
976
|
+
break
|
|
977
|
+
yield (name, buf_at + start, buf[start:nl])
|
|
978
|
+
start = nl + 1
|
|
979
|
+
if start:
|
|
980
|
+
buf = buf[start:]
|
|
981
|
+
buf_at += start
|
|
964
982
|
|
|
965
983
|
|
|
966
|
-
def
|
|
967
|
-
"""
|
|
984
|
+
def _read_segment_lines(seg_path, lo: int, hi: int) -> list[tuple[str, int, bytes]]:
|
|
985
|
+
"""Materialized form of :func:`_iter_segment_lines` (see it for the
|
|
986
|
+
contract). Callers that walk a whole range at once should prefer
|
|
987
|
+
:func:`iter_range`; this list form is retained for the ingest cycle, which
|
|
988
|
+
needs the batch as an indexable sequence."""
|
|
989
|
+
return list(_iter_segment_lines(seg_path, lo, hi))
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
def iter_range(cursor, hw):
|
|
993
|
+
"""Stream `cursor -> HW` across segments in canonical order (spec §5.2.2).
|
|
968
994
|
|
|
969
995
|
Prior segments (before HW's) are immutable and read to their full size;
|
|
970
996
|
HW's segment is read only up to the snapshot size, so appends past HW
|
|
971
997
|
belong to the next cycle.
|
|
998
|
+
|
|
999
|
+
Streaming, not list-building: a caller that only needs to fold each record
|
|
1000
|
+
into a table (the Codex attribution rehydration) must not put a transient
|
|
1001
|
+
the size of the whole journal on the hot path. `_read_range` remains the
|
|
1002
|
+
materialized form for the ingest cycle, which genuinely needs the batch as
|
|
1003
|
+
an indexable sequence (prefix-stop indices address into it).
|
|
972
1004
|
"""
|
|
973
1005
|
hw_seg, hw_size = hw
|
|
974
1006
|
segments = list_segments()
|
|
975
1007
|
if hw_seg not in segments:
|
|
976
|
-
return
|
|
1008
|
+
return
|
|
977
1009
|
hw_idx = segments.index(hw_seg)
|
|
978
1010
|
if cursor is None:
|
|
979
1011
|
start_idx, start_off = 0, 0
|
|
@@ -983,7 +1015,6 @@ def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
|
|
|
983
1015
|
start_idx, start_off = segments.index(cur_seg), cur_off
|
|
984
1016
|
else:
|
|
985
1017
|
start_idx, start_off = 0, 0
|
|
986
|
-
lines: list[tuple[str, int, bytes]] = []
|
|
987
1018
|
for idx in range(start_idx, hw_idx + 1):
|
|
988
1019
|
seg = segments[idx]
|
|
989
1020
|
seg_path = _cctally_core.JOURNAL_DIR / seg
|
|
@@ -991,8 +1022,12 @@ def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
|
|
|
991
1022
|
hi = hw_size if idx == hw_idx else os.path.getsize(seg_path)
|
|
992
1023
|
if lo >= hi:
|
|
993
1024
|
continue
|
|
994
|
-
|
|
995
|
-
|
|
1025
|
+
yield from _iter_segment_lines(seg_path, lo, hi)
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
|
|
1029
|
+
"""Materialized `cursor -> HW` (see :func:`iter_range`)."""
|
|
1030
|
+
return list(iter_range(cursor, hw))
|
|
996
1031
|
|
|
997
1032
|
|
|
998
1033
|
def journal_prefix_hash(high_water) -> "str | None":
|
|
@@ -1053,7 +1088,7 @@ def _capture_protocol_prefix_evidence(record, prior_high_water, evidence) -> Non
|
|
|
1053
1088
|
|
|
1054
1089
|
_QUOTA_OBS_KIND = "quota_window_snapshot"
|
|
1055
1090
|
|
|
1056
|
-
|
|
1091
|
+
_QUOTA_SNAPSHOT_INSERT_LEGACY = (
|
|
1057
1092
|
"INSERT OR IGNORE INTO quota_window_snapshots "
|
|
1058
1093
|
"(source, source_root_key, source_path, line_offset, captured_at_utc, "
|
|
1059
1094
|
" observed_slot, logical_limit_key, limit_id, limit_name, window_minutes, "
|
|
@@ -1062,6 +1097,15 @@ _QUOTA_SNAPSHOT_INSERT = (
|
|
|
1062
1097
|
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
1063
1098
|
)
|
|
1064
1099
|
|
|
1100
|
+
_QUOTA_SNAPSHOT_INSERT = (
|
|
1101
|
+
"INSERT OR IGNORE INTO quota_window_snapshots "
|
|
1102
|
+
"(source, source_root_key, source_path, line_offset, captured_at_utc, "
|
|
1103
|
+
" observed_slot, logical_limit_key, limit_id, limit_name, window_minutes, "
|
|
1104
|
+
" used_percent, resets_at_utc, plan_type, individual_limit_json, "
|
|
1105
|
+
" reached_type, observed_model, account_key, canonical_resets_at_utc) "
|
|
1106
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
1107
|
+
)
|
|
1108
|
+
|
|
1065
1109
|
_QUOTA_SNAPSHOT_COLS = (
|
|
1066
1110
|
"source", "source_root_key", "source_path", "line_offset", "captured_at_utc",
|
|
1067
1111
|
"observed_slot", "logical_limit_key", "limit_id", "limit_name",
|
|
@@ -1070,15 +1114,23 @@ _QUOTA_SNAPSHOT_COLS = (
|
|
|
1070
1114
|
)
|
|
1071
1115
|
|
|
1072
1116
|
|
|
1073
|
-
def _quota_snapshot_values(rec: dict) -> tuple:
|
|
1117
|
+
def _quota_snapshot_values(rec: dict, anchor: "str | None" = None) -> tuple:
|
|
1074
1118
|
"""Build the INSERT values tuple for one Codex quota obs. account_key (#341)
|
|
1075
1119
|
rides the obs TOP-LEVEL ``account`` field (obs stamp shape), not the payload,
|
|
1076
1120
|
so an unstamped/sentinel obs re-materializes cache.db with NULL account_key
|
|
1077
1121
|
(``NULL ≡ unattributed`` on the read path). first-stamp-wins via the
|
|
1078
1122
|
``INSERT OR IGNORE`` natural key — account_key is a stamped attribute, never
|
|
1079
|
-
part of the identity.
|
|
1123
|
+
part of the identity.
|
|
1124
|
+
|
|
1125
|
+
``anchor`` is the #416 §4.2 canonical reset. It is NOT journaled: it is a
|
|
1126
|
+
property of the observation's POPULATION, not of the observation, so it is
|
|
1127
|
+
re-resolved by whichever writer materializes the row. NULL leaves every
|
|
1128
|
+
reader on the raw-reset fallback, i.e. exactly today's behaviour."""
|
|
1080
1129
|
p = rec.get("payload") or {}
|
|
1081
|
-
return
|
|
1130
|
+
return (
|
|
1131
|
+
tuple(p.get(col) for col in _QUOTA_SNAPSHOT_COLS)
|
|
1132
|
+
+ (rec.get("account"), anchor)
|
|
1133
|
+
)
|
|
1082
1134
|
|
|
1083
1135
|
|
|
1084
1136
|
def _is_codex_quota_obs(rec: dict) -> bool:
|
|
@@ -1089,25 +1141,537 @@ def _is_codex_quota_obs(rec: dict) -> bool:
|
|
|
1089
1141
|
)
|
|
1090
1142
|
|
|
1091
1143
|
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1144
|
+
# --------------------------------------------------------------------------
|
|
1145
|
+
# #416: the second family this leg carries — the durable Codex attribution
|
|
1146
|
+
# decision (`codex_file_account` op, spec §3.3). It shares the leg rather than
|
|
1147
|
+
# getting its own applier because `run_stats_ingest` invokes exactly ONE applier
|
|
1148
|
+
# and truncates the batch afterwards, while an applier commits everything it
|
|
1149
|
+
# handled BEFORE returning a stop index. Two independent prefix-stopping
|
|
1150
|
+
# appliers could therefore commit past each other's stop, exposing suffix
|
|
1151
|
+
# effects beyond the retained prefix and violating the scalar-cursor rule
|
|
1152
|
+
# (docs/journal-gotchas.md; #416 review F1).
|
|
1153
|
+
# --------------------------------------------------------------------------
|
|
1154
|
+
|
|
1155
|
+
_FILE_ACCOUNT_OP_KIND = "codex_file_account"
|
|
1156
|
+
# Byte prefilter for the streamed replay: the canonical encoder is
|
|
1157
|
+
# `json.dumps(..., ensure_ascii=False)`, which never escapes an ASCII token, so
|
|
1158
|
+
# every genuine op carries the kind verbatim.
|
|
1159
|
+
_FILE_ACCOUNT_KIND_MARKER = f'"{_FILE_ACCOUNT_OP_KIND}"'.encode("ascii")
|
|
1160
|
+
|
|
1161
|
+
# FIRST-WINS at a contended primary key (Slice 1 closeout review C1). Spec §3.3
|
|
1162
|
+
# — "a mid-file account change appends a second range-qualified op; the first is
|
|
1163
|
+
# never rewritten" — and §3.5 — "a genuine correction is expressed as an explicit
|
|
1164
|
+
# new range decision, not by mutating history". Ops apply in journal order, so
|
|
1165
|
+
# `DO NOTHING` retains the FIRST decision at that key.
|
|
1166
|
+
#
|
|
1167
|
+
# This reverses the fix-round's last-op-wins. The #374 concern that motivated it
|
|
1168
|
+
# (a fold applier is an inserter, not a convergence operator) does not apply
|
|
1169
|
+
# here, because the path that must converge — `cache-sync --rebuild` — runs
|
|
1170
|
+
# `rehydrate_codex_file_accounts(authoritative=True)`, which DELETEs the table
|
|
1171
|
+
# before replaying. After an authoritative clear, `DO NOTHING` is first-WINS on
|
|
1172
|
+
# an empty table, not a no-op, so clear-then-replay still repairs a drifted row
|
|
1173
|
+
# (pinned by `test_authoritative_replay_still_repairs_a_drifted_row`).
|
|
1174
|
+
#
|
|
1175
|
+
# Last-op-wins was actively wrong on the one path where duplicates are
|
|
1176
|
+
# reachable: a failed rehydration lets the walk re-decide from the live
|
|
1177
|
+
# `auth.json` and mint a second op at the same key (plan candidate 10). Under
|
|
1178
|
+
# last-op-wins the documented remedy would then CEMENT that newer
|
|
1179
|
+
# live-auth-derived value rather than restore the original — the inverse of
|
|
1180
|
+
# acceptance criterion 4. The disagreement is REPORTED (see
|
|
1181
|
+
# `_apply_file_account_records`) rather than applied silently.
|
|
1182
|
+
#
|
|
1183
|
+
# `OR IGNORE` is deliberate, exactly as on `_QUOTA_SNAPSHOT_UPSERT`: SQLite
|
|
1184
|
+
# gives the named upsert clause precedence for the conflict it names, so the
|
|
1185
|
+
# first-wins policy is unaffected, while a record violating some OTHER
|
|
1186
|
+
# constraint is dropped instead of raising — an `IntegrityError` here would
|
|
1187
|
+
# prefix-stop `_cache_applier`, and the scalar cursor could never advance past
|
|
1188
|
+
# that record, wedging the whole journal ingest cycle for every provider.
|
|
1189
|
+
_FILE_ACCOUNT_INSERT = (
|
|
1190
|
+
"INSERT OR IGNORE INTO codex_file_accounts "
|
|
1191
|
+
"(file_identity, incarnation, from_offset, root_scope, account_key, "
|
|
1192
|
+
" decided_at_utc) "
|
|
1193
|
+
"VALUES (?,?,?,?,?,?) "
|
|
1194
|
+
"ON CONFLICT(file_identity, incarnation, from_offset) DO NOTHING"
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
# The incarnation high-water rides the same replay. MAX-set, never an increment,
|
|
1198
|
+
# so replaying the same op converges instead of drifting.
|
|
1199
|
+
#
|
|
1200
|
+
# `OR IGNORE` for the same reason its sibling carries it (closeout review C2):
|
|
1201
|
+
# an `IntegrityError` raised HERE prefix-stops `_cache_applier` just as surely,
|
|
1202
|
+
# and the scalar cursor then never advances past the record. The named upsert
|
|
1203
|
+
# clause still takes precedence for the primary-key conflict, so the MAX-set
|
|
1204
|
+
# convergence is unaffected. `_apply_file_account_records` additionally refuses
|
|
1205
|
+
# to reach this statement for a record the map insert dropped, so the two guards
|
|
1206
|
+
# are belt-and-suspenders over disjoint failure modes: this one covers any
|
|
1207
|
+
# constraint the incarnation table might gain that the map table lacks.
|
|
1208
|
+
_FILE_INCARNATION_INSERT = (
|
|
1209
|
+
"INSERT OR IGNORE INTO codex_file_incarnations "
|
|
1210
|
+
"(file_identity, incarnation, updated_at_utc) "
|
|
1211
|
+
"VALUES (?,?,?) "
|
|
1212
|
+
"ON CONFLICT(file_identity) DO UPDATE SET "
|
|
1213
|
+
" incarnation = MAX(codex_file_incarnations.incarnation, excluded.incarnation), "
|
|
1214
|
+
" updated_at_utc = excluded.updated_at_utc"
|
|
1215
|
+
)
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
def _is_codex_file_account_op(rec: dict) -> bool:
|
|
1219
|
+
return (
|
|
1220
|
+
rec.get("t") == "op"
|
|
1221
|
+
and (rec.get("payload") or {}).get("kind") == _FILE_ACCOUNT_OP_KIND
|
|
1222
|
+
)
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
def _file_account_values(rec: dict) -> tuple:
|
|
1226
|
+
"""INSERT values for one attribution decision.
|
|
1227
|
+
|
|
1228
|
+
``account_key`` is read with ``.get`` because the stably-absent sentinel
|
|
1229
|
+
OMITS the field (two-shaped stamp), and the absence must materialize as SQL
|
|
1230
|
+
NULL — the literal string is never stored.
|
|
1231
|
+
"""
|
|
1232
|
+
p = rec.get("payload") or {}
|
|
1233
|
+
return (
|
|
1234
|
+
p.get("file_identity"), p.get("incarnation"), p.get("from_offset"),
|
|
1235
|
+
p.get("root_scope"), p.get("account_key"), rec.get("at"),
|
|
1236
|
+
)
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
def _apply_file_account_records(cache, records) -> "tuple[int, int]":
|
|
1240
|
+
"""Materialize the given ``codex_file_account`` ops into an OPEN cache.db
|
|
1241
|
+
transaction; return ``(restored, conflicts)`` — how many rows were ABSENT
|
|
1242
|
+
before and actually landed (a genuine restore) and how many CONTRADICTED a
|
|
1243
|
+
different account already recorded at the same primary key (and were
|
|
1244
|
+
therefore declined, first-wins). Never opens or commits a transaction itself
|
|
1245
|
+
— every call site owns the flocks and the single ``BEGIN IMMEDIATE`` so the
|
|
1246
|
+
two families stay atomic.
|
|
1247
|
+
|
|
1248
|
+
``restored`` deliberately excludes a no-op replay of a row that is already
|
|
1249
|
+
present and already says the same thing. The rehydration re-reads the ops
|
|
1250
|
+
its OWN previous sync appended (the cursor is snapshotted before the walk,
|
|
1251
|
+
which is exactly what recovers a failed write), so counting those would
|
|
1252
|
+
make every second sync claim to have rehydrated something — and that claim
|
|
1253
|
+
is printed on stderr, where several golden harnesses read it.
|
|
1254
|
+
|
|
1255
|
+
The ``prior is None`` probe is NOT sufficient on its own to call a record
|
|
1256
|
+
restored (closeout review C3): ``_FILE_ACCOUNT_INSERT`` carries ``OR
|
|
1257
|
+
IGNORE``, so a record violating some OTHER constraint is silently dropped
|
|
1258
|
+
and nothing was restored at all. ``total_changes`` after the statement is
|
|
1259
|
+
the only honest witness. A dropped record must also NOT raise the
|
|
1260
|
+
incarnation high-water — an inflated counter is the DANGEROUS direction,
|
|
1261
|
+
because ranges resolve at exactly the walk's current incarnation, so the
|
|
1262
|
+
range list comes back empty, ``covered`` is False, and a plain sync falls
|
|
1263
|
+
straight through to the live ``auth.json``.
|
|
1264
|
+
|
|
1265
|
+
Both counts are returned rather than printed here because this is called
|
|
1266
|
+
once per record by the streamed rehydration; the caller collapses a run
|
|
1267
|
+
into one line AFTER its commit (closeout review C5) — a rollback must not
|
|
1268
|
+
leave the operator told about a decline that never happened."""
|
|
1269
|
+
restored = conflicts = 0
|
|
1270
|
+
for rec in records:
|
|
1271
|
+
values = _file_account_values(rec)
|
|
1272
|
+
prior = cache.execute(
|
|
1273
|
+
"SELECT account_key FROM codex_file_accounts "
|
|
1274
|
+
"WHERE file_identity = ? AND incarnation = ? AND from_offset = ?",
|
|
1275
|
+
values[:3],
|
|
1276
|
+
).fetchone()
|
|
1277
|
+
before = cache.total_changes
|
|
1278
|
+
cache.execute(_FILE_ACCOUNT_INSERT, values)
|
|
1279
|
+
landed = cache.total_changes > before
|
|
1280
|
+
if prior is None:
|
|
1281
|
+
if not landed:
|
|
1282
|
+
# Dropped by a constraint the map table carries and the
|
|
1283
|
+
# incarnation table does not (`root_scope`, `decided_at_utc`).
|
|
1284
|
+
# Nothing was restored and nothing may be advanced.
|
|
1285
|
+
continue
|
|
1286
|
+
restored += 1
|
|
1287
|
+
elif prior[0] != values[4]:
|
|
1288
|
+
conflicts += 1
|
|
1289
|
+
cache.execute(_FILE_INCARNATION_INSERT, (values[0], values[1], values[5]))
|
|
1290
|
+
return restored, conflicts
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
def _report_file_account_conflicts(conflicts: int) -> None:
|
|
1294
|
+
"""One stderr line for a run of replayed decisions that contradicted a
|
|
1295
|
+
different account already recorded at the same
|
|
1296
|
+
``(file_identity, incarnation, from_offset)`` and were therefore DECLINED
|
|
1297
|
+
(first-wins, spec §3.3). Two ops at one primary key means one of them was
|
|
1298
|
+
minted without seeing the other, which is a real (if rare) condition with a
|
|
1299
|
+
real remedy, so it is reported rather than applied silently — the #374 rule.
|
|
1300
|
+
|
|
1301
|
+
Every call site must invoke this AFTER its commit (closeout review C5): a
|
|
1302
|
+
rolled-back transaction applied nothing, so reporting from inside it would
|
|
1303
|
+
tell the operator about a decline that did not happen."""
|
|
1304
|
+
if conflicts > 0:
|
|
1305
|
+
print(
|
|
1306
|
+
f"[ingest] codex attribution replay declined {conflicts} "
|
|
1307
|
+
"contradicting decision(s); the first journalled decision for each "
|
|
1308
|
+
"byte range is retained; run "
|
|
1309
|
+
"`cctally cache-sync --source codex --rebuild` if the "
|
|
1310
|
+
"attribution still looks wrong",
|
|
1311
|
+
file=sys.stderr,
|
|
1312
|
+
)
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
# --------------------------------------------------------------------------
|
|
1316
|
+
# #416 spec §3.5 (review F5): which record is AUTHORITATIVE.
|
|
1317
|
+
#
|
|
1318
|
+
# "The journal always wins" is the wrong authority. Journaled quota obs are
|
|
1319
|
+
# deduplicated on a natural key that EXCLUDES the account
|
|
1320
|
+
# (`_codex_quota_natural_key`) and later records for that key are discarded by
|
|
1321
|
+
# `append_record`, so the retained observation is FIRST-STAMP-WINS and may
|
|
1322
|
+
# preserve the known late-ingest guess — bytes written under one login but
|
|
1323
|
+
# ingested after a switch (spec §1.7). An unconditional upsert from it would
|
|
1324
|
+
# overwrite a corrected file-range decision.
|
|
1325
|
+
#
|
|
1326
|
+
# Precedence: the durable file/range DECISION is authoritative; the observation
|
|
1327
|
+
# stamp is used only where no range decision covers those bytes. Where the two
|
|
1328
|
+
# disagree the row keeps the decision and the disagreement is REPORTED rather
|
|
1329
|
+
# than silently applied — a genuine correction is expressed as an explicit new
|
|
1330
|
+
# range decision, never by mutating history.
|
|
1331
|
+
# --------------------------------------------------------------------------
|
|
1332
|
+
|
|
1333
|
+
# Converging form, used ONLY for a row whose bytes a decision covers. Repeating
|
|
1334
|
+
# a deterministic DO UPDATE is idempotent, which is what preserves crash-replay;
|
|
1335
|
+
# an uncovered obs keeps the first-write-wins INSERT OR IGNORE above.
|
|
1336
|
+
#
|
|
1337
|
+
# `INSERT OR IGNORE` is RETAINED here, not replaced by a plain `INSERT`. SQLite
|
|
1338
|
+
# gives the upsert clause precedence for the conflict it names, so the targeted
|
|
1339
|
+
# `DO UPDATE` still fires and convergence is unaffected — while every OTHER
|
|
1340
|
+
# constraint on `quota_window_snapshots` (four CHECKs and several NOT NULLs)
|
|
1341
|
+
# keeps the silent-drop tolerance the uncovered path has. Without it a violating
|
|
1342
|
+
# record raises `IntegrityError`, `_cache_applier` catches it as `sqlite3.Error`
|
|
1343
|
+
# and prefix-stops, and the scalar cursor can never advance past that record —
|
|
1344
|
+
# so ONE permanently-violating row would wedge the whole journal ingest cycle
|
|
1345
|
+
# forever, for every provider, not just Codex.
|
|
1346
|
+
_QUOTA_SNAPSHOT_UPSERT_CLAUSE = (
|
|
1347
|
+
" ON CONFLICT(source, source_path, line_offset, logical_limit_key) "
|
|
1348
|
+
"DO UPDATE SET account_key = excluded.account_key"
|
|
1349
|
+
)
|
|
1350
|
+
|
|
1351
|
+
_QUOTA_SNAPSHOT_UPSERT = _QUOTA_SNAPSHOT_INSERT + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
|
|
1352
|
+
_QUOTA_SNAPSHOT_UPSERT_LEGACY = (
|
|
1353
|
+
_QUOTA_SNAPSHOT_INSERT_LEGACY + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
|
|
1354
|
+
)
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
class _CodexAttributionOracle:
|
|
1358
|
+
"""Resolve ``(root_scope, source_path, line_offset)`` to the authoritative
|
|
1359
|
+
decision, memoised per file for one transaction.
|
|
1360
|
+
|
|
1361
|
+
The map is keyed on the durable file identity, not on the path, so the path
|
|
1362
|
+
is canonicalized through the SAME helper the ingest used
|
|
1363
|
+
(``_cctally_cache._canonical_codex_path``) to reach it. When that lookup
|
|
1364
|
+
cannot be made — the sibling is unavailable, or the map holds MORE THAN ONE
|
|
1365
|
+
incarnation for the file — the oracle DECLINES. Declining matters most for
|
|
1366
|
+
the multi-incarnation case: a truncation reuses offsets from zero, so an
|
|
1367
|
+
observation's byte offset no longer identifies which incarnation it belongs
|
|
1368
|
+
to, and guessing would attribute pre-truncation bytes to the replacement
|
|
1369
|
+
file's account. Declining falls back to the observation stamp, which is
|
|
1370
|
+
exactly the documented "no range decision covers those bytes" branch.
|
|
1371
|
+
"""
|
|
1372
|
+
|
|
1373
|
+
def __init__(self, cache):
|
|
1374
|
+
self._cache = cache
|
|
1375
|
+
self._cache_by_path: dict = {}
|
|
1376
|
+
self._canonicalize = None
|
|
1377
|
+
self._available = None
|
|
1378
|
+
|
|
1379
|
+
def _ensure_loaded(self) -> bool:
|
|
1380
|
+
if self._available is not None:
|
|
1381
|
+
return self._available
|
|
1382
|
+
try:
|
|
1383
|
+
probe = self._cache.execute(
|
|
1384
|
+
"SELECT 1 FROM codex_file_accounts LIMIT 1").fetchone()
|
|
1385
|
+
except sqlite3.Error:
|
|
1386
|
+
self._available = False
|
|
1387
|
+
return False
|
|
1388
|
+
if probe is None:
|
|
1389
|
+
self._available = False
|
|
1390
|
+
return False
|
|
1391
|
+
try:
|
|
1392
|
+
import _cctally_cache as _cc
|
|
1393
|
+
from _lib_source_identity import codex_file_key
|
|
1394
|
+
except Exception: # pragma: no cover — sibling unavailable
|
|
1395
|
+
self._available = False
|
|
1396
|
+
return False
|
|
1397
|
+
self._canonicalize = (_cc._canonical_codex_path, codex_file_key)
|
|
1398
|
+
self._available = True
|
|
1399
|
+
return True
|
|
1400
|
+
|
|
1401
|
+
def _ranges_for(self, root_scope, source_path):
|
|
1402
|
+
key = (root_scope, source_path)
|
|
1403
|
+
if key in self._cache_by_path:
|
|
1404
|
+
return self._cache_by_path[key]
|
|
1405
|
+
ranges: list = []
|
|
1406
|
+
canonical, file_key = self._canonicalize
|
|
1407
|
+
try:
|
|
1408
|
+
identity = file_key(root_scope, str(canonical(pathlib.Path(source_path))))
|
|
1409
|
+
except Exception:
|
|
1410
|
+
self._cache_by_path[key] = ranges
|
|
1411
|
+
return ranges
|
|
1412
|
+
rows = self._cache.execute(
|
|
1413
|
+
"SELECT DISTINCT incarnation FROM codex_file_accounts "
|
|
1414
|
+
"WHERE file_identity = ?", (identity,)).fetchall()
|
|
1415
|
+
if len(rows) == 1:
|
|
1416
|
+
ranges = [
|
|
1417
|
+
(int(r[0]), r[1]) for r in self._cache.execute(
|
|
1418
|
+
"SELECT from_offset, account_key FROM codex_file_accounts "
|
|
1419
|
+
"WHERE file_identity = ? AND incarnation = ? "
|
|
1420
|
+
"ORDER BY from_offset ASC", (identity, int(rows[0][0])))
|
|
1421
|
+
]
|
|
1422
|
+
self._cache_by_path[key] = ranges
|
|
1423
|
+
return ranges
|
|
1424
|
+
|
|
1425
|
+
def resolve(self, rec) -> "tuple[bool, str | None]":
|
|
1426
|
+
"""``(covered, account_key)`` for one quota obs."""
|
|
1427
|
+
if not self._ensure_loaded():
|
|
1428
|
+
return False, None
|
|
1429
|
+
payload = rec.get("payload") or {}
|
|
1430
|
+
root_scope = payload.get("source_root_key")
|
|
1431
|
+
source_path = payload.get("source_path")
|
|
1432
|
+
offset = payload.get("line_offset")
|
|
1433
|
+
if not root_scope or not source_path or offset is None:
|
|
1434
|
+
return False, None
|
|
1435
|
+
covered, account_key = False, None
|
|
1436
|
+
for from_offset, decided in self._ranges_for(root_scope, source_path):
|
|
1437
|
+
if from_offset > offset:
|
|
1438
|
+
break
|
|
1439
|
+
covered, account_key = True, decided
|
|
1440
|
+
return covered, account_key
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
def _cache_has_anchor_column(cache) -> bool:
|
|
1444
|
+
"""Whether this cache.db carries ``quota_window_snapshots.canonical_resets_at_utc``.
|
|
1445
|
+
|
|
1446
|
+
Probed rather than assumed. This leg opens cache.db RAW (no dispatcher, no
|
|
1447
|
+
schema apply), so it can meet a cache that has not yet gained the column. A
|
|
1448
|
+
column-count mismatch would raise ``sqlite3.OperationalError``, which
|
|
1449
|
+
``_cache_applier`` catches as a write failure and turns into a PREFIX-STOP —
|
|
1450
|
+
and the scalar cursor could then never advance past that record, wedging the
|
|
1451
|
+
journal ingest cycle for every provider. Same reasoning as the ``OR IGNORE``
|
|
1452
|
+
on ``_FILE_ACCOUNT_INSERT``; the walk's own writer needs no such guard
|
|
1453
|
+
because it only ever runs on a dispatcher-opened connection.
|
|
1454
|
+
"""
|
|
1455
|
+
try:
|
|
1456
|
+
return "canonical_resets_at_utc" in {
|
|
1457
|
+
str(row[1]) for row in cache.execute(
|
|
1458
|
+
"PRAGMA table_info(quota_window_snapshots)")
|
|
1459
|
+
}
|
|
1460
|
+
except sqlite3.Error: # pragma: no cover — unreadable schema
|
|
1461
|
+
return False
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
def _codex_anchor_resolver(cache):
|
|
1465
|
+
"""A ``CodexResetAnchorResolver`` over this connection, or ``None`` when the
|
|
1466
|
+
sibling is unavailable. Degrading to ``None`` leaves the anchor column NULL,
|
|
1467
|
+
which every reader treats as "use the raw reset" — today's behaviour."""
|
|
1468
|
+
try:
|
|
1469
|
+
import _cctally_cache as _cc
|
|
1470
|
+
except Exception: # pragma: no cover — sibling unavailable
|
|
1471
|
+
return None
|
|
1472
|
+
try:
|
|
1473
|
+
return _cc.CodexResetAnchorResolver(cache)
|
|
1474
|
+
except Exception: # pragma: no cover — older sibling without the resolver
|
|
1475
|
+
return None
|
|
1476
|
+
|
|
1098
1477
|
|
|
1099
|
-
|
|
1478
|
+
def _resolve_obs_anchor(resolver, rec: dict) -> "str | None":
|
|
1479
|
+
if resolver is None:
|
|
1480
|
+
return None
|
|
1481
|
+
p = rec.get("payload") or {}
|
|
1482
|
+
root = p.get("source_root_key")
|
|
1483
|
+
slot = p.get("observed_slot")
|
|
1484
|
+
key = p.get("logical_limit_key")
|
|
1485
|
+
if not isinstance(root, str) or not isinstance(slot, str) or not isinstance(key, str):
|
|
1486
|
+
return None
|
|
1487
|
+
try:
|
|
1488
|
+
return resolver.resolve(
|
|
1489
|
+
source_root_key=root, observed_slot=slot, logical_limit_key=key,
|
|
1490
|
+
window_minutes=p.get("window_minutes"),
|
|
1491
|
+
resets_at_utc=p.get("resets_at_utc"),
|
|
1492
|
+
)
|
|
1493
|
+
except Exception: # pragma: no cover — never fail an ingest over a label
|
|
1494
|
+
return None
|
|
1495
|
+
|
|
1496
|
+
|
|
1497
|
+
def _apply_quota_records(cache, records) -> None:
|
|
1498
|
+
"""Materialize Codex quota obs into an OPEN cache.db transaction, applying
|
|
1499
|
+
the §3.5 precedence rule. Callers must apply the batch's file-account
|
|
1500
|
+
decisions FIRST, so a decision arriving in the same batch already governs
|
|
1501
|
+
the observations it covers."""
|
|
1502
|
+
oracle = _CodexAttributionOracle(cache)
|
|
1503
|
+
# One line per FILE PER BATCH, not per record. A mid-file account switch
|
|
1504
|
+
# legitimately produces a run of observations whose first-stamp-wins account
|
|
1505
|
+
# disagrees with the range decision now governing those bytes, and an
|
|
1506
|
+
# unthrottled warning would emit one line per row for the whole run. The set
|
|
1507
|
+
# is deliberately local, so a file whose conflicting run SPANS several
|
|
1508
|
+
# ingest batches reports once per batch — a per-cycle or per-process set
|
|
1509
|
+
# would have to outlive the transaction that may roll back, and repeating a
|
|
1510
|
+
# standing condition a handful of times is the cheaper error. The condition
|
|
1511
|
+
# is worth reporting at all because a genuine correction is expressed as an
|
|
1512
|
+
# explicit new range decision, never by mutating history.
|
|
1513
|
+
reported_conflicts: set = set()
|
|
1514
|
+
# #416 spec §4.2: this leg is a genuine INGEST into cache.db (it materializes
|
|
1515
|
+
# observations whose source rollout may have evaporated), so it must resolve
|
|
1516
|
+
# the canonical anchor too. Without it, a journal-replayed row lands with a
|
|
1517
|
+
# NULL anchor, a later walk's `INSERT OR IGNORE` cannot correct it, and that
|
|
1518
|
+
# window stays fragmented forever.
|
|
1519
|
+
has_anchor = _cache_has_anchor_column(cache)
|
|
1520
|
+
anchors = _codex_anchor_resolver(cache) if has_anchor else None
|
|
1521
|
+
insert_sql = _QUOTA_SNAPSHOT_INSERT if has_anchor else _QUOTA_SNAPSHOT_INSERT_LEGACY
|
|
1522
|
+
upsert_sql = _QUOTA_SNAPSHOT_UPSERT if has_anchor else _QUOTA_SNAPSHOT_UPSERT_LEGACY
|
|
1523
|
+
for rec in records:
|
|
1524
|
+
covered, decided = oracle.resolve(rec)
|
|
1525
|
+
anchor = _resolve_obs_anchor(anchors, rec)
|
|
1526
|
+
row_values = _quota_snapshot_values(rec, anchor)
|
|
1527
|
+
if not has_anchor:
|
|
1528
|
+
row_values = row_values[:-1]
|
|
1529
|
+
if not covered:
|
|
1530
|
+
cache.execute(insert_sql, row_values)
|
|
1531
|
+
continue
|
|
1532
|
+
observed = rec.get("account")
|
|
1533
|
+
payload = rec.get("payload") or {}
|
|
1534
|
+
conflict_key = (payload.get("source_root_key"), payload.get("source_path"))
|
|
1535
|
+
if (observed is not None and observed != decided
|
|
1536
|
+
and conflict_key not in reported_conflicts):
|
|
1537
|
+
reported_conflicts.add(conflict_key)
|
|
1538
|
+
print(
|
|
1539
|
+
"[ingest] codex attribution conflict: "
|
|
1540
|
+
f"{payload.get('source_path')}@{payload.get('line_offset')} "
|
|
1541
|
+
f"observation stamped {observed} but the durable decision says "
|
|
1542
|
+
f"{decided if decided is not None else 'unattributed'}; "
|
|
1543
|
+
"keeping the decision",
|
|
1544
|
+
file=sys.stderr,
|
|
1545
|
+
)
|
|
1546
|
+
values = list(row_values)
|
|
1547
|
+
values[16] = decided
|
|
1548
|
+
cache.execute(upsert_sql, tuple(values))
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
def rehydrate_codex_file_accounts(
|
|
1552
|
+
cache_conn, *, authoritative: bool = False, since=None,
|
|
1553
|
+
) -> "tuple[int, tuple[str, int] | None, int]":
|
|
1554
|
+
"""Replay journaled ``codex_file_account`` ops into an open cache.db
|
|
1555
|
+
connection; return ``(applied_count, high_water, declined_conflicts)``
|
|
1556
|
+
(#416 spec §3.4).
|
|
1557
|
+
|
|
1558
|
+
The conflict count is RETURNED rather than reported here (closeout review
|
|
1559
|
+
C5). This function runs inside the caller's transaction, and that caller
|
|
1560
|
+
rolls back on failure — reporting from in here would tell the operator about
|
|
1561
|
+
a decline that was undone. The two ``_apply_file_account_records`` call
|
|
1562
|
+
sites in the appliers already report post-commit; this one now matches.
|
|
1563
|
+
|
|
1564
|
+
``since`` is the ``(segment, offset)`` journal cursor the caller last
|
|
1565
|
+
replayed, or ``None`` for "from the beginning". The returned high-water is
|
|
1566
|
+
what the caller must persist so the NEXT call replays only the delta — the
|
|
1567
|
+
two together are what make this affordable on the hot path AND what recovers
|
|
1568
|
+
a decision that was journaled but never materialized (spec §3.6: "a crash
|
|
1569
|
+
after append but before the cache-map commit is recovered by replaying
|
|
1570
|
+
pending journal state under the same locked operation BEFORE ``auth.json``
|
|
1571
|
+
is consulted on retry"). A one-shot "already rehydrated" marker cannot do
|
|
1572
|
+
that: the failing sync's own op lands AFTER the marker was written, so the
|
|
1573
|
+
retry would never replay it and would re-decide from a possibly-changed
|
|
1574
|
+
identity instead.
|
|
1575
|
+
|
|
1576
|
+
``authoritative`` ignores ``since`` — a clear-then-replay is only correct
|
|
1577
|
+
from the beginning of the journal.
|
|
1578
|
+
|
|
1579
|
+
The caller owns the flocks, the transaction and the commit — this function
|
|
1580
|
+
only executes the idempotent upserts, so it can run inside
|
|
1581
|
+
``sync_codex_cache``'s already-locked phases without violating the
|
|
1582
|
+
lock-order law.
|
|
1583
|
+
|
|
1584
|
+
Why an explicit phase exists at all: the ordinary journal-to-cache replay
|
|
1585
|
+
runs only inside ``rebuild_stats_index``, whereas ``cache-sync`` clears (on
|
|
1586
|
+
``--rebuild``) and begins the rollout walk with NO applier in front of it. A
|
|
1587
|
+
recreated cache.db (corruption recovery, a manual ``rm cache.db``) therefore
|
|
1588
|
+
starts with an empty map, and the walk would fall straight back to the live
|
|
1589
|
+
``auth.json`` for every file — which is the defect (review F2). Note this is
|
|
1590
|
+
NOT rebuild-only: every production Codex call site syncs with
|
|
1591
|
+
``rebuild=False``, and the corruption auto-heal recreates the cache.db family
|
|
1592
|
+
and then re-runs the ORDINARY sync, so a rebuild-only wiring leaves the
|
|
1593
|
+
defect reachable by a shorter road.
|
|
1594
|
+
|
|
1595
|
+
``authoritative=True`` makes the replay a CONVERGENCE operator rather than an
|
|
1596
|
+
inserter: it clears ``codex_file_accounts`` first, so a row that has drifted
|
|
1597
|
+
away from the journal is corrected instead of being silently preserved by the
|
|
1598
|
+
``DO NOTHING`` conflict clause (the #374 fold-applier defect class — see
|
|
1599
|
+
``docs/journal-gotchas.md``). This is lossless because the ingest's
|
|
1600
|
+
fail-closed append journals the decision BEFORE any accounting DML or map
|
|
1601
|
+
write for that file, so every map row has a journal op behind it, and the
|
|
1602
|
+
journal is append-only with no segment pruning. It is the documented remedy
|
|
1603
|
+
(``cache-sync --rebuild``), so it must actually be able to repair.
|
|
1604
|
+
|
|
1605
|
+
``codex_file_incarnations`` is deliberately NOT cleared even under
|
|
1606
|
+
``authoritative``, and the reason is NOT that a clear would be conservative.
|
|
1607
|
+
The op is journaled BEFORE the batch that persists the incarnation, so every
|
|
1608
|
+
committed incarnation is ``<=`` the highest incarnation any op carries — a
|
|
1609
|
+
re-derivation from ops can therefore never LOWER the counter, only raise it
|
|
1610
|
+
above what any committed batch used. And too high is the DANGEROUS
|
|
1611
|
+
direction, not the safe one: ranges are resolved at exactly the walk's
|
|
1612
|
+
current incarnation, so an inflated counter loads an EMPTY range list,
|
|
1613
|
+
``covered`` is False, and a plain sync falls straight through to the live
|
|
1614
|
+
``auth.json`` branch and re-decides — the original defect. Since the MAX-set
|
|
1615
|
+
upsert already converges the counter, a clear has no upside and that
|
|
1616
|
+
downside.
|
|
1617
|
+
"""
|
|
1618
|
+
hw = journal_high_water()
|
|
1619
|
+
if hw is None:
|
|
1620
|
+
if authoritative:
|
|
1621
|
+
# No journal at all: an authoritative pass still says "the journal
|
|
1622
|
+
# is the truth", and the truth is that there are no decisions.
|
|
1623
|
+
cache_conn.execute("DELETE FROM codex_file_accounts")
|
|
1624
|
+
return 0, None, 0
|
|
1625
|
+
if authoritative:
|
|
1626
|
+
cache_conn.execute("DELETE FROM codex_file_accounts")
|
|
1627
|
+
since = None
|
|
1628
|
+
applied = 0
|
|
1629
|
+
conflicts = 0
|
|
1630
|
+
# Streamed, never materialized: this runs on the FIRST ordinary sync of
|
|
1631
|
+
# every cache.db (hook-tick, the dashboard, the corruption auto-heal's
|
|
1632
|
+
# re-sync) while both cache flocks are held, so a whole-journal transient
|
|
1633
|
+
# here is a multi-second global cache-writer stall — itself a
|
|
1634
|
+
# `database is locked` trigger. The cheap byte prefilter skips the JSON
|
|
1635
|
+
# decode for every non-decision line; the canonical encoder is
|
|
1636
|
+
# `json.dumps(..., ensure_ascii=False)`, which never escapes an ASCII kind
|
|
1637
|
+
# token, so a genuine op always carries this substring verbatim. A false
|
|
1638
|
+
# positive is harmless — it is decoded and rejected by the real predicate.
|
|
1639
|
+
for _seg, _off, raw in iter_range(since, hw):
|
|
1640
|
+
if _FILE_ACCOUNT_KIND_MARKER not in raw:
|
|
1641
|
+
continue
|
|
1642
|
+
rec = _lib_journal.decode_line(raw)
|
|
1643
|
+
if rec is not None and _is_codex_file_account_op(rec):
|
|
1644
|
+
_restored, _conflicts = _apply_file_account_records(cache_conn, (rec,))
|
|
1645
|
+
applied += _restored
|
|
1646
|
+
conflicts += _conflicts
|
|
1647
|
+
return applied, hw, conflicts
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def _cache_applier(decoded) -> int | None:
|
|
1651
|
+
"""Composite cache leg (spec §5.2 step 3 + #416 spec §3.4): materialize this
|
|
1652
|
+
batch's Codex quota obs into `quota_window_snapshots` AND its
|
|
1653
|
+
`codex_file_account` ops into the attribution map, under the NON-BLOCKING
|
|
1654
|
+
global cache writer lock followed by `cache.db.codex.lock`, in ONE
|
|
1655
|
+
`BEGIN IMMEDIATE`. Contract (journal seam): `(decoded) -> stop | None`,
|
|
1656
|
+
`decoded = [(record, segment, offset), ...]` in canonical order.
|
|
1657
|
+
|
|
1658
|
+
- Neither family present in the batch → return None (no flock taken).
|
|
1100
1659
|
- Busy global/provider flock, OR a cache write it cannot complete → PREFIX-STOP:
|
|
1101
|
-
return the index
|
|
1102
|
-
`decoded[:stop]` and advances the cursor to
|
|
1103
|
-
retrying the remainder next cycle (the scalar
|
|
1104
|
-
unmaterialized
|
|
1105
|
-
- Flock acquired +
|
|
1660
|
+
return the EARLIEST index across BOTH families having committed NEITHER, so
|
|
1661
|
+
the cycle processes only `decoded[:stop]` and advances the cursor to
|
|
1662
|
+
`decoded[stop]`'s offset, retrying the remainder next cycle (the scalar
|
|
1663
|
+
cursor never advances past an unmaterialized record — spec §5.2 step 3).
|
|
1664
|
+
- Flock acquired + everything upserted → return None (full consumption).
|
|
1106
1665
|
"""
|
|
1107
1666
|
quota_idx = [i for i, (rec, _s, _o) in enumerate(decoded)
|
|
1108
1667
|
if _is_codex_quota_obs(rec)]
|
|
1109
|
-
|
|
1668
|
+
file_idx = [i for i, (rec, _s, _o) in enumerate(decoded)
|
|
1669
|
+
if _is_codex_file_account_op(rec)]
|
|
1670
|
+
if not quota_idx and not file_idx:
|
|
1110
1671
|
return None
|
|
1672
|
+
# All-or-nothing across the two families: one stop, the earliest of either.
|
|
1673
|
+
stop_idx = min(quota_idx[0] if quota_idx else file_idx[0],
|
|
1674
|
+
file_idx[0] if file_idx else quota_idx[0])
|
|
1111
1675
|
from _lib_cache_writer_lock import (
|
|
1112
1676
|
acquire_cache_writer_flocks,
|
|
1113
1677
|
release_cache_writer_flocks,
|
|
@@ -1120,31 +1684,36 @@ def _quota_applier(decoded) -> int | None:
|
|
|
1120
1684
|
_cctally_core.CACHE_LOCK_CODEX_PATH,
|
|
1121
1685
|
)
|
|
1122
1686
|
except OSError:
|
|
1123
|
-
return
|
|
1687
|
+
return stop_idx
|
|
1124
1688
|
if held is None:
|
|
1125
|
-
return
|
|
1689
|
+
return stop_idx
|
|
1126
1690
|
try:
|
|
1127
1691
|
try:
|
|
1128
1692
|
cache = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH), timeout=15.0)
|
|
1129
1693
|
except sqlite3.Error as exc: # pragma: no cover — cache.db unopenable
|
|
1130
|
-
print(f"[ingest]
|
|
1131
|
-
return
|
|
1694
|
+
print(f"[ingest] cache leg connect failed: {exc}", file=sys.stderr)
|
|
1695
|
+
return stop_idx
|
|
1132
1696
|
try:
|
|
1133
1697
|
cache.execute("PRAGMA busy_timeout=15000")
|
|
1134
1698
|
cache.execute("BEGIN IMMEDIATE")
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1699
|
+
# Decisions FIRST: §3.5 makes the file/range decision authoritative
|
|
1700
|
+
# over the observation stamp, so a decision arriving in this batch
|
|
1701
|
+
# must already govern the observations it covers.
|
|
1702
|
+
_, _file_conflicts = _apply_file_account_records(
|
|
1703
|
+
cache, [decoded[i][0] for i in file_idx])
|
|
1704
|
+
_apply_quota_records(cache, [decoded[i][0] for i in quota_idx])
|
|
1138
1705
|
cache.commit()
|
|
1706
|
+
_report_file_account_conflicts(_file_conflicts)
|
|
1139
1707
|
except sqlite3.Error as exc:
|
|
1140
1708
|
try:
|
|
1141
1709
|
cache.rollback()
|
|
1142
1710
|
except sqlite3.Error:
|
|
1143
1711
|
pass
|
|
1144
1712
|
# Could not materialize -> prefix-stop so the cursor holds and the
|
|
1145
|
-
# next cycle retries (the
|
|
1146
|
-
|
|
1147
|
-
|
|
1713
|
+
# next cycle retries (the records stay durable in the journal
|
|
1714
|
+
# regardless). NEITHER family is committed.
|
|
1715
|
+
print(f"[ingest] cache leg write failed: {exc}", file=sys.stderr)
|
|
1716
|
+
return stop_idx
|
|
1148
1717
|
finally:
|
|
1149
1718
|
cache.close()
|
|
1150
1719
|
return None
|
|
@@ -1152,10 +1721,14 @@ def _quota_applier(decoded) -> int | None:
|
|
|
1152
1721
|
release_cache_writer_flocks(held)
|
|
1153
1722
|
|
|
1154
1723
|
|
|
1724
|
+
# Back-compat alias: the leg was Codex-quota-only until #416 widened it.
|
|
1725
|
+
_quota_applier = _cache_applier
|
|
1726
|
+
|
|
1155
1727
|
# Wire the seam (declared None near the top as the contract stub). Always-on:
|
|
1156
|
-
# a Claude-only cycle's scan finds
|
|
1157
|
-
#
|
|
1158
|
-
|
|
1728
|
+
# a Claude-only cycle's scan finds neither family and returns None before any
|
|
1729
|
+
# flock/DB touch, so the cost is two list comprehensions over the batch.
|
|
1730
|
+
CACHE_APPLIER = _cache_applier
|
|
1731
|
+
QUOTA_APPLIER = _cache_applier
|
|
1159
1732
|
|
|
1160
1733
|
|
|
1161
1734
|
# --------------------------------------------------------------------------
|
|
@@ -1468,11 +2041,21 @@ _EVT_KIND_PROVIDER = {
|
|
|
1468
2041
|
"projected": "claude",
|
|
1469
2042
|
"project_budget": "claude",
|
|
1470
2043
|
"quota_alert_arming": "codex",
|
|
2044
|
+
"quota_threshold_event": "codex",
|
|
1471
2045
|
}
|
|
1472
2046
|
|
|
1473
2047
|
# Op kinds that are accounts-machinery (recognised, never classified as legacy).
|
|
2048
|
+
# `codex_file_account` (#416 spec §3.3) joins them: it is the durable Codex
|
|
2049
|
+
# attribution DECISION, and its sentinel form deliberately OMITS `account_key`
|
|
2050
|
+
# — exactly the shape the legacy classifier keys on — so registration here is
|
|
2051
|
+
# what keeps `_normalize_legacy_account_stamp` from ever retro-stamping it.
|
|
2052
|
+
# Registering a kind here also feeds `_cctally_rederive.plan_claude_usage`'s
|
|
2053
|
+
# `op_kinds` set, so the kind MUST additionally carry a
|
|
2054
|
+
# `_lib_rederive._OP_CLASSIFICATIONS` entry or the re-derive planner raises
|
|
2055
|
+
# `RederiveConflict` on every run.
|
|
1474
2056
|
_ACCOUNTS_MACHINERY_KINDS = frozenset(
|
|
1475
|
-
("account_observe", "account_label", "accounts_cutover"
|
|
2057
|
+
("account_observe", "account_label", "accounts_cutover",
|
|
2058
|
+
"codex_file_account"))
|
|
1476
2059
|
|
|
1477
2060
|
# Legacy-classifier exhaustiveness guard (#341, review finding P2-1). EVERY evt
|
|
1478
2061
|
# kind in `_EVT_SPECS` and every harvest kind in `_HARVEST_SPECS` must carry a
|
|
@@ -1769,6 +2352,75 @@ def _apply_quota_alert_arming(conn, evt):
|
|
|
1769
2352
|
return None
|
|
1770
2353
|
|
|
1771
2354
|
|
|
2355
|
+
def _apply_quota_threshold_event(conn, evt):
|
|
2356
|
+
"""Fold a `quota_threshold_event` evt (#416 spec §7.2, review F13).
|
|
2357
|
+
|
|
2358
|
+
`quota_threshold_events` is TERMINAL alert evidence: each row records that a
|
|
2359
|
+
threshold was crossed and either alerted or was suppressed as backfill, with
|
|
2360
|
+
the exact moment it happened. It is in NEITHER `_HARVEST_SPECS` (eight
|
|
2361
|
+
families, none of them this one) nor — before #416 — `_CUTOVER_SPECS`, and
|
|
2362
|
+
`rematerialize_quota_projection_for_rebuild` runs with
|
|
2363
|
+
`alert_eligible_roots=frozenset()`, so a rebuild could not recreate an
|
|
2364
|
+
`alerted` row at all. Every rebuild silently discarded the evidence that an
|
|
2365
|
+
alert had already fired, which is what would let it fire again.
|
|
2366
|
+
|
|
2367
|
+
Convergence is a natural-key UPSERT, exactly like `_apply_quota_alert_arming`
|
|
2368
|
+
— the table has no `journal_id` column, so idempotence cannot ride an
|
|
2369
|
+
`INSERT OR IGNORE` on the journal id. The upsert restores `disposition`,
|
|
2370
|
+
`alerted_at` and `suppressed_at` VERBATIM, which matters because the rebuild's
|
|
2371
|
+
re-materialization pass can legitimately re-derive the same crossing as a
|
|
2372
|
+
fresh `suppressed_backfill` row: whichever of the two runs second, the
|
|
2373
|
+
journaled terminal fact is what stands.
|
|
2374
|
+
|
|
2375
|
+
`orphaned_at` is deliberately NOT journaled and NOT touched here. It marks a
|
|
2376
|
+
window whose evidence has since vanished, and `_orphan_unseen` re-derives it
|
|
2377
|
+
from the current projection on every pass — replaying a stale value would
|
|
2378
|
+
fight that.
|
|
2379
|
+
"""
|
|
2380
|
+
p = evt.get("payload") or {}
|
|
2381
|
+
account_key = p.get("account_key") or _lib_accounts.UNATTRIBUTED
|
|
2382
|
+
disposition = p.get("disposition")
|
|
2383
|
+
alerted_at = p.get("alerted_at")
|
|
2384
|
+
suppressed_at = p.get("suppressed_at")
|
|
2385
|
+
# The table's CHECK pairs disposition with exactly one timestamp. Normalize
|
|
2386
|
+
# rather than trust the payload, so a malformed record cannot raise here and
|
|
2387
|
+
# prefix-stop the whole fold.
|
|
2388
|
+
if disposition == "alerted":
|
|
2389
|
+
suppressed_at = None
|
|
2390
|
+
alerted_at = alerted_at or p.get("created_at_utc")
|
|
2391
|
+
elif disposition == "suppressed_backfill":
|
|
2392
|
+
alerted_at = None
|
|
2393
|
+
suppressed_at = suppressed_at or p.get("created_at_utc")
|
|
2394
|
+
else:
|
|
2395
|
+
return None
|
|
2396
|
+
conn.execute(
|
|
2397
|
+
"INSERT OR IGNORE INTO quota_threshold_events "
|
|
2398
|
+
"(source, source_root_key, logical_limit_key, observed_slot, "
|
|
2399
|
+
" window_minutes, resets_at_utc, threshold, qualifying_kind, "
|
|
2400
|
+
" qualifying_percent, projected_percent, severity, created_at_utc, "
|
|
2401
|
+
" disposition, alerted_at, suppressed_at, account_key) "
|
|
2402
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "
|
|
2403
|
+
"ON CONFLICT(source, source_root_key, account_key, logical_limit_key, "
|
|
2404
|
+
" observed_slot, window_minutes, resets_at_utc, threshold) "
|
|
2405
|
+
"DO UPDATE SET "
|
|
2406
|
+
" qualifying_kind=excluded.qualifying_kind, "
|
|
2407
|
+
" qualifying_percent=excluded.qualifying_percent, "
|
|
2408
|
+
" projected_percent=excluded.projected_percent, "
|
|
2409
|
+
" severity=excluded.severity, "
|
|
2410
|
+
" created_at_utc=excluded.created_at_utc, "
|
|
2411
|
+
" disposition=excluded.disposition, "
|
|
2412
|
+
" alerted_at=excluded.alerted_at, "
|
|
2413
|
+
" suppressed_at=excluded.suppressed_at",
|
|
2414
|
+
(p.get("source"), p.get("source_root_key"), p.get("logical_limit_key"),
|
|
2415
|
+
p.get("observed_slot"), p.get("window_minutes"), p.get("resets_at_utc"),
|
|
2416
|
+
p.get("threshold"), p.get("qualifying_kind"),
|
|
2417
|
+
p.get("qualifying_percent"), p.get("projected_percent"),
|
|
2418
|
+
p.get("severity"), p.get("created_at_utc"), disposition,
|
|
2419
|
+
alerted_at, suppressed_at, account_key),
|
|
2420
|
+
)
|
|
2421
|
+
return None
|
|
2422
|
+
|
|
2423
|
+
|
|
1772
2424
|
def _apply_block_close(conn, evt):
|
|
1773
2425
|
"""Fold one authoritative frozen-block fact.
|
|
1774
2426
|
|
|
@@ -1957,6 +2609,14 @@ _EVT_SPECS = {
|
|
|
1957
2609
|
# upsert applier. order is arbitrary among evts (no cross-family FK).
|
|
1958
2610
|
"quota_alert_arming": _EvtSpec(
|
|
1959
2611
|
None, order=45, applier=_apply_quota_alert_arming),
|
|
2612
|
+
# Terminal quota alert evidence (#416 spec §7.2). Order 44 — BEFORE
|
|
2613
|
+
# `quota_alert_arming` (45) and before the quota projection
|
|
2614
|
+
# re-materialization, so the journaled terminal fact is already in place
|
|
2615
|
+
# when the rebuild's re-derivation runs. Both directions are safe anyway:
|
|
2616
|
+
# this applier converges by natural-key upsert and the re-derivation's own
|
|
2617
|
+
# insert is `INSERT OR IGNORE`, so neither can clobber the other's row.
|
|
2618
|
+
"quota_threshold_event": _EvtSpec(
|
|
2619
|
+
None, order=44, applier=_apply_quota_threshold_event),
|
|
1960
2620
|
}
|
|
1961
2621
|
for _hs in _HARVEST_SPECS:
|
|
1962
2622
|
if _hs.children:
|
|
@@ -3972,16 +4632,25 @@ def _publish_rebuilt_stats_index(
|
|
|
3972
4632
|
|
|
3973
4633
|
|
|
3974
4634
|
def _rebuild_quota_cache_leg(records) -> None:
|
|
3975
|
-
"""Re-materialize cache.db `quota_window_snapshots`
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
the
|
|
4635
|
+
"""Re-materialize cache.db `quota_window_snapshots` AND the #416 Codex
|
|
4636
|
+
attribution map from the journal (spec §5.4 + #416 spec §3.4).
|
|
4637
|
+
|
|
4638
|
+
The journal records are the DURABLE source (§1 latent data-loss hole — the
|
|
4639
|
+
rollout JSONL evaporates); this INSERT-OR-IGNOREs the quota obs on their
|
|
4640
|
+
natural key and the attribution decisions on theirs, mirroring
|
|
4641
|
+
`_cache_applier` family for family. The map half is not optional: without it
|
|
4642
|
+
a rebuild would leave the map empty and the following rollout walk would
|
|
4643
|
+
have nothing to replay, sending every file back to the live `auth.json` —
|
|
4644
|
+
the exact defect this mechanism exists to prevent.
|
|
4645
|
+
|
|
4646
|
+
Runs BEFORE any stats transaction, under the global cache writer lock
|
|
4647
|
+
followed by the `cache.db.codex.lock` provider flock (lock-order law).
|
|
4648
|
+
Best-effort: a missing/busy cache.db is a clean skip (the records stay
|
|
4649
|
+
durable in the journal; the stats quota projection pass then degrades
|
|
4650
|
+
cleanly)."""
|
|
3983
4651
|
quota_obs = [r for r in records if _is_codex_quota_obs(r)]
|
|
3984
|
-
if
|
|
4652
|
+
file_accounts = [r for r in records if _is_codex_file_account_op(r)]
|
|
4653
|
+
if not quota_obs and not file_accounts:
|
|
3985
4654
|
return
|
|
3986
4655
|
cache_path = _cctally_core.CACHE_DB_PATH
|
|
3987
4656
|
if not cache_path.exists():
|
|
@@ -4013,10 +4682,11 @@ def _rebuild_quota_cache_leg(records) -> None:
|
|
|
4013
4682
|
try:
|
|
4014
4683
|
cache.execute("PRAGMA busy_timeout=15000")
|
|
4015
4684
|
cache.execute("BEGIN IMMEDIATE")
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4685
|
+
# Decisions FIRST — same §3.5 precedence ordering as `_cache_applier`.
|
|
4686
|
+
_, _file_conflicts = _apply_file_account_records(cache, file_accounts)
|
|
4687
|
+
_apply_quota_records(cache, quota_obs)
|
|
4019
4688
|
cache.commit()
|
|
4689
|
+
_report_file_account_conflicts(_file_conflicts)
|
|
4020
4690
|
except sqlite3.Error as exc:
|
|
4021
4691
|
try:
|
|
4022
4692
|
cache.rollback()
|
|
@@ -4387,6 +5057,23 @@ _CUTOVER_SPECS = (
|
|
|
4387
5057
|
"logical_limit_key", "observed_slot",
|
|
4388
5058
|
"window_minutes", "rule_fingerprint",
|
|
4389
5059
|
"activated_at_utc")),
|
|
5060
|
+
_CutoverSpec("quota_threshold_events", "quota_threshold_event", "evt",
|
|
5061
|
+
"created_at_utc", stamp=False, natural_key_prefix="qte",
|
|
5062
|
+
natural_key_id=("source", "source_root_key", "account_key",
|
|
5063
|
+
"logical_limit_key", "observed_slot",
|
|
5064
|
+
"window_minutes", "resets_at_utc",
|
|
5065
|
+
"threshold")),
|
|
5066
|
+
# quota_threshold_events (#416 spec §7.2, review F13) — TERMINAL alert
|
|
5067
|
+
# evidence, modelled exactly on the quota_alert_arming precedent above: no
|
|
5068
|
+
# `journal_id` column -> NOT stamped, and the fold applier converges by
|
|
5069
|
+
# natural key. The `qte:` id mirrors the table's own UNIQUE key so one
|
|
5070
|
+
# crossing is one event forever, and it MUST match the live emitter in
|
|
5071
|
+
# `_cctally_quota._codex_leg._emit_terminal_event`.
|
|
5072
|
+
#
|
|
5073
|
+
# This spec covers a legacy install that has not yet cut over. An install
|
|
5074
|
+
# already past the cutover carries no history here — which is exactly why
|
|
5075
|
+
# the live emitter exists: without it, only pre-cutover rows would ever be
|
|
5076
|
+
# replayable, and every row written since would still be lost on a rebuild.
|
|
4390
5077
|
)
|
|
4391
5078
|
|
|
4392
5079
|
def _export_stats_table(conn, spec) -> list:
|
|
@@ -4420,7 +5107,7 @@ def _export_stats_table(conn, spec) -> list:
|
|
|
4420
5107
|
payload[payload_key] = [
|
|
4421
5108
|
{k: cr[k] for k in cr.keys() if k not in ("id", "block_id")}
|
|
4422
5109
|
for cr in child_rows]
|
|
4423
|
-
if spec.kind
|
|
5110
|
+
if spec.kind in ("quota_alert_arming", "quota_threshold_event"):
|
|
4424
5111
|
payload["journal_identity_version"] = 2
|
|
4425
5112
|
if spec.natural_key_id:
|
|
4426
5113
|
# §5.3 "state" family: the evt id is the state-instance form (matching
|