cctally 1.83.0 → 1.84.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -532,7 +532,7 @@ def journal_high_water() -> tuple[str, int] | None:
532
532
  #
533
533
  # 1. HW snapshot (leaf lock, µs). journal_high_water()
534
534
  # 2. read+decode cursor -> HW, counting malformed. _read_range()
535
- # 3. cache leg (Codex quota) BEFORE the stats txn. QUOTA_APPLIER seam
535
+ # 3. cache leg (Codex quota + attribution) before stats. CACHE_APPLIER seam
536
536
  # 4. ONE `BEGIN IMMEDIATE`:
537
537
  # a. replay journal evt lines (apply-only, NO alerts). _apply_evt()
538
538
  # b. per-record sequential PIPELINE over obs/op. PIPELINE hooks
@@ -548,14 +548,19 @@ def journal_high_water() -> tuple[str, int] | None:
548
548
  # milestones, resets/credits, cost snapshots, budgets); the
549
549
  # built-in `_pipeline_op_weekly_credit_floor` op fold ships
550
550
  # here (spec §5.3 "fold op").
551
- # QUOTA_APPLIER the Codex quota cache leg (Task 7; wired to `_quota_applier`
552
- # below). Contract: (decoded) -> stop_index | None. `decoded`
553
- # is the ordered list of (record, segment, offset); a non-None
554
- # int is a prefix-stop boundary (busy global or Codex cache
555
- # writer flock at a quota line): the cycle processes
556
- # decoded[:stop] and
557
- # advances the cursor to decoded[stop]'s offset (spec §5.2
558
- # step 3). Always-on: a Claude-only batch scans + returns None.
551
+ # CACHE_APPLIER the composite Codex cache leg (Task 7; #416 widened it from
552
+ # quota-only to quota + `codex_file_account` attribution ops,
553
+ # wired to `_cache_applier` below; `QUOTA_APPLIER` remains as
554
+ # its back-compat alias). Contract: (decoded) -> stop_index |
555
+ # None. `decoded` is the ordered list of (record, segment,
556
+ # offset); a non-None int is a prefix-stop boundary (busy
557
+ # global or Codex cache writer flock, or an incomplete cache
558
+ # write): the cycle processes decoded[:stop] and advances the
559
+ # cursor to decoded[stop]'s offset (spec §5.2 step 3). ONE
560
+ # applier, ONE `BEGIN IMMEDIATE`, ONE stop across BOTH
561
+ # families — see the #416 review-F1 note at `_cache_applier`
562
+ # for why a second independent applier is unsafe here.
563
+ # Always-on: a Claude-only batch scans + returns None.
559
564
  # codex_apply per-cycle `(ctx) -> None` closure (Task 7, a `run_stats_
560
565
  # ingest` arg, not a module global) run in step 4b'' on
561
566
  # ctx.conn — the seam every Codex on-demand stats.db writer
@@ -944,36 +949,63 @@ def _write_cursor(conn: sqlite3.Connection, segment: str, offset: int) -> None:
944
949
  )
945
950
 
946
951
 
947
- def _read_segment_lines(seg_path, lo: int, hi: int) -> list[tuple[str, int, bytes]]:
948
- """`(basename, absolute-offset, raw-line-without-newline)` for every
949
- complete line in `[lo, hi)`. `hi` is a line boundary (a HW snapshot size or
950
- an immutable prior segment's full size), so no partial trailing line
951
- appears."""
952
+ _SEGMENT_READ_CHUNK = 256 * 1024
953
+
954
+
955
+ def _iter_segment_lines(seg_path, lo: int, hi: int):
956
+ """Stream `(basename, absolute-offset, raw-line-without-newline)` for every
957
+ complete line in `[lo, hi)`, holding at most one chunk plus one partial line
958
+ in memory. `hi` is a line boundary (a HW snapshot size or an immutable prior
959
+ segment's full size), so no partial trailing line appears."""
960
+ name = seg_path.name
952
961
  with open(seg_path, "rb") as fh:
953
962
  fh.seek(lo)
954
- data = fh.read(hi - lo)
955
- out = []
956
- start = 0
957
- while True:
958
- nl = data.find(b"\n", start)
959
- if nl == -1:
960
- break
961
- out.append((seg_path.name, lo + start, data[start:nl]))
962
- start = nl + 1
963
- return out
963
+ pos = lo
964
+ buf = b""
965
+ buf_at = lo
966
+ while pos < hi:
967
+ data = fh.read(min(_SEGMENT_READ_CHUNK, hi - pos))
968
+ if not data:
969
+ break
970
+ pos += len(data)
971
+ buf = buf + data if buf else data
972
+ start = 0
973
+ while True:
974
+ nl = buf.find(b"\n", start)
975
+ if nl == -1:
976
+ break
977
+ yield (name, buf_at + start, buf[start:nl])
978
+ start = nl + 1
979
+ if start:
980
+ buf = buf[start:]
981
+ buf_at += start
964
982
 
965
983
 
966
- def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
967
- """Read `cursor -> HW` across segments in canonical order (spec §5.2.2).
984
+ def _read_segment_lines(seg_path, lo: int, hi: int) -> list[tuple[str, int, bytes]]:
985
+ """Materialized form of :func:`_iter_segment_lines` (see it for the
986
+ contract). Callers that walk a whole range at once should prefer
987
+ :func:`iter_range`; this list form is retained for the ingest cycle, which
988
+ needs the batch as an indexable sequence."""
989
+ return list(_iter_segment_lines(seg_path, lo, hi))
990
+
991
+
992
+ def iter_range(cursor, hw):
993
+ """Stream `cursor -> HW` across segments in canonical order (spec §5.2.2).
968
994
 
969
995
  Prior segments (before HW's) are immutable and read to their full size;
970
996
  HW's segment is read only up to the snapshot size, so appends past HW
971
997
  belong to the next cycle.
998
+
999
+ Streaming, not list-building: a caller that only needs to fold each record
1000
+ into a table (the Codex attribution rehydration) must not put a transient
1001
+ the size of the whole journal on the hot path. `_read_range` remains the
1002
+ materialized form for the ingest cycle, which genuinely needs the batch as
1003
+ an indexable sequence (prefix-stop indices address into it).
972
1004
  """
973
1005
  hw_seg, hw_size = hw
974
1006
  segments = list_segments()
975
1007
  if hw_seg not in segments:
976
- return []
1008
+ return
977
1009
  hw_idx = segments.index(hw_seg)
978
1010
  if cursor is None:
979
1011
  start_idx, start_off = 0, 0
@@ -983,7 +1015,6 @@ def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
983
1015
  start_idx, start_off = segments.index(cur_seg), cur_off
984
1016
  else:
985
1017
  start_idx, start_off = 0, 0
986
- lines: list[tuple[str, int, bytes]] = []
987
1018
  for idx in range(start_idx, hw_idx + 1):
988
1019
  seg = segments[idx]
989
1020
  seg_path = _cctally_core.JOURNAL_DIR / seg
@@ -991,8 +1022,12 @@ def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
991
1022
  hi = hw_size if idx == hw_idx else os.path.getsize(seg_path)
992
1023
  if lo >= hi:
993
1024
  continue
994
- lines.extend(_read_segment_lines(seg_path, lo, hi))
995
- return lines
1025
+ yield from _iter_segment_lines(seg_path, lo, hi)
1026
+
1027
+
1028
+ def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
1029
+ """Materialized `cursor -> HW` (see :func:`iter_range`)."""
1030
+ return list(iter_range(cursor, hw))
996
1031
 
997
1032
 
998
1033
  def journal_prefix_hash(high_water) -> "str | None":
@@ -1053,7 +1088,7 @@ def _capture_protocol_prefix_evidence(record, prior_high_water, evidence) -> Non
1053
1088
 
1054
1089
  _QUOTA_OBS_KIND = "quota_window_snapshot"
1055
1090
 
1056
- _QUOTA_SNAPSHOT_INSERT = (
1091
+ _QUOTA_SNAPSHOT_INSERT_LEGACY = (
1057
1092
  "INSERT OR IGNORE INTO quota_window_snapshots "
1058
1093
  "(source, source_root_key, source_path, line_offset, captured_at_utc, "
1059
1094
  " observed_slot, logical_limit_key, limit_id, limit_name, window_minutes, "
@@ -1062,6 +1097,15 @@ _QUOTA_SNAPSHOT_INSERT = (
1062
1097
  "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
1063
1098
  )
1064
1099
 
1100
+ _QUOTA_SNAPSHOT_INSERT = (
1101
+ "INSERT OR IGNORE INTO quota_window_snapshots "
1102
+ "(source, source_root_key, source_path, line_offset, captured_at_utc, "
1103
+ " observed_slot, logical_limit_key, limit_id, limit_name, window_minutes, "
1104
+ " used_percent, resets_at_utc, plan_type, individual_limit_json, "
1105
+ " reached_type, observed_model, account_key, canonical_resets_at_utc) "
1106
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
1107
+ )
1108
+
1065
1109
  _QUOTA_SNAPSHOT_COLS = (
1066
1110
  "source", "source_root_key", "source_path", "line_offset", "captured_at_utc",
1067
1111
  "observed_slot", "logical_limit_key", "limit_id", "limit_name",
@@ -1070,15 +1114,23 @@ _QUOTA_SNAPSHOT_COLS = (
1070
1114
  )
1071
1115
 
1072
1116
 
1073
- def _quota_snapshot_values(rec: dict) -> tuple:
1117
+ def _quota_snapshot_values(rec: dict, anchor: "str | None" = None) -> tuple:
1074
1118
  """Build the INSERT values tuple for one Codex quota obs. account_key (#341)
1075
1119
  rides the obs TOP-LEVEL ``account`` field (obs stamp shape), not the payload,
1076
1120
  so an unstamped/sentinel obs re-materializes cache.db with NULL account_key
1077
1121
  (``NULL ≡ unattributed`` on the read path). first-stamp-wins via the
1078
1122
  ``INSERT OR IGNORE`` natural key — account_key is a stamped attribute, never
1079
- part of the identity."""
1123
+ part of the identity.
1124
+
1125
+ ``anchor`` is the #416 §4.2 canonical reset. It is NOT journaled: it is a
1126
+ property of the observation's POPULATION, not of the observation, so it is
1127
+ re-resolved by whichever writer materializes the row. NULL leaves every
1128
+ reader on the raw-reset fallback, i.e. exactly today's behaviour."""
1080
1129
  p = rec.get("payload") or {}
1081
- return tuple(p.get(col) for col in _QUOTA_SNAPSHOT_COLS) + (rec.get("account"),)
1130
+ return (
1131
+ tuple(p.get(col) for col in _QUOTA_SNAPSHOT_COLS)
1132
+ + (rec.get("account"), anchor)
1133
+ )
1082
1134
 
1083
1135
 
1084
1136
  def _is_codex_quota_obs(rec: dict) -> bool:
@@ -1089,25 +1141,537 @@ def _is_codex_quota_obs(rec: dict) -> bool:
1089
1141
  )
1090
1142
 
1091
1143
 
1092
- def _quota_applier(decoded) -> int | None:
1093
- """Cache leg (spec §5.2 step 3): materialize this batch's Codex quota obs
1094
- into cache.db `quota_window_snapshots`, under the NON-BLOCKING global cache
1095
- writer lock followed by `cache.db.codex.lock`. Contract (journal seam):
1096
- `(decoded) -> stop | None`, `decoded = [(record, segment, offset), ...]` in
1097
- canonical order.
1144
+ # --------------------------------------------------------------------------
1145
+ # #416: the second family this leg carries the durable Codex attribution
1146
+ # decision (`codex_file_account` op, spec §3.3). It shares the leg rather than
1147
+ # getting its own applier because `run_stats_ingest` invokes exactly ONE applier
1148
+ # and truncates the batch afterwards, while an applier commits everything it
1149
+ # handled BEFORE returning a stop index. Two independent prefix-stopping
1150
+ # appliers could therefore commit past each other's stop, exposing suffix
1151
+ # effects beyond the retained prefix and violating the scalar-cursor rule
1152
+ # (docs/journal-gotchas.md; #416 review F1).
1153
+ # --------------------------------------------------------------------------
1154
+
1155
+ _FILE_ACCOUNT_OP_KIND = "codex_file_account"
1156
+ # Byte prefilter for the streamed replay: the canonical encoder is
1157
+ # `json.dumps(..., ensure_ascii=False)`, which never escapes an ASCII token, so
1158
+ # every genuine op carries the kind verbatim.
1159
+ _FILE_ACCOUNT_KIND_MARKER = f'"{_FILE_ACCOUNT_OP_KIND}"'.encode("ascii")
1160
+
1161
+ # FIRST-WINS at a contended primary key (Slice 1 closeout review C1). Spec §3.3
1162
+ # — "a mid-file account change appends a second range-qualified op; the first is
1163
+ # never rewritten" — and §3.5 — "a genuine correction is expressed as an explicit
1164
+ # new range decision, not by mutating history". Ops apply in journal order, so
1165
+ # `DO NOTHING` retains the FIRST decision at that key.
1166
+ #
1167
+ # This reverses the fix-round's last-op-wins. The #374 concern that motivated it
1168
+ # (a fold applier is an inserter, not a convergence operator) does not apply
1169
+ # here, because the path that must converge — `cache-sync --rebuild` — runs
1170
+ # `rehydrate_codex_file_accounts(authoritative=True)`, which DELETEs the table
1171
+ # before replaying. After an authoritative clear, `DO NOTHING` is first-WINS on
1172
+ # an empty table, not a no-op, so clear-then-replay still repairs a drifted row
1173
+ # (pinned by `test_authoritative_replay_still_repairs_a_drifted_row`).
1174
+ #
1175
+ # Last-op-wins was actively wrong on the one path where duplicates are
1176
+ # reachable: a failed rehydration lets the walk re-decide from the live
1177
+ # `auth.json` and mint a second op at the same key (plan candidate 10). Under
1178
+ # last-op-wins the documented remedy would then CEMENT that newer
1179
+ # live-auth-derived value rather than restore the original — the inverse of
1180
+ # acceptance criterion 4. The disagreement is REPORTED (see
1181
+ # `_apply_file_account_records`) rather than applied silently.
1182
+ #
1183
+ # `OR IGNORE` is deliberate, exactly as on `_QUOTA_SNAPSHOT_UPSERT`: SQLite
1184
+ # gives the named upsert clause precedence for the conflict it names, so the
1185
+ # first-wins policy is unaffected, while a record violating some OTHER
1186
+ # constraint is dropped instead of raising — an `IntegrityError` here would
1187
+ # prefix-stop `_cache_applier`, and the scalar cursor could never advance past
1188
+ # that record, wedging the whole journal ingest cycle for every provider.
1189
+ _FILE_ACCOUNT_INSERT = (
1190
+ "INSERT OR IGNORE INTO codex_file_accounts "
1191
+ "(file_identity, incarnation, from_offset, root_scope, account_key, "
1192
+ " decided_at_utc) "
1193
+ "VALUES (?,?,?,?,?,?) "
1194
+ "ON CONFLICT(file_identity, incarnation, from_offset) DO NOTHING"
1195
+ )
1196
+
1197
+ # The incarnation high-water rides the same replay. MAX-set, never an increment,
1198
+ # so replaying the same op converges instead of drifting.
1199
+ #
1200
+ # `OR IGNORE` for the same reason its sibling carries it (closeout review C2):
1201
+ # an `IntegrityError` raised HERE prefix-stops `_cache_applier` just as surely,
1202
+ # and the scalar cursor then never advances past the record. The named upsert
1203
+ # clause still takes precedence for the primary-key conflict, so the MAX-set
1204
+ # convergence is unaffected. `_apply_file_account_records` additionally refuses
1205
+ # to reach this statement for a record the map insert dropped, so the two guards
1206
+ # are belt-and-suspenders over disjoint failure modes: this one covers any
1207
+ # constraint the incarnation table might gain that the map table lacks.
1208
+ _FILE_INCARNATION_INSERT = (
1209
+ "INSERT OR IGNORE INTO codex_file_incarnations "
1210
+ "(file_identity, incarnation, updated_at_utc) "
1211
+ "VALUES (?,?,?) "
1212
+ "ON CONFLICT(file_identity) DO UPDATE SET "
1213
+ " incarnation = MAX(codex_file_incarnations.incarnation, excluded.incarnation), "
1214
+ " updated_at_utc = excluded.updated_at_utc"
1215
+ )
1216
+
1217
+
1218
+ def _is_codex_file_account_op(rec: dict) -> bool:
1219
+ return (
1220
+ rec.get("t") == "op"
1221
+ and (rec.get("payload") or {}).get("kind") == _FILE_ACCOUNT_OP_KIND
1222
+ )
1223
+
1224
+
1225
+ def _file_account_values(rec: dict) -> tuple:
1226
+ """INSERT values for one attribution decision.
1227
+
1228
+ ``account_key`` is read with ``.get`` because the stably-absent sentinel
1229
+ OMITS the field (two-shaped stamp), and the absence must materialize as SQL
1230
+ NULL — the literal string is never stored.
1231
+ """
1232
+ p = rec.get("payload") or {}
1233
+ return (
1234
+ p.get("file_identity"), p.get("incarnation"), p.get("from_offset"),
1235
+ p.get("root_scope"), p.get("account_key"), rec.get("at"),
1236
+ )
1237
+
1238
+
1239
+ def _apply_file_account_records(cache, records) -> "tuple[int, int]":
1240
+ """Materialize the given ``codex_file_account`` ops into an OPEN cache.db
1241
+ transaction; return ``(restored, conflicts)`` — how many rows were ABSENT
1242
+ before and actually landed (a genuine restore) and how many CONTRADICTED a
1243
+ different account already recorded at the same primary key (and were
1244
+ therefore declined, first-wins). Never opens or commits a transaction itself
1245
+ — every call site owns the flocks and the single ``BEGIN IMMEDIATE`` so the
1246
+ two families stay atomic.
1247
+
1248
+ ``restored`` deliberately excludes a no-op replay of a row that is already
1249
+ present and already says the same thing. The rehydration re-reads the ops
1250
+ its OWN previous sync appended (the cursor is snapshotted before the walk,
1251
+ which is exactly what recovers a failed write), so counting those would
1252
+ make every second sync claim to have rehydrated something — and that claim
1253
+ is printed on stderr, where several golden harnesses read it.
1254
+
1255
+ The ``prior is None`` probe is NOT sufficient on its own to call a record
1256
+ restored (closeout review C3): ``_FILE_ACCOUNT_INSERT`` carries ``OR
1257
+ IGNORE``, so a record violating some OTHER constraint is silently dropped
1258
+ and nothing was restored at all. ``total_changes`` after the statement is
1259
+ the only honest witness. A dropped record must also NOT raise the
1260
+ incarnation high-water — an inflated counter is the DANGEROUS direction,
1261
+ because ranges resolve at exactly the walk's current incarnation, so the
1262
+ range list comes back empty, ``covered`` is False, and a plain sync falls
1263
+ straight through to the live ``auth.json``.
1264
+
1265
+ Both counts are returned rather than printed here because this is called
1266
+ once per record by the streamed rehydration; the caller collapses a run
1267
+ into one line AFTER its commit (closeout review C5) — a rollback must not
1268
+ leave the operator told about a decline that never happened."""
1269
+ restored = conflicts = 0
1270
+ for rec in records:
1271
+ values = _file_account_values(rec)
1272
+ prior = cache.execute(
1273
+ "SELECT account_key FROM codex_file_accounts "
1274
+ "WHERE file_identity = ? AND incarnation = ? AND from_offset = ?",
1275
+ values[:3],
1276
+ ).fetchone()
1277
+ before = cache.total_changes
1278
+ cache.execute(_FILE_ACCOUNT_INSERT, values)
1279
+ landed = cache.total_changes > before
1280
+ if prior is None:
1281
+ if not landed:
1282
+ # Dropped by a constraint the map table carries and the
1283
+ # incarnation table does not (`root_scope`, `decided_at_utc`).
1284
+ # Nothing was restored and nothing may be advanced.
1285
+ continue
1286
+ restored += 1
1287
+ elif prior[0] != values[4]:
1288
+ conflicts += 1
1289
+ cache.execute(_FILE_INCARNATION_INSERT, (values[0], values[1], values[5]))
1290
+ return restored, conflicts
1291
+
1292
+
1293
+ def _report_file_account_conflicts(conflicts: int) -> None:
1294
+ """One stderr line for a run of replayed decisions that contradicted a
1295
+ different account already recorded at the same
1296
+ ``(file_identity, incarnation, from_offset)`` and were therefore DECLINED
1297
+ (first-wins, spec §3.3). Two ops at one primary key means one of them was
1298
+ minted without seeing the other, which is a real (if rare) condition with a
1299
+ real remedy, so it is reported rather than applied silently — the #374 rule.
1300
+
1301
+ Every call site must invoke this AFTER its commit (closeout review C5): a
1302
+ rolled-back transaction applied nothing, so reporting from inside it would
1303
+ tell the operator about a decline that did not happen."""
1304
+ if conflicts > 0:
1305
+ print(
1306
+ f"[ingest] codex attribution replay declined {conflicts} "
1307
+ "contradicting decision(s); the first journalled decision for each "
1308
+ "byte range is retained; run "
1309
+ "`cctally cache-sync --source codex --rebuild` if the "
1310
+ "attribution still looks wrong",
1311
+ file=sys.stderr,
1312
+ )
1313
+
1314
+
1315
+ # --------------------------------------------------------------------------
1316
+ # #416 spec §3.5 (review F5): which record is AUTHORITATIVE.
1317
+ #
1318
+ # "The journal always wins" is the wrong authority. Journaled quota obs are
1319
+ # deduplicated on a natural key that EXCLUDES the account
1320
+ # (`_codex_quota_natural_key`) and later records for that key are discarded by
1321
+ # `append_record`, so the retained observation is FIRST-STAMP-WINS and may
1322
+ # preserve the known late-ingest guess — bytes written under one login but
1323
+ # ingested after a switch (spec §1.7). An unconditional upsert from it would
1324
+ # overwrite a corrected file-range decision.
1325
+ #
1326
+ # Precedence: the durable file/range DECISION is authoritative; the observation
1327
+ # stamp is used only where no range decision covers those bytes. Where the two
1328
+ # disagree the row keeps the decision and the disagreement is REPORTED rather
1329
+ # than silently applied — a genuine correction is expressed as an explicit new
1330
+ # range decision, never by mutating history.
1331
+ # --------------------------------------------------------------------------
1332
+
1333
+ # Converging form, used ONLY for a row whose bytes a decision covers. Repeating
1334
+ # a deterministic DO UPDATE is idempotent, which is what preserves crash-replay;
1335
+ # an uncovered obs keeps the first-write-wins INSERT OR IGNORE above.
1336
+ #
1337
+ # `INSERT OR IGNORE` is RETAINED here, not replaced by a plain `INSERT`. SQLite
1338
+ # gives the upsert clause precedence for the conflict it names, so the targeted
1339
+ # `DO UPDATE` still fires and convergence is unaffected — while every OTHER
1340
+ # constraint on `quota_window_snapshots` (four CHECKs and several NOT NULLs)
1341
+ # keeps the silent-drop tolerance the uncovered path has. Without it a violating
1342
+ # record raises `IntegrityError`, `_cache_applier` catches it as `sqlite3.Error`
1343
+ # and prefix-stops, and the scalar cursor can never advance past that record —
1344
+ # so ONE permanently-violating row would wedge the whole journal ingest cycle
1345
+ # forever, for every provider, not just Codex.
1346
+ _QUOTA_SNAPSHOT_UPSERT_CLAUSE = (
1347
+ " ON CONFLICT(source, source_path, line_offset, logical_limit_key) "
1348
+ "DO UPDATE SET account_key = excluded.account_key"
1349
+ )
1350
+
1351
+ _QUOTA_SNAPSHOT_UPSERT = _QUOTA_SNAPSHOT_INSERT + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
1352
+ _QUOTA_SNAPSHOT_UPSERT_LEGACY = (
1353
+ _QUOTA_SNAPSHOT_INSERT_LEGACY + _QUOTA_SNAPSHOT_UPSERT_CLAUSE
1354
+ )
1355
+
1356
+
1357
+ class _CodexAttributionOracle:
1358
+ """Resolve ``(root_scope, source_path, line_offset)`` to the authoritative
1359
+ decision, memoised per file for one transaction.
1360
+
1361
+ The map is keyed on the durable file identity, not on the path, so the path
1362
+ is canonicalized through the SAME helper the ingest used
1363
+ (``_cctally_cache._canonical_codex_path``) to reach it. When that lookup
1364
+ cannot be made — the sibling is unavailable, or the map holds MORE THAN ONE
1365
+ incarnation for the file — the oracle DECLINES. Declining matters most for
1366
+ the multi-incarnation case: a truncation reuses offsets from zero, so an
1367
+ observation's byte offset no longer identifies which incarnation it belongs
1368
+ to, and guessing would attribute pre-truncation bytes to the replacement
1369
+ file's account. Declining falls back to the observation stamp, which is
1370
+ exactly the documented "no range decision covers those bytes" branch.
1371
+ """
1372
+
1373
+ def __init__(self, cache):
1374
+ self._cache = cache
1375
+ self._cache_by_path: dict = {}
1376
+ self._canonicalize = None
1377
+ self._available = None
1378
+
1379
+ def _ensure_loaded(self) -> bool:
1380
+ if self._available is not None:
1381
+ return self._available
1382
+ try:
1383
+ probe = self._cache.execute(
1384
+ "SELECT 1 FROM codex_file_accounts LIMIT 1").fetchone()
1385
+ except sqlite3.Error:
1386
+ self._available = False
1387
+ return False
1388
+ if probe is None:
1389
+ self._available = False
1390
+ return False
1391
+ try:
1392
+ import _cctally_cache as _cc
1393
+ from _lib_source_identity import codex_file_key
1394
+ except Exception: # pragma: no cover — sibling unavailable
1395
+ self._available = False
1396
+ return False
1397
+ self._canonicalize = (_cc._canonical_codex_path, codex_file_key)
1398
+ self._available = True
1399
+ return True
1400
+
1401
+ def _ranges_for(self, root_scope, source_path):
1402
+ key = (root_scope, source_path)
1403
+ if key in self._cache_by_path:
1404
+ return self._cache_by_path[key]
1405
+ ranges: list = []
1406
+ canonical, file_key = self._canonicalize
1407
+ try:
1408
+ identity = file_key(root_scope, str(canonical(pathlib.Path(source_path))))
1409
+ except Exception:
1410
+ self._cache_by_path[key] = ranges
1411
+ return ranges
1412
+ rows = self._cache.execute(
1413
+ "SELECT DISTINCT incarnation FROM codex_file_accounts "
1414
+ "WHERE file_identity = ?", (identity,)).fetchall()
1415
+ if len(rows) == 1:
1416
+ ranges = [
1417
+ (int(r[0]), r[1]) for r in self._cache.execute(
1418
+ "SELECT from_offset, account_key FROM codex_file_accounts "
1419
+ "WHERE file_identity = ? AND incarnation = ? "
1420
+ "ORDER BY from_offset ASC", (identity, int(rows[0][0])))
1421
+ ]
1422
+ self._cache_by_path[key] = ranges
1423
+ return ranges
1424
+
1425
+ def resolve(self, rec) -> "tuple[bool, str | None]":
1426
+ """``(covered, account_key)`` for one quota obs."""
1427
+ if not self._ensure_loaded():
1428
+ return False, None
1429
+ payload = rec.get("payload") or {}
1430
+ root_scope = payload.get("source_root_key")
1431
+ source_path = payload.get("source_path")
1432
+ offset = payload.get("line_offset")
1433
+ if not root_scope or not source_path or offset is None:
1434
+ return False, None
1435
+ covered, account_key = False, None
1436
+ for from_offset, decided in self._ranges_for(root_scope, source_path):
1437
+ if from_offset > offset:
1438
+ break
1439
+ covered, account_key = True, decided
1440
+ return covered, account_key
1441
+
1442
+
1443
+ def _cache_has_anchor_column(cache) -> bool:
1444
+ """Whether this cache.db carries ``quota_window_snapshots.canonical_resets_at_utc``.
1445
+
1446
+ Probed rather than assumed. This leg opens cache.db RAW (no dispatcher, no
1447
+ schema apply), so it can meet a cache that has not yet gained the column. A
1448
+ column-count mismatch would raise ``sqlite3.OperationalError``, which
1449
+ ``_cache_applier`` catches as a write failure and turns into a PREFIX-STOP —
1450
+ and the scalar cursor could then never advance past that record, wedging the
1451
+ journal ingest cycle for every provider. Same reasoning as the ``OR IGNORE``
1452
+ on ``_FILE_ACCOUNT_INSERT``; the walk's own writer needs no such guard
1453
+ because it only ever runs on a dispatcher-opened connection.
1454
+ """
1455
+ try:
1456
+ return "canonical_resets_at_utc" in {
1457
+ str(row[1]) for row in cache.execute(
1458
+ "PRAGMA table_info(quota_window_snapshots)")
1459
+ }
1460
+ except sqlite3.Error: # pragma: no cover — unreadable schema
1461
+ return False
1462
+
1463
+
1464
+ def _codex_anchor_resolver(cache):
1465
+ """A ``CodexResetAnchorResolver`` over this connection, or ``None`` when the
1466
+ sibling is unavailable. Degrading to ``None`` leaves the anchor column NULL,
1467
+ which every reader treats as "use the raw reset" — today's behaviour."""
1468
+ try:
1469
+ import _cctally_cache as _cc
1470
+ except Exception: # pragma: no cover — sibling unavailable
1471
+ return None
1472
+ try:
1473
+ return _cc.CodexResetAnchorResolver(cache)
1474
+ except Exception: # pragma: no cover — older sibling without the resolver
1475
+ return None
1476
+
1098
1477
 
1099
- - No Codex quota obs in the batch → return None (no flock taken).
1478
+ def _resolve_obs_anchor(resolver, rec: dict) -> "str | None":
1479
+ if resolver is None:
1480
+ return None
1481
+ p = rec.get("payload") or {}
1482
+ root = p.get("source_root_key")
1483
+ slot = p.get("observed_slot")
1484
+ key = p.get("logical_limit_key")
1485
+ if not isinstance(root, str) or not isinstance(slot, str) or not isinstance(key, str):
1486
+ return None
1487
+ try:
1488
+ return resolver.resolve(
1489
+ source_root_key=root, observed_slot=slot, logical_limit_key=key,
1490
+ window_minutes=p.get("window_minutes"),
1491
+ resets_at_utc=p.get("resets_at_utc"),
1492
+ )
1493
+ except Exception: # pragma: no cover — never fail an ingest over a label
1494
+ return None
1495
+
1496
+
1497
+ def _apply_quota_records(cache, records) -> None:
1498
+ """Materialize Codex quota obs into an OPEN cache.db transaction, applying
1499
+ the §3.5 precedence rule. Callers must apply the batch's file-account
1500
+ decisions FIRST, so a decision arriving in the same batch already governs
1501
+ the observations it covers."""
1502
+ oracle = _CodexAttributionOracle(cache)
1503
+ # One line per FILE PER BATCH, not per record. A mid-file account switch
1504
+ # legitimately produces a run of observations whose first-stamp-wins account
1505
+ # disagrees with the range decision now governing those bytes, and an
1506
+ # unthrottled warning would emit one line per row for the whole run. The set
1507
+ # is deliberately local, so a file whose conflicting run SPANS several
1508
+ # ingest batches reports once per batch — a per-cycle or per-process set
1509
+ # would have to outlive the transaction that may roll back, and repeating a
1510
+ # standing condition a handful of times is the cheaper error. The condition
1511
+ # is worth reporting at all because a genuine correction is expressed as an
1512
+ # explicit new range decision, never by mutating history.
1513
+ reported_conflicts: set = set()
1514
+ # #416 spec §4.2: this leg is a genuine INGEST into cache.db (it materializes
1515
+ # observations whose source rollout may have evaporated), so it must resolve
1516
+ # the canonical anchor too. Without it, a journal-replayed row lands with a
1517
+ # NULL anchor, a later walk's `INSERT OR IGNORE` cannot correct it, and that
1518
+ # window stays fragmented forever.
1519
+ has_anchor = _cache_has_anchor_column(cache)
1520
+ anchors = _codex_anchor_resolver(cache) if has_anchor else None
1521
+ insert_sql = _QUOTA_SNAPSHOT_INSERT if has_anchor else _QUOTA_SNAPSHOT_INSERT_LEGACY
1522
+ upsert_sql = _QUOTA_SNAPSHOT_UPSERT if has_anchor else _QUOTA_SNAPSHOT_UPSERT_LEGACY
1523
+ for rec in records:
1524
+ covered, decided = oracle.resolve(rec)
1525
+ anchor = _resolve_obs_anchor(anchors, rec)
1526
+ row_values = _quota_snapshot_values(rec, anchor)
1527
+ if not has_anchor:
1528
+ row_values = row_values[:-1]
1529
+ if not covered:
1530
+ cache.execute(insert_sql, row_values)
1531
+ continue
1532
+ observed = rec.get("account")
1533
+ payload = rec.get("payload") or {}
1534
+ conflict_key = (payload.get("source_root_key"), payload.get("source_path"))
1535
+ if (observed is not None and observed != decided
1536
+ and conflict_key not in reported_conflicts):
1537
+ reported_conflicts.add(conflict_key)
1538
+ print(
1539
+ "[ingest] codex attribution conflict: "
1540
+ f"{payload.get('source_path')}@{payload.get('line_offset')} "
1541
+ f"observation stamped {observed} but the durable decision says "
1542
+ f"{decided if decided is not None else 'unattributed'}; "
1543
+ "keeping the decision",
1544
+ file=sys.stderr,
1545
+ )
1546
+ values = list(row_values)
1547
+ values[16] = decided
1548
+ cache.execute(upsert_sql, tuple(values))
1549
+
1550
+
1551
+ def rehydrate_codex_file_accounts(
1552
+ cache_conn, *, authoritative: bool = False, since=None,
1553
+ ) -> "tuple[int, tuple[str, int] | None, int]":
1554
+ """Replay journaled ``codex_file_account`` ops into an open cache.db
1555
+ connection; return ``(applied_count, high_water, declined_conflicts)``
1556
+ (#416 spec §3.4).
1557
+
1558
+ The conflict count is RETURNED rather than reported here (closeout review
1559
+ C5). This function runs inside the caller's transaction, and that caller
1560
+ rolls back on failure — reporting from in here would tell the operator about
1561
+ a decline that was undone. The two ``_apply_file_account_records`` call
1562
+ sites in the appliers already report post-commit; this one now matches.
1563
+
1564
+ ``since`` is the ``(segment, offset)`` journal cursor the caller last
1565
+ replayed, or ``None`` for "from the beginning". The returned high-water is
1566
+ what the caller must persist so the NEXT call replays only the delta — the
1567
+ two together are what make this affordable on the hot path AND what recovers
1568
+ a decision that was journaled but never materialized (spec §3.6: "a crash
1569
+ after append but before the cache-map commit is recovered by replaying
1570
+ pending journal state under the same locked operation BEFORE ``auth.json``
1571
+ is consulted on retry"). A one-shot "already rehydrated" marker cannot do
1572
+ that: the failing sync's own op lands AFTER the marker was written, so the
1573
+ retry would never replay it and would re-decide from a possibly-changed
1574
+ identity instead.
1575
+
1576
+ ``authoritative`` ignores ``since`` — a clear-then-replay is only correct
1577
+ from the beginning of the journal.
1578
+
1579
+ The caller owns the flocks, the transaction and the commit — this function
1580
+ only executes the idempotent upserts, so it can run inside
1581
+ ``sync_codex_cache``'s already-locked phases without violating the
1582
+ lock-order law.
1583
+
1584
+ Why an explicit phase exists at all: the ordinary journal-to-cache replay
1585
+ runs only inside ``rebuild_stats_index``, whereas ``cache-sync`` clears (on
1586
+ ``--rebuild``) and begins the rollout walk with NO applier in front of it. A
1587
+ recreated cache.db (corruption recovery, a manual ``rm cache.db``) therefore
1588
+ starts with an empty map, and the walk would fall straight back to the live
1589
+ ``auth.json`` for every file — which is the defect (review F2). Note this is
1590
+ NOT rebuild-only: every production Codex call site syncs with
1591
+ ``rebuild=False``, and the corruption auto-heal recreates the cache.db family
1592
+ and then re-runs the ORDINARY sync, so a rebuild-only wiring leaves the
1593
+ defect reachable by a shorter road.
1594
+
1595
+ ``authoritative=True`` makes the replay a CONVERGENCE operator rather than an
1596
+ inserter: it clears ``codex_file_accounts`` first, so a row that has drifted
1597
+ away from the journal is corrected instead of being silently preserved by the
1598
+ ``DO NOTHING`` conflict clause (the #374 fold-applier defect class — see
1599
+ ``docs/journal-gotchas.md``). This is lossless because the ingest's
1600
+ fail-closed append journals the decision BEFORE any accounting DML or map
1601
+ write for that file, so every map row has a journal op behind it, and the
1602
+ journal is append-only with no segment pruning. It is the documented remedy
1603
+ (``cache-sync --rebuild``), so it must actually be able to repair.
1604
+
1605
+ ``codex_file_incarnations`` is deliberately NOT cleared even under
1606
+ ``authoritative``, and the reason is NOT that a clear would be conservative.
1607
+ The op is journaled BEFORE the batch that persists the incarnation, so every
1608
+ committed incarnation is ``<=`` the highest incarnation any op carries — a
1609
+ re-derivation from ops can therefore never LOWER the counter, only raise it
1610
+ above what any committed batch used. And too high is the DANGEROUS
1611
+ direction, not the safe one: ranges are resolved at exactly the walk's
1612
+ current incarnation, so an inflated counter loads an EMPTY range list,
1613
+ ``covered`` is False, and a plain sync falls straight through to the live
1614
+ ``auth.json`` branch and re-decides — the original defect. Since the MAX-set
1615
+ upsert already converges the counter, a clear has no upside and that
1616
+ downside.
1617
+ """
1618
+ hw = journal_high_water()
1619
+ if hw is None:
1620
+ if authoritative:
1621
+ # No journal at all: an authoritative pass still says "the journal
1622
+ # is the truth", and the truth is that there are no decisions.
1623
+ cache_conn.execute("DELETE FROM codex_file_accounts")
1624
+ return 0, None, 0
1625
+ if authoritative:
1626
+ cache_conn.execute("DELETE FROM codex_file_accounts")
1627
+ since = None
1628
+ applied = 0
1629
+ conflicts = 0
1630
+ # Streamed, never materialized: this runs on the FIRST ordinary sync of
1631
+ # every cache.db (hook-tick, the dashboard, the corruption auto-heal's
1632
+ # re-sync) while both cache flocks are held, so a whole-journal transient
1633
+ # here is a multi-second global cache-writer stall — itself a
1634
+ # `database is locked` trigger. The cheap byte prefilter skips the JSON
1635
+ # decode for every non-decision line; the canonical encoder is
1636
+ # `json.dumps(..., ensure_ascii=False)`, which never escapes an ASCII kind
1637
+ # token, so a genuine op always carries this substring verbatim. A false
1638
+ # positive is harmless — it is decoded and rejected by the real predicate.
1639
+ for _seg, _off, raw in iter_range(since, hw):
1640
+ if _FILE_ACCOUNT_KIND_MARKER not in raw:
1641
+ continue
1642
+ rec = _lib_journal.decode_line(raw)
1643
+ if rec is not None and _is_codex_file_account_op(rec):
1644
+ _restored, _conflicts = _apply_file_account_records(cache_conn, (rec,))
1645
+ applied += _restored
1646
+ conflicts += _conflicts
1647
+ return applied, hw, conflicts
1648
+
1649
+
1650
+ def _cache_applier(decoded) -> int | None:
1651
+ """Composite cache leg (spec §5.2 step 3 + #416 spec §3.4): materialize this
1652
+ batch's Codex quota obs into `quota_window_snapshots` AND its
1653
+ `codex_file_account` ops into the attribution map, under the NON-BLOCKING
1654
+ global cache writer lock followed by `cache.db.codex.lock`, in ONE
1655
+ `BEGIN IMMEDIATE`. Contract (journal seam): `(decoded) -> stop | None`,
1656
+ `decoded = [(record, segment, offset), ...]` in canonical order.
1657
+
1658
+ - Neither family present in the batch → return None (no flock taken).
1100
1659
  - Busy global/provider flock, OR a cache write it cannot complete → PREFIX-STOP:
1101
- return the index of the FIRST codex quota obs, so the cycle processes only
1102
- `decoded[:stop]` and advances the cursor to `decoded[stop]`'s offset,
1103
- retrying the remainder next cycle (the scalar cursor never advances past an
1104
- unmaterialized obs — spec §5.2 step 3).
1105
- - Flock acquired + all obs upserted → return None (full consumption).
1660
+ return the EARLIEST index across BOTH families having committed NEITHER, so
1661
+ the cycle processes only `decoded[:stop]` and advances the cursor to
1662
+ `decoded[stop]`'s offset, retrying the remainder next cycle (the scalar
1663
+ cursor never advances past an unmaterialized record — spec §5.2 step 3).
1664
+ - Flock acquired + everything upserted → return None (full consumption).
1106
1665
  """
1107
1666
  quota_idx = [i for i, (rec, _s, _o) in enumerate(decoded)
1108
1667
  if _is_codex_quota_obs(rec)]
1109
- if not quota_idx:
1668
+ file_idx = [i for i, (rec, _s, _o) in enumerate(decoded)
1669
+ if _is_codex_file_account_op(rec)]
1670
+ if not quota_idx and not file_idx:
1110
1671
  return None
1672
+ # All-or-nothing across the two families: one stop, the earliest of either.
1673
+ stop_idx = min(quota_idx[0] if quota_idx else file_idx[0],
1674
+ file_idx[0] if file_idx else quota_idx[0])
1111
1675
  from _lib_cache_writer_lock import (
1112
1676
  acquire_cache_writer_flocks,
1113
1677
  release_cache_writer_flocks,
@@ -1120,31 +1684,36 @@ def _quota_applier(decoded) -> int | None:
1120
1684
  _cctally_core.CACHE_LOCK_CODEX_PATH,
1121
1685
  )
1122
1686
  except OSError:
1123
- return quota_idx[0]
1687
+ return stop_idx
1124
1688
  if held is None:
1125
- return quota_idx[0]
1689
+ return stop_idx
1126
1690
  try:
1127
1691
  try:
1128
1692
  cache = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH), timeout=15.0)
1129
1693
  except sqlite3.Error as exc: # pragma: no cover — cache.db unopenable
1130
- print(f"[ingest] quota cache leg connect failed: {exc}", file=sys.stderr)
1131
- return quota_idx[0]
1694
+ print(f"[ingest] cache leg connect failed: {exc}", file=sys.stderr)
1695
+ return stop_idx
1132
1696
  try:
1133
1697
  cache.execute("PRAGMA busy_timeout=15000")
1134
1698
  cache.execute("BEGIN IMMEDIATE")
1135
- for i in quota_idx:
1136
- cache.execute(_QUOTA_SNAPSHOT_INSERT,
1137
- _quota_snapshot_values(decoded[i][0]))
1699
+ # Decisions FIRST: §3.5 makes the file/range decision authoritative
1700
+ # over the observation stamp, so a decision arriving in this batch
1701
+ # must already govern the observations it covers.
1702
+ _, _file_conflicts = _apply_file_account_records(
1703
+ cache, [decoded[i][0] for i in file_idx])
1704
+ _apply_quota_records(cache, [decoded[i][0] for i in quota_idx])
1138
1705
  cache.commit()
1706
+ _report_file_account_conflicts(_file_conflicts)
1139
1707
  except sqlite3.Error as exc:
1140
1708
  try:
1141
1709
  cache.rollback()
1142
1710
  except sqlite3.Error:
1143
1711
  pass
1144
1712
  # Could not materialize -> prefix-stop so the cursor holds and the
1145
- # next cycle retries (the obs stay durable in the journal regardless).
1146
- print(f"[ingest] quota cache leg write failed: {exc}", file=sys.stderr)
1147
- return quota_idx[0]
1713
+ # next cycle retries (the records stay durable in the journal
1714
+ # regardless). NEITHER family is committed.
1715
+ print(f"[ingest] cache leg write failed: {exc}", file=sys.stderr)
1716
+ return stop_idx
1148
1717
  finally:
1149
1718
  cache.close()
1150
1719
  return None
@@ -1152,10 +1721,14 @@ def _quota_applier(decoded) -> int | None:
1152
1721
  release_cache_writer_flocks(held)
1153
1722
 
1154
1723
 
1724
+ # Back-compat alias: the leg was Codex-quota-only until #416 widened it.
1725
+ _quota_applier = _cache_applier
1726
+
1155
1727
  # Wire the seam (declared None near the top as the contract stub). Always-on:
1156
- # a Claude-only cycle's scan finds no Codex quota obs and returns None before
1157
- # any flock/DB touch, so the cost is one list comprehension over the batch.
1158
- QUOTA_APPLIER = _quota_applier
1728
+ # a Claude-only cycle's scan finds neither family and returns None before any
1729
+ # flock/DB touch, so the cost is two list comprehensions over the batch.
1730
+ CACHE_APPLIER = _cache_applier
1731
+ QUOTA_APPLIER = _cache_applier
1159
1732
 
1160
1733
 
1161
1734
  # --------------------------------------------------------------------------
@@ -1468,11 +2041,21 @@ _EVT_KIND_PROVIDER = {
1468
2041
  "projected": "claude",
1469
2042
  "project_budget": "claude",
1470
2043
  "quota_alert_arming": "codex",
2044
+ "quota_threshold_event": "codex",
1471
2045
  }
1472
2046
 
1473
2047
  # Op kinds that are accounts-machinery (recognised, never classified as legacy).
2048
+ # `codex_file_account` (#416 spec §3.3) joins them: it is the durable Codex
2049
+ # attribution DECISION, and its sentinel form deliberately OMITS `account_key`
2050
+ # — exactly the shape the legacy classifier keys on — so registration here is
2051
+ # what keeps `_normalize_legacy_account_stamp` from ever retro-stamping it.
2052
+ # Registering a kind here also feeds `_cctally_rederive.plan_claude_usage`'s
2053
+ # `op_kinds` set, so the kind MUST additionally carry a
2054
+ # `_lib_rederive._OP_CLASSIFICATIONS` entry or the re-derive planner raises
2055
+ # `RederiveConflict` on every run.
1474
2056
  _ACCOUNTS_MACHINERY_KINDS = frozenset(
1475
- ("account_observe", "account_label", "accounts_cutover"))
2057
+ ("account_observe", "account_label", "accounts_cutover",
2058
+ "codex_file_account"))
1476
2059
 
1477
2060
  # Legacy-classifier exhaustiveness guard (#341, review finding P2-1). EVERY evt
1478
2061
  # kind in `_EVT_SPECS` and every harvest kind in `_HARVEST_SPECS` must carry a
@@ -1769,6 +2352,75 @@ def _apply_quota_alert_arming(conn, evt):
1769
2352
  return None
1770
2353
 
1771
2354
 
2355
+ def _apply_quota_threshold_event(conn, evt):
2356
+ """Fold a `quota_threshold_event` evt (#416 spec §7.2, review F13).
2357
+
2358
+ `quota_threshold_events` is TERMINAL alert evidence: each row records that a
2359
+ threshold was crossed and either alerted or was suppressed as backfill, with
2360
+ the exact moment it happened. It is in NEITHER `_HARVEST_SPECS` (eight
2361
+ families, none of them this one) nor — before #416 — `_CUTOVER_SPECS`, and
2362
+ `rematerialize_quota_projection_for_rebuild` runs with
2363
+ `alert_eligible_roots=frozenset()`, so a rebuild could not recreate an
2364
+ `alerted` row at all. Every rebuild silently discarded the evidence that an
2365
+ alert had already fired, which is what would let it fire again.
2366
+
2367
+ Convergence is a natural-key UPSERT, exactly like `_apply_quota_alert_arming`
2368
+ — the table has no `journal_id` column, so idempotence cannot ride an
2369
+ `INSERT OR IGNORE` on the journal id. The upsert restores `disposition`,
2370
+ `alerted_at` and `suppressed_at` VERBATIM, which matters because the rebuild's
2371
+ re-materialization pass can legitimately re-derive the same crossing as a
2372
+ fresh `suppressed_backfill` row: whichever of the two runs second, the
2373
+ journaled terminal fact is what stands.
2374
+
2375
+ `orphaned_at` is deliberately NOT journaled and NOT touched here. It marks a
2376
+ window whose evidence has since vanished, and `_orphan_unseen` re-derives it
2377
+ from the current projection on every pass — replaying a stale value would
2378
+ fight that.
2379
+ """
2380
+ p = evt.get("payload") or {}
2381
+ account_key = p.get("account_key") or _lib_accounts.UNATTRIBUTED
2382
+ disposition = p.get("disposition")
2383
+ alerted_at = p.get("alerted_at")
2384
+ suppressed_at = p.get("suppressed_at")
2385
+ # The table's CHECK pairs disposition with exactly one timestamp. Normalize
2386
+ # rather than trust the payload, so a malformed record cannot raise here and
2387
+ # prefix-stop the whole fold.
2388
+ if disposition == "alerted":
2389
+ suppressed_at = None
2390
+ alerted_at = alerted_at or p.get("created_at_utc")
2391
+ elif disposition == "suppressed_backfill":
2392
+ alerted_at = None
2393
+ suppressed_at = suppressed_at or p.get("created_at_utc")
2394
+ else:
2395
+ return None
2396
+ conn.execute(
2397
+ "INSERT OR IGNORE INTO quota_threshold_events "
2398
+ "(source, source_root_key, logical_limit_key, observed_slot, "
2399
+ " window_minutes, resets_at_utc, threshold, qualifying_kind, "
2400
+ " qualifying_percent, projected_percent, severity, created_at_utc, "
2401
+ " disposition, alerted_at, suppressed_at, account_key) "
2402
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "
2403
+ "ON CONFLICT(source, source_root_key, account_key, logical_limit_key, "
2404
+ " observed_slot, window_minutes, resets_at_utc, threshold) "
2405
+ "DO UPDATE SET "
2406
+ " qualifying_kind=excluded.qualifying_kind, "
2407
+ " qualifying_percent=excluded.qualifying_percent, "
2408
+ " projected_percent=excluded.projected_percent, "
2409
+ " severity=excluded.severity, "
2410
+ " created_at_utc=excluded.created_at_utc, "
2411
+ " disposition=excluded.disposition, "
2412
+ " alerted_at=excluded.alerted_at, "
2413
+ " suppressed_at=excluded.suppressed_at",
2414
+ (p.get("source"), p.get("source_root_key"), p.get("logical_limit_key"),
2415
+ p.get("observed_slot"), p.get("window_minutes"), p.get("resets_at_utc"),
2416
+ p.get("threshold"), p.get("qualifying_kind"),
2417
+ p.get("qualifying_percent"), p.get("projected_percent"),
2418
+ p.get("severity"), p.get("created_at_utc"), disposition,
2419
+ alerted_at, suppressed_at, account_key),
2420
+ )
2421
+ return None
2422
+
2423
+
1772
2424
  def _apply_block_close(conn, evt):
1773
2425
  """Fold one authoritative frozen-block fact.
1774
2426
 
@@ -1957,6 +2609,14 @@ _EVT_SPECS = {
1957
2609
  # upsert applier. order is arbitrary among evts (no cross-family FK).
1958
2610
  "quota_alert_arming": _EvtSpec(
1959
2611
  None, order=45, applier=_apply_quota_alert_arming),
2612
+ # Terminal quota alert evidence (#416 spec §7.2). Order 44 — BEFORE
2613
+ # `quota_alert_arming` (45) and before the quota projection
2614
+ # re-materialization, so the journaled terminal fact is already in place
2615
+ # when the rebuild's re-derivation runs. Both directions are safe anyway:
2616
+ # this applier converges by natural-key upsert and the re-derivation's own
2617
+ # insert is `INSERT OR IGNORE`, so neither can clobber the other's row.
2618
+ "quota_threshold_event": _EvtSpec(
2619
+ None, order=44, applier=_apply_quota_threshold_event),
1960
2620
  }
1961
2621
  for _hs in _HARVEST_SPECS:
1962
2622
  if _hs.children:
@@ -3972,16 +4632,25 @@ def _publish_rebuilt_stats_index(
3972
4632
 
3973
4633
 
3974
4634
  def _rebuild_quota_cache_leg(records) -> None:
3975
- """Re-materialize cache.db `quota_window_snapshots` from the journal's Codex
3976
- quota obs (spec §5.4). The journal obs are the DURABLE source (§1 latent
3977
- data-loss hole — the rollout JSONL evaporates); this INSERT OR IGNOREs them
3978
- on the natural key, mirroring `_quota_applier`. Runs BEFORE any stats
3979
- transaction, under the global cache writer lock followed by the
3980
- `cache.db.codex.lock` provider flock (lock-order law). Best-effort: a
3981
- missing/busy cache.db is a clean skip (the obs stay durable in the journal;
3982
- the stats quota projection pass then degrades cleanly)."""
4635
+ """Re-materialize cache.db `quota_window_snapshots` AND the #416 Codex
4636
+ attribution map from the journal (spec §5.4 + #416 spec §3.4).
4637
+
4638
+ The journal records are the DURABLE source (§1 latent data-loss hole the
4639
+ rollout JSONL evaporates); this INSERT-OR-IGNOREs the quota obs on their
4640
+ natural key and the attribution decisions on theirs, mirroring
4641
+ `_cache_applier` family for family. The map half is not optional: without it
4642
+ a rebuild would leave the map empty and the following rollout walk would
4643
+ have nothing to replay, sending every file back to the live `auth.json` —
4644
+ the exact defect this mechanism exists to prevent.
4645
+
4646
+ Runs BEFORE any stats transaction, under the global cache writer lock
4647
+ followed by the `cache.db.codex.lock` provider flock (lock-order law).
4648
+ Best-effort: a missing/busy cache.db is a clean skip (the records stay
4649
+ durable in the journal; the stats quota projection pass then degrades
4650
+ cleanly)."""
3983
4651
  quota_obs = [r for r in records if _is_codex_quota_obs(r)]
3984
- if not quota_obs:
4652
+ file_accounts = [r for r in records if _is_codex_file_account_op(r)]
4653
+ if not quota_obs and not file_accounts:
3985
4654
  return
3986
4655
  cache_path = _cctally_core.CACHE_DB_PATH
3987
4656
  if not cache_path.exists():
@@ -4013,10 +4682,11 @@ def _rebuild_quota_cache_leg(records) -> None:
4013
4682
  try:
4014
4683
  cache.execute("PRAGMA busy_timeout=15000")
4015
4684
  cache.execute("BEGIN IMMEDIATE")
4016
- for r in quota_obs:
4017
- cache.execute(_QUOTA_SNAPSHOT_INSERT,
4018
- _quota_snapshot_values(r))
4685
+ # Decisions FIRST — same §3.5 precedence ordering as `_cache_applier`.
4686
+ _, _file_conflicts = _apply_file_account_records(cache, file_accounts)
4687
+ _apply_quota_records(cache, quota_obs)
4019
4688
  cache.commit()
4689
+ _report_file_account_conflicts(_file_conflicts)
4020
4690
  except sqlite3.Error as exc:
4021
4691
  try:
4022
4692
  cache.rollback()
@@ -4387,6 +5057,23 @@ _CUTOVER_SPECS = (
4387
5057
  "logical_limit_key", "observed_slot",
4388
5058
  "window_minutes", "rule_fingerprint",
4389
5059
  "activated_at_utc")),
5060
+ _CutoverSpec("quota_threshold_events", "quota_threshold_event", "evt",
5061
+ "created_at_utc", stamp=False, natural_key_prefix="qte",
5062
+ natural_key_id=("source", "source_root_key", "account_key",
5063
+ "logical_limit_key", "observed_slot",
5064
+ "window_minutes", "resets_at_utc",
5065
+ "threshold")),
5066
+ # quota_threshold_events (#416 spec §7.2, review F13) — TERMINAL alert
5067
+ # evidence, modelled exactly on the quota_alert_arming precedent above: no
5068
+ # `journal_id` column -> NOT stamped, and the fold applier converges by
5069
+ # natural key. The `qte:` id mirrors the table's own UNIQUE key so one
5070
+ # crossing is one event forever, and it MUST match the live emitter in
5071
+ # `_cctally_quota._codex_leg._emit_terminal_event`.
5072
+ #
5073
+ # This spec covers a legacy install that has not yet cut over. An install
5074
+ # already past the cutover carries no history here — which is exactly why
5075
+ # the live emitter exists: without it, only pre-cutover rows would ever be
5076
+ # replayable, and every row written since would still be lost on a rebuild.
4390
5077
  )
4391
5078
 
4392
5079
  def _export_stats_table(conn, spec) -> list:
@@ -4420,7 +5107,7 @@ def _export_stats_table(conn, spec) -> list:
4420
5107
  payload[payload_key] = [
4421
5108
  {k: cr[k] for k in cr.keys() if k not in ("id", "block_id")}
4422
5109
  for cr in child_rows]
4423
- if spec.kind == "quota_alert_arming":
5110
+ if spec.kind in ("quota_alert_arming", "quota_threshold_event"):
4424
5111
  payload["journal_identity_version"] = 2
4425
5112
  if spec.natural_key_id:
4426
5113
  # §5.3 "state" family: the evt id is the state-instance form (matching