cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
  import datetime as dt
5
5
  import hashlib
6
6
  import json
7
+ import itertools
7
8
  import os
8
9
  import pathlib
9
10
  import sqlite3
@@ -14,7 +15,7 @@ from collections import OrderedDict
14
15
  from collections.abc import Mapping
15
16
  from dataclasses import dataclass, replace
16
17
  from types import MappingProxyType, SimpleNamespace
17
- from typing import Any, Iterable
18
+ from typing import Any, Iterable, Sequence
18
19
  from zoneinfo import ZoneInfo
19
20
 
20
21
  from _cctally_core import get_week_start_name
@@ -28,28 +29,33 @@ from _cctally_quota import (
28
29
  load_codex_quota_projection_certificate,
29
30
  )
30
31
  from _cctally_source_analytics import (
32
+ CodexProjectMetadataHealth,
33
+ CodexThreadMetadataInventory,
31
34
  QualifiedMetadataUnavailable,
35
+ RootedCodexAccountingEntry,
32
36
  has_cached_codex_accounting_entries,
33
37
  load_cached_rooted_codex_accounting_entries,
34
38
  load_codex_project_metadata_health,
39
+ load_codex_thread_metadata_inventory,
35
40
  load_qualified_codex_entries,
41
+ probe_codex_detail_metadata_health,
42
+ )
43
+ from _lib_codex_metadata import (
44
+ codex_metadata_is_malformed,
45
+ decode_codex_project_metadata,
36
46
  )
37
47
  import _lib_log
38
48
  import _lib_accounts
39
49
  import _lib_perf
40
50
  import _lib_snapshot_cache
41
- from _lib_retained_size import (
42
- RETAINED_SIZE_WORK_LOCK,
43
- RETAINED_SIZE_WORKER_DUTY,
44
- RetainedSizeCancelled,
45
- retained_size_bytes,
46
- )
51
+ from _lib_retained_size import retained_size_bytes
47
52
  from _lib_dashboard_sources import (
48
53
  CapabilityRecord,
49
54
  ProjectionCoherence,
50
55
  SourceDashboardState,
51
56
  SourceDashboardWarning,
52
57
  assess_codex_projection_coherence,
58
+ build_metadata_health,
53
59
  canonical_alerted_at,
54
60
  canonical_alerted_at_sql,
55
61
  claude_stats_digest,
@@ -80,6 +86,7 @@ from _lib_fmt import stable_sum
80
86
  from _lib_aggregators import _aggregate_codex_buckets, codex_path_scope
81
87
  from _lib_five_hour import _FIVE_HOUR_JITTER_FLOOR_SECONDS
82
88
  from _lib_source_analytics import (
89
+ QualifiedCodexEntry,
83
90
  assign_collision_safe_project_labels,
84
91
  build_codex_project_result,
85
92
  collision_safe_project_label_map,
@@ -115,57 +122,359 @@ def _mark_codex_source_accelerators_dirty() -> None:
115
122
  _CODEX_SOURCE_ACCELERATOR_GENERATION += 1
116
123
 
117
124
 
125
+ #: Every mutation surface `_ObservedCacheMixin` accounts for, and the three it
126
+ #: deliberately classifies rather than accounts for.
127
+ #: `tests/test_retained_byte_accounting.py` DERIVES the surface it checks these
128
+ #: against from `dir(dict)` and `dir(OrderedDict)` on the running interpreter,
129
+ #: compared to a frozen snapshot of the names those types carried when this set
130
+ #: was written, so a mutator a future interpreter version adds is a test
131
+ #: failure rather than a silent desynchronisation between the running total and
132
+ #: the retained graph.
133
+ _OBSERVED_CACHE_ACCOUNTED_MUTATORS = frozenset({
134
+ "__setitem__", "__delitem__", "clear", "pop", "popitem", "setdefault",
135
+ "update",
136
+ })
137
+ #: Reordering changes eviction order and nothing else, so it adjusts no byte
138
+ #: total and marks nothing dirty — a memo HIT calls `move_to_end`, and dirtying
139
+ #: the accelerators on a read would make every read a generation. It DOES stamp
140
+ #: recency, because a reordering is the memo saying which entry it just used,
141
+ #: and an eviction order that could not see that would evict the hottest entry.
142
+ _OBSERVED_CACHE_REORDERING_MUTATORS = frozenset({"move_to_end"})
143
+ #: Refused outright. `dict.__ior__` mutates in place without routing through
144
+ #: `update`, so a `cache |= other` would move the retained graph while the
145
+ #: running total stood still. There is no caller; the refusal is what keeps it
146
+ #: that way.
147
+ _OBSERVED_CACHE_PROHIBITED_MUTATORS = frozenset({"__ior__"})
148
+ #: Population rather than mutation: `__init__` runs the C-level constructor,
149
+ #: which fills the mapping without routing through `__setitem__`. It is safe
150
+ #: only because the ledgers are initialised in the SAME call, immediately
151
+ #: before that constructor runs, so a fresh object's total and contents agree.
152
+ #: Re-invoking it on a live cache is what the classification forbids, and
153
+ #: `_ObservedCacheMixin.__init__` refuses that outright rather than leaving a
154
+ #: zeroed ledger beside a populated mapping.
155
+ _OBSERVED_CACHE_CONSTRUCTION_SURFACES = frozenset({"__init__"})
156
+ #: The names `dict` and `OrderedDict` carried when the classification above was
157
+ #: written, on CPython 3.13. The completeness test compares the RUNNING
158
+ #: interpreter's `dir()` against this, so a name either type gains later has to
159
+ #: be classified before the estate goes green again.
160
+ _OBSERVED_CACHE_KNOWN_MAPPING_NAMES = frozenset({
161
+ "__class__", "__class_getitem__", "__contains__", "__delattr__",
162
+ "__delitem__", "__dict__", "__dir__", "__doc__", "__eq__", "__format__",
163
+ "__ge__", "__getattribute__", "__getitem__", "__getstate__", "__gt__",
164
+ "__hash__", "__init__", "__init_subclass__", "__ior__", "__iter__",
165
+ "__le__", "__len__", "__lt__", "__ne__", "__new__", "__or__",
166
+ "__reduce__", "__reduce_ex__", "__repr__", "__reversed__", "__ror__",
167
+ "__setattr__", "__setitem__", "__sizeof__", "__str__",
168
+ "__subclasshook__", "clear", "copy", "fromkeys", "get", "items", "keys",
169
+ "move_to_end", "pop", "popitem", "setdefault", "update", "values",
170
+ })
171
+
172
+
173
+ #: One monotonic clock for cache recency, so entries from different caches are
174
+ #: comparable when the budget has to choose between them.
175
+ _CODEX_SOURCE_RECENCY = 0
176
+ _CODEX_SOURCE_RECENCY_LOCK = threading.Lock()
177
+
178
+
179
+ def _next_cache_recency() -> int:
180
+ global _CODEX_SOURCE_RECENCY
181
+ with _CODEX_SOURCE_RECENCY_LOCK:
182
+ _CODEX_SOURCE_RECENCY += 1
183
+ return _CODEX_SOURCE_RECENCY
184
+
185
+
186
+ #: The build whose journal this thread's cache mutations belong to. A thread
187
+ #: with no open overlay journals nothing, which is what keeps eviction, reset
188
+ #: and test setup out of a build's rollback.
189
+ _CODEX_SOURCE_OVERLAY_STATE = threading.local()
190
+
191
+
192
+ def _journal_source_mutation(cache, key) -> None:
193
+ """Record one key's prior state for the open build on this thread."""
194
+ overlay = getattr(_CODEX_SOURCE_OVERLAY_STATE, "overlay", None)
195
+ if overlay is not None:
196
+ overlay.record(cache, key)
197
+
198
+
199
+ def _journal_source_removed(cache, key, prior) -> None:
200
+ """Record a key whose prior value is only known after its removal."""
201
+ overlay = getattr(_CODEX_SOURCE_OVERLAY_STATE, "overlay", None)
202
+ if overlay is not None:
203
+ overlay.record_removed(cache, key, prior)
204
+
205
+
206
+ def _journal_source_clear(cache) -> None:
207
+ overlay = getattr(_CODEX_SOURCE_OVERLAY_STATE, "overlay", None)
208
+ if overlay is not None:
209
+ overlay.record_all(cache)
210
+
211
+
118
212
  class _ObservedCacheMixin:
119
- """Make every retained-cache mutation visible to aggregate admission."""
213
+ """Make every retained-cache mutation visible to aggregate admission.
214
+
215
+ Two things happen at each mutation. The generation counter is bumped, which
216
+ is what schedules aggregate admission, and the owner's RETAINED-BYTE TOTAL
217
+ is adjusted by the delta this mutation makes. The second is what replaced
218
+ the per-generation recursive traversal: the byte total is now known when
219
+ the mutation returns, so nothing has to walk the whole retained graph
220
+ afterwards to find it out.
221
+
222
+ Each entry is charged ONCE, when it is admitted, by its owner's declared
223
+ charge function (see `_CODEX_SOURCE_CACHE_OWNERS`), and the charge is
224
+ retained beside the entry. A replacement subtracts the stored charge before
225
+ it adds the new one, so a same-key overwrite cannot leave the old charge
226
+ standing. A removal subtracts. An owner with no declared charge function
227
+ accounts nothing and reports zero, which is the state of a cache during
228
+ module import before `_declare_codex_source_cache_owner` reaches it.
229
+ """
230
+
231
+ #: Set by `_declare_codex_source_cache_owner`. `None` means "not a declared
232
+ #: owner", which accounts nothing rather than guessing a charge.
233
+ _cache_owner_name: str | None = None
234
+ _cache_owner_charge = None
235
+
236
+ def __init__(self, *args, **kwargs):
237
+ # `dict.__init__` populates at C level without routing through
238
+ # `__setitem__`, so it is a POPULATION surface rather than an accounted
239
+ # mutation. It is sound exactly once, because the ledgers below are
240
+ # created in the same call immediately before it runs. Re-invoking it
241
+ # on a live cache would leave a zeroed ledger beside a populated
242
+ # mapping — the silent desynchronisation the whole surface
243
+ # classification exists to prevent — so it is refused rather than
244
+ # left implicit. See `_OBSERVED_CACHE_CONSTRUCTION_SURFACES`.
245
+ if "_cache_entry_bytes" in vars(self):
246
+ raise TypeError(
247
+ f"{type(self).__name__} refuses a second __init__; it "
248
+ "populates without charging, which would zero the byte ledger "
249
+ "beside a populated mapping")
250
+ self._cache_entry_bytes: dict[object, int] = {}
251
+ self._cache_entry_recency: dict[object, int] = {}
252
+ self._cache_retained_bytes = 0
253
+ super().__init__(*args, **kwargs)
254
+
255
+ # ── recency, for byte-weighted segmented eviction ──────────────────────
256
+
257
+ def touch(self, key) -> None:
258
+ """Mark one entry as recently used.
259
+
260
+ Recency is stamped on every WRITE, and read recency has to be declared
261
+ by the reuse sites, because stamping it inside `get` would add a dict
262
+ write per read and the adapter cache is read once per accounting row
263
+ per build. The sites that return an entry without rewriting it are the
264
+ ones that call this: a cache whose hits never say so would age while
265
+ it was in constant use and be evicted first.
266
+ """
267
+ if key in self._cache_entry_bytes or key in self:
268
+ self._cache_entry_recency[key] = _next_cache_recency()
269
+
270
+ # ── the byte ledger ────────────────────────────────────────────────────
271
+
272
+ @property
273
+ def retained_owner_bytes(self) -> int:
274
+ """This owner's charge for what it currently retains."""
275
+ return int(self._cache_retained_bytes)
276
+
277
+ def _charge(self, key, value) -> None:
278
+ # RECENCY IS STAMPED FIRST, and unconditionally. A self-reported owner
279
+ # has no per-entry charge, but it still has to age: when the stamp sat
280
+ # after this early return, every key of every self-reported owner
281
+ # carried recency 0 and therefore sorted as the coldest thing in the
282
+ # process — so the quota memo, written last, and the whole visible
283
+ # population were the first two buckets any over-budget pass evicted,
284
+ # ahead of genuinely older and far cheaper buckets.
285
+ self._cache_entry_recency[key] = _next_cache_recency()
286
+ charge = self._cache_owner_charge
287
+ if charge is None:
288
+ return
289
+ try:
290
+ entry_bytes = int(charge(key, value))
291
+ except Exception: # noqa: BLE001 - a charge never fails a mutation
292
+ # A charge that cannot be computed must not take the write with it.
293
+ # Charging zero is the conservative direction: it understates the
294
+ # total, and admission compares the total against a ceiling. The
295
+ # failure is COUNTED rather than swallowed, on the same counter the
296
+ # background verifier used to report its own failures on, so a
297
+ # charge that has stopped working is visible in
298
+ # `/api/debug/backend` instead of quietly deflating the total.
299
+ global _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS
300
+ _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS += 1
301
+ entry_bytes = 0
302
+ self._cache_entry_bytes[key] = entry_bytes
303
+ self._cache_retained_bytes += entry_bytes
304
+
305
+ def _discharge(self, key) -> None:
306
+ self._cache_entry_recency.pop(key, None)
307
+ if self._cache_owner_charge is None:
308
+ return
309
+ self._cache_retained_bytes -= self._cache_entry_bytes.pop(key, 0)
310
+
311
+ def _discharge_all(self) -> None:
312
+ self._cache_entry_recency.clear()
313
+ if self._cache_owner_charge is None:
314
+ return
315
+ self._cache_entry_bytes.clear()
316
+ self._cache_retained_bytes = 0
317
+
318
+ def eviction_candidates(self) -> "list[tuple[int, object, int]]":
319
+ """`(recency, key, bytes)` for everything this owner could give up."""
320
+ return [
321
+ (int(self._cache_entry_recency.get(key, 0)), key,
322
+ int(self._cache_entry_bytes.get(key, 0)))
323
+ for key in tuple(self)
324
+ ]
325
+
326
+ # ── the accounted mutation surface ─────────────────────────────────────
327
+
328
+ def _refuse_if_oversized(self, key) -> bool:
329
+ """Never admit a single entry that alone exceeds the whole budget.
330
+
331
+ This is the accelerator's counterpart to the quota memo's own
332
+ never-admit branch. Admitting an oversized entry and discovering it on
333
+ the NEXT admission pass is not the same thing: between the two the
334
+ cache is over the ceiling, and the pass that finds it has to evict
335
+ other buckets first because it works coldest-first and the oversized
336
+ entry is the one just written. Refusing at the write means the result
337
+ is computed and used by the caller that asked for it, and simply not
338
+ retained — which is what "computed without admission" says.
339
+
340
+ A self-reported owner has no per-entry charge and cannot answer this;
341
+ it is bounded by its own admission meter instead.
342
+ """
343
+ if self._cache_owner_charge is None:
344
+ return False
345
+ if self._cache_entry_bytes.get(key, 0) <= (
346
+ _CODEX_SOURCE_ACCELERATOR_MAX_BYTES):
347
+ return False
348
+ global _CODEX_SOURCE_ACCELERATOR_ADMISSION_REFUSALS
349
+ _CODEX_SOURCE_ACCELERATOR_ADMISSION_REFUSALS += 1
350
+ self._discharge(key)
351
+ super().__delitem__(key)
352
+ return True
120
353
 
121
354
  def __setitem__(self, key, value):
355
+ _journal_source_mutation(self, key)
356
+ # Subtract before adding: a same-key overwrite otherwise leaves the
357
+ # superseded entry's charge standing in the running total forever.
358
+ self._discharge(key)
122
359
  super().__setitem__(key, value)
360
+ self._charge(key, value)
361
+ self._refuse_if_oversized(key)
123
362
  _mark_codex_source_accelerators_dirty()
124
363
 
125
364
  def __delitem__(self, key):
365
+ _journal_source_mutation(self, key)
126
366
  super().__delitem__(key)
367
+ self._discharge(key)
127
368
  _mark_codex_source_accelerators_dirty()
128
369
 
129
370
  def clear(self):
130
371
  if self:
372
+ _journal_source_clear(self)
131
373
  super().clear()
374
+ self._discharge_all()
132
375
  _mark_codex_source_accelerators_dirty()
133
376
 
134
377
  def pop(self, key, default=_CACHE_MISSING):
135
378
  existed = key in self
379
+ if existed:
380
+ _journal_source_mutation(self, key)
136
381
  if default is _CACHE_MISSING:
137
382
  value = super().pop(key)
138
383
  else:
139
384
  value = super().pop(key, default)
140
385
  if existed:
386
+ self._discharge(key)
141
387
  _mark_codex_source_accelerators_dirty()
142
388
  return value
143
389
 
144
390
  def popitem(self, *args, **kwargs):
391
+ # The key is not known until the call returns, so the journal has to
392
+ # take it afterwards. Recording an already-removed key still restores
393
+ # it, because the prior value comes back with the journal entry.
145
394
  value = super().popitem(*args, **kwargs)
395
+ _journal_source_removed(self, value[0], value[1])
396
+ self._discharge(value[0])
146
397
  _mark_codex_source_accelerators_dirty()
147
398
  return value
148
399
 
149
400
  def setdefault(self, key, default=None):
150
401
  existed = key in self
402
+ if not existed:
403
+ _journal_source_mutation(self, key)
151
404
  value = super().setdefault(key, default)
152
405
  if not existed:
406
+ self._charge(key, value)
407
+ self._refuse_if_oversized(key)
153
408
  _mark_codex_source_accelerators_dirty()
154
409
  return value
155
410
 
156
411
  def update(self, *args, **kwargs):
157
412
  before = len(self)
158
- super().update(*args, **kwargs)
413
+ # The keys are enumerated WITHOUT materialising a copy of the source
414
+ # MAPPING. `dict(other)` here would reintroduce a full-container copy
415
+ # on the restore path, which is one of the two copies this tranche
416
+ # removes.
417
+ #
418
+ # A NON-MAPPING SOURCE IS MATERIALISED ONCE, because `dict.update`
419
+ # accepts any iterable of pairs and a one-shot iterator can only be
420
+ # walked once. Enumerating the keys out of the iterator and then
421
+ # handing the exhausted iterator to `super().update` inserted nothing
422
+ # and then raised `KeyError` charging keys that were never there —
423
+ # `update(iter([("a", 1)]))` failed that way. Rebinding the argument to
424
+ # the materialised pairs keeps `dict.update`'s documented contract.
425
+ sources: list = []
426
+ changed: list = []
427
+ for source in args:
428
+ if hasattr(source, "keys"):
429
+ sources.append(source)
430
+ changed.extend(source.keys())
431
+ else:
432
+ pairs = list(source)
433
+ sources.append(pairs)
434
+ changed.extend(key for key, _value in pairs)
435
+ changed.extend(kwargs)
436
+ # ONE KEY, ONE CHARGE. `dict.update` accepts a key in the positional
437
+ # source AND in `kwargs`, and it appeared twice in `changed`: the
438
+ # discharge loop subtracted it once and then subtracted nothing, while
439
+ # the charge loop ADDED it twice against a `_cache_entry_bytes` that
440
+ # holds it once. The running total then stood permanently above the
441
+ # graph it describes and the caches evicted early for the life of the
442
+ # process, and nothing raised. `dict.fromkeys` deduplicates in
443
+ # first-seen order, which is the order the journal and the discharge
444
+ # loop want anyway.
445
+ changed = list(dict.fromkeys(changed))
446
+ for key in changed:
447
+ _journal_source_mutation(self, key)
448
+ self._discharge(key)
449
+ super().update(*sources, **kwargs)
450
+ for key in changed:
451
+ self._charge(key, self[key])
159
452
  if args or kwargs or len(self) != before:
160
453
  _mark_codex_source_accelerators_dirty()
161
454
 
455
+ def __ior__(self, other):
456
+ raise TypeError(
457
+ f"{type(self).__name__} refuses |=; use update() so the retained-"
458
+ "byte total moves with the entry")
459
+
162
460
 
163
461
  class _ObservedDict(_ObservedCacheMixin, dict):
164
462
  pass
165
463
 
166
464
 
167
465
  class _ObservedOrderedDict(_ObservedCacheMixin, OrderedDict):
168
- pass
466
+ def move_to_end(self, key, last=True):
467
+ """Reorder, and stamp recency — a reordering IS a use.
468
+
469
+ `OrderedDict.move_to_end` is how the quota memo records a hit, and it
470
+ is the only signal that memo emits on one. Leaving it unstamped made
471
+ every memoized entry look untouched since it was written, which is what
472
+ put the most recently used quota window at the head of the eviction
473
+ order. It adjusts no byte total and marks nothing dirty, because
474
+ dirtying the accelerators on a read would make every read a generation.
475
+ """
476
+ super().move_to_end(key, last)
477
+ self._cache_entry_recency[key] = _next_cache_recency()
169
478
 
170
479
 
171
480
  _CODEX_QUOTA_OBSERVATION_MAX_ENTRIES = 32
@@ -212,13 +521,26 @@ def codex_quota_observation_cache_stats() -> Mapping[str, int]:
212
521
  })
213
522
 
214
523
 
215
- def _quota_observation_cache_checkpoint() -> tuple[dict, int, int, int]:
524
+ def _quota_observation_cache_checkpoint() -> tuple[dict, int, int, int, tuple]:
525
+ """The memo's byte ledger, its counters, AND its LRU key order.
526
+
527
+ THE ORDER IS CHECKPOINTED OUTRIGHT, because the journal cannot reconstruct
528
+ it. Replaying a journal in reverse reinstates the prior order only when the
529
+ build's removals were last-in-first-out, and this memo evicts with
530
+ `popitem(last=False)`, which takes the OLDEST key; re-assigning that key
531
+ afterwards appends it at the young end. Worse, a memo HIT calls
532
+ `move_to_end`, which is a reordering the journal does not record at all, so
533
+ a build that only READ the memo and was then discarded would leave it
534
+ permanently reordered. Thirty-two keys is a cheap thing to remember, and
535
+ remembering it is the only way the overlay bundle's ordering claim is true.
536
+ """
216
537
  with _CODEX_QUOTA_OBSERVATION_CACHE_LOCK:
217
538
  return (
218
539
  dict(_CODEX_QUOTA_OBSERVATION_CACHE_SIZES),
219
540
  int(_CODEX_QUOTA_OBSERVATION_CACHE_BYTES),
220
541
  int(_CODEX_QUOTA_OBSERVATION_CACHE_EVICTIONS),
221
542
  int(_CODEX_QUOTA_OBSERVATION_CACHE_FALLBACKS),
543
+ tuple(_CODEX_QUOTA_OBSERVATION_CACHE),
222
544
  )
223
545
 
224
546
 
@@ -226,13 +548,22 @@ def _restore_quota_observation_cache_checkpoint(checkpoint) -> None:
226
548
  global _CODEX_QUOTA_OBSERVATION_CACHE_BYTES
227
549
  global _CODEX_QUOTA_OBSERVATION_CACHE_EVICTIONS
228
550
  global _CODEX_QUOTA_OBSERVATION_CACHE_FALLBACKS
229
- sizes, retained_bytes, evictions, fallbacks = checkpoint
551
+ sizes, retained_bytes, evictions, fallbacks, order = checkpoint
230
552
  with _CODEX_QUOTA_OBSERVATION_CACHE_LOCK:
231
553
  _CODEX_QUOTA_OBSERVATION_CACHE_SIZES.clear()
232
554
  _CODEX_QUOTA_OBSERVATION_CACHE_SIZES.update(sizes)
233
555
  _CODEX_QUOTA_OBSERVATION_CACHE_BYTES = int(retained_bytes)
234
556
  _CODEX_QUOTA_OBSERVATION_CACHE_EVICTIONS = int(evictions)
235
557
  _CODEX_QUOTA_OBSERVATION_CACHE_FALLBACKS = int(fallbacks)
558
+ # Reinstate the eviction order the build started from. The journal has
559
+ # already restored WHICH keys are resident; this restores the order
560
+ # they were in, for the two reasons `_quota_observation_cache_checkpoint`
561
+ # gives. Keys the checkpoint does not name are left where they are
562
+ # rather than dropped, because membership is the journal's business and
563
+ # a restore that also deleted would undo a key this build never wrote.
564
+ for key in order:
565
+ if key in _CODEX_QUOTA_OBSERVATION_CACHE:
566
+ _CODEX_QUOTA_OBSERVATION_CACHE.move_to_end(key)
236
567
 
237
568
 
238
569
  def _codex_quota_reuse_identity(
@@ -1260,13 +1591,81 @@ class DashboardReadContext:
1260
1591
  raise ValueError("codex_budget must be a mapping or None")
1261
1592
 
1262
1593
 
1594
+ #: #769 S6 / #809. Handle-lifetime tokens for connections with no file
1595
+ #: identity. ``id()`` is an ADDRESS, not an identity: CPython hands a released
1596
+ #: connection's address straight to the next allocation of the same size —
1597
+ #: measured, nineteen of twenty sequential create/close/drop cycles share one
1598
+ #: ``id()`` — so an in-memory corpus opened after the previous handle was
1599
+ #: released matched the stale process memo. That is exactly the collision #809
1600
+ #: exists to close, moved from file-backed stores to in-memory ones.
1601
+ #:
1602
+ #: The registry holds a STRONG reference to each connection it has issued a
1603
+ #: token for, which is what makes the token an identity: while the entry lives
1604
+ #: the object cannot be freed, so its address cannot be handed to anything
1605
+ #: else. ``sqlite3.Connection`` supports neither weak references nor attribute
1606
+ #: assignment, so there is no lighter binding available.
1607
+ #:
1608
+ #: Pruning of entries whose connection has been CLOSED runs on the MISS path
1609
+ #: only, immediately before a new token is issued. A hit returns at
1610
+ #: ``move_to_end`` without pruning, so a closed connection's entry — and the
1611
+ #: strong reference it holds — stays resident until the next miss prunes it or
1612
+ #: the capacity below evicts it. That is a bounded retention rather than a
1613
+ #: prompt release, and it is deliberate: pruning is O(registry) and a hit is
1614
+ #: the common call. Tokens are drawn from a monotonic counter and never
1615
+ #: reused, so a pruned or evicted connection that is used again simply
1616
+ #: receives a NEW token and misses the memo — fail-closed in both directions.
1617
+ _CONNECTION_IDENTITY_TOKENS: "OrderedDict[int, tuple[sqlite3.Connection, str]]" = (
1618
+ OrderedDict()
1619
+ )
1620
+ _CONNECTION_IDENTITY_LOCK = threading.Lock()
1621
+ _CONNECTION_IDENTITY_SEQUENCE = itertools.count(1)
1622
+ #: Bounds what the registry retains. Only in-memory and unreadable-path stores
1623
+ #: reach it, so this is generous; eviction is safe because it can only cause a
1624
+ #: miss, never a false hit.
1625
+ _CONNECTION_IDENTITY_CAPACITY = 64
1626
+
1627
+
1628
+ def _connection_is_closed(conn: sqlite3.Connection) -> bool:
1629
+ """True when the handle has been closed. Never raises."""
1630
+ try:
1631
+ conn.total_changes
1632
+ except sqlite3.ProgrammingError:
1633
+ return True
1634
+ except Exception: # noqa: BLE001
1635
+ return False
1636
+ return False
1637
+
1638
+
1639
+ def _connection_identity_token(conn: sqlite3.Connection) -> str:
1640
+ """A token bound to this handle's lifetime, never to its address."""
1641
+ key = id(conn)
1642
+ with _CONNECTION_IDENTITY_LOCK:
1643
+ entry = _CONNECTION_IDENTITY_TOKENS.get(key)
1644
+ if entry is not None and entry[0] is conn:
1645
+ _CONNECTION_IDENTITY_TOKENS.move_to_end(key)
1646
+ return entry[1]
1647
+ for stale in [
1648
+ stale_key
1649
+ for stale_key, (held, _token) in _CONNECTION_IDENTITY_TOKENS.items()
1650
+ if _connection_is_closed(held)
1651
+ ]:
1652
+ _CONNECTION_IDENTITY_TOKENS.pop(stale, None)
1653
+ while len(_CONNECTION_IDENTITY_TOKENS) >= _CONNECTION_IDENTITY_CAPACITY:
1654
+ _CONNECTION_IDENTITY_TOKENS.popitem(last=False)
1655
+ token = f"handle-{next(_CONNECTION_IDENTITY_SEQUENCE)}"
1656
+ _CONNECTION_IDENTITY_TOKENS[key] = (conn, token)
1657
+ return token
1658
+
1659
+
1263
1660
  def _cache_database_identity(conn: sqlite3.Connection) -> tuple[object, ...]:
1264
1661
  """Identity of the cache.db file currently backing ``conn``.
1265
1662
 
1266
1663
  A pathname is not an identity: rebuild publishes a replacement at the same
1267
1664
  pathname. The device/inode pair makes every process memo fail closed on
1268
- that replacement. In-memory and unreadable paths use the connection
1269
- object, which intentionally prevents reuse across a newly-opened handle.
1665
+ that replacement. In-memory and unreadable paths use a handle-lifetime
1666
+ token, which intentionally prevents reuse across a newly-opened handle —
1667
+ see ``_connection_identity_token`` for why the connection's address is not
1668
+ usable for this.
1270
1669
  """
1271
1670
  try:
1272
1671
  db_path = next(
@@ -1274,16 +1673,174 @@ def _cache_database_identity(conn: sqlite3.Connection) -> tuple[object, ...]:
1274
1673
  if str(row[1]) == "main"
1275
1674
  )
1276
1675
  except (sqlite3.Error, StopIteration):
1277
- return ("connection", id(conn))
1676
+ return ("connection", _connection_identity_token(conn))
1278
1677
  if not db_path:
1279
- return ("connection", id(conn))
1678
+ return ("connection", _connection_identity_token(conn))
1280
1679
  try:
1281
1680
  stat = os.stat(db_path)
1282
1681
  except OSError:
1283
- return ("path-unreadable", db_path, id(conn))
1682
+ return ("path-unreadable", db_path, _connection_identity_token(conn))
1284
1683
  return ("file", db_path, int(stat.st_dev), int(stat.st_ino))
1285
1684
 
1286
1685
 
1686
+ #: #769 S6 / #753. The named per-account hero operands, and nothing else. The
1687
+ #: cohort copies exactly these keys out of an account card; a card also carries
1688
+ #: a label, a plan, a percentage, a reset and freshness disclosure, none of
1689
+ #: which is a hero operand and all of which belong to the current generation.
1690
+ _HERO_CARD_OPERANDS = (
1691
+ "spendUsd", "inputTokens", "cachedInputTokens", "outputTokens",
1692
+ "reasoningOutputTokens", "totalTokens",
1693
+ )
1694
+
1695
+ #: The provider hero's own accounting operands.
1696
+ _HERO_PROVIDER_OPERANDS = (
1697
+ "cost_usd", "input_tokens", "cached_input_tokens", "output_tokens",
1698
+ "reasoning_output_tokens", "total_tokens",
1699
+ )
1700
+
1701
+
1702
+ def _codex_cycle_vector(cycles: "Iterable[CodexCycleBoundary]") -> tuple:
1703
+ """The per-account cycle boundary vector, as the cohort's validity input.
1704
+
1705
+ Boundaries only. The spend a cycle bounds is the thing being retained, so
1706
+ folding it in here would invalidate the cohort on exactly the ticks the
1707
+ retention exists to survive.
1708
+ """
1709
+ return tuple(sorted(
1710
+ (
1711
+ str(
1712
+ cycle.quota_identity.account_key
1713
+ if cycle.quota_identity is not None
1714
+ else _lib_accounts.UNATTRIBUTED
1715
+ ),
1716
+ int(cycle.window_minutes),
1717
+ cycle.start_at.astimezone(UTC).isoformat(),
1718
+ cycle.resets_at.astimezone(UTC).isoformat(),
1719
+ )
1720
+ for cycle in cycles
1721
+ ))
1722
+
1723
+
1724
+ def _codex_hero_cohort_validity(
1725
+ context: DashboardReadContext,
1726
+ *,
1727
+ cycle_vector: tuple,
1728
+ ) -> "dict[str, object] | None":
1729
+ """The retained hero cohort's validity key, or None when unresolvable.
1730
+
1731
+ R8 decoration turns on at more than one REAL account, so a count or a
1732
+ decoration boolean cannot distinguish the account set A and B from A and C,
1733
+ nor a label change, nor a change of active account — and a cohort retained
1734
+ across any of those would show an account or a decoration the current
1735
+ generation should not. The key is therefore the existing
1736
+ ``accounts_identity_digest``, which already covers account keys, provider,
1737
+ labels and active identities, together with the full per-account cycle
1738
+ vector and BOTH store identities.
1739
+
1740
+ ``None`` means the key could not be resolved, and the caller drops the
1741
+ cohort and renders Pending. The retention fails closed.
1742
+ """
1743
+ try:
1744
+ # `accounts_identity_digest` swallows a read failure and returns "",
1745
+ # which is also exactly what a no-account install returns. Probing the
1746
+ # registry here is what separates the two, so an unreadable registry
1747
+ # fails closed rather than matching every other unreadable one.
1748
+ context.stats_conn.execute("SELECT COUNT(*) FROM accounts").fetchone()
1749
+ if context.stats_identity is not None:
1750
+ accounts_digest = str(context.stats_identity[1])
1751
+ else:
1752
+ accounts_digest = accounts_identity_digest(context.stats_conn)
1753
+ return {
1754
+ "accounts_digest": accounts_digest,
1755
+ "cycle_vector": cycle_vector,
1756
+ "cache_identity": _cache_database_identity(context.cache_conn),
1757
+ # The same helper, on the other store. It reads `PRAGMA
1758
+ # database_list` off whatever connection it is given, so it is
1759
+ # generic over SQLite handles despite its name naming the store it
1760
+ # was written for.
1761
+ "stats_identity": _cache_database_identity(context.stats_conn),
1762
+ }
1763
+ except Exception: # noqa: BLE001
1764
+ return None
1765
+
1766
+
1767
+ def _retained_hero_cohort(
1768
+ prior_hero_cohort: "Mapping[str, object] | None",
1769
+ validity: "Mapping[str, object] | None",
1770
+ ) -> "Mapping[str, object] | None":
1771
+ """The prior cohort, but only when this generation still certifies it."""
1772
+ if not isinstance(prior_hero_cohort, Mapping) or validity is None:
1773
+ return None
1774
+ prior_validity = prior_hero_cohort.get("validity")
1775
+ if not isinstance(prior_validity, Mapping):
1776
+ return None
1777
+ if _cohort_validity_key(prior_validity) != _cohort_validity_key(validity):
1778
+ return None
1779
+ return prior_hero_cohort
1780
+
1781
+
1782
+ def _cohort_validity_key(validity: "Mapping[str, object]") -> str:
1783
+ """One canonical spelling of a validity key.
1784
+
1785
+ The prior cohort was frozen when its state was constructed, so its tuples
1786
+ arrive as read-only mappings and sequences while the freshly-derived key
1787
+ holds plain dicts and tuples. Rendering both through the same serializer is
1788
+ what makes the comparison an equality of VALUES rather than of containers.
1789
+ """
1790
+ def normalize(value: object) -> object:
1791
+ # `_freeze` turns the retained key's dicts into read-only mappings and
1792
+ # `json.dumps` refuses those, so a plain dump would fall through to
1793
+ # `default=str` on one side and produce a repr that no freshly-derived
1794
+ # key can ever equal. Sequences are levelled to lists for the same
1795
+ # reason: a tuple and a list of the same values must compare equal.
1796
+ if isinstance(value, Mapping):
1797
+ return {str(key): normalize(item) for key, item in value.items()}
1798
+ if isinstance(value, (list, tuple)):
1799
+ return [normalize(item) for item in value]
1800
+ if value is None or isinstance(value, (str, int, float, bool)):
1801
+ return value
1802
+ return str(value)
1803
+
1804
+ return json.dumps(
1805
+ normalize(validity), sort_keys=True, allow_nan=False,
1806
+ separators=(",", ":"),
1807
+ )
1808
+
1809
+
1810
+ def _codex_accounting_period_signature(
1811
+ cache_conn: sqlite3.Connection,
1812
+ range_start: dt.datetime,
1813
+ metadata_incomplete: bool,
1814
+ ) -> tuple[object, ...]:
1815
+ """The accounting memo's semantic signature — the ONE construction site.
1816
+
1817
+ #769 S6 / #809. This signature has SEVEN direct consumers — the daily,
1818
+ monthly, weekly, session and project views, the cache-report wire and the
1819
+ account scope, which passes it on to the account-scoped session memo — and
1820
+ until this change it carried no store identity. Two stores built in one
1821
+ process that agreed on ``range_start`` and ``metadata_incomplete`` shared
1822
+ one memo entry, and because the incremental path evicts a cached row only
1823
+ when it ages past the cutoff or appears in ``changed_old`` / ``changed_new``,
1824
+ a row simply ABSENT from the second corpus survived into that corpus's
1825
+ published rows.
1826
+
1827
+ Its sibling ``("codex-visible-population-v1", _cache_database_identity(…),
1828
+ …)`` already folds store identity in, as does the accounting loader's
1829
+ ``extra_signature``. This signature was the outlier in its own family, and
1830
+ the element is inserted in the same position the sibling puts it.
1831
+
1832
+ The identity never reaches the wire, so #782's single-store behaviour and
1833
+ published bytes are unchanged: on a single store the added element is
1834
+ constant.
1835
+ """
1836
+ return (
1837
+ "codex-accounting-v1",
1838
+ _cache_database_identity(cache_conn),
1839
+ range_start,
1840
+ metadata_incomplete,
1841
+ )
1842
+
1843
+
1287
1844
  _RESOURCE_ROWS = {
1288
1845
  "session": ("sessions", "rows"),
1289
1846
  "project": ("projects", "rows"),
@@ -1602,6 +2159,7 @@ def _cached_codex_session_view(
1602
2159
  values = tuple(entries)
1603
2160
  signature = (semantic_signature, tz_name, speed)
1604
2161
  state = _CODEX_SESSION_VIEW_CACHE.get(cache_key)
2162
+ _CODEX_SESSION_VIEW_CACHE.touch(cache_key)
1605
2163
  if state is None or state[0] != signature:
1606
2164
  view = build_codex_session_view(
1607
2165
  values, now_utc=now_utc, tz_name=tz_name, speed=speed,
@@ -1716,6 +2274,7 @@ def _cached_codex_period_view(
1716
2274
  values = tuple(entries)
1717
2275
  signature = (semantic_signature, kind, tz_name, speed)
1718
2276
  state = _CODEX_PERIOD_VIEW_CACHE.get((cache_key, kind))
2277
+ _CODEX_PERIOD_VIEW_CACHE.touch((cache_key, kind))
1719
2278
  builder = build_codex_daily_view if kind == "daily" else build_codex_monthly_view
1720
2279
  if state is None or state[0] != signature:
1721
2280
  view = builder(values, now_utc=now_utc, tz_name=tz_name, speed=speed)
@@ -1799,6 +2358,8 @@ def _codex_cache_report_wire(
1799
2358
  entries: Iterable[object],
1800
2359
  *,
1801
2360
  metadata: Mapping[tuple[str, str], Mapping[str, object]],
2361
+ by_accounting_identity: "Mapping[tuple[str, str, str], Mapping[str, object]] | None" = None,
2362
+ withhold_projects: bool = False,
1802
2363
  now_utc: dt.datetime,
1803
2364
  display_tz_name: str | None,
1804
2365
  speed: str,
@@ -1824,15 +2385,14 @@ def _codex_cache_report_wire(
1824
2385
  cutoff = now_utc - dt.timedelta(days=window_days)
1825
2386
  bucket_tz = crk._resolve_bucket_tz(display_tz)
1826
2387
 
1827
- def _tiered_cost(tokens: int, pricing: Mapping[str, object], base: str, above: str) -> float:
2388
+ def _tiered_cost(tokens: int, long_context: bool,
2389
+ pricing: Mapping[str, object], base: str, above: str) -> float:
1828
2390
  if tokens <= 0:
1829
2391
  return 0.0
1830
2392
  base_rate = float(pricing.get(base, 0.0) or 0.0)
1831
2393
  above_rate = pricing.get(above)
1832
- threshold = int(c.CODEX_TIERED_THRESHOLD)
1833
- if tokens > threshold and above_rate is not None:
1834
- return threshold * base_rate + (tokens - threshold) * float(above_rate)
1835
- return tokens * base_rate
2394
+ rate = float(above_rate) if long_context and above_rate is not None else base_rate
2395
+ return tokens * rate
1836
2396
 
1837
2397
  def _wrap_entry(entry: object) -> object | None:
1838
2398
  timestamp = getattr(entry, "timestamp", None)
@@ -1842,23 +2402,34 @@ def _codex_cache_report_wire(
1842
2402
  input_tokens = int(getattr(entry, "input_tokens", 0))
1843
2403
  cached_tokens = min(input_tokens, int(getattr(entry, "cached_input_tokens", 0)))
1844
2404
  uncached_tokens = max(0, input_tokens - cached_tokens)
2405
+ long_context = input_tokens > int(c.CODEX_TIERED_THRESHOLD)
1845
2406
  pricing, _is_fallback = c._resolve_codex_pricing(model)
1846
2407
  pricing = pricing or {}
1847
2408
  uncached_counterfactual = _tiered_cost(
1848
- cached_tokens, pricing,
2409
+ cached_tokens, long_context, pricing,
1849
2410
  "input_cost_per_token", "input_cost_per_token_above_272k_tokens",
1850
2411
  )
1851
2412
  cached_actual = _tiered_cost(
1852
- cached_tokens, pricing,
2413
+ cached_tokens, long_context, pricing,
1853
2414
  "cache_read_input_token_cost", "cache_read_input_token_cost_above_272k_tokens",
1854
2415
  )
1855
2416
  multiplier = c._codex_fast_multiplier(model) if speed == "fast" else 1.0
1856
2417
  saved = max(0.0, uncached_counterfactual - cached_actual) * multiplier
1857
- identity = (
1858
- str(getattr(entry, "source_root_key", "") or ""),
1859
- str(getattr(entry, "source_path", "") or ""),
1860
- )
1861
- item_metadata = metadata.get(identity) or {}
2418
+ # #845 §4.4: in the partial generation the lookup is by accounting
2419
+ # identity, so one rollout path hosting a healthy and a malformed
2420
+ # conversation does not publish one decision for both. An entry the
2421
+ # identity map leaves unattributed lands in this fold's existing
2422
+ # `(unknown)` bucket, which is its not-attributed bucket rather than a
2423
+ # project.
2424
+ if by_accounting_identity is not None:
2425
+ item_metadata = by_accounting_identity.get(
2426
+ _codex_accounting_identity(entry)) or {}
2427
+ else:
2428
+ identity = (
2429
+ str(getattr(entry, "source_root_key", "") or ""),
2430
+ str(getattr(entry, "source_path", "") or ""),
2431
+ )
2432
+ item_metadata = metadata.get(identity) or {}
1862
2433
  project = (
1863
2434
  str(getattr(entry, "project_label", "") or "").strip()
1864
2435
  or str(item_metadata.get("project_label") or "").strip()
@@ -1968,6 +2539,7 @@ def _codex_cache_report_wire(
1968
2539
  ]
1969
2540
  else:
1970
2541
  state = _CODEX_CACHE_REPORT_ROWS.get(cache_key)
2542
+ _CODEX_CACHE_REPORT_ROWS.touch(cache_key)
1971
2543
  if state is None or len(state) != 4 or state[0] != signature:
1972
2544
  cached_rows = {}
1973
2545
  for entry in values:
@@ -2015,9 +2587,30 @@ def _codex_cache_report_wire(
2015
2587
  if not cache_entry_id:
2016
2588
  continue
2017
2589
  wrapped_entry = _wrap_entry(entry)
2590
+ # A replaced row's PREVIOUS day has to be retired with it
2591
+ # (#769 S4 #782). `groups` files each id under the day it was
2592
+ # last frozen at, and only `old_ids` removes it from there, so
2593
+ # replacing a row whose day moved left the id in both groups.
2594
+ # Refreezing the old day then handed `_aggregate_cache_by_day`
2595
+ # two dates at once and it raised
2596
+ # "one cache-report day produced multiple rows" — and when the
2597
+ # old day was NOT refrozen, it silently kept publishing a row
2598
+ # that no longer belonged to it. The same applies when the
2599
+ # replacement wraps to None (the row aged past the window):
2600
+ # without this the id stays in `groups` while `cached_rows` no
2601
+ # longer holds it, and the next refreeze of that day raises
2602
+ # KeyError while sorting.
2603
+ prior = cached_rows.get(cache_entry_id)
2018
2604
  if wrapped_entry is None:
2605
+ if prior is not None:
2606
+ old_ids.add(cache_entry_id)
2607
+ affected_days.add(prior._cache_day)
2019
2608
  cached_rows.pop(cache_entry_id, None)
2020
2609
  else:
2610
+ if (prior is not None
2611
+ and prior._cache_day != wrapped_entry._cache_day):
2612
+ old_ids.add(cache_entry_id)
2613
+ affected_days.add(prior._cache_day)
2021
2614
  cached_rows[cache_entry_id] = wrapped_entry
2022
2615
  affected_days.add(wrapped_entry._cache_day)
2023
2616
  for day in affected_days:
@@ -2202,6 +2795,12 @@ def _codex_cache_report_wire(
2202
2795
 
2203
2796
  by_project = _finish_breakdown("project")
2204
2797
  by_model = _finish_breakdown("model")
2798
+ if withhold_projects:
2799
+ # #845 §4.5. The conversation-metadata read failed, so no entry has a
2800
+ # project identity at all. Publishing an empty collection says that;
2801
+ # converting every entry to `(unknown)` would report a bucket the
2802
+ # reader cannot distinguish from a real absence of attribution.
2803
+ by_project = ()
2205
2804
  seven = days[:7]
2206
2805
  saved_total = stable_sum(float(row["saved_usd"]) for row in days)
2207
2806
  wasted_total = stable_sum(float(row["wasted_usd"]) for row in days)
@@ -2270,9 +2869,127 @@ _CODEX_FILE_ALIAS_SQL = (
2270
2869
  )
2271
2870
 
2272
2871
 
2872
+ @dataclass(frozen=True)
2873
+ class _CodexConversationMetadataRead:
2874
+ """The typed result of the conversation-metadata read (#845 §4.5).
2875
+
2876
+ ``metadata`` is the legacy path map, keyed ``(source_root_key,
2877
+ source_path)`` and built exactly as it was. ``by_accounting_identity`` is
2878
+ the per-conversation map of §4.4, keyed ``(source_root_key,
2879
+ conversation_key or "", source_path)`` and materialized only for the
2880
+ partial generation. ``error`` distinguishes a store with no Codex
2881
+ conversations, which returns two empty maps and ``None``, from a read that
2882
+ failed, which returns two empty maps and the exception: without it the two
2883
+ were byte-identical afterwards and the second published as the first.
2884
+ """
2885
+
2886
+ metadata: "Mapping[tuple[str, str], Mapping[str, object]]"
2887
+ by_accounting_identity: "Mapping[tuple[str, str, str], Mapping[str, object]]"
2888
+ error: "BaseException | None" = None
2889
+
2890
+
2891
+ def _codex_accounting_identity(entry: object) -> tuple[str, str, str]:
2892
+ """The identity map's key for one accounting entry, with no fallback.
2893
+
2894
+ The exact key for every entry: a sentinel fallback between this key and
2895
+ the path-keyed one would silently attribute a refused entry through a
2896
+ neighbour that shares its path.
2897
+ """
2898
+ return (
2899
+ str(getattr(entry, "source_root_key", "") or ""),
2900
+ str(getattr(entry, "conversation_key", "") or ""),
2901
+ str(getattr(entry, "source_path", "") or ""),
2902
+ )
2903
+
2904
+
2905
+ def _codex_identity_map(
2906
+ cache_conn: sqlite3.Connection,
2907
+ accounting_entries: "Iterable[object]",
2908
+ inventory: "CodexThreadMetadataInventory | None" = None,
2909
+ ) -> dict[tuple[str, str, str], dict[str, object]]:
2910
+ """Decide project attribution per accounting identity (§4.4).
2911
+
2912
+ The decision order is the qualified reader's, so the map, the counter and
2913
+ the reader agree on every rooted entry. First the reader's two refusals,
2914
+ which precede any metadata: an empty ``conversation_key``, whatever its
2915
+ path's alias winner holds, and a nonempty key with neither a thread row nor
2916
+ an alias winner. A refused item carries ``None`` for both fields; no
2917
+ resolver is called and the predicate is not consulted.
2918
+
2919
+ Every other entry is classified through ``codex_metadata_is_malformed``
2920
+ with its direct row and its path's inherited winner, and a malformed one
2921
+ carries ``None`` for both fields too. Otherwise the entry is attributed
2922
+ exactly as the qualified reader would attribute it: the merged fields
2923
+ ``cwd = direct.cwd or inherited.cwd`` and ``git_json = direct.git_json or
2924
+ inherited.git_json``, with the same ``inherited`` operand the decision
2925
+ used. An alias-only identity therefore attributes through the inventory's
2926
+ ordering-only winner rather than through the path map's native-id
2927
+ inheritance, so an unrelated malformed row can never relabel or remove a
2928
+ healthy project row.
2929
+
2930
+ The reader's first refusal, an empty ``source_root_key``, is not mirrored
2931
+ here: the rooted loader raises for such a row before any entry reaches this
2932
+ map (§4.6 rule 3).
2933
+ """
2934
+ from _cctally_cache import _codex_conversation_project_attribution
2935
+
2936
+ if inventory is None:
2937
+ inventory = load_codex_thread_metadata_inventory(cache_conn)
2938
+ decided: dict[tuple[str, str, str], dict[str, object]] = {}
2939
+ for entry in accounting_entries:
2940
+ identity = _codex_accounting_identity(entry)
2941
+ if identity in decided:
2942
+ continue
2943
+ root_key, conversation_key, source_path = identity
2944
+ direct = inventory.direct_by_conversation.get((root_key, conversation_key))
2945
+ inherited = inventory.inherited_by_path.get((root_key, source_path))
2946
+ refused = not conversation_key or (direct is None and inherited is None)
2947
+ if refused or codex_metadata_is_malformed(
2948
+ direct if conversation_key else None, inherited,
2949
+ ):
2950
+ decided[identity] = {"project_key": None, "project_label": None}
2951
+ continue
2952
+ cwd = (direct.cwd if direct is not None else None) or (
2953
+ inherited.cwd if inherited is not None else None)
2954
+ git_json = (direct.git_json if direct is not None else None) or (
2955
+ inherited.git_json if inherited is not None else None)
2956
+ project_key, project_label = _codex_conversation_project_attribution(
2957
+ root_key, cwd, git_json)
2958
+ decided[identity] = {
2959
+ "project_key": project_key, "project_label": project_label,
2960
+ }
2961
+ return decided
2962
+
2963
+
2964
+ def codex_unattributed_paths(
2965
+ accounting_entries: "Iterable[object]",
2966
+ by_accounting_identity: "Mapping[tuple[str, str, str], Mapping[str, object]]",
2967
+ ) -> "frozenset[tuple[str, str]]":
2968
+ """The ``(root, path)`` set whose session rows must publish NULL projects.
2969
+
2970
+ Computed per caller from the entries that built THAT session view — the
2971
+ parent over every fallback entry, each account child over its own
2972
+ partition — so a child whose entries carry no unattributed identity on a
2973
+ shared path keeps that path's session project while a child whose entries
2974
+ do receives NULL. A session row's single project field cannot represent a
2975
+ mixed path.
2976
+ """
2977
+ unattributed: set[tuple[str, str]] = set()
2978
+ for entry in accounting_entries:
2979
+ identity = _codex_accounting_identity(entry)
2980
+ item = by_accounting_identity.get(identity)
2981
+ if item is None or not item.get("project_key"):
2982
+ unattributed.add((identity[0], identity[2]))
2983
+ return frozenset(unattributed)
2984
+
2985
+
2273
2986
  def _codex_conversation_metadata(
2274
2987
  cache_conn: sqlite3.Connection,
2275
- ) -> dict[tuple[str, str], dict[str, object]]:
2988
+ *,
2989
+ accounting_entries: "Iterable[object]" = (),
2990
+ build_identity_map: bool = False,
2991
+ inventory: "CodexThreadMetadataInventory | None" = None,
2992
+ ) -> _CodexConversationMetadataRead:
2276
2993
  """Read task short names and cached project metadata by rooted rollout.
2277
2994
 
2278
2995
  ``state_5.sqlite.threads.title`` is Codex's persisted user-facing task name.
@@ -2292,7 +3009,9 @@ def _codex_conversation_metadata(
2292
3009
  ") "
2293
3010
  "SELECT t.source_root_key, t.source_path, t.native_thread_id, "
2294
3011
  "e.session_id AS accounting_session_id, "
2295
- "t.cwd, t.git_json, a.started_at, t.last_seen_utc "
3012
+ "CAST(t.cwd AS BLOB) AS cwd_blob, "
3013
+ "CAST(t.git_json AS BLOB) AS git_json_blob, "
3014
+ "a.started_at, t.last_seen_utc "
2296
3015
  "FROM codex_conversation_threads AS t "
2297
3016
  "LEFT JOIN accounting AS a "
2298
3017
  "ON a.source_root_key=t.source_root_key AND a.source_path=t.source_path "
@@ -2300,15 +3019,27 @@ def _codex_conversation_metadata(
2300
3019
  "ORDER BY t.last_seen_utc DESC, t.conversation_key DESC"
2301
3020
  ))
2302
3021
  from _cctally_cache import _codex_conversation_project_attribution
3022
+
3023
+ def _attribution(root_key, cwd_blob, git_json_blob):
3024
+ # #845 §4.1. The two columns arrive as BLOBs; a thread whose
3025
+ # resolver path reaches an undecodable field carries no attribution
3026
+ # at all rather than the `(unassigned)` its decoded `None` fields
3027
+ # would otherwise produce.
3028
+ decoded = decode_codex_project_metadata(cwd_blob, git_json_blob)
3029
+ if codex_metadata_is_malformed(decoded, None):
3030
+ return (None, None)
3031
+ return _codex_conversation_project_attribution(
3032
+ root_key, decoded.cwd, decoded.git_json)
3033
+
2303
3034
  rows = tuple(
2304
3035
  (
2305
3036
  root_key, source_path, native_thread_id, accounting_session_id,
2306
- *_codex_conversation_project_attribution(root_key, cwd, git_json),
3037
+ *_attribution(root_key, cwd_blob, git_json_blob),
2307
3038
  first_seen_at,
2308
3039
  )
2309
3040
  for (
2310
3041
  root_key, source_path, native_thread_id, accounting_session_id,
2311
- cwd, git_json, first_seen_at, _last_seen_at,
3042
+ cwd_blob, git_json_blob, first_seen_at, _last_seen_at,
2312
3043
  ) in core_rows
2313
3044
  )
2314
3045
  file_aliases = tuple(cache_conn.execute(_CODEX_FILE_ALIAS_SQL))
@@ -2407,9 +3138,16 @@ def _codex_conversation_metadata(
2407
3138
  "project_label": (inherited or {}).get("project_label"),
2408
3139
  "started_at": started_at or (inherited or {}).get("started_at"),
2409
3140
  }
2410
- except sqlite3.Error:
2411
- return {}
2412
- return metadata
3141
+ except sqlite3.Error as exc:
3142
+ return _CodexConversationMetadataRead({}, {}, exc)
3143
+ by_identity: dict[tuple[str, str, str], dict[str, object]] = {}
3144
+ if build_identity_map:
3145
+ try:
3146
+ by_identity = _codex_identity_map(
3147
+ cache_conn, accounting_entries, inventory)
3148
+ except sqlite3.Error as exc:
3149
+ return _CodexConversationMetadataRead({}, {}, exc)
3150
+ return _CodexConversationMetadataRead(metadata, by_identity, None)
2413
3151
 
2414
3152
 
2415
3153
  def _session_wire(
@@ -2417,7 +3155,17 @@ def _session_wire(
2417
3155
  *,
2418
3156
  metadata: Mapping[tuple[str, str], Mapping[str, object]] | None = None,
2419
3157
  private_labels: dict[str, str] | None = None,
3158
+ unattributed_paths: "frozenset[tuple[str, str]]" = frozenset(),
2420
3159
  ) -> dict[str, object]:
3160
+ """Fold one session view; ``unattributed_paths`` nulls a row's project.
3161
+
3162
+ #845 §4.4: a row's project fields are NULL exactly when its
3163
+ ``(source_root_key, source_path)`` is in that set, because the row's single
3164
+ project field cannot represent a path that hosts one healthy and one
3165
+ malformed conversation. Otherwise they come from the path map as they do
3166
+ today, so a healthy path's session row is unchanged in the partial
3167
+ generation. Titles and ``started_at`` always come from the path map.
3168
+ """
2421
3169
  rows = []
2422
3170
  for row in view.rows:
2423
3171
  # The Codex session aggregator intentionally splits equal relative
@@ -2449,7 +3197,15 @@ def _session_wire(
2449
3197
  if row_metadata and row_metadata.get("title") is not None
2450
3198
  else None
2451
3199
  )
2452
- project = str(row_metadata.get("project_label") or "").strip() if row_metadata else ""
3200
+ unattributed = (
3201
+ str(row.codex_root or ""), str(row.session_id_path),
3202
+ ) in unattributed_paths
3203
+ project = (
3204
+ ""
3205
+ if unattributed
3206
+ else str(row_metadata.get("project_label") or "").strip()
3207
+ if row_metadata else ""
3208
+ )
2453
3209
  started_at = row_metadata.get("started_at") if row_metadata else None
2454
3210
  duration_min = None
2455
3211
  if isinstance(started_at, str):
@@ -2467,7 +3223,10 @@ def _session_wire(
2467
3223
  "key": key,
2468
3224
  "source": "codex",
2469
3225
  "project": project or None,
2470
- "project_key": row_metadata.get("project_key") if row_metadata else None,
3226
+ "project_key": (
3227
+ None if unattributed
3228
+ else row_metadata.get("project_key") if row_metadata else None
3229
+ ),
2471
3230
  "started_at": started_at,
2472
3231
  "duration_min": duration_min,
2473
3232
  "last_activity": row.last_activity.astimezone(UTC).isoformat(),
@@ -3625,6 +4384,17 @@ def refresh_codex_source_clock(
3625
4384
  # dropping it here would withhold the aggregate on any idle tick whose
3626
4385
  # clock moved.
3627
4386
  aggregate_scope=state.aggregate_scope,
4387
+ # #769 S6 / #753: this constructor lists every field explicitly, so an
4388
+ # omission drops the retained hero cohort on the first idle tick that
4389
+ # moved a clock — and the next incoherent build would then render
4390
+ # Pending on a process that had published a coherent hero.
4391
+ hero_cohort=state.hero_cohort,
4392
+ # #834 S2 (#828, #829): this clock publishes the SAME rows over
4393
+ # refreshed presentation axes, so it carries the health result that
4394
+ # described those rows. Re-deriving it here would need provider I/O
4395
+ # the idle path must not perform, and dropping it would report the
4396
+ # metadata health as unknown on the first idle tick.
4397
+ metadata_health=state.metadata_health,
3628
4398
  )
3629
4399
  return state if refreshed_state == state else refreshed_state
3630
4400
 
@@ -3743,19 +4513,108 @@ def _alerts_wire(
3743
4513
 
3744
4514
  _CODEX_PROJECT_LABEL_CACHE: dict[object, dict[str, object]] = _ObservedDict()
3745
4515
 
4516
+ #: Bumped when `assign_collision_safe_project_labels` changes what it produces
4517
+ #: for the same population. It joins the accounting population signature so a
4518
+ #: retained label set is invalidated by an algorithm change as well as by a
4519
+ #: data change.
4520
+ _CODEX_PROJECT_LABEL_ALGORITHM_VERSION = "collision-safe-labels-v1"
4521
+
4522
+
4523
+ def _codex_project_label_signature(base: tuple | None) -> tuple | None:
4524
+ """Extend one accounting population name with the labelling algorithm's.
4525
+
4526
+ The accounting carrier names the population; the label caches also depend
4527
+ on HOW labels are assigned, so a change to
4528
+ `assign_collision_safe_project_labels` has to invalidate every retained
4529
+ label set even though not one row moved. Appending the algorithm version is
4530
+ what makes that true, and it is a separate function from the reader below
4531
+ so a caller holding its own build's signature can extend that one rather
4532
+ than re-reading a process-wide global.
4533
+ """
4534
+ if base is None:
4535
+ return None
4536
+ return (*base, _CODEX_PROJECT_LABEL_ALGORITHM_VERSION)
4537
+
4538
+
4539
+ def _codex_project_population_signature() -> tuple | None:
4540
+ """The identity the project and label caches compare instead of scanning.
4541
+
4542
+ `None` means the accounting carrier could not establish an identity, and
4543
+ every consumer must then prove its hit the old way rather than assume one.
4544
+ """
4545
+ return _codex_project_label_signature(
4546
+ _lib_snapshot_cache.current_codex_population_signature())
4547
+
4548
+
4549
+ def _label_pair_counts(entries: Iterable[object]) -> dict[tuple[str, str], int]:
4550
+ counts: dict[tuple[str, str], int] = {}
4551
+ for entry in entries:
4552
+ pair = (str(entry.project_key), str(entry.project_label))
4553
+ counts[pair] = counts.get(pair, 0) + 1
4554
+ return counts
4555
+
3746
4556
 
3747
4557
  def _cached_project_labeled_entries(
3748
4558
  entries: tuple[object, ...], cache_key: object | None,
4559
+ *,
4560
+ population_signature: tuple | None = None,
4561
+ changed_old: Iterable[object] = (),
4562
+ changed_new: Iterable[object] = (),
3749
4563
  ) -> tuple[object, ...]:
3750
- """Reuse per-scope display-label annotation for unchanged accounting rows."""
4564
+ """Reuse per-scope display-label annotation for unchanged accounting rows.
4565
+
4566
+ A CLEAN HIT VISITS NO ROW. Proving the hit used to mean building a
4567
+ `frozenset` of every entry's `(project_key, project_label)` pair and then
4568
+ iterating every entry again to assemble the answer, so the nominal reuse
4569
+ path cost the whole population on every tick — 891,584 pairs visited across
4570
+ one run on the sealed 222,888-row measurement corpus. When the accounting
4571
+ carrier can name the population, comparing that name answers the same
4572
+ question without touching a row, and the labelled tuple it produced last
4573
+ time is still the right answer.
4574
+
4575
+ Without a signature the old proof stands. `None` is "cannot establish an
4576
+ identity", and the fail-safe for that is to look.
4577
+ """
4578
+ state = _CODEX_PROJECT_LABEL_CACHE.get(cache_key)
4579
+ if (
4580
+ cache_key is not None
4581
+ and population_signature is not None
4582
+ and state is not None
4583
+ and state.get("population_signature") == population_signature
4584
+ ):
4585
+ labelled = state.get("labeled")
4586
+ if labelled is not None:
4587
+ # A hit that rewrites nothing must still say it was used, or the
4588
+ # bucket ages while it is in constant use and is evicted first.
4589
+ _CODEX_PROJECT_LABEL_CACHE.touch(cache_key)
4590
+ return labelled
3751
4591
  if cache_key is None or any(
3752
4592
  not int(getattr(entry, "cache_entry_id", 0) or 0) for entry in entries
3753
4593
  ):
3754
4594
  return tuple(assign_collision_safe_project_labels(entries))
3755
- pairs = frozenset(
3756
- (str(entry.project_key), str(entry.project_label)) for entry in entries
3757
- )
3758
- state = _CODEX_PROJECT_LABEL_CACHE.get(cache_key)
4595
+ if (
4596
+ population_signature is not None
4597
+ and state is not None
4598
+ and (changed_old or changed_new)
4599
+ and isinstance(state.get("label_counts"), dict)
4600
+ ):
4601
+ # The delta lists say exactly which pairs left and which arrived, so
4602
+ # the multiset moves in time proportional to the change rather than to
4603
+ # the population.
4604
+ counts = dict(state["label_counts"])
4605
+ for entry in changed_old:
4606
+ pair = (str(entry.project_key), str(entry.project_label))
4607
+ remaining = counts.get(pair, 0) - 1
4608
+ if remaining > 0:
4609
+ counts[pair] = remaining
4610
+ else:
4611
+ counts.pop(pair, None)
4612
+ for entry in changed_new:
4613
+ pair = (str(entry.project_key), str(entry.project_label))
4614
+ counts[pair] = counts.get(pair, 0) + 1
4615
+ else:
4616
+ counts = _label_pair_counts(entries)
4617
+ pairs = frozenset(counts)
3759
4618
  if state is None or state.get("pairs") != pairs:
3760
4619
  labeled = tuple(assign_collision_safe_project_labels(entries))
3761
4620
  else:
@@ -3773,6 +4632,9 @@ def _cached_project_labeled_entries(
3773
4632
  )
3774
4633
  _CODEX_PROJECT_LABEL_CACHE[cache_key] = {
3775
4634
  "pairs": pairs,
4635
+ "label_counts": counts,
4636
+ "population_signature": population_signature,
4637
+ "labeled": labeled,
3776
4638
  "labels": {
3777
4639
  str(entry.project_key): entry.display_label for entry in labeled
3778
4640
  },
@@ -3791,10 +4653,15 @@ def _projects_wire(
3791
4653
  *,
3792
4654
  accounting_end: dt.datetime,
3793
4655
  cache_key: object | None = None,
4656
+ population_signature: tuple | None = None,
4657
+ changed_old: Iterable[object] = (),
4658
+ changed_new: Iterable[object] = (),
3794
4659
  ) -> dict[str, object]:
3795
4660
  """Adapt S3's already-qualified attribution result without re-formulas."""
3796
4661
  qualified_entries = _cached_project_labeled_entries(
3797
4662
  tuple(entries), cache_key,
4663
+ population_signature=population_signature,
4664
+ changed_old=changed_old, changed_new=changed_new,
3798
4665
  )
3799
4666
  result = build_codex_project_result(
3800
4667
  qualified_entries,
@@ -3838,24 +4705,40 @@ def _cached_projects_wire(
3838
4705
  accounting_end: dt.datetime,
3839
4706
  cache_key: object,
3840
4707
  semantic_signature: object,
4708
+ population_signature: tuple | None = None,
3841
4709
  ) -> dict[str, object]:
3842
- """Rebuild only project groups touched by accounting changes."""
4710
+ """Rebuild only project groups touched by accounting changes.
4711
+
4712
+ The population half of the key is the accounting carrier's own signature
4713
+ when it can name one, and the per-entry `(project_key, project_label)`
4714
+ multiset only when it cannot. The multiset answered the same question by
4715
+ visiting every row, which made the nominal reuse path proportional to the
4716
+ whole population on every tick.
4717
+ """
3843
4718
  values = tuple(entries)
3844
- pairs = frozenset(
3845
- (str(entry.project_key), str(entry.project_label)) for entry in values
3846
- )
4719
+ changed_old = tuple(changed_old)
4720
+ changed_new = tuple(changed_new)
4721
+ if population_signature is not None:
4722
+ population_key: object = population_signature
4723
+ else:
4724
+ population_key = frozenset(
4725
+ (str(entry.project_key), str(entry.project_label))
4726
+ for entry in values
4727
+ )
3847
4728
  # `entries` is already the complete half-open population for the advancing
3848
4729
  # upper bound. A moving wall clock is therefore not an aggregation
3849
4730
  # semantic: `build_cached_codex_accounting` emits newly-visible rows in the
3850
4731
  # delta when its upper bound advances. Keeping `accounting_end` here made
3851
4732
  # every live dirty tick discard every project group before that delta could
3852
4733
  # be spliced.
3853
- signature = (semantic_signature, pairs, context.range_start)
4734
+ signature = (semantic_signature, population_key, context.range_start)
3854
4735
  state = _CODEX_PROJECT_WIRE_CACHE.get(cache_key)
3855
4736
  if state is None or state[0] != signature:
3856
4737
  value = _projects_wire(
3857
4738
  context, quota_observations, values,
3858
4739
  accounting_end=accounting_end, cache_key=cache_key,
4740
+ population_signature=population_signature,
4741
+ changed_old=changed_old, changed_new=changed_new,
3859
4742
  )
3860
4743
  groups: dict[tuple[str, str], list[object]] = {}
3861
4744
  for entry in values:
@@ -3868,9 +4751,10 @@ def _cached_projects_wire(
3868
4751
  )
3869
4752
  return value
3870
4753
 
4754
+ _CODEX_PROJECT_WIRE_CACHE.touch(cache_key)
3871
4755
  affected = {
3872
4756
  (str(entry.source_root_key), str(entry.project_key))
3873
- for entry in (*tuple(changed_old), *tuple(changed_new))
4757
+ for entry in (*changed_old, *changed_new)
3874
4758
  }
3875
4759
  if not affected:
3876
4760
  return state[1]
@@ -3946,21 +4830,22 @@ def _cached_projects_wire(
3946
4830
 
3947
4831
  def _partial_projects_wire(
3948
4832
  entries: Iterable[object],
3949
- metadata: Mapping[tuple[str, str], Mapping[str, object]],
4833
+ by_accounting_identity: "Mapping[tuple[str, str, str], Mapping[str, object]]",
3950
4834
  ) -> dict[str, object]:
3951
4835
  """Aggregate the qualified subset when older accounting metadata is mixed.
3952
4836
 
3953
- Rows without a cached conversation/project identity are omitted and remain
3954
- covered by the Projects-domain warning. Valid projects stay visible; their
3955
- totals never include an unqualified accounting row.
4837
+ #845 §4.4: the lookup is by ACCOUNTING IDENTITY, not by path. One rollout
4838
+ path can host a healthy conversation and a malformed one, and a path-keyed
4839
+ map would publish one decision for both. An entry the identity map leaves
4840
+ unattributed — refused or malformed — carries a ``None`` item and is
4841
+ omitted here, where the Projects-domain warning covers it. Every other
4842
+ entry keeps the project key the healthy fold publishes for it.
3956
4843
  """
3957
4844
  groups: dict[tuple[str, str], dict[str, object]] = {}
3958
4845
  for entry in entries:
3959
- identity = (
3960
- str(getattr(entry, "source_root_key", "") or ""),
3961
- str(getattr(entry, "source_path", "") or ""),
3962
- )
3963
- row_metadata = metadata.get(identity)
4846
+ accounting_identity = _codex_accounting_identity(entry)
4847
+ identity = (accounting_identity[0], accounting_identity[2])
4848
+ row_metadata = by_accounting_identity.get(accounting_identity)
3964
4849
  project_key = str(row_metadata.get("project_key") or "").strip() if row_metadata else ""
3965
4850
  project_label = str(row_metadata.get("project_label") or "").strip() if row_metadata else ""
3966
4851
  if not project_key or not project_label:
@@ -4062,6 +4947,7 @@ def _codex_entries_from_accounting(entries: Iterable[object]) -> list[CodexEntry
4062
4947
  cache_entry_id = int(getattr(entry, "cache_entry_id", 0) or 0)
4063
4948
  cached = _CODEX_ENTRY_ADAPTER_CACHE.get(cache_entry_id)
4064
4949
  if cache_entry_id and cached is not None and cached[0] == entry:
4950
+ _CODEX_ENTRY_ADAPTER_CACHE.touch(cache_entry_id)
4065
4951
  converted.append(cached[1])
4066
4952
  continue
4067
4953
  source_path = str(getattr(entry, "source_path", "") or "")
@@ -4240,6 +5126,7 @@ def _cached_codex_native_weekly_view(
4240
5126
  account_key, include_account_keys,
4241
5127
  )
4242
5128
  state = _CODEX_WEEKLY_VIEW_CACHE.get(cache_key)
5129
+ _CODEX_WEEKLY_VIEW_CACHE.touch(cache_key)
4243
5130
  if state is None or state[0] != signature:
4244
5131
  view = _build_codex_native_weekly_view(
4245
5132
  stats_conn, values, source_root_keys=roots,
@@ -4542,6 +5429,7 @@ def _codex_accounts_wire(
4542
5429
  context.speed,
4543
5430
  ) if account_versions is not None and population_signature is not None else None
4544
5431
  cached = _CODEX_ACCOUNT_CARD_TOTALS_CACHE.get(account_key)
5432
+ _CODEX_ACCOUNT_CARD_TOTALS_CACHE.touch(account_key)
4545
5433
  if signature is not None and cached is not None and cached[0] == signature:
4546
5434
  return cached[1]
4547
5435
  entries = _codex_entries_from_accounting(rows)
@@ -4809,7 +5697,7 @@ def _cached_codex_visible_rows(
4809
5697
  changed_old: tuple[object, ...],
4810
5698
  changed_new: tuple[object, ...],
4811
5699
  semantic_signature: object,
4812
- ) -> "tuple[list[CodexEntry], dict[str, tuple[object, ...]], dict[str, tuple[CodexEntry, ...]], dict[str, int]]":
5700
+ ) -> "tuple[Sequence[CodexEntry], dict[str, tuple[object, ...]], dict[str, tuple[CodexEntry, ...]], dict[str, int]]":
4813
5701
  """Reuse the immutable visible population, updating only dirty accounts.
4814
5702
 
4815
5703
  The durable accounting ledger supplies exact old/new path populations. A
@@ -4834,8 +5722,15 @@ def _cached_codex_visible_rows(
4834
5722
  for row in (*changed_old, *changed_new))
4835
5723
  )
4836
5724
  if reusable and not changed_old and not changed_new:
5725
+ state.touch("entries")
4837
5726
  return (
4838
- list(state["entries"]),
5727
+ # The RETAINED TUPLE, not a copy of it. `list(state["entries"])`
5728
+ # here rebuilt a 222,932-element list on every clean tick on the
5729
+ # measurement corpus, and no consumer mutates what this returns.
5730
+ # The three per-account maps beside it are keyed by account, so
5731
+ # copying them costs a handful of slots and keeps the state's own
5732
+ # mappings out of a caller's hands.
5733
+ state["entries"],
4839
5734
  dict(state["rows_by_account"]),
4840
5735
  dict(state["entries_by_account"]),
4841
5736
  dict(state["account_versions"]),
@@ -4977,7 +5872,7 @@ def _cached_codex_visible_rows(
4977
5872
  "account_count": len(rows_by_account),
4978
5873
  "fallback_count": prior_fallbacks,
4979
5874
  })
4980
- return list(entries), rows_by_account, entries_by_account, account_versions
5875
+ return entries, rows_by_account, entries_by_account, account_versions
4981
5876
 
4982
5877
 
4983
5878
  _CODEX_ACCOUNT_SCOPE_CACHE: dict[
@@ -5014,6 +5909,17 @@ def _codex_account_scopes_wire(
5014
5909
  accounting_end: dt.datetime,
5015
5910
  metadata_incomplete: bool,
5016
5911
  conversation_metadata: Mapping[tuple[str, str], Mapping[str, object]],
5912
+ #: #845 §4.4/§4.5. The per-conversation decisions of the partial
5913
+ #: generation, and whether the conversation-metadata read itself failed.
5914
+ #: Both default to the healthy shape so a direct caller that has neither
5915
+ #: keeps today's behaviour.
5916
+ conversation_identity_metadata: "Mapping[tuple[str, str, str], Mapping[str, object]]" = (
5917
+ MappingProxyType({})),
5918
+ conversation_metadata_failed: bool = False,
5919
+ #: §4.6 rule 1: a transient generation publishes NO Codex project rows on
5920
+ #: any child either, so a focused account cannot show rows the parent is
5921
+ #: withholding.
5922
+ metadata_transient: bool = False,
5017
5923
  alerts: Iterable[Mapping[str, object]],
5018
5924
  budget_milestones: Iterable[Mapping[str, object]],
5019
5925
  projected_budget_milestones: Iterable[Mapping[str, object]],
@@ -5022,6 +5928,7 @@ def _codex_account_scopes_wire(
5022
5928
  hero_failure: bool = False,
5023
5929
  dirty_accounts: Iterable[str] = (),
5024
5930
  scope_signature: object | None = None,
5931
+ project_population_signature: tuple | None = None,
5025
5932
  changed_old_by_account: Mapping[str, tuple[CodexEntry, ...]] | None = None,
5026
5933
  changed_new_by_account: Mapping[str, tuple[CodexEntry, ...]] | None = None,
5027
5934
  changed_old_rows_by_account: Mapping[str, tuple[object, ...]] | None = None,
@@ -5173,9 +6080,19 @@ def _codex_account_scopes_wire(
5173
6080
  "sessions": _session_wire(
5174
6081
  sessions_view, metadata=conversation_metadata,
5175
6082
  private_labels=private_session_labels,
6083
+ # Each child computes the set over its OWN partition, so a
6084
+ # child whose entries carry no unattributed identity on a
6085
+ # shared path keeps that path's session project.
6086
+ unattributed_paths=(
6087
+ codex_unattributed_paths(
6088
+ rows, conversation_identity_metadata)
6089
+ if metadata_incomplete else frozenset()
6090
+ ),
5176
6091
  ),
5177
6092
  "projects": (
5178
- _partial_projects_wire(rows, conversation_metadata)
6093
+ _partial_projects_wire((), {})
6094
+ if metadata_transient
6095
+ else _partial_projects_wire(rows, conversation_identity_metadata)
5179
6096
  if metadata_incomplete else _cached_projects_wire(
5180
6097
  context, account_observations, rows,
5181
6098
  changed_old=changed_old_rows_by_account.get(key, ()),
@@ -5183,10 +6100,16 @@ def _codex_account_scopes_wire(
5183
6100
  accounting_end=accounting_end,
5184
6101
  cache_key=("account", key),
5185
6102
  semantic_signature=scope_signature,
6103
+ population_signature=project_population_signature,
5186
6104
  )
5187
6105
  ),
5188
6106
  "cache_report": _codex_cache_report_wire(
5189
- rows, metadata=conversation_metadata, now_utc=context.now_utc,
6107
+ rows, metadata=conversation_metadata,
6108
+ by_accounting_identity=(
6109
+ conversation_identity_metadata if metadata_incomplete
6110
+ else None),
6111
+ withhold_projects=conversation_metadata_failed,
6112
+ now_utc=context.now_utc,
5190
6113
  display_tz_name=context.display_tz_name, speed=context.speed,
5191
6114
  anomaly_threshold_pp=context.cache_report_anomaly_threshold_pp,
5192
6115
  cache_key=("account", key),
@@ -5266,6 +6189,7 @@ def _codex_account_scopes_wire(
5266
6189
  hero_failure,
5267
6190
  )
5268
6191
  cached = _CODEX_ACCOUNT_SCOPE_CACHE.get(key)
6192
+ _CODEX_ACCOUNT_SCOPE_CACHE.touch(key)
5269
6193
  if (
5270
6194
  scope_signature is not None
5271
6195
  and key not in dirty_account_keys
@@ -5358,12 +6282,45 @@ def _claude_accounts_wire(
5358
6282
  ordered_keys = [r["account_key"] for r in reg]
5359
6283
 
5360
6284
  def _latest_usage(key: str):
5361
- return stats_conn.execute(
5362
- "SELECT weekly_percent, five_hour_percent, week_end_at "
6285
+ # SPLIT PER AXIS (#769 S11, #824). This card publishes both axes, so
6286
+ # one row cannot serve it: the weekly figure comes from the latest
6287
+ # GENUINELY OBSERVED row and the five-hour figure from the latest row
6288
+ # of either kind, which is the held one whenever a weekly-clamped tick
6289
+ # carried a fresher five-hour reading.
6290
+ #
6291
+ # An earlier pass kept this held-inclusive, reasoning that a held row's
6292
+ # weekly value is the same account's latest genuine reading because the
6293
+ # fold copies it from a basis selected by account and week. That is
6294
+ # true of every row this binary writes, so the two queries agree in
6295
+ # production — but the browser gate rendered a divergent held row and
6296
+ # the card showed 91% beside a trend panel and an alert that both said
6297
+ # 63%, on one screen. The split is what stops that copy from being
6298
+ # load-bearing, which is the same reason every other weekly consumer
6299
+ # excludes held rows rather than trusting it.
6300
+ #
6301
+ # None is still returned only when the account retained NO row at all,
6302
+ # because the caller uses that to decide whether the unattributed
6303
+ # bucket appears.
6304
+ weekly = stats_conn.execute(
6305
+ "SELECT weekly_percent, week_end_at "
5363
6306
  "FROM weekly_usage_snapshots WHERE account_key=? "
6307
+ " AND weekly_observation_held = 0 "
5364
6308
  "ORDER BY captured_at_utc DESC LIMIT 1",
5365
6309
  (key,),
5366
6310
  ).fetchone()
6311
+ five_hour = stats_conn.execute(
6312
+ "SELECT five_hour_percent FROM weekly_usage_snapshots "
6313
+ "WHERE account_key=? "
6314
+ "ORDER BY captured_at_utc DESC LIMIT 1",
6315
+ (key,),
6316
+ ).fetchone()
6317
+ if weekly is None and five_hour is None:
6318
+ return None
6319
+ return (
6320
+ None if weekly is None else weekly[0],
6321
+ None if five_hour is None else five_hour[0],
6322
+ None if weekly is None else weekly[1],
6323
+ )
5367
6324
 
5368
6325
  def _latest_cost(key: str):
5369
6326
  row = stats_conn.execute(
@@ -5525,6 +6482,419 @@ def _codex_source_caches() -> tuple[dict, ...]:
5525
6482
  )
5526
6483
 
5527
6484
 
6485
+ # ── the retained-byte ownership graph (#769 S6 T2a, #716 Task B) ───────────
6486
+ #
6487
+ # The retained-size oracle counts a shared object identity ONCE across
6488
+ # everything it walks. A sum of per-cache charges only agrees with that if each
6489
+ # shared payload has exactly one cache carrying its charge, so the graph below
6490
+ # names that cache for every payload the Codex source caches repeat.
6491
+ #
6492
+ # Three payload classes are shared, and each has one owner:
6493
+ #
6494
+ # accounting rows (`QualifiedCodexEntry`, `RootedCodexAccountingEntry`) are
6495
+ # retained by `_lib_snapshot_cache._CODEX_ACCOUNTING_CACHE_STATE` and are
6496
+ # BORROWED by every cache here. That state is charged by the snapshot
6497
+ # cache's own accelerator, which is a separate owner with its own ceiling;
6498
+ # charging the rows here as well would count one population twice.
6499
+ #
6500
+ # adapted `CodexEntry` values are OWNED by `_CODEX_ENTRY_ADAPTER_CACHE`,
6501
+ # which is where they are constructed and keyed by durable row id, and are
6502
+ # borrowed by the visible-population fold and every view built from it.
6503
+ #
6504
+ # labelled row copies (`replace(row, display_label=...)`) are OWNED by
6505
+ # `_CODEX_PROJECT_LABEL_CACHE`. They are the same class as an accounting
6506
+ # row and cannot be told apart by type, so that owner names its borrowed
6507
+ # rows BY IDENTITY out of the entry it is charging.
6508
+ #
6509
+ # A cache whose charge is `None` is SELF-REPORTED: it already maintains an
6510
+ # incremental byte estimate of its own, and a second charge computed here would
6511
+ # be the same number derived twice. Those readers are named beside the owner.
6512
+ #
6513
+ # WHAT THE TOTAL IS, AND WHAT IT IS NOT. It is the sum of eleven DECLARED
6514
+ # CHARGES. It is not a measurement of process memory, and
6515
+ # `_CODEX_SOURCE_ACCELERATOR_MAX_BYTES` is not a cap on process memory: it is a
6516
+ # budget over these charges, and the charges depart from the retained graph in
6517
+ # both directions.
6518
+ #
6519
+ # UPWARD, and bounded. A charge is computed per entry, so two entries of the
6520
+ # SAME owner that share an ordinary string — a model name, a session id, both
6521
+ # of which the view models carry through by reference — pay for it twice, where
6522
+ # one walk of the whole graph would pay once. The predicate is type-based and
6523
+ # cannot see that a `str` belongs to a row. This direction errs toward evicting
6524
+ # early, which is the safe direction for a budget.
6525
+ #
6526
+ # DOWNWARD, and NOT bounded. FOUR owners are charged in CLOSED FORM rather
6527
+ # than by a walk, deliberately, because walking them would make an ordinary
6528
+ # tick proportional to the whole population again — which is the cost this
6529
+ # tranche exists to remove. `_charge_adapter_entry` charges two shells and a
6530
+ # float, `_charge_project_wire_entry` charges the group containers and not the
6531
+ # rows inside them, and `_visible_population_owner_bytes` charges four
6532
+ # references per indexed row rather than anything the fold reaches. Each is a
6533
+ # reasoned decision and each understates. Nothing in this file bounds by how
6534
+ # much, so a comparison of the total against the ceiling cannot be read as a
6535
+ # statement about how much memory the caches hold.
6536
+ #
6537
+ # `CODEX_SOURCE_CHARGE_FIDELITY` below is what makes the split checkable, and
6538
+ # `tests/test_source_cache_ownership.py` holds the WALKED owners to a two-sided
6539
+ # band against one walk of their own cache — the upper half is what caught the
6540
+ # adapter walking its row's strings once per row — while asserting only the
6541
+ # upper bound for the four closed-form owners and asserting that the
6542
+ # closed-form set is exactly the four named here.
6543
+
6544
+
6545
+ def _is_codex_accounting_row(obj) -> bool:
6546
+ """Is this a payload the accounting carrier retains and charges?"""
6547
+ return isinstance(obj, (QualifiedCodexEntry, RootedCodexAccountingEntry))
6548
+
6549
+
6550
+ def _is_borrowed_codex_payload(obj) -> bool:
6551
+ """Accounting rows and adapted entries, both charged by another owner."""
6552
+ return isinstance(
6553
+ obj,
6554
+ (QualifiedCodexEntry, RootedCodexAccountingEntry, CodexEntry),
6555
+ )
6556
+
6557
+
6558
+ def _charge_borrowing_payloads(_key, value) -> int:
6559
+ """Charge one entry's own structure, never a payload another owner holds."""
6560
+ return retained_size_bytes(value, borrowed=_is_borrowed_codex_payload)
6561
+
6562
+
6563
+ def _charge_adapter_entry(_key, value) -> int:
6564
+ """`(source_row, CodexEntry)` — the row is borrowed, the entry is owned.
6565
+
6566
+ The charge is CLOSED FORM rather than a walk, and the reason is that the
6567
+ adapted entry shares every string it has with the row beside it: the
6568
+ adapter builds each field with `str(x)`, and `str` on a `str` returns the
6569
+ same object. Walking the entry would therefore charge the accounting
6570
+ carrier's strings to this cache once per adapted row — a systematic double
6571
+ charge proportional to the whole population, which is the failure this
6572
+ graph exists to prevent. What the entry actually owns is its own shell, the
6573
+ pair holding it, and the one float `float(entry.cost_usd)` constructs.
6574
+ """
6575
+ total = sys.getsizeof(value) + sys.getsizeof(value[1])
6576
+ cost = getattr(value[1], "cost_usd", None)
6577
+ if cost is not None:
6578
+ total += sys.getsizeof(cost)
6579
+ return total
6580
+
6581
+
6582
+ def _charge_project_label_entry(_key, value) -> int:
6583
+ """Container shells, plus one shell and one label string per copy.
6584
+
6585
+ A labelled copy is `replace(row, display_label=...)`, so every field except
6586
+ `display_label` is the SAME object as the borrowed row's. Charging it by a
6587
+ walk would descend into fields the accounting carrier owns; charging the
6588
+ copy's own shell and its one new string is both exact and constant per row.
6589
+ """
6590
+ total = sys.getsizeof(value)
6591
+ total += retained_size_bytes(
6592
+ value.get("pairs", frozenset()), borrowed=_is_borrowed_codex_payload)
6593
+ total += retained_size_bytes(
6594
+ value.get("labels", {}), borrowed=_is_borrowed_codex_payload)
6595
+ total += retained_size_bytes(
6596
+ value.get("label_counts") or {}, borrowed=_is_borrowed_codex_payload)
6597
+ total += retained_size_bytes(
6598
+ value.get("population_signature") or (),
6599
+ borrowed=_is_borrowed_codex_payload)
6600
+ # The labelled tuple holds the SAME copies the `entries` map does, so only
6601
+ # its own shell is owed here; charging its members would pay twice.
6602
+ total += sys.getsizeof(value.get("labeled") or ())
6603
+ entries = value.get("entries") or {}
6604
+ total += sys.getsizeof(entries)
6605
+ for _raw, labelled in entries.values():
6606
+ total += sys.getsizeof(labelled) + 8 * 2
6607
+ label = getattr(labelled, "display_label", None)
6608
+ if label is not None:
6609
+ total += sys.getsizeof(label)
6610
+ return total
6611
+
6612
+
6613
+ def _charge_project_wire_entry(_key, value) -> int:
6614
+ """`(signature, wire, groups)` — the wire is owned, the groups are shells.
6615
+
6616
+ `groups` maps a project identity to a tuple of BORROWED accounting rows.
6617
+ Its own charge is the mapping plus one tuple shell per project, which is
6618
+ constant in the number of projects; descending into the tuples to ask about
6619
+ each row would make an ordinary tick proportional to the whole population
6620
+ again, which is the cost this tranche exists to remove.
6621
+ """
6622
+ signature, wire, groups = value[0], value[1], value[2]
6623
+ total = sys.getsizeof(value)
6624
+ total += retained_size_bytes(
6625
+ signature, borrowed=_is_borrowed_codex_payload)
6626
+ total += retained_size_bytes(wire, borrowed=_is_borrowed_codex_payload)
6627
+ total += sys.getsizeof(groups)
6628
+ for key, group in groups.items():
6629
+ total += sys.getsizeof(key) + sys.getsizeof(group)
6630
+ return total
6631
+
6632
+
6633
+ #: How faithfully each owner's declared charge tracks what one walk of that
6634
+ #: owner's own cache would reach. `walked` means the charge descends the entry
6635
+ #: and departs from a whole-graph walk only by the shared-string residual
6636
+ #: described above. `closed-form` means the charge is a constant-per-entry
6637
+ #: formula that DELIBERATELY UNDERSTATES, with no stated bound on the gap; the
6638
+ #: value beside the name is why that trade was taken.
6639
+ #:
6640
+ #: The budget is compared against a sum that includes the four closed-form
6641
+ #: owners, so the budget is a heuristic over declared charges rather than a cap
6642
+ #: on memory. Adding a fifth closed-form owner without adding it here is a
6643
+ #: test failure, which is the only reason this constant exists.
6644
+ CODEX_SOURCE_CHARGE_FIDELITY = MappingProxyType({
6645
+ "quota_observations": "walked",
6646
+ "period_views": "walked",
6647
+ "cache_report_rows": "walked",
6648
+ "session_views": "walked",
6649
+ "project_labels": (
6650
+ "closed-form: the labelled copies' shells and their one new label "
6651
+ "string each, because a labelled copy is `replace(row, "
6652
+ "display_label=...)` and every other field is the borrowed row's own "
6653
+ "object"),
6654
+ "project_wire": (
6655
+ "closed-form: the group containers, never the borrowed rows inside "
6656
+ "them, because descending into them makes an ordinary tick "
6657
+ "proportional to the whole population"),
6658
+ "entry_adapters": (
6659
+ "closed-form: two object shells and one float, because the adapted "
6660
+ "entry shares every string with the row beside it and walking it "
6661
+ "would charge the accounting carrier's strings once per adapted row"),
6662
+ "visible_population": (
6663
+ "closed-form: four references per indexed row, one container per "
6664
+ "account and the state mapping, because every payload the fold "
6665
+ "indexes is charged by the owner that constructed it"),
6666
+ "account_card_totals": "walked",
6667
+ "weekly_views": "walked",
6668
+ "account_scopes": "walked",
6669
+ })
6670
+
6671
+
6672
+ @dataclass(frozen=True)
6673
+ class _CodexCacheOwner:
6674
+ """One declared owner in the retained-byte graph."""
6675
+
6676
+ name: str
6677
+ cache: object
6678
+ charge: object
6679
+ reader: object
6680
+ owns: str
6681
+ borrows: str
6682
+ #: Whether this owner's keys are independently rebuildable buckets. An
6683
+ #: indivisible owner is evicted whole, because its keys are the internal
6684
+ #: fields of one result rather than separate results.
6685
+ indivisible: bool = False
6686
+ #: Per-key byte reader for a SELF-REPORTED owner that keeps its own ledger
6687
+ #: beside the mapping. Without it every one of that owner's keys reported
6688
+ #: zero bytes to the eviction sort, so the byte weighting was meaningless
6689
+ #: there and `bucket_bytes > budget` could never fire for it.
6690
+ entry_reader: object = None
6691
+ #: The whole eviction, for an owner whose byte ledger lives beside the
6692
+ #: mapping under its own lock. Popping the key and adjusting the ledger in
6693
+ #: two steps crosses that lock's discipline; this runs both inside it.
6694
+ evict_fn: object = None
6695
+
6696
+ def retained_bytes(self) -> int:
6697
+ if self.reader is not None:
6698
+ return int(self.reader())
6699
+ return int(getattr(self.cache, "retained_owner_bytes", 0))
6700
+
6701
+ def entry_bytes(self, key) -> int:
6702
+ """This owner's charge for one key, however that owner keeps it."""
6703
+ if self.entry_reader is not None:
6704
+ return int(self.entry_reader(key))
6705
+ if self.reader is not None:
6706
+ # An indivisible self-reported owner is one bucket, so the whole
6707
+ # owner IS the entry the budget is deciding about.
6708
+ return self.retained_bytes()
6709
+ return int(getattr(self.cache, "_cache_entry_bytes", {}).get(key, 0))
6710
+
6711
+ def evict(self, key) -> None:
6712
+ if self.evict_fn is not None:
6713
+ self.evict_fn(key)
6714
+ return
6715
+ self.cache.pop(key, None)
6716
+
6717
+ def evict_all(self) -> int:
6718
+ removed = len(self.cache)
6719
+ for key in tuple(self.cache):
6720
+ self.evict(key)
6721
+ return removed
6722
+
6723
+
6724
+ def _quota_observation_owner_bytes() -> int:
6725
+ with _CODEX_QUOTA_OBSERVATION_CACHE_LOCK:
6726
+ return int(_CODEX_QUOTA_OBSERVATION_CACHE_BYTES)
6727
+
6728
+
6729
+ def _quota_observation_entry_bytes(key) -> int:
6730
+ """One memoized quota window's charge, out of the memo's own ledger."""
6731
+ with _CODEX_QUOTA_OBSERVATION_CACHE_LOCK:
6732
+ return int(_CODEX_QUOTA_OBSERVATION_CACHE_SIZES.get(key, 0))
6733
+
6734
+
6735
+ def _quota_observation_evicted(key) -> None:
6736
+ """Evict one memoized window, mapping and byte ledger together.
6737
+
6738
+ This memo predates the ownership graph and keeps its sizes beside the
6739
+ mapping rather than inside the mixin, so an eviction that only popped the
6740
+ key would leave its byte total standing and the budget would never come
6741
+ back inside the ceiling. BOTH steps run under the memo's own lock, because
6742
+ `_codex_quota_observations_memoized` does its `get`, `popitem` and
6743
+ `__setitem__` under that lock: popping outside it and adjusting the ledger
6744
+ inside it afterwards leaves a window in which a concurrent memo write sees
6745
+ a mapping and a byte total that disagree.
6746
+ """
6747
+ global _CODEX_QUOTA_OBSERVATION_CACHE_BYTES
6748
+ with _CODEX_QUOTA_OBSERVATION_CACHE_LOCK:
6749
+ _CODEX_QUOTA_OBSERVATION_CACHE.pop(key, None)
6750
+ _CODEX_QUOTA_OBSERVATION_CACHE_BYTES -= (
6751
+ _CODEX_QUOTA_OBSERVATION_CACHE_SIZES.pop(key, 0))
6752
+ if _CODEX_QUOTA_OBSERVATION_CACHE_BYTES < 0:
6753
+ _CODEX_QUOTA_OBSERVATION_CACHE_BYTES = 0
6754
+
6755
+
6756
+ def _visible_population_owner_bytes() -> int:
6757
+ """The fold's OWN containers and references, never its borrowed payloads.
6758
+
6759
+ `_codex_visible_population_estimated_bytes` is this cache's admission
6760
+ meter, and it deliberately charges the shell of every row and every adapted
6761
+ entry so that the 128 MiB cap it guards is a bound on the whole fold. Those
6762
+ shells are charged elsewhere in the ownership graph — the rows by the
6763
+ accounting carrier, the adapted entries by the adapter cache — so reusing
6764
+ that number here would pay for one population twice. The remaining terms of
6765
+ the same formula are exactly the fold's own share: four references per
6766
+ indexed row, one per-account container, and the state mapping itself.
6767
+ """
6768
+ state = _CODEX_VISIBLE_POPULATION_CACHE
6769
+ if state.get("signature") is None:
6770
+ return 0
6771
+ return (
6772
+ int(state.get("entry_count", 0)) * 8 * 4
6773
+ + int(state.get("account_count", 0)) * 1024
6774
+ + 4096
6775
+ )
6776
+
6777
+
6778
+ def _declare_codex_source_cache_owners() -> tuple[_CodexCacheOwner, ...]:
6779
+ """Bind each cache to its charge, in the order `_codex_source_caches` lists.
6780
+
6781
+ Declared as a function rather than as a literal so the binding happens
6782
+ after every cache object exists, and so the ordering assertion below is
6783
+ evaluated once at import: an owner list that has drifted from the
6784
+ checkpointed set is a desynchronised byte total, not a cosmetic mismatch.
6785
+ """
6786
+ owners = (
6787
+ _CodexCacheOwner(
6788
+ "quota_observations", _CODEX_QUOTA_OBSERVATION_CACHE, None,
6789
+ _quota_observation_owner_bytes,
6790
+ owns="the frozen quota observations it memoizes",
6791
+ borrows="nothing",
6792
+ entry_reader=_quota_observation_entry_bytes,
6793
+ evict_fn=_quota_observation_evicted,
6794
+ ),
6795
+ _CodexCacheOwner(
6796
+ "period_views", _CODEX_PERIOD_VIEW_CACHE,
6797
+ _charge_borrowing_payloads, None,
6798
+ owns="the rendered daily/monthly view models",
6799
+ borrows="accounting rows and adapted entries",
6800
+ ),
6801
+ _CodexCacheOwner(
6802
+ "cache_report_rows", _CODEX_CACHE_REPORT_ROWS,
6803
+ _charge_borrowing_payloads, None,
6804
+ owns="the rendered cache-report rows",
6805
+ borrows="accounting rows and adapted entries",
6806
+ ),
6807
+ _CodexCacheOwner(
6808
+ "session_views", _CODEX_SESSION_VIEW_CACHE,
6809
+ _charge_borrowing_payloads, None,
6810
+ owns="the rendered session view models",
6811
+ borrows="accounting rows and adapted entries",
6812
+ ),
6813
+ _CodexCacheOwner(
6814
+ "project_labels", _CODEX_PROJECT_LABEL_CACHE,
6815
+ _charge_project_label_entry, None,
6816
+ owns="the labelled row copies and the label map",
6817
+ borrows="the raw accounting rows, named by identity",
6818
+ ),
6819
+ _CodexCacheOwner(
6820
+ "project_wire", _CODEX_PROJECT_WIRE_CACHE,
6821
+ _charge_project_wire_entry, None,
6822
+ owns="the rendered project wire and the group container shells",
6823
+ borrows="the accounting rows inside those groups",
6824
+ ),
6825
+ _CodexCacheOwner(
6826
+ "entry_adapters", _CODEX_ENTRY_ADAPTER_CACHE,
6827
+ _charge_adapter_entry, None,
6828
+ owns="the adapted CodexEntry values",
6829
+ borrows="the source accounting row beside each one",
6830
+ ),
6831
+ _CodexCacheOwner(
6832
+ "visible_population", _CODEX_VISIBLE_POPULATION_CACHE, None,
6833
+ _visible_population_owner_bytes,
6834
+ owns="the fold's own containers and references",
6835
+ borrows="every accounting row and adapted entry it indexes",
6836
+ # Its keys are the internal fields of ONE ordered population, not
6837
+ # separate results. See
6838
+ # `CODEX_VISIBLE_POPULATION_INDIVISIBLE_REASON`.
6839
+ indivisible=True,
6840
+ ),
6841
+ _CodexCacheOwner(
6842
+ "account_card_totals", _CODEX_ACCOUNT_CARD_TOTALS_CACHE,
6843
+ _charge_borrowing_payloads, None,
6844
+ owns="one finalized totals mapping per account",
6845
+ borrows="nothing, though the borrowing charge is used so a future "
6846
+ "payload cannot silently be charged twice",
6847
+ ),
6848
+ _CodexCacheOwner(
6849
+ "weekly_views", _CODEX_WEEKLY_VIEW_CACHE,
6850
+ _charge_borrowing_payloads, None,
6851
+ owns="the rendered native weekly view models",
6852
+ borrows="accounting rows and adapted entries",
6853
+ ),
6854
+ _CodexCacheOwner(
6855
+ "account_scopes", _CODEX_ACCOUNT_SCOPE_CACHE,
6856
+ _charge_borrowing_payloads, None,
6857
+ owns="the finalized per-account scope mappings",
6858
+ borrows="accounting rows and adapted entries",
6859
+ ),
6860
+ )
6861
+ caches = _codex_source_caches()
6862
+ if tuple(owner.cache for owner in owners) != caches:
6863
+ raise RuntimeError(
6864
+ "the retained-byte ownership graph does not describe the "
6865
+ "checkpointed cache set")
6866
+ for owner in owners:
6867
+ owner.cache._cache_owner_name = owner.name
6868
+ owner.cache._cache_owner_charge = owner.charge
6869
+ return owners
6870
+
6871
+
6872
+ _CODEX_SOURCE_CACHE_OWNERS = _declare_codex_source_cache_owners()
6873
+
6874
+
6875
+ def codex_source_retained_bytes() -> int:
6876
+ """The eleven Codex source caches' DECLARED CHARGES, summed.
6877
+
6878
+ Read it as "what these caches say they cost", not as "how much memory
6879
+ these caches hold". FOUR of the eleven are charged in closed form and
6880
+ understate without a stated bound — see `CODEX_SOURCE_CHARGE_FIDELITY` —
6881
+ so this figure is the input to a heuristic budget rather than a
6882
+ measurement of the retained graph.
6883
+
6884
+ Maintained at mutation, so this is a read rather than a traversal. It
6885
+ replaced a background worker that walked two full container copies of the
6886
+ whole cache set once per generation: on the sealed 222,888-row measurement
6887
+ corpus that walk visited over eleven million objects across one run and
6888
+ never once completed a generation, so the only verdict it ever published
6889
+ was the over-budget sentinel that cleared every cache at once.
6890
+ """
6891
+ return sum(owner.retained_bytes() for owner in _CODEX_SOURCE_CACHE_OWNERS)
6892
+
6893
+
6894
+ #: The budget the declared charges are held to. NOT a bound on process memory:
6895
+ #: it is compared against `codex_source_retained_bytes()`, four of whose
6896
+ #: eleven terms are closed-form under-charges with no stated bound on the gap.
6897
+ #: See `CODEX_SOURCE_CHARGE_FIDELITY`.
5528
6898
  _CODEX_SOURCE_ACCELERATOR_MAX_BYTES = 768 * 1024 * 1024
5529
6899
  _CODEX_SOURCE_ACCELERATOR_MAX_ENTRIES = 500_000
5530
6900
  _CODEX_SOURCE_ACCELERATOR_STATS_LOCK = threading.Lock()
@@ -5534,300 +6904,246 @@ _CODEX_SOURCE_ACCELERATOR_EVICTIONS = 0
5534
6904
  _CODEX_SOURCE_ACCELERATOR_FALLBACKS = 0
5535
6905
  _CODEX_SOURCE_ACCELERATOR_MEASURED_GENERATION = -1
5536
6906
  _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS = 0
6907
+ #: Entries refused at the write because one of them alone exceeds the whole
6908
+ #: budget. Counted separately from `_CODEX_SOURCE_ACCELERATOR_FALLBACKS`, which
6909
+ #: counts an admission pass that found an oversized bucket already resident.
6910
+ _CODEX_SOURCE_ACCELERATOR_ADMISSION_REFUSALS = 0
6911
+ #: Evict to this fraction of the budget rather than to the budget itself, so a
6912
+ #: permanently above-budget population is not one entry over again on the very
6913
+ #: next build and paying a full candidate pass over every resident key on every
6914
+ #: tick. The margin buys the headroom that lets
6915
+ #: `enforce_codex_source_accelerator_bounds` take its early return instead.
6916
+ _CODEX_SOURCE_ACCELERATOR_LOW_WATER = 0.9
6917
+ #: …and it applies ONLY above this many resident entries. The margin exists to
6918
+ #: avoid an O(resident) pass per tick; below this the pass is already cheap, and
6919
+ #: evicting past the budget there would discard buckets that fit for no gain —
6920
+ #: which is a worse trade, not a smaller one.
6921
+ _CODEX_SOURCE_ACCELERATOR_LOW_WATER_MIN_ENTRIES = 1024
5537
6922
  _CODEX_SOURCE_MEMORY_COMPLETION_LOCK = None
5538
6923
 
5539
6924
 
5540
- class _CodexSourceMemoryWorker:
5541
- """One latest-wins, cancellable retained-size verifier.
5542
-
5543
- The source caches are shallow-copied at a successful build boundary. The
5544
- worker traverses that immutable container snapshot so its multi-second
5545
- production-scale admission pass neither blocks the five-second publisher
5546
- nor races a live dict mutation. A newer generation cancels the old walk;
5547
- shutdown uses the same cooperative cancellation path.
6925
+ def set_codex_source_memory_completion_lock(lock) -> None:
6926
+ """Retained no-op: there is no longer a background verifier to bind.
6927
+
6928
+ The dashboard binds this at startup and unbinds it at shutdown. It existed
6929
+ to let a background retained-size walk evict an over-budget generation
6930
+ under the publisher's own lock without waiting for another tick. That walk
6931
+ is gone, so the completion it serialized no longer exists and the function
6932
+ stays only to keep the dashboard's startup and shutdown paths in shape.
6933
+
6934
+ WHAT IS NOT SERIALIZED ANY MORE, stated rather than implied. Admission now
6935
+ runs inline at the end of whichever build finished, and there are two
6936
+ builders: the synchronized dashboard tick and the share render on an HTTP
6937
+ request thread (`bin/_cctally_dashboard_share.py`). So a share render's
6938
+ admission pass can evict entries a concurrent dashboard fold is about to
6939
+ read. The consequence is an unnecessary MISS and never a wrong answer —
6940
+ every miss rebuilds from the current immutable accounting generation — but
6941
+ it is a real behaviour difference from binding the publisher's lock, and
6942
+ "there is no completion to serialize" was true of the deleted worker rather
6943
+ than of the two builders.
5548
6944
  """
5549
6945
 
5550
- def __init__(self) -> None:
5551
- self._condition = threading.Condition()
5552
- self._pending: tuple[int, tuple[dict, ...]] | None = None
5553
- self._result: tuple[int, int | None, str | None] | None = None
5554
- self._latest_generation = -1
5555
- self._active_generation: int | None = None
5556
- self._stop = False
5557
- self._thread: threading.Thread | None = None
5558
- self._duty_wall = 0.0
5559
- self._duty_cpu = 0.0
5560
-
5561
- def submit(self, generation: int, snapshot: tuple[dict, ...]) -> None:
5562
- with self._condition:
5563
- if self._stop:
5564
- return
5565
- self._latest_generation = int(generation)
5566
- self._pending = (int(generation), snapshot)
5567
- if self._thread is None or not self._thread.is_alive():
5568
- self._thread = threading.Thread(
5569
- target=self._run,
5570
- name="cctally-source-memory",
5571
- daemon=True,
5572
- )
5573
- self._thread.start()
5574
- self._condition.notify()
5575
-
5576
- def take_result(self) -> tuple[int, int | None, str | None] | None:
5577
- with self._condition:
5578
- result = self._result
5579
- self._result = None
5580
- return result
5581
-
5582
- def _cooperate(self) -> bool:
5583
- with self._condition:
5584
- cancelled = (
5585
- self._stop
5586
- or self._active_generation != self._latest_generation
5587
- )
5588
- if cancelled:
5589
- return True
5590
- cpu_elapsed = time.thread_time() - self._duty_cpu
5591
- wall_elapsed = time.monotonic() - self._duty_wall
5592
- delay = (cpu_elapsed / RETAINED_SIZE_WORKER_DUTY) - wall_elapsed
5593
- if delay > 0:
5594
- time.sleep(min(delay, 0.01))
5595
- with self._condition:
5596
- return (
5597
- self._stop
5598
- or self._active_generation != self._latest_generation
5599
- )
5600
6946
 
5601
- def _run(self) -> None:
5602
- while True:
5603
- with self._condition:
5604
- while self._pending is None and not self._stop:
5605
- self._condition.wait()
5606
- if self._stop:
5607
- return
5608
- generation, snapshot = self._pending
5609
- self._pending = None
5610
- self._active_generation = generation
5611
- estimated: int | None = None
5612
- error: str | None = None
5613
- entry_count = sum(len(cache) for cache in snapshot)
5614
- try:
5615
- while not RETAINED_SIZE_WORK_LOCK.acquire(timeout=0.05):
5616
- if self._cooperate():
5617
- raise RetainedSizeCancelled()
5618
- try:
5619
- self._duty_wall = time.monotonic()
5620
- self._duty_cpu = time.thread_time()
5621
- estimated = retained_size_bytes(
5622
- snapshot,
5623
- stop_after=_CODEX_SOURCE_ACCELERATOR_MAX_BYTES,
5624
- cancelled=self._cooperate,
5625
- )
5626
- finally:
5627
- RETAINED_SIZE_WORK_LOCK.release()
5628
- except RetainedSizeCancelled:
5629
- continue
5630
- except Exception as exc: # noqa: BLE001 - keep the sole worker alive
5631
- error = f"{type(exc).__name__}: {exc}"
5632
- finally:
5633
- snapshot = ()
5634
- published = (
5635
- error is None
5636
- and estimated is not None
5637
- and _publish_codex_source_measurement(
5638
- generation, int(estimated), entry_count)
5639
- )
5640
- enforced = False
5641
- completion_lock = _CODEX_SOURCE_MEMORY_COMPLETION_LOCK
5642
- if not published and completion_lock is not None:
5643
- with completion_lock:
5644
- enforced = _enforce_codex_source_unsafe_completion(
5645
- generation,
5646
- None if estimated is None else int(estimated),
5647
- error,
5648
- )
5649
- with self._condition:
5650
- self._active_generation = None
5651
- if (
5652
- not published and not enforced
5653
- and not self._stop
5654
- and generation == self._latest_generation
5655
- ):
5656
- self._result = (
5657
- generation,
5658
- None if estimated is None else int(estimated),
5659
- error,
5660
- )
6947
+ def shutdown_codex_source_memory_worker() -> None:
6948
+ """Retained no-op: see `set_codex_source_memory_completion_lock`.
5661
6949
 
5662
- def shutdown(self) -> bool:
5663
- with self._condition:
5664
- self._stop = True
5665
- self._pending = None
5666
- thread = self._thread
5667
- self._condition.notify_all()
5668
- if thread is not None:
5669
- thread.join(timeout=1.0)
5670
- return thread is None or not thread.is_alive()
6950
+ Shutdown used to have to cancel and join a walk that could be seconds from
6951
+ finishing. There is no thread to stop, so there is nothing that can fail to
6952
+ stop, and this raises nothing.
6953
+ """
5671
6954
 
5672
- def is_alive(self) -> bool:
5673
- with self._condition:
5674
- return self._thread is not None and self._thread.is_alive()
5675
6955
 
6956
+ def _mark_codex_source_accelerators_clean() -> None:
6957
+ global _CODEX_SOURCE_ACCELERATOR_DIRTY
6958
+ with _CODEX_SOURCE_ACCELERATOR_DIRTY_LOCK:
6959
+ _CODEX_SOURCE_ACCELERATOR_DIRTY = False
5676
6960
 
5677
- _CODEX_SOURCE_MEMORY_WORKER = _CodexSourceMemoryWorker()
5678
6961
 
6962
+ #: The one cap this tranche leaves all-or-nothing, and why. Recorded rather
6963
+ #: than quietly segmented, because a partition that buys nothing is worse than
6964
+ #: an admitted limit: it looks like degradation and behaves like a clear.
6965
+ CODEX_VISIBLE_POPULATION_INDIVISIBLE_REASON = (
6966
+ "The visible-population fold's product is ONE ordered population, "
6967
+ "reconstructed by integer-id lookup over `entries_by_id` so that "
6968
+ "downstream floating-point accumulation order stays byte-identical. A "
6969
+ "partial index fails that lookup, and the fold already falls closed to a "
6970
+ "complete cold rebuild when it does, so evicting some accounts costs the "
6971
+ "same rebuild as clearing the state and loses the eviction bookkeeping as "
6972
+ "well. Segmenting it needs the fold restructured to produce per-account "
6973
+ "populations that compose, which is a change to what it emits rather than "
6974
+ "to how much of it is kept."
6975
+ )
5679
6976
 
5680
- def set_codex_source_memory_completion_lock(lock) -> None:
5681
- """Bind unsafe verifier completions to the dashboard publisher lock."""
5682
- global _CODEX_SOURCE_MEMORY_COMPLETION_LOCK
5683
- _CODEX_SOURCE_MEMORY_COMPLETION_LOCK = lock
5684
6977
 
6978
+ def _evict_codex_source_buckets(caches) -> tuple[int, bool]:
6979
+ """Evict byte-weighted least-recently-used buckets until within budget.
5685
6980
 
5686
- def _enforce_codex_source_unsafe_completion(
5687
- generation: int, estimated: int | None, error: str | None,
5688
- ) -> bool:
5689
- """Evict a current over-cap/error result without awaiting another tick."""
5690
- if error is None and (
5691
- estimated is None or estimated <= _CODEX_SOURCE_ACCELERATOR_MAX_BYTES
5692
- ):
5693
- return False
5694
- with _CODEX_SOURCE_ACCELERATOR_DIRTY_LOCK:
5695
- if (
5696
- _CODEX_SOURCE_ACCELERATOR_DIRTY
5697
- or generation != _CODEX_SOURCE_ACCELERATOR_GENERATION
5698
- ):
5699
- return False
5700
- caches = _codex_source_caches()
5701
- entry_count = sum(len(cache) for cache in caches)
5702
- global _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES
5703
- global _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT
5704
- global _CODEX_SOURCE_ACCELERATOR_EVICTIONS
5705
- global _CODEX_SOURCE_ACCELERATOR_FALLBACKS
5706
- global _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS
5707
- with _CODEX_SOURCE_ACCELERATOR_STATS_LOCK:
5708
- reset_codex_quota_observation_cache()
5709
- for cache in caches[1:]:
5710
- cache.clear()
5711
- _CODEX_SOURCE_ACCELERATOR_EVICTIONS += entry_count
5712
- _CODEX_SOURCE_ACCELERATOR_FALLBACKS += 1
5713
- if error is not None:
5714
- _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS += 1
5715
- _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES = 0
5716
- _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT = 0
5717
- return True
5718
-
5719
-
5720
- def _publish_codex_source_measurement(
5721
- generation: int, estimated: int, entry_count: int,
5722
- ) -> bool:
5723
- """Publish an under-cap current generation without waiting for a tick."""
5724
- if estimated > _CODEX_SOURCE_ACCELERATOR_MAX_BYTES:
5725
- return False
5726
- with _CODEX_SOURCE_ACCELERATOR_DIRTY_LOCK:
5727
- if (
5728
- _CODEX_SOURCE_ACCELERATOR_DIRTY
5729
- or generation != _CODEX_SOURCE_ACCELERATOR_GENERATION
5730
- ):
5731
- return False
5732
- global _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES
5733
- global _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT
5734
- global _CODEX_SOURCE_ACCELERATOR_MEASURED_GENERATION
5735
- with _CODEX_SOURCE_ACCELERATOR_STATS_LOCK:
5736
- _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES = int(estimated)
5737
- _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT = int(entry_count)
5738
- _CODEX_SOURCE_ACCELERATOR_MEASURED_GENERATION = int(generation)
5739
- return True
6981
+ Returns `(evicted_entries, saw_an_indivisible_oversized_bucket)`.
5740
6982
 
6983
+ Ten of the eleven caches are already keyed by stable semantic units —
6984
+ `("account", key)`, `("parent",)`, a project identity, a session identity,
6985
+ a physical row id — so those keys ARE the buckets and no new partition is
6986
+ needed. The eleventh is declared indivisible and is evicted whole; see
6987
+ `CODEX_VISIBLE_POPULATION_INDIVISIBLE_REASON`.
5741
6988
 
5742
- def _take_codex_source_accelerator_dirty() -> int | None:
5743
- """Claim one dirty generation for a serialized source-build admission."""
5744
- global _CODEX_SOURCE_ACCELERATOR_DIRTY
5745
- with _CODEX_SOURCE_ACCELERATOR_DIRTY_LOCK:
5746
- if not _CODEX_SOURCE_ACCELERATOR_DIRTY:
5747
- return None
5748
- _CODEX_SOURCE_ACCELERATOR_DIRTY = False
5749
- return int(_CODEX_SOURCE_ACCELERATOR_GENERATION)
6989
+ Cold buckets go first and everything else stays resident, which is what
6990
+ makes an above-cap steady state rebuild only what it lost. Clearing all
6991
+ eleven caches, which is what this replaced, made a population permanently
6992
+ above a cap rebuild and discard on every generation: measured on the sealed
6993
+ #786 v3 `current` corpus, one run cleared 222,932 entries at a stroke.
5750
6994
 
6995
+ A bucket that alone exceeds the whole budget cannot be split further and is
6996
+ computed WITHOUT admission: it is dropped and counted as a fallback, rather
6997
+ than sitting above the ceiling or taking the rest of the cache with it.
6998
+ """
6999
+ owners = _CODEX_SOURCE_CACHE_OWNERS
7000
+ budget = int(_CODEX_SOURCE_ACCELERATOR_MAX_BYTES)
7001
+ max_entries = int(_CODEX_SOURCE_ACCELERATOR_MAX_ENTRIES)
7002
+ total = int(codex_source_retained_bytes())
7003
+ count = sum(len(cache) for cache in caches)
7004
+ if total <= budget and count <= max_entries:
7005
+ return 0, False
7006
+ # Evict PAST the ceiling, to a low-water mark. Stopping exactly at the
7007
+ # budget leaves the next build one entry over it again, and every tick then
7008
+ # pays the candidate build below over every resident key — 222,932 of them
7009
+ # on the measurement corpus. Freeing headroom instead lets the early return
7010
+ # above answer the next several ticks for eleven integer reads.
7011
+ if count >= int(_CODEX_SOURCE_ACCELERATOR_LOW_WATER_MIN_ENTRIES):
7012
+ low_water = min(
7013
+ budget, int(budget * _CODEX_SOURCE_ACCELERATOR_LOW_WATER))
7014
+ low_water_entries = min(
7015
+ max_entries, int(max_entries * _CODEX_SOURCE_ACCELERATOR_LOW_WATER))
7016
+ else:
7017
+ low_water, low_water_entries = budget, max_entries
5751
7018
 
5752
- def _mark_codex_source_accelerators_clean() -> None:
5753
- global _CODEX_SOURCE_ACCELERATOR_DIRTY
5754
- with _CODEX_SOURCE_ACCELERATOR_DIRTY_LOCK:
5755
- _CODEX_SOURCE_ACCELERATOR_DIRTY = False
7019
+ candidates: list[tuple[int, int, int, object, int]] = []
7020
+ position = 0
7021
+ for index, owner in enumerate(owners):
7022
+ entries = owner.cache.eviction_candidates()
7023
+ if not entries:
7024
+ continue
7025
+ if owner.indivisible:
7026
+ # One bucket, weighted by the whole owner and aged by its MOST
7027
+ # recently used field. `min` was wrong: every field of this state
7028
+ # is written in one `update` and then only the reuse path touches
7029
+ # one of them, so aging the bucket by its coldest field reported
7030
+ # the write time forever and sorted a population in constant use
7031
+ # ahead of buckets nothing had read in hours.
7032
+ #
7033
+ # WHAT `max` COSTS, WHERE A READER WILL FIND IT. The clean-hit path
7034
+ # calls `state.touch("entries")` on EVERY reuse
7035
+ # (`_cached_codex_visible_rows`), so on a dashboard that keeps
7036
+ # reusing its population this bucket is the youngest thing in the
7037
+ # process on every tick and eviction pressure never reaches it.
7038
+ # That is the same owner the budget under-counts most — its charge
7039
+ # is four references per indexed row and nothing the fold reaches
7040
+ # (`CODEX_SOURCE_CHARGE_FIDELITY`). Both decisions are right on
7041
+ # their own: a bucket in constant use should not be evicted first,
7042
+ # and charging what the fold reaches would make an ordinary tick
7043
+ # proportional to the whole population. Together they mean the
7044
+ # owner with the largest under-charge is also the last one this
7045
+ # pass will ever reach, so freeing memory by lowering the budget
7046
+ # acts on the other ten owners first.
7047
+ candidates.append((
7048
+ max(recency for recency, _key, _bytes in entries),
7049
+ index, position, _CACHE_MISSING, owner.retained_bytes(),
7050
+ ))
7051
+ position += 1
7052
+ continue
7053
+ # A charged owner's per-key figure IS the one the mixin just returned.
7054
+ # A self-reported owner keeps its ledger beside the mapping, and the
7055
+ # mixin reports zero for every one of its keys, so ask the owner —
7056
+ # otherwise the byte weighting is meaningless there and
7057
+ # `bucket_bytes > budget` can never fire for it.
7058
+ reads_its_own = owner.entry_reader is not None
7059
+ for recency, key, mixin_bytes in entries:
7060
+ candidates.append((
7061
+ recency, index, position, key,
7062
+ owner.entry_bytes(key) if reads_its_own else mixin_bytes,
7063
+ ))
7064
+ position += 1
7065
+ # Coldest first. The owner index and the enumeration position keep the
7066
+ # order total and stable when two buckets were stamped in the same tick.
7067
+ # The position replaced `repr(key)`, which formatted every resident key on
7068
+ # every over-budget tick to buy the same tie-break an integer already
7069
+ # provides.
7070
+ candidates.sort(key=lambda item: item[:3])
7071
+
7072
+ evicted = 0
7073
+ oversized = False
7074
+ for _recency, index, _position, key, bucket_bytes in candidates:
7075
+ if total <= low_water and count <= low_water_entries:
7076
+ break
7077
+ owner = owners[index]
7078
+ if bucket_bytes > budget:
7079
+ oversized = True
7080
+ before_owner_bytes = owner.retained_bytes()
7081
+ if key is _CACHE_MISSING:
7082
+ removed = owner.evict_all()
7083
+ elif key in owner.cache:
7084
+ owner.evict(key)
7085
+ removed = 1
7086
+ else:
7087
+ continue
7088
+ evicted += removed
7089
+ count -= removed
7090
+ # Re-read ONE owner rather than summing all eleven after every single
7091
+ # eviction. A charged owner's total moves by exactly this bucket, and a
7092
+ # self-reported one answers for itself; either way the arithmetic below
7093
+ # is checked against a full re-read once, after the loop.
7094
+ total -= before_owner_bytes - owner.retained_bytes()
7095
+
7096
+ # THE ONE CLEAR-ALL BRANCH THAT SURVIVES, and it is a backstop rather than
7097
+ # the ordinary path. The loop above tracks the total by owner delta, and a
7098
+ # self-reported owner reports what it holds rather than what it was
7099
+ # charged, so the arithmetic is checked against a full re-read once here.
7100
+ # Anything still over the ceiling after every candidate has been offered is
7101
+ # over it for a reason this pass cannot see, and the fail-closed answer is
7102
+ # the cold rebuild the caps always had. `CHANGELOG.md` names this branch
7103
+ # rather than claiming eviction replaced clearing outright.
7104
+ if int(codex_source_retained_bytes()) > budget:
7105
+ for owner in owners:
7106
+ if owner.cache:
7107
+ oversized = True
7108
+ evicted += owner.evict_all()
7109
+ return evicted, oversized
5756
7110
 
5757
7111
 
5758
7112
  def enforce_codex_source_accelerator_bounds() -> None:
5759
- """Schedule byte admission and apply completed results without cadence I/O."""
7113
+ """Apply byte and entry admission from the incrementally maintained total.
7114
+
7115
+ The byte total is a sum of eleven integers each owner keeps current, so
7116
+ admission is a read. It used to be a background traversal of two full
7117
+ container copies of the whole cache set, once per generation, in a worker
7118
+ limited to a quarter of one core — which on the measurement corpus never
7119
+ finished a generation at all, so the caches were admitted against a byte
7120
+ figure that stayed at zero until the walk finally returned the over-budget
7121
+ sentinel and cleared all eleven at once.
7122
+
7123
+ Publication is unconditional now rather than gated on the dirty flag,
7124
+ because there is nothing left to schedule: an unchanged generation costs
7125
+ the same eleven reads and publishes the same number.
7126
+ """
5760
7127
  caches = _codex_source_caches()
5761
- entry_count = sum(len(cache) for cache in caches)
5762
7128
  global _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES
5763
7129
  global _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT
5764
7130
  global _CODEX_SOURCE_ACCELERATOR_EVICTIONS
5765
7131
  global _CODEX_SOURCE_ACCELERATOR_FALLBACKS
5766
7132
  global _CODEX_SOURCE_ACCELERATOR_MEASURED_GENERATION
5767
- global _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS
5768
- completed = _CODEX_SOURCE_MEMORY_WORKER.take_result()
5769
- with _CODEX_SOURCE_ACCELERATOR_DIRTY_LOCK:
5770
- current_generation = int(_CODEX_SOURCE_ACCELERATOR_GENERATION)
5771
- currently_dirty = bool(_CODEX_SOURCE_ACCELERATOR_DIRTY)
5772
- measured = (
5773
- None if completed is None or completed[1] is None
5774
- else int(completed[1])
5775
- )
5776
- measured_generation = -1 if completed is None else int(completed[0])
5777
- measurement_error = None if completed is None else completed[2]
5778
- measurement_is_current = (
5779
- completed is not None
5780
- and measured_generation == current_generation
5781
- and not currently_dirty
5782
- )
5783
7133
  with _CODEX_SOURCE_ACCELERATOR_STATS_LOCK:
5784
- if measurement_is_current and measurement_error is not None:
5785
- reset_codex_quota_observation_cache()
5786
- for cache in caches[1:]:
5787
- cache.clear()
5788
- _CODEX_SOURCE_ACCELERATOR_EVICTIONS += entry_count
7134
+ evicted, oversized = _evict_codex_source_buckets(caches)
7135
+ if evicted:
7136
+ _CODEX_SOURCE_ACCELERATOR_EVICTIONS += evicted
7137
+ if oversized:
5789
7138
  _CODEX_SOURCE_ACCELERATOR_FALLBACKS += 1
5790
- _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS += 1
5791
- _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES = 0
5792
- _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT = 0
5793
- elif (
5794
- (measurement_is_current
5795
- and measured is not None
5796
- and measured > _CODEX_SOURCE_ACCELERATOR_MAX_BYTES)
5797
- or entry_count > _CODEX_SOURCE_ACCELERATOR_MAX_ENTRIES
5798
- ):
5799
- # The caller already owns the completed SourceDashboardState. A
5800
- # cold next build preserves wire semantics and prevents any one
5801
- # history/account/root churn pattern from growing process state.
5802
- reset_codex_quota_observation_cache()
5803
- for cache in caches[1:]:
5804
- cache.clear()
5805
- _CODEX_SOURCE_ACCELERATOR_EVICTIONS += entry_count
5806
- _CODEX_SOURCE_ACCELERATOR_FALLBACKS += 1
5807
- _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES = 0
5808
- _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT = 0
5809
- elif measured is not None:
5810
- _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES = measured
5811
- _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT = int(entry_count)
5812
- _CODEX_SOURCE_ACCELERATOR_MEASURED_GENERATION = measured_generation
5813
- else:
5814
- # Entry count is exact immediately; bytes remain the last complete
5815
- # generation until the latest-wins verifier publishes a result.
5816
- _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT = int(entry_count)
5817
-
5818
- generation = _take_codex_source_accelerator_dirty()
5819
- if generation is not None:
5820
- _CODEX_SOURCE_MEMORY_WORKER.submit(
5821
- generation, tuple(dict(cache) for cache in caches))
5822
-
5823
-
5824
- def shutdown_codex_source_memory_worker() -> None:
5825
- """Cancel and join the one aggregate verifier during dashboard shutdown."""
5826
- global _CODEX_SOURCE_MEMORY_WORKER
5827
- worker = _CODEX_SOURCE_MEMORY_WORKER
5828
- if not worker.shutdown():
5829
- raise RuntimeError("Codex source memory verifier did not stop")
5830
- _CODEX_SOURCE_MEMORY_WORKER = _CodexSourceMemoryWorker()
7139
+ _CODEX_SOURCE_ACCELERATOR_ESTIMATED_BYTES = int(
7140
+ codex_source_retained_bytes())
7141
+ _CODEX_SOURCE_ACCELERATOR_ENTRY_COUNT = sum(
7142
+ len(cache) for cache in caches)
7143
+ with _CODEX_SOURCE_ACCELERATOR_DIRTY_LOCK:
7144
+ _CODEX_SOURCE_ACCELERATOR_MEASURED_GENERATION = int(
7145
+ _CODEX_SOURCE_ACCELERATOR_GENERATION)
7146
+ _mark_codex_source_accelerators_clean()
5831
7147
 
5832
7148
 
5833
7149
  def codex_source_accelerator_memory_stats() -> Mapping[str, int]:
@@ -5840,6 +7156,8 @@ def codex_source_accelerator_memory_stats() -> Mapping[str, int]:
5840
7156
  "maxEntries": int(_CODEX_SOURCE_ACCELERATOR_MAX_ENTRIES),
5841
7157
  "evictionCount": int(_CODEX_SOURCE_ACCELERATOR_EVICTIONS),
5842
7158
  "fallbackCount": int(_CODEX_SOURCE_ACCELERATOR_FALLBACKS),
7159
+ "admissionRefusalCount": int(
7160
+ _CODEX_SOURCE_ACCELERATOR_ADMISSION_REFUSALS),
5843
7161
  "measurementErrorCount": int(
5844
7162
  _CODEX_SOURCE_ACCELERATOR_MEASUREMENT_ERRORS),
5845
7163
  "measuredGeneration": int(
@@ -5849,9 +7167,10 @@ def codex_source_accelerator_memory_stats() -> Mapping[str, int]:
5849
7167
  _CODEX_SOURCE_ACCELERATOR_MEASURED_GENERATION
5850
7168
  != _CODEX_SOURCE_ACCELERATOR_GENERATION
5851
7169
  ),
5852
- "workerAlive": int(
5853
- getattr(_CODEX_SOURCE_MEMORY_WORKER, "is_alive", lambda: False)()
5854
- ),
7170
+ # Retained at zero rather than removed: `/api/debug/backend`
7171
+ # publishes this key and a client that reads it must keep reading
7172
+ # something. There is no verifier thread to be alive.
7173
+ "workerAlive": 0,
5855
7174
  })
5856
7175
 
5857
7176
 
@@ -5880,6 +7199,219 @@ def reset_codex_source_caches() -> None:
5880
7199
  _CODEX_SOURCE_ACCELERATOR_GENERATION)
5881
7200
 
5882
7201
 
7202
+ # ── the per-build generation overlay (#769 S6 T2a, #716 Task B) ────────────
7203
+
7204
+ #: What a discard restores, enumerated rather than assumed. `capture` and its
7205
+ #: restore covered these pieces; a piece added later and left out of this map
7206
+ #: would survive a discard, which is a generation the build was supposed to
7207
+ #: throw away still being served.
7208
+ CODEX_SOURCE_OVERLAY_BUNDLE = MappingProxyType({
7209
+ "mappings": tuple(owner.name for owner in _CODEX_SOURCE_CACHE_OWNERS),
7210
+ "extra_state": (
7211
+ # The quota memo's size map, byte total, eviction and fallback
7212
+ # counters, and its key order.
7213
+ "quota_observation_cache_stats",
7214
+ # The account-card totals fallback counter.
7215
+ "account_card_totals_fallbacks",
7216
+ # `_lib_snapshot_cache`'s accounting population, cursor, generation and
7217
+ # signature — a handful of keys, so it stays a plain shallow copy.
7218
+ "snapshot_accounting_cache_state",
7219
+ ),
7220
+ # LRU ORDER is transactional state too: the quota memo is an OrderedDict
7221
+ # whose eviction order decides what survives the next admission. It is
7222
+ # covered by an EXPLICIT ORDER CHECKPOINT taken beside the byte ledger, not
7223
+ # by the journal.
7224
+ #
7225
+ # An earlier version of this entry claimed the journal covered it, on the
7226
+ # ground that re-assigning each key in reverse reinstates the order the
7227
+ # keys were in. That is false in both directions. Reverse replay reinstates
7228
+ # the order only when the build's removals were last-in-first-out, and this
7229
+ # memo evicts with `popitem(last=False)`, which takes the OLDEST key:
7230
+ #
7231
+ # start ['a', 'b', 'c']
7232
+ # in-build ['b', 'c', 'e'] # popitem(last=False) took 'a'
7233
+ # replayed ['b', 'c', 'a'] # not ['a', 'b', 'c']
7234
+ #
7235
+ # And `move_to_end` — which is how a memo HIT records itself — is
7236
+ # classified as reordering and journaled not at all, so a build that only
7237
+ # read the memo and was then discarded left it permanently reordered.
7238
+ # `_quota_observation_cache_checkpoint` remembers the order instead.
7239
+ "ordering": ("quota_observation_lru_order",),
7240
+ "not_rolled_back": (
7241
+ # A rolled-back build still HAPPENED. These count work the process did,
7242
+ # and a counter that forgot a discarded generation would under-report
7243
+ # it. The eviction and fallback counters are the exception and ARE
7244
+ # rolled back, because they describe what the caches currently hold.
7245
+ "codex_source_accelerator_generation",
7246
+ "codex_source_accelerator_measurement_errors",
7247
+ ),
7248
+ })
7249
+
7250
+ #: Advanced by every build that publishes. A build records the value it started
7251
+ #: from and refuses to publish if it has moved, which is the compare-and-swap
7252
+ #: that keeps a share render and a dashboard tick from publishing into each
7253
+ #: other's generation.
7254
+ _CODEX_SOURCE_PUBLISH_EPOCH = 0
7255
+ _CODEX_SOURCE_PUBLISH_LOCK = threading.Lock()
7256
+
7257
+
7258
+ class _CodexSourceBuildOverlay:
7259
+ """One build's tentative generation over the eleven source caches.
7260
+
7261
+ THE JOURNAL IS THE OVERLAY. Rather than hold changed keys in a separate
7262
+ mapping every read would have to consult, each mutation records the key's
7263
+ PRIOR state once, before it applies. Publishing is then free — the
7264
+ mutations are already the base generation — and discarding replays the
7265
+ journal in reverse, which restores exactly the keys this build touched and
7266
+ leaves every other key alone. That is what makes an invalidation of one
7267
+ physical path cost the one path instead of a complete container copy of
7268
+ every retained cache.
7269
+
7270
+ Recording happens once per key, on FIRST touch, so a key written repeatedly
7271
+ inside one build costs one journal entry and restores to what it held
7272
+ before the build rather than to an intermediate value.
7273
+ """
7274
+
7275
+ __slots__ = ("base_epoch", "_journal", "_touched", "_open", "_published")
7276
+
7277
+ def __init__(self, base_epoch: int) -> None:
7278
+ self.base_epoch = int(base_epoch)
7279
+ self._journal: list[tuple[object, object, object]] = []
7280
+ self._touched: set[tuple[int, object]] = set()
7281
+ self._open = True
7282
+ self._published = False
7283
+
7284
+ @property
7285
+ def open(self) -> bool:
7286
+ return self._open
7287
+
7288
+ @property
7289
+ def published(self) -> bool:
7290
+ return self._published
7291
+
7292
+ def journal_size(self) -> int:
7293
+ return len(self._journal)
7294
+
7295
+ def record(self, cache, key) -> None:
7296
+ if not self._open:
7297
+ return
7298
+ identity = (id(cache), key)
7299
+ if identity in self._touched:
7300
+ return
7301
+ self._touched.add(identity)
7302
+ self._journal.append((cache, key, cache.get(key, _CACHE_MISSING)))
7303
+
7304
+ def record_removed(self, cache, key, prior) -> None:
7305
+ if not self._open:
7306
+ return
7307
+ identity = (id(cache), key)
7308
+ if identity in self._touched:
7309
+ return
7310
+ self._touched.add(identity)
7311
+ self._journal.append((cache, key, prior))
7312
+
7313
+ def record_all(self, cache) -> None:
7314
+ if not self._open:
7315
+ return
7316
+ for key in tuple(cache):
7317
+ self.record(cache, key)
7318
+
7319
+ def _close(self) -> None:
7320
+ self._open = False
7321
+ if getattr(_CODEX_SOURCE_OVERLAY_STATE, "overlay", None) is self:
7322
+ _CODEX_SOURCE_OVERLAY_STATE.overlay = None
7323
+
7324
+ def publish(self) -> bool:
7325
+ """Commit this build's generation, or refuse because the base moved."""
7326
+ global _CODEX_SOURCE_PUBLISH_EPOCH
7327
+ self._close()
7328
+ with _CODEX_SOURCE_PUBLISH_LOCK:
7329
+ if _CODEX_SOURCE_PUBLISH_EPOCH != self.base_epoch:
7330
+ return False
7331
+ _CODEX_SOURCE_PUBLISH_EPOCH += 1
7332
+ self._published = True
7333
+ self._journal.clear()
7334
+ self._touched.clear()
7335
+ return True
7336
+
7337
+ def base_moved(self) -> bool:
7338
+ """Has another build published into the generation this one started from?"""
7339
+ with _CODEX_SOURCE_PUBLISH_LOCK:
7340
+ return _CODEX_SOURCE_PUBLISH_EPOCH != self.base_epoch
7341
+
7342
+ def abandon(self) -> None:
7343
+ """Close without replaying, for a base generation that has moved.
7344
+
7345
+ Replaying the journal writes each key's PRIOR value back, and those
7346
+ prior values predate this build. When another build has published in
7347
+ between, a key it legitimately republished would be overwritten with a
7348
+ value older than both builds — which is a stale hit, not a rollback.
7349
+ The caller clears the affected caches instead.
7350
+ """
7351
+ self._close()
7352
+ self._journal.clear()
7353
+ self._touched.clear()
7354
+
7355
+ def discard(self) -> None:
7356
+ """Undo exactly the keys this build touched, newest first.
7357
+
7358
+ ONLY SOUND WHILE THE BASE GENERATION HAS NOT MOVED; see `abandon`.
7359
+ Every caller has to compare the publish epoch before choosing this over
7360
+ `abandon`, and there are three: `_restore_codex_source_capture`,
7361
+ `capture_codex_source_state`'s exception path, and
7362
+ `_open_codex_source_overlay` when it closes a journal a caller opened
7363
+ and never built from. An earlier version of this docstring named the
7364
+ first of those as the one caller that decides, and the third did not
7365
+ decide at all — which left the T11 stale-hit defect standing on that
7366
+ path while the other two were fixed.
7367
+ """
7368
+ self._close()
7369
+ for cache, key, prior in reversed(self._journal):
7370
+ if prior is _CACHE_MISSING:
7371
+ cache.pop(key, None)
7372
+ else:
7373
+ cache[key] = prior
7374
+ self._journal.clear()
7375
+ self._touched.clear()
7376
+
7377
+
7378
+ def _open_codex_source_overlay() -> _CodexSourceBuildOverlay:
7379
+ """Open this thread's build journal, discarding an abandoned one first.
7380
+
7381
+ `capture_codex_source_state` and `build_codex_source_state_from_capture`
7382
+ are two calls with caller code between them (`bin/_cctally_tui.py:3868` and
7383
+ `:4030`), so a caller can capture and never build. An abandoned journal left
7384
+ open would keep recording another build's mutations and roll them back if
7385
+ it were ever discarded, so opening a new one closes it first.
7386
+
7387
+ AND IT TAKES THE SAME EPOCH DECISION EVERY OTHER DISCARD PATH TAKES. An
7388
+ abandoned journal's prior values predate the build that recorded them, so
7389
+ when another build has published in between, replaying them writes values
7390
+ older than BOTH builds over keys the winner legitimately republished —
7391
+ which is a stale hit rather than a rollback, and is exactly the defect the
7392
+ other paths fixed. Not reachable today, because only `bin/_cctally_tui.py`
7393
+ splits capture from build and nothing else publishes into these module
7394
+ globals in that process; it is closed here so the rule holds on every path
7395
+ rather than on four of five.
7396
+
7397
+ The account-card fallback counter has no pre-build checkpoint to restore on
7398
+ this path, because the caller that captured it never came back to hand one
7399
+ over. The CURRENT value is carried across the reset instead of being zeroed
7400
+ by it, which keeps a counter that describes work the process really did.
7401
+ """
7402
+ stale = getattr(_CODEX_SOURCE_OVERLAY_STATE, "overlay", None)
7403
+ if stale is not None and stale.open:
7404
+ if stale.base_moved():
7405
+ _drop_codex_source_generation(
7406
+ stale, _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS)
7407
+ else:
7408
+ stale.discard()
7409
+ with _CODEX_SOURCE_PUBLISH_LOCK:
7410
+ overlay = _CodexSourceBuildOverlay(_CODEX_SOURCE_PUBLISH_EPOCH)
7411
+ _CODEX_SOURCE_OVERLAY_STATE.overlay = overlay
7412
+ return overlay
7413
+
7414
+
5883
7415
  @dataclass(frozen=True)
5884
7416
  class _CodexAccountingCapture:
5885
7417
  qualified_entries: tuple[object, ...]
@@ -5888,6 +7420,86 @@ class _CodexAccountingCapture:
5888
7420
  changed_new: tuple[object, ...]
5889
7421
  entries: tuple[object, ...]
5890
7422
  metadata_incomplete: bool
7423
+ #: #834 S2 (#829). Whether THIS capture's own qualified read failed, which
7424
+ #: `metadata_incomplete` above cannot say: that boolean is true both for a
7425
+ #: row that is deterministically unqualifiable and for a read that failed,
7426
+ #: and those want different explanations and different remedies.
7427
+ metadata_read_failed: bool = False
7428
+ #: #845/#846 §4.6. The fixed internal vocabulary of metadata legs that
7429
+ #: failed with a genuine `sqlite3.Error` or a `transient=True` refusal:
7430
+ #: `accounting_health`, `detail_probe`, `qualified_accounting`,
7431
+ #: `fallback_accounting`, `conversation_metadata`. Nothing is added to the wire; the combined
7432
+ #: verdict is derived from this tuple.
7433
+ failed_legs: tuple[str, ...] = ()
7434
+ #: The name of the population THIS capture read, taken from the accounting
7435
+ #: result rather than re-read from `_lib_snapshot_cache`'s module global at
7436
+ #: fold time. The global is process-wide, so a concurrent share render's
7437
+ #: capture can overwrite it between this build's capture and its fold, and
7438
+ #: a build would then stamp another build's population name onto its own
7439
+ #: entries. No stale hit was reachable through that, because the
7440
+ #: compare-and-swap refuses to publish a build whose base moved — but the
7441
+ #: safety came from the compare-and-swap rather than from the signature,
7442
+ #: and carrying the value makes the signature answer for itself.
7443
+ population_signature: tuple | None = None
7444
+
7445
+
7446
+ #: The detail routes' horizon: one year ending at the generation instant. The
7447
+ #: carrier probe covers its union with the accounting window, which may begin
7448
+ #: earlier for a configured budget.
7449
+ CODEX_DETAIL_HORIZON = dt.timedelta(days=365)
7450
+
7451
+
7452
+ def _codex_build_metadata_inventory(
7453
+ cache_conn: sqlite3.Connection,
7454
+ ) -> "CodexThreadMetadataInventory | None":
7455
+ """The ONE thread-and-alias inventory this build's consumers share (§4.3).
7456
+
7457
+ The health counter, the detail probe's counter, the qualified reader and
7458
+ the identity map each used to load it for themselves, so a build paid the
7459
+ whole-table thread pass three or four times over. They now receive this
7460
+ one.
7461
+
7462
+ A read failure answers ``None`` rather than raising. Each consumer then
7463
+ loads the inventory itself and fails inside its OWN classification `try`,
7464
+ exactly as it did before this call existed, so the set of failed legs
7465
+ §4.6 rule 1 composes is unchanged by the sharing.
7466
+ """
7467
+ try:
7468
+ return load_codex_thread_metadata_inventory(cache_conn)
7469
+ except (sqlite3.Error, RuntimeError):
7470
+ return None
7471
+
7472
+
7473
+ def _probe_codex_detail_metadata_health_leg(
7474
+ context: DashboardReadContext,
7475
+ inventory: "CodexThreadMetadataInventory | None" = None,
7476
+ accounting_start: dt.datetime | None = None,
7477
+ accounting_end: dt.datetime | None = None,
7478
+ ) -> "tuple[Mapping[str, object], BaseException | None]":
7479
+ """The probe leg, with its failure REPORTED rather than absorbed.
7480
+
7481
+ #846 §4.6: a failure here is one of four legs the combined verdict weighs,
7482
+ so the exception is handed back instead of being turned into a carrier on
7483
+ the spot. The published carrier for the failed case is the same
7484
+ `transient_read_failure` this leg published on its own before.
7485
+ """
7486
+ try:
7487
+ probe = probe_codex_detail_metadata_health(
7488
+ cache_conn=context.cache_conn,
7489
+ start=min(context.now_utc - CODEX_DETAIL_HORIZON, accounting_start)
7490
+ if accounting_start is not None
7491
+ else context.now_utc - CODEX_DETAIL_HORIZON,
7492
+ end=max(context.now_utc, accounting_end)
7493
+ if accounting_end is not None else context.now_utc,
7494
+ inventory=inventory,
7495
+ )
7496
+ except (sqlite3.Error, RuntimeError) as exc:
7497
+ return build_metadata_health("transient_read_failure"), exc
7498
+ if probe.incomplete_rows > 0:
7499
+ return build_metadata_health(
7500
+ "malformed_row_partial", incomplete_rows=probe.incomplete_rows,
7501
+ ), None
7502
+ return build_metadata_health("healthy"), None
5891
7503
 
5892
7504
 
5893
7505
  def _capture_codex_accounting(
@@ -5897,14 +7509,52 @@ def _capture_codex_accounting(
5897
7509
  accounting_end: dt.datetime,
5898
7510
  active_roots: tuple[str, ...],
5899
7511
  path_scope: object,
5900
- metadata_incomplete: bool,
7512
+ incomplete_rows: int,
7513
+ failed_legs: tuple[str, ...] = (),
7514
+ leg_errors: "list[tuple[str, BaseException]] | None" = None,
7515
+ inventory: "CodexThreadMetadataInventory | None" = None,
5901
7516
  ) -> _CodexAccountingCapture:
5902
- """Read the incremental accounting carrier without folding public views."""
7517
+ """Read the incremental accounting carrier without folding public views.
7518
+
7519
+ #834 S2 (#828, #829): the caller passes the health read's ROW COUNT rather than
7520
+ a boolean, because the typed health result this capture returns has to
7521
+ report the count a malformed-row partial measured, and a boolean has
7522
+ already thrown it away.
7523
+ """
7524
+ metadata_incomplete = incomplete_rows > 0
7525
+ #: #834 S2 (#829). Set only when THIS capture's qualified read failed. The
7526
+ #: published health comes from the detail-horizon probe; a failure here
7527
+ #: overrides it, because a build whose own qualified read raised has not
7528
+ #: established anything about the rows.
7529
+ metadata_read_failed = False
5903
7530
  qualified_entries: tuple[object, ...] = ()
5904
7531
  dirty_accounts: tuple[str, ...] = ()
5905
7532
  changed_old: tuple[object, ...] = ()
5906
7533
  changed_new: tuple[object, ...] = ()
5907
- if not metadata_incomplete:
7534
+ population_signature: tuple | None = None
7535
+
7536
+ def fallback_entries() -> tuple[object, ...]:
7537
+ nonlocal metadata_read_failed, failed_legs
7538
+ try:
7539
+ return load_cached_rooted_codex_accounting_entries(
7540
+ accounting_start, accounting_end, speed=context.speed,
7541
+ cache_conn=context.cache_conn,
7542
+ )
7543
+ except QualifiedMetadataUnavailable as exc:
7544
+ # A failed SQLite read has no trustworthy rows. Keep the typed
7545
+ # transient carrier; deterministic identity refusals retain their
7546
+ # existing fail-loud path.
7547
+ if not isinstance(exc.__cause__, sqlite3.Error):
7548
+ raise
7549
+ metadata_read_failed = True
7550
+ failed_legs = (*failed_legs, "fallback_accounting")
7551
+ if leg_errors is not None:
7552
+ leg_errors.append(("fallback_accounting", exc))
7553
+ return ()
7554
+ # §4.6 rule 1: the qualified read is skipped when a leg before it has
7555
+ # already failed. Its result could not be published anyway, and issuing it
7556
+ # over a store whose metadata legs are failing only costs the tick.
7557
+ if not metadata_incomplete and not failed_legs:
5908
7558
  try:
5909
7559
  cached_accounting = _lib_snapshot_cache.build_cached_codex_accounting(
5910
7560
  cache_conn=context.cache_conn,
@@ -5922,6 +7572,7 @@ def _capture_codex_accounting(
5922
7572
  speed=context.speed,
5923
7573
  sync=False,
5924
7574
  cache_conn=context.cache_conn,
7575
+ inventory=inventory,
5925
7576
  ),
5926
7577
  load_paths=lambda identities: load_qualified_codex_entries(
5927
7578
  accounting_start,
@@ -5930,6 +7581,7 @@ def _capture_codex_accounting(
5930
7581
  sync=False,
5931
7582
  cache_conn=context.cache_conn,
5932
7583
  source_identities=identities,
7584
+ inventory=inventory,
5933
7585
  ),
5934
7586
  path_of=lambda entry: (
5935
7587
  str(entry.source_root_key), str(entry.source_path),
@@ -5947,19 +7599,27 @@ def _capture_codex_accounting(
5947
7599
  dirty_accounts = cached_accounting.dirty_accounts
5948
7600
  changed_old = cached_accounting.changed_old
5949
7601
  changed_new = cached_accounting.changed_new
7602
+ population_signature = cached_accounting.population_signature
5950
7603
  entries: tuple[object, ...] = qualified_entries
5951
- except QualifiedMetadataUnavailable:
7604
+ except QualifiedMetadataUnavailable as exc:
5952
7605
  _lib_log.get_logger("dashboard").warning(
5953
7606
  "Codex qualified metadata read became unavailable; using cache-only accounting fallback"
5954
7607
  )
5955
7608
  metadata_incomplete = True
7609
+ # #834 S2 (#829): only a TRANSIENT refusal overrides the published
7610
+ # health. A failure of the read established nothing about the rows,
7611
+ # so reporting a malformed-row partial would tell the reader to
7612
+ # rebuild a cache that is not the problem. A DETERMINISTIC refusal
7613
+ # does not override: the detail-horizon probe measures exactly that
7614
+ # class over a superset of this window, so its count is the
7615
+ # measured answer and this exception carries none.
7616
+ metadata_read_failed = getattr(exc, "transient", True)
7617
+ if metadata_read_failed:
7618
+ failed_legs = (*failed_legs, "qualified_accounting")
7619
+ if leg_errors is not None:
7620
+ leg_errors.append(("qualified_accounting", exc))
5956
7621
  _lib_snapshot_cache.reset_codex_accounting_cache_state()
5957
- entries = load_cached_rooted_codex_accounting_entries(
5958
- accounting_start,
5959
- accounting_end,
5960
- speed=context.speed,
5961
- cache_conn=context.cache_conn,
5962
- )
7622
+ entries = fallback_entries()
5963
7623
  dirty_accounts = tuple(sorted({
5964
7624
  str(getattr(entry, "account_key", "") or
5965
7625
  _lib_accounts.UNATTRIBUTED)
@@ -5967,12 +7627,7 @@ def _capture_codex_accounting(
5967
7627
  }))
5968
7628
  else:
5969
7629
  _lib_snapshot_cache.reset_codex_accounting_cache_state()
5970
- entries = load_cached_rooted_codex_accounting_entries(
5971
- accounting_start,
5972
- accounting_end,
5973
- speed=context.speed,
5974
- cache_conn=context.cache_conn,
5975
- )
7630
+ entries = fallback_entries()
5976
7631
  dirty_accounts = tuple(sorted({
5977
7632
  str(getattr(entry, "account_key", "") or
5978
7633
  _lib_accounts.UNATTRIBUTED)
@@ -5985,9 +7640,86 @@ def _capture_codex_accounting(
5985
7640
  changed_new=changed_new,
5986
7641
  entries=tuple(entries),
5987
7642
  metadata_incomplete=metadata_incomplete,
7643
+ metadata_read_failed=metadata_read_failed,
7644
+ failed_legs=failed_legs,
7645
+ population_signature=(
7646
+ None if metadata_incomplete else population_signature),
5988
7647
  )
5989
7648
 
5990
7649
 
7650
+ @dataclass(frozen=True)
7651
+ class AccountRegistryReading:
7652
+ """One provider's REAL account count, or the failure that hid it (#819).
7653
+
7654
+ Exactly one of the two fields is set. The distinction is the whole point:
7655
+ the decoration gate used to resolve through a ``try`` whose bare
7656
+ ``except Exception`` returned ``False``, so a registry read that failed and
7657
+ a registry holding one account produced the identical answer and the two
7658
+ states were indistinguishable afterwards. #819's investigation needed that
7659
+ exception text and there was none to read.
7660
+
7661
+ Both consumers of the registry read this one object: the R8 decoration gate
7662
+ in the Codex source build, and the ``account_scope`` the combined figure
7663
+ composes from. Two separate reads could disagree inside one tick — one
7664
+ decorating the envelope while the other withheld the combined figure for
7665
+ ``account_scope_unresolved`` — and
7666
+ ``test_one_codex_registry_reading_feeds_decoration_and_the_account_scope``
7667
+ measured exactly two Codex reads per tick before this existed.
7668
+ """
7669
+
7670
+ count: int | None
7671
+ error: BaseException | None = None
7672
+
7673
+ @property
7674
+ def decorated(self) -> bool:
7675
+ """R8: more than one REAL account, evaluated on the ESTABLISHED count.
7676
+
7677
+ The threshold is ``_cctally_account.provider_is_decorated``'s own rule
7678
+ applied to the count this reading already holds, rather than a second
7679
+ query. An unresolved count is never decorated, because there is nothing
7680
+ to decorate by — but that withholding now travels with `error` and the
7681
+ named warning the build publishes, instead of looking like a healthy
7682
+ single-account install.
7683
+ """
7684
+ return self.count is not None and self.count > 1
7685
+
7686
+ @property
7687
+ def account_scope(self) -> "dict[str, object] | None":
7688
+ """The published ``SourceDashboardState.account_scope``, or None.
7689
+
7690
+ ``None`` is the fail-closed reading `_lib_dashboard_sources` turns into
7691
+ ``account_scope_unresolved``: composition withholds the combined figure
7692
+ rather than assuming a single account.
7693
+ """
7694
+ if self.count is None:
7695
+ return None
7696
+ return {"real_account_count": int(self.count)}
7697
+
7698
+
7699
+ def read_account_registry(stats_conn, provider: str) -> AccountRegistryReading:
7700
+ """Read one provider's REAL account count once, reporting any failure.
7701
+
7702
+ The ONE implementation of this read and of its catch policy. The catch is
7703
+ narrow on purpose, and the reasoning is the one
7704
+ ``_cctally_tui._tui_resolve_account_scope`` recorded when it was the only
7705
+ surface that refused to swallow: ``real_account_count`` is a single SELECT
7706
+ over ``accounts`` with no file I/O and no label lookups, so
7707
+ ``sqlite3.Error`` covers every failure the CALL can produce, and anything
7708
+ else is a real defect that must not be reported as an unreadable registry.
7709
+ ``ImportError`` is caught beside it because the deferred import is a
7710
+ different failure class from the query — an unimportable module is exactly
7711
+ the "count cannot be read" state — and letting it propagate would fail the
7712
+ whole tick instead of withholding one figure.
7713
+ """
7714
+ try:
7715
+ import _cctally_account
7716
+ return AccountRegistryReading(
7717
+ count=int(_cctally_account.real_account_count(stats_conn, provider)),
7718
+ )
7719
+ except (ImportError, sqlite3.Error) as exc:
7720
+ return AccountRegistryReading(count=None, error=exc)
7721
+
7722
+
5991
7723
  @dataclass(frozen=True)
5992
7724
  class CodexSourceCapture:
5993
7725
  """Every cache-backed input for a post-transaction Codex source build."""
@@ -6000,24 +7732,113 @@ class CodexSourceCapture:
6000
7732
  accounting_start: dt.datetime
6001
7733
  accounting_end: dt.datetime
6002
7734
  health: object
7735
+ #: #834 S2 (#828). The ONE detail-horizon probe result this generation
7736
+ #: carries, kept SEPARATE from `health` above: that one bounds the ~30-day
7737
+ #: accounting window and decides Projects availability, while this one
7738
+ #: bounds the year both detail routes read. Folding them would let a
7739
+ #: year-old malformed row withhold a thirty-day project ranking no figure
7740
+ #: of which ever counted it.
7741
+ detail_metadata_health: Mapping[str, object]
6003
7742
  accounting: _CodexAccountingCapture
6004
7743
  accounting_exists: bool
6005
7744
  accounting_exists_error: Exception | None
6006
- conversation_metadata: Mapping
6007
- decorated: bool
7745
+ #: #845 §4.5: the typed result, not a bare map. An empty store and a
7746
+ #: failed read produce identical maps, and the composition of §4.6 needs to
7747
+ #: tell them apart.
7748
+ conversation_metadata: "_CodexConversationMetadataRead"
7749
+ #: #819: the count OR the exception, never a boolean that cannot tell them
7750
+ #: apart. `AccountRegistryReading` states why, and the build threads this
7751
+ #: one object into both the decoration gate and the published
7752
+ #: `account_scope`.
7753
+ account_registry: AccountRegistryReading
6008
7754
  card_population: tuple[object, ...]
6009
7755
  breakdowns: Mapping
6010
- cache_checkpoint: tuple[dict, ...]
7756
+ overlay: _CodexSourceBuildOverlay
6011
7757
  quota_cache_stats_checkpoint: tuple[dict, int, int, int]
6012
7758
  account_card_fallbacks_checkpoint: int
6013
7759
  accounting_checkpoint: object
6014
7760
 
6015
7761
 
7762
+ def _drop_codex_source_generation(
7763
+ overlay: "_CodexSourceBuildOverlay", account_card_fallbacks: int,
7764
+ ) -> None:
7765
+ """Take this build's generation cold, without writing anything back.
7766
+
7767
+ Used when the base generation has moved — a refused compare-and-swap, or an
7768
+ exception raised while another build published in between. The journal is
7769
+ ABANDONED rather than replayed, because every value in it predates both
7770
+ builds and replaying it would overwrite whatever the other build
7771
+ legitimately published.
7772
+
7773
+ ALL ELEVEN CACHES GO, not only the ones this build wrote to, and that is a
7774
+ correctness requirement rather than caution. Ten of them are fed by the
7775
+ accounting carrier's DELTA lists, not by its population: a build hands
7776
+ `_cached_codex_period_view` and its siblings `changed_old` and
7777
+ `changed_new`, and they apply that delta to what they already hold. The
7778
+ accounting state itself has to be reset here, because the checkpoint this
7779
+ build captured predates the winning build and restoring it would be a stale
7780
+ hit. A reset accounting state then reports its next reconstruction as cold
7781
+ with `changed_old` EMPTY and `changed_new` the whole population — so a
7782
+ period view that survived would add the entire population to a generation
7783
+ it already held and publish exactly double. Measured, not reasoned about:
7784
+ `tests/test_source_cache_degradation.py` compares published envelope bytes,
7785
+ and a partial clear published `cost_usd` and every token count at twice
7786
+ their value.
7787
+
7788
+ The cost of that is real and is not reduced here. Under sustained
7789
+ concurrent share rendering and dashboard ticking, every refused publication
7790
+ costs both parties a full cold rebuild. Reducing it needs the derived
7791
+ caches to consume a population rather than a delta, or the accounting state
7792
+ and its consumers to move under one generation stamp — neither of which is
7793
+ a bounded change to this function. Recorded in spec section 5.2.
7794
+ """
7795
+ global _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS
7796
+ overlay.abandon()
7797
+ reset_codex_source_caches()
7798
+ # `_CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS` is enumerated `extra_state`, so it
7799
+ # is RESTORED from the checkpoint rather than left at the zero
7800
+ # `reset_codex_source_caches` writes. The value restored is this build's own
7801
+ # pre-build one, so an increment the winning build made in between is lost;
7802
+ # that is a diagnostic counter off by at most one build's fallbacks,
7803
+ # against a counter that otherwise described entries no longer resident.
7804
+ #
7805
+ # THE QUOTA MEMO'S OWN EVICTION AND FALLBACK COUNTERS ARE DELIBERATELY NOT
7806
+ # RESTORED HERE, and the asymmetry is a decision rather than an oversight.
7807
+ # `reset_codex_quota_observation_cache` zeroes
7808
+ # `_CODEX_QUOTA_OBSERVATION_CACHE_EVICTIONS` and `..._FALLBACKS`, and this
7809
+ # path leaves them at zero. Those two describe what that memo currently
7810
+ # holds — how many windows it has given up and how many it declined to
7811
+ # admit — and this path has just emptied it, so zero is the true reading
7812
+ # for an empty memo. The account-card counter is different in kind: it
7813
+ # counts BUILDS that fell back, not entries that are resident, so zeroing
7814
+ # it would erase work the process really did. `CODEX_SOURCE_OVERLAY_BUNDLE`
7815
+ # states the same split from the other side, under `not_rolled_back`.
7816
+ _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS = int(account_card_fallbacks)
7817
+
7818
+
6016
7819
  def _restore_codex_source_capture(capture: CodexSourceCapture) -> None:
7820
+ """Discard this build's generation, restoring what preceded it.
7821
+
7822
+ The mappings are restored from the build's own journal, so only the keys it
7823
+ touched move. The three pieces of extra state named in
7824
+ `CODEX_SOURCE_OVERLAY_BUNDLE` are small enough to have been copied outright
7825
+ at capture and are restored the way they always were.
7826
+
7827
+ UNLESS THE BASE GENERATION HAS MOVED, in which case there is nothing to
7828
+ restore to. The exception path and the refused-publication path used to
7829
+ differ here: the second reset the caches and the first replayed its journal
7830
+ over whatever the other build had published. Every reachable case
7831
+ self-corrected to an unnecessary miss rather than a stale hit, but only
7832
+ because the signature-bearing caches store their signature inside the value
7833
+ and compare it on read — the safety lived two functions away rather than in
7834
+ the discard. It lives here now, and the two paths take the same decision.
7835
+ """
6017
7836
  global _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS
6018
- for cache, prior in zip(_codex_source_caches(), capture.cache_checkpoint):
6019
- cache.clear()
6020
- cache.update(prior)
7837
+ if capture.overlay.base_moved():
7838
+ _drop_codex_source_generation(
7839
+ capture.overlay, capture.account_card_fallbacks_checkpoint)
7840
+ return
7841
+ capture.overlay.discard()
6021
7842
  _restore_quota_observation_cache_checkpoint(
6022
7843
  capture.quota_cache_stats_checkpoint)
6023
7844
  _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS = (
@@ -6033,8 +7854,7 @@ def capture_codex_source_state(
6033
7854
  ) -> CodexSourceCapture:
6034
7855
  """Capture Codex cache evidence without constructing public view models."""
6035
7856
  global _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS
6036
- caches = _codex_source_caches()
6037
- cache_checkpoint = tuple(dict(cache) for cache in caches)
7857
+ overlay = _open_codex_source_overlay()
6038
7858
  quota_cache_stats_checkpoint = _quota_observation_cache_checkpoint()
6039
7859
  account_card_fallbacks_checkpoint = _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS
6040
7860
  accounting_checkpoint = (
@@ -6074,18 +7894,45 @@ def capture_codex_source_state(
6074
7894
  else:
6075
7895
  accounting_start = min(accounting_start, budget_start)
6076
7896
  with _lib_perf.phase("source.accounting"):
6077
- health = load_codex_project_metadata_health(
6078
- cache_conn=context.cache_conn,
6079
- start=accounting_start,
6080
- end=accounting_end,
6081
- )
7897
+ leg_errors: "list[tuple[str, BaseException]]" = []
7898
+ failed_legs: tuple[str, ...] = ()
7899
+ inventory = _codex_build_metadata_inventory(context.cache_conn)
7900
+ # §4.6, the first taxonomy hole. `load_codex_project_metadata_health`
7901
+ # had no local classification `try`, so a SQLite failure here
7902
+ # propagated to the broad handler below and failed the provider
7903
+ # outright. It is a leg now.
7904
+ try:
7905
+ health = load_codex_project_metadata_health(
7906
+ cache_conn=context.cache_conn,
7907
+ start=accounting_start,
7908
+ end=accounting_end,
7909
+ inventory=inventory,
7910
+ )
7911
+ except sqlite3.Error as exc:
7912
+ failed_legs = (*failed_legs, "accounting_health")
7913
+ leg_errors.append(("accounting_health", exc))
7914
+ health = CodexProjectMetadataHealth(
7915
+ total_rows=0, qualified_rows=0,
7916
+ missing_conversation_key_rows=0,
7917
+ missing_thread_join_rows=0,
7918
+ undecodable_metadata_rows=0,
7919
+ )
7920
+ detail_metadata_health, probe_error = (
7921
+ _probe_codex_detail_metadata_health_leg(
7922
+ context, inventory, accounting_start, accounting_end))
7923
+ if probe_error is not None:
7924
+ failed_legs = (*failed_legs, "detail_probe")
7925
+ leg_errors.append(("detail_probe", probe_error))
6082
7926
  accounting = _capture_codex_accounting(
6083
7927
  context,
6084
7928
  accounting_start=accounting_start,
6085
7929
  accounting_end=accounting_end,
6086
7930
  active_roots=active_roots,
6087
7931
  path_scope=path_scope,
6088
- metadata_incomplete=health.incomplete_rows > 0,
7932
+ incomplete_rows=health.incomplete_rows,
7933
+ failed_legs=failed_legs,
7934
+ leg_errors=leg_errors,
7935
+ inventory=inventory,
6089
7936
  )
6090
7937
  try:
6091
7938
  accounting_exists = has_cached_codex_accounting_entries(
@@ -6097,15 +7944,39 @@ def capture_codex_source_state(
6097
7944
  # value, then defer it until the build knows that branch is live.
6098
7945
  accounting_exists = False
6099
7946
  accounting_exists_error = exc
6100
- conversation_metadata = MappingProxyType(
6101
- _codex_conversation_metadata(context.cache_conn))
7947
+ conversation_metadata = _codex_conversation_metadata(
7948
+ context.cache_conn,
7949
+ inventory=inventory,
7950
+ accounting_entries=accounting.entries,
7951
+ # §4.4: the identity map's trigger is the accounting-window gate,
7952
+ # not the carrier. It is consulted exactly when the qualified read
7953
+ # was skipped and the partial fold runs, and a healthy generation
7954
+ # never consults it.
7955
+ build_identity_map=accounting.metadata_incomplete,
7956
+ )
7957
+ if conversation_metadata.error is not None:
7958
+ accounting = replace(
7959
+ accounting,
7960
+ failed_legs=(*accounting.failed_legs, "conversation_metadata"),
7961
+ )
7962
+ leg_errors.append(
7963
+ ("conversation_metadata", conversation_metadata.error))
7964
+ if accounting.failed_legs:
7965
+ # ONE server-side line naming the sorted failed legs and their
7966
+ # exception classes, so a transient generation can be diagnosed
7967
+ # without guessing which read raised.
7968
+ _lib_log.get_logger("dashboard").warning(
7969
+ "Codex metadata legs failed (%s); publishing a retryable "
7970
+ "transient health result for this generation",
7971
+ ", ".join(
7972
+ f"{leg}:{type(error).__name__}"
7973
+ for leg, error in sorted(leg_errors, key=lambda pair: pair[0])
7974
+ ),
7975
+ )
6102
7976
  with _lib_perf.phase("source.accounts"):
6103
- try:
6104
- import _cctally_account
6105
- decorated = _cctally_account.provider_is_decorated(
6106
- context.stats_conn, "codex")
6107
- except Exception:
6108
- decorated = False
7977
+ account_registry = read_account_registry(
7978
+ context.stats_conn, "codex")
7979
+ decorated = account_registry.decorated
6109
7980
  card_population_start = accounting_start
6110
7981
  if decorated:
6111
7982
  card_population_start = min(
@@ -6149,23 +8020,28 @@ def capture_codex_source_state(
6149
8020
  accounting_start=accounting_start,
6150
8021
  accounting_end=accounting_end,
6151
8022
  health=health,
8023
+ detail_metadata_health=detail_metadata_health,
6152
8024
  accounting=accounting,
6153
8025
  accounting_exists=accounting_exists,
6154
8026
  accounting_exists_error=accounting_exists_error,
6155
8027
  conversation_metadata=conversation_metadata,
6156
- decorated=decorated,
8028
+ account_registry=account_registry,
6157
8029
  card_population=tuple(card_population),
6158
8030
  breakdowns=breakdowns,
6159
- cache_checkpoint=cache_checkpoint,
8031
+ overlay=overlay,
6160
8032
  quota_cache_stats_checkpoint=quota_cache_stats_checkpoint,
6161
8033
  account_card_fallbacks_checkpoint=(
6162
8034
  account_card_fallbacks_checkpoint),
6163
8035
  accounting_checkpoint=accounting_checkpoint,
6164
8036
  )
6165
8037
  except Exception:
6166
- for cache, prior in zip(caches, cache_checkpoint):
6167
- cache.clear()
6168
- cache.update(prior)
8038
+ # The same decision `_restore_codex_source_capture` takes, made here
8039
+ # because the capture object does not exist yet to pass to it.
8040
+ if overlay.base_moved():
8041
+ _drop_codex_source_generation(
8042
+ overlay, account_card_fallbacks_checkpoint)
8043
+ raise
8044
+ overlay.discard()
6169
8045
  _restore_quota_observation_cache_checkpoint(
6170
8046
  quota_cache_stats_checkpoint)
6171
8047
  _CODEX_ACCOUNT_CARD_TOTALS_FALLBACKS = (
@@ -6180,6 +8056,7 @@ def build_codex_source_state_from_capture(
6180
8056
  *,
6181
8057
  data_version: str,
6182
8058
  path_scope: object,
8059
+ prior_hero_cohort: "Mapping[str, object] | None" = None,
6183
8060
  ) -> SourceDashboardState:
6184
8061
  """Fold one captured Codex generation without issuing cache SQL."""
6185
8062
  try:
@@ -6189,18 +8066,32 @@ def build_codex_source_state_from_capture(
6189
8066
  data_version=data_version,
6190
8067
  path_scope=path_scope,
6191
8068
  capture=capture,
8069
+ prior_hero_cohort=prior_hero_cohort,
6192
8070
  )
6193
- enforce_codex_source_accelerator_bounds()
6194
- return result
6195
8071
  except Exception:
6196
8072
  _restore_codex_source_capture(capture)
6197
8073
  raise
8074
+ if not capture.overlay.publish():
8075
+ # Another build published into the base generation this one started
8076
+ # from — the share render on an HTTP request thread reaching the same
8077
+ # module-global caches as the synchronized builder. Neither generation
8078
+ # is trustworthy as a mixture, so the caches this build wrote to go
8079
+ # cold and the next build reconstructs them. The RESULT is still
8080
+ # returned, because it was folded from this build's own captured
8081
+ # evidence and is internally consistent.
8082
+ _drop_codex_source_generation(
8083
+ capture.overlay, capture.account_card_fallbacks_checkpoint)
8084
+ # Admission runs AFTER publication, so its eviction is a change to the
8085
+ # published generation rather than a tentative one a discard would undo.
8086
+ enforce_codex_source_accelerator_bounds()
8087
+ return result
6198
8088
 
6199
8089
 
6200
8090
  def build_codex_source_state(
6201
8091
  context: DashboardReadContext,
6202
8092
  *,
6203
8093
  data_version: str,
8094
+ prior_hero_cohort: "Mapping[str, object] | None" = None,
6204
8095
  ) -> SourceDashboardState:
6205
8096
  """Build Codex data strictly from the coordinated cache/stats reads.
6206
8097
 
@@ -6224,7 +8115,8 @@ def build_codex_source_state(
6224
8115
  capture = capture_codex_source_state(
6225
8116
  context, path_scope=path_scope)
6226
8117
  return build_codex_source_state_from_capture(
6227
- capture, data_version=data_version, path_scope=path_scope)
8118
+ capture, data_version=data_version, path_scope=path_scope,
8119
+ prior_hero_cohort=prior_hero_cohort)
6228
8120
 
6229
8121
 
6230
8122
  def _build_codex_cycle_accounting(
@@ -6251,6 +8143,7 @@ def _build_codex_source_state(
6251
8143
  data_version: str,
6252
8144
  path_scope: object,
6253
8145
  capture: "CodexSourceCapture | None" = None,
8146
+ prior_hero_cohort: "Mapping[str, object] | None" = None,
6254
8147
  ) -> SourceDashboardState:
6255
8148
  active_roots = (
6256
8149
  capture.active_roots
@@ -6353,41 +8246,139 @@ def _build_codex_source_state(
6353
8246
  _warn_codex_budget_window_once("accounting_range")
6354
8247
  else:
6355
8248
  accounting_start = min(accounting_start, budget_start)
6356
- health = (
6357
- capture.health
6358
- if capture is not None
6359
- else load_codex_project_metadata_health(
6360
- cache_conn=context.cache_conn,
6361
- start=accounting_start,
6362
- end=accounting_end,
6363
- )
6364
- )
6365
- metadata_incomplete = health.incomplete_rows > 0
6366
- metadata_warning_message = (
6367
- f"{health.incomplete_rows} Codex accounting row(s) lack project metadata; "
6368
- "run `cctally cache-sync --source codex --rebuild`."
6369
- if metadata_incomplete
6370
- else "Codex project metadata could not be read; "
6371
- "run `cctally cache-sync --source codex --rebuild`."
6372
- )
6373
- accounting_capture = (
6374
- capture.accounting
6375
- if capture is not None
6376
- else _capture_codex_accounting(
8249
+ builder_leg_errors: "list[tuple[str, BaseException]]" = []
8250
+ builder_failed_legs: tuple[str, ...] = ()
8251
+ builder_inventory: "CodexThreadMetadataInventory | None" = None
8252
+ if capture is not None:
8253
+ health = capture.health
8254
+ detail_metadata_health = capture.detail_metadata_health
8255
+ accounting_capture = capture.accounting
8256
+ else:
8257
+ builder_inventory = _codex_build_metadata_inventory(context.cache_conn)
8258
+ try:
8259
+ health = load_codex_project_metadata_health(
8260
+ cache_conn=context.cache_conn,
8261
+ start=accounting_start,
8262
+ end=accounting_end,
8263
+ inventory=builder_inventory,
8264
+ )
8265
+ except sqlite3.Error as exc:
8266
+ builder_failed_legs = (*builder_failed_legs, "accounting_health")
8267
+ builder_leg_errors.append(("accounting_health", exc))
8268
+ health = CodexProjectMetadataHealth(
8269
+ total_rows=0, qualified_rows=0,
8270
+ missing_conversation_key_rows=0, missing_thread_join_rows=0,
8271
+ undecodable_metadata_rows=0,
8272
+ )
8273
+ detail_metadata_health, builder_probe_error = (
8274
+ _probe_codex_detail_metadata_health_leg(
8275
+ context, builder_inventory, accounting_start, accounting_end))
8276
+ if builder_probe_error is not None:
8277
+ builder_failed_legs = (*builder_failed_legs, "detail_probe")
8278
+ builder_leg_errors.append(("detail_probe", builder_probe_error))
8279
+ accounting_capture = _capture_codex_accounting(
6377
8280
  context,
6378
8281
  accounting_start=accounting_start,
6379
8282
  accounting_end=accounting_end,
6380
8283
  active_roots=active_roots,
6381
8284
  path_scope=path_scope,
6382
- metadata_incomplete=metadata_incomplete,
8285
+ incomplete_rows=health.incomplete_rows,
8286
+ failed_legs=builder_failed_legs,
8287
+ leg_errors=builder_leg_errors,
8288
+ inventory=builder_inventory,
6383
8289
  )
6384
- )
6385
8290
  qualified_entries = accounting_capture.qualified_entries
6386
8291
  accounting_dirty_accounts = accounting_capture.dirty_accounts
6387
8292
  accounting_changed_old = accounting_capture.changed_old
6388
8293
  accounting_changed_new = accounting_capture.changed_new
6389
8294
  accounting_entries = accounting_capture.entries
6390
8295
  metadata_incomplete = accounting_capture.metadata_incomplete
8296
+
8297
+ conversation_metadata_read = (
8298
+ capture.conversation_metadata
8299
+ if capture is not None
8300
+ else _codex_conversation_metadata(
8301
+ context.cache_conn,
8302
+ inventory=builder_inventory,
8303
+ accounting_entries=accounting_entries,
8304
+ build_identity_map=metadata_incomplete,
8305
+ )
8306
+ )
8307
+ # Restored with the typed result of §4.5: this map used to be handed
8308
+ # to the folds as a `MappingProxyType`, and the read now returns a
8309
+ # dataclass whose `metadata` member is the same dict, so the
8310
+ # read-only view moves here, where both the captured and the
8311
+ # capture-less path consume it.
8312
+ conversation_metadata = MappingProxyType(
8313
+ conversation_metadata_read.metadata)
8314
+ conversation_identity_metadata = (
8315
+ conversation_metadata_read.by_accounting_identity)
8316
+ conversation_metadata_failed = conversation_metadata_read.error is not None
8317
+ # §4.6 rule 1. A transient generation is one where ANY metadata leg failed
8318
+ # with a genuine `sqlite3.Error` or a `transient=True` refusal. It
8319
+ # publishes no Codex project rows anywhere, says so on the chip, and is
8320
+ # never retained by a reuse or retention gate, so the next tick rebuilds.
8321
+ codex_failed_legs = tuple(accounting_capture.failed_legs)
8322
+ if conversation_metadata_failed and (
8323
+ "conversation_metadata" not in codex_failed_legs
8324
+ ):
8325
+ codex_failed_legs = (*codex_failed_legs, "conversation_metadata")
8326
+ builder_leg_errors.append(
8327
+ ("conversation_metadata", conversation_metadata_read.error))
8328
+ if capture is None and codex_failed_legs:
8329
+ # The capture path emits this line itself; on the direct builder path
8330
+ # it is emitted here, once, naming the sorted failed legs and their
8331
+ # exception classes.
8332
+ _lib_log.get_logger("dashboard").warning(
8333
+ "Codex metadata legs failed (%s); publishing a retryable "
8334
+ "transient health result for this generation",
8335
+ ", ".join(
8336
+ f"{leg}:{type(error).__name__}"
8337
+ for leg, error in sorted(
8338
+ builder_leg_errors, key=lambda pair: pair[0])
8339
+ ) or ", ".join(sorted(codex_failed_legs)),
8340
+ )
8341
+ metadata_transient = bool(codex_failed_legs)
8342
+
8343
+ # Derived HERE rather than beside `health`, because `metadata_incomplete`
8344
+ # is reassigned on the line above and the message has to answer for the
8345
+ # state that is actually published. Computed early, a transient read
8346
+ # failure — which leaves `health.incomplete_rows` at zero — selected the
8347
+ # rebuild sentence, so the chip told the reader to rebuild a Codex cache
8348
+ # for a SQLite error that will retry on its own. That is the exact
8349
+ # conflation #829 exists to end, and the detail note already avoided it
8350
+ # while this message did not.
8351
+ #
8352
+ # `metadata_read_failed` is tested FIRST, and that ordering is currently
8353
+ # DEFENSIVE rather than load-bearing: `_capture_codex_accounting` issues
8354
+ # the qualified read only under `if not metadata_incomplete:`, so a window
8355
+ # holding malformed rows never attempts the read that could fail, and the
8356
+ # two conditions are mutually exclusive on every path today. Both orders
8357
+ # therefore produce identical output over every reachable state, verified
8358
+ # in a browser. The order is stated this way round so that REMOVING that
8359
+ # gate — making the pair reachable — cannot silently make the chip advise
8360
+ # a rebuild while the carrier it sits beside reports a retryable transient
8361
+ # failure. NO TEST COVERS THE PAIR, deliberately: the gate decides before
8362
+ # the call, so patching `load_qualified_codex_entries` does not reach it
8363
+ # either, and a test that tries asserts a state nothing can produce. One
8364
+ # was written during #834 S2 and removed for exactly that reason.
8365
+ #
8366
+ # The count is the accounting window's, matching this warning's own scope;
8367
+ # the carrier's count comes from the detail-horizon probe over a superset,
8368
+ # so the two are deliberately not the same number.
8369
+ metadata_warning_message = (
8370
+ "Codex project metadata could not be read for this build; it will "
8371
+ "retry on the next refresh."
8372
+ if metadata_transient
8373
+ else f"{health.incomplete_rows} Codex accounting row(s) could not be "
8374
+ "used; run `cctally cache-sync --source codex --rebuild`."
8375
+ if health.malformed_accounting_rows > 0
8376
+ else f"{health.incomplete_rows} Codex accounting row(s) lack project "
8377
+ "metadata; run `cctally cache-sync --source codex --rebuild`."
8378
+ if health.incomplete_rows > 0
8379
+ else "Codex project metadata could not be read; "
8380
+ "run `cctally cache-sync --source codex --rebuild`."
8381
+ )
6391
8382
  cycles_all: list[CodexCycleBoundary] = []
6392
8383
  try:
6393
8384
  # Per-account list (#341 Task 2). ``cycles_all`` drives the per-account
@@ -6536,9 +8527,27 @@ def _build_codex_source_state(
6536
8527
  # discard every clean period/session/project/account group. The accounting
6537
8528
  # population cache above owns upper-bound, root and speed invalidation; the
6538
8529
  # individual builders add their own tz/speed/period/cycle dimensions.
6539
- period_signature = (
6540
- "codex-accounting-v1", context.range_start, metadata_incomplete,
8530
+ period_signature = _codex_accounting_period_signature(
8531
+ context.cache_conn, context.range_start, metadata_incomplete,
6541
8532
  )
8533
+ # The accounting carrier's own exact name for the population these domains
8534
+ # are built from. The project and label caches compare it instead of
8535
+ # scanning every row to prove a hit; `None` means it could not establish
8536
+ # one, and they then prove the hit the old way.
8537
+ # Taken from THIS build's own accounting capture where there is one, and
8538
+ # only re-read from the process-wide module global on the uncaptured path.
8539
+ # See `_CodexAccountingCapture.population_signature`.
8540
+ if capture is not None:
8541
+ project_population_signature = (
8542
+ None if metadata_incomplete
8543
+ else _codex_project_label_signature(
8544
+ capture.accounting.population_signature)
8545
+ )
8546
+ else:
8547
+ project_population_signature = (
8548
+ None if metadata_incomplete
8549
+ else _codex_project_population_signature()
8550
+ )
6542
8551
  daily = _cached_codex_period_view(
6543
8552
  entries, changed_old=changed_old_entries,
6544
8553
  changed_new=changed_new_entries, kind="daily", cache_key=("parent",),
@@ -6554,15 +8563,17 @@ def _build_codex_source_state(
6554
8563
  # R8 gate, resolved once before the parent weekly projection so that only
6555
8564
  # a decorated merged row gains the additive account axis. Focused children
6556
8565
  # and <=1-real-account providers remain byte-identical.
6557
- if capture is not None:
6558
- _codex_decorated = capture.decorated
6559
- else:
6560
- try:
6561
- import _cctally_account
6562
- _codex_decorated = _cctally_account.provider_is_decorated(
6563
- context.stats_conn, "codex")
6564
- except Exception:
6565
- _codex_decorated = False
8566
+ #
8567
+ # #819: ONE reading, carried rather than re-derived. It gates decoration
8568
+ # below, it is published as this state's `account_scope`, and a failure to
8569
+ # read it becomes a named warning instead of a silent `False`. The
8570
+ # uncaptured branch — the share path and any direct builder call — performs
8571
+ # the same single read, so both entry points publish the same contract.
8572
+ account_registry = (
8573
+ capture.account_registry if capture is not None
8574
+ else read_account_registry(context.stats_conn, "codex")
8575
+ )
8576
+ _codex_decorated = account_registry.decorated
6566
8577
  weekly = _cached_codex_native_weekly_view(
6567
8578
  context.stats_conn,
6568
8579
  visible_accounting_entries,
@@ -6636,14 +8647,12 @@ def _build_codex_source_state(
6636
8647
  context, budget_entries, cost_events=budget_cost_events,
6637
8648
  )
6638
8649
  configured_budget = configured_budget_domain["status"]
6639
- conversation_metadata = (
6640
- capture.conversation_metadata
6641
- if capture is not None
6642
- else _codex_conversation_metadata(context.cache_conn)
6643
- )
6644
8650
  cache_report = _codex_cache_report_wire(
6645
8651
  visible_accounting_entries,
6646
8652
  metadata=conversation_metadata,
8653
+ by_accounting_identity=(
8654
+ conversation_identity_metadata if metadata_incomplete else None),
8655
+ withhold_projects=conversation_metadata_failed,
6647
8656
  now_utc=context.now_utc,
6648
8657
  display_tz_name=context.display_tz_name,
6649
8658
  speed=context.speed,
@@ -6654,7 +8663,13 @@ def _build_codex_source_state(
6654
8663
  semantic_signature=period_signature,
6655
8664
  )
6656
8665
  projects = (
6657
- _partial_projects_wire(visible_accounting_entries, conversation_metadata)
8666
+ # §4.6 rule 1: empty on the parent and on every account child. A
8667
+ # generation whose metadata legs failed has established nothing about
8668
+ # the rows, so publishing any of them would be a guess.
8669
+ _partial_projects_wire((), {})
8670
+ if metadata_transient
8671
+ else _partial_projects_wire(
8672
+ visible_accounting_entries, conversation_identity_metadata)
6658
8673
  if metadata_incomplete else _cached_projects_wire(
6659
8674
  context,
6660
8675
  quota_observations,
@@ -6664,6 +8679,7 @@ def _build_codex_source_state(
6664
8679
  accounting_end=accounting_end,
6665
8680
  cache_key=("parent",),
6666
8681
  semantic_signature=period_signature,
8682
+ population_signature=project_population_signature,
6667
8683
  )
6668
8684
  )
6669
8685
  alerts = _alerts_wire(context.stats_conn, decorated=_codex_decorated)
@@ -6675,9 +8691,37 @@ def _build_codex_source_state(
6675
8691
  sessions,
6676
8692
  metadata=conversation_metadata,
6677
8693
  private_labels=private_session_labels,
8694
+ # §4.4: computed from the entries that built THIS view.
8695
+ unattributed_paths=(
8696
+ codex_unattributed_paths(
8697
+ visible_accounting_entries, conversation_identity_metadata)
8698
+ if metadata_incomplete else frozenset()
8699
+ ),
6678
8700
  )
8701
+ # #819: an unreadable account registry makes this build PARTIAL. The
8702
+ # source itself is intact — every non-account domain below is built from
8703
+ # cache evidence the registry never touches — but the three R8-gated
8704
+ # subtrees are withheld, and publishing that as `ok` is precisely the
8705
+ # silent degradation this axis exists to end. `partial` keeps the provider
8706
+ # COMPOSABLE (`_coherent_provider` admits it), so the combined figure still
8707
+ # reports `account_scope_unresolved` rather than the blunter
8708
+ # `provider_incoherent`.
8709
+ #
8710
+ # It does stop the state being REUSED, since #834 S2 (#830):
8711
+ # `_reusable_provider` refuses every `partial` prior, which is what the
8712
+ # reuse docstring always claimed and what the Codex reuse version — an
8713
+ # identity digest a transient registry failure does not move — could never
8714
+ # achieve on its own. That refusal replaced the hand-maintained
8715
+ # `_CODEX_NON_REUSABLE_WARNING_CODES` allowlist in `_cctally_tui`, which
8716
+ # named this warning's code for exactly that reason and named nothing
8717
+ # else. This cause is deliberately NOT retainable under the retry kernel
8718
+ # either: the reuse branch re-resolves the account scope from the current
8719
+ # registry, so retaining this generation across a registry that recovered
8720
+ # would publish a scope naming two accounts beside a `data` carrying none.
6679
8721
  availability = (
6680
- "partial" if metadata_incomplete or hero_failure
8722
+ "partial"
8723
+ if metadata_incomplete or metadata_transient or hero_failure
8724
+ or account_registry.error is not None
6681
8725
  else ("ok" if (entries or quota_blocks or budget_rows) else "empty")
6682
8726
  )
6683
8727
  hero_input = None if hero_failure else sum(entry.input_tokens for entry in cycle_entries)
@@ -6686,7 +8730,7 @@ def _build_codex_source_state(
6686
8730
  hero_reasoning = None if hero_failure else sum(entry.reasoning_output_tokens for entry in cycle_entries)
6687
8731
  hero_total = None if hero_failure else sum(entry.total_tokens for entry in cycle_entries)
6688
8732
  warnings: list[SourceDashboardWarning] = []
6689
- if metadata_incomplete:
8733
+ if metadata_incomplete or metadata_transient:
6690
8734
  warnings.append(SourceDashboardWarning(
6691
8735
  "codex_metadata_incomplete",
6692
8736
  metadata_warning_message,
@@ -6704,6 +8748,19 @@ def _build_codex_source_state(
6704
8748
  "Codex native reset cycle is unavailable.",
6705
8749
  "hero",
6706
8750
  ))
8751
+ if account_registry.error is not None:
8752
+ # #819. The named disclosure that replaces the silent `False`. It is
8753
+ # deliberately its own code rather than the generic
8754
+ # `source_build_failed` the TUI's outer catch publishes: the build
8755
+ # SUCCEEDED and every non-account domain below is real, so a reader
8756
+ # must be able to tell "the account registry could not be read" from
8757
+ # "this source could not be built at all".
8758
+ warnings.append(SourceDashboardWarning(
8759
+ "codex_account_scope_unresolved",
8760
+ "Codex account registry could not be read, so per-account "
8761
+ "detail is withheld.",
8762
+ "accounts",
8763
+ ))
6707
8764
  # #341 Task 4: the conditional per-account wire. Built ONLY when the Codex
6708
8765
  # provider has >1 REAL account (R8) — a <=1-real-account install adds nothing
6709
8766
  # so its envelope is byte-identical to today. The array ships EVERY account's
@@ -6833,6 +8890,9 @@ def _build_codex_source_state(
6833
8890
  accounting_end=accounting_end,
6834
8891
  metadata_incomplete=metadata_incomplete,
6835
8892
  conversation_metadata=conversation_metadata,
8893
+ conversation_identity_metadata=conversation_identity_metadata,
8894
+ conversation_metadata_failed=conversation_metadata_failed,
8895
+ metadata_transient=metadata_transient,
6836
8896
  alerts=alerts,
6837
8897
  budget_milestones=budget_rows,
6838
8898
  projected_budget_milestones=projected_budget_rows,
@@ -6852,6 +8912,7 @@ def _build_codex_source_state(
6852
8912
  # accounting-only semantic key here so an unrelated account's
6853
8913
  # fresh quota sample cannot evict every clean child.
6854
8914
  scope_signature=period_signature,
8915
+ project_population_signature=project_population_signature,
6855
8916
  )
6856
8917
  # #416 QA P1-A — the "All accounts" Blocks panel is the UNION of
6857
8918
  # every account's 5-hour blocks. `_quota_wire` filters
@@ -6948,6 +9009,99 @@ def _build_codex_source_state(
6948
9009
  hero_reasoning = sum(
6949
9010
  int(card["reasoningOutputTokens"]) for card in accounts_wire)
6950
9011
  hero_total = sum(int(card["totalTokens"]) for card in accounts_wire)
9012
+ # #769 S6 / #753 — the retained hero cohort.
9013
+ #
9014
+ # The Codex hook commits cache ingestion with `quota_reconcile="defer"`,
9015
+ # closes that handle, and then runs `reconcile_codex_quota_projection`,
9016
+ # whose stats projection commits before the cache-side certificate is
9017
+ # stamped. A dashboard read landing in that window sees new cache facts
9018
+ # against an old stats projection, `codex_projection_coherence` correctly
9019
+ # rejects it, and this source used to publish that rejection destructively:
9020
+ # a null hero spend over still-populated account cards.
9021
+ #
9022
+ # A coherent generation publishes its cohort. An incoherent one republishes
9023
+ # the last coherent cohort — the provider hero operands AND each card's own
9024
+ # hero operands, so the hero and the card it sits above agree by
9025
+ # construction — together with the projection-update warning and a quiet
9026
+ # `update_state` marker (D4). When no coherent cohort has ever been
9027
+ # published in this process, or the retained one is no longer certified,
9028
+ # the hero renders an explicit Pending state (D5) rather than a zero, a
9029
+ # dash, or a blank.
9030
+ #
9031
+ # Publishing the whole cohort atomically also preserves latest-wins SSE,
9032
+ # because each frame carries exactly one coherent set.
9033
+ hero_cohort_validity = _codex_hero_cohort_validity(
9034
+ context, cycle_vector=_codex_cycle_vector(cycles_all),
9035
+ )
9036
+ hero_update_state: str | None = None
9037
+ published_hero_cohort: "Mapping[str, object] | None" = None
9038
+ retained_cycle: object | None = None
9039
+ if not hero_failure:
9040
+ if hero_cohort_validity is not None:
9041
+ published_hero_cohort = {
9042
+ "hero": {
9043
+ "cost_usd": cycle_cost_usd,
9044
+ "input_tokens": hero_input,
9045
+ "cached_input_tokens": hero_cached,
9046
+ "output_tokens": hero_output,
9047
+ "reasoning_output_tokens": hero_reasoning,
9048
+ "total_tokens": hero_total,
9049
+ "cycle": (
9050
+ {
9051
+ "window_minutes": cycle.window_minutes,
9052
+ "start_at": cycle.start_at.astimezone(UTC).isoformat(),
9053
+ "resets_at": cycle.resets_at.astimezone(UTC).isoformat(),
9054
+ }
9055
+ if cycle is not None else None
9056
+ ),
9057
+ },
9058
+ "accounts": {
9059
+ str(card["accountKey"]): {
9060
+ name: card[name] for name in _HERO_CARD_OPERANDS
9061
+ if name in card
9062
+ }
9063
+ for card in accounts_wire
9064
+ },
9065
+ "validity": hero_cohort_validity,
9066
+ }
9067
+ elif not projection_incoherent:
9068
+ # #769 S6 / #753 (T2b closing review W1). Retention is scoped to the
9069
+ # PROJECTION gap, which is the transient reconciliation window §6
9070
+ # describes. A cycle failure is not that gap: `cycle_failure` requires
9071
+ # a missing cycle AND existing accounting, so an install with neither
9072
+ # publishes a COHERENT cohort whose spend is 0.0 over an empty cycle
9073
+ # vector, and the user's first Codex session then flips only
9074
+ # `cycle_failure` while every element of the validity key stays put.
9075
+ # Retaining there republished that 0.0 with `update_state="updating"`
9076
+ # beside a `SPENT THIS WEEK` label, and suppressed the
9077
+ # `codex_cycle_unavailable` reason the client surfaces only when
9078
+ # `cost_usd` is null. Beside that label a zero is a claim, so a cycle
9079
+ # failure takes the Pending path D5 exists for.
9080
+ hero_update_state = "pending"
9081
+ else:
9082
+ retained = _retained_hero_cohort(
9083
+ prior_hero_cohort, hero_cohort_validity)
9084
+ if retained is None:
9085
+ hero_update_state = "pending"
9086
+ else:
9087
+ published_hero_cohort = retained
9088
+ hero_update_state = "updating"
9089
+ retained_hero = retained.get("hero") or {}
9090
+ cycle_cost_usd = retained_hero.get("cost_usd")
9091
+ hero_input = retained_hero.get("input_tokens")
9092
+ hero_cached = retained_hero.get("cached_input_tokens")
9093
+ hero_output = retained_hero.get("output_tokens")
9094
+ hero_reasoning = retained_hero.get("reasoning_output_tokens")
9095
+ hero_total = retained_hero.get("total_tokens")
9096
+ retained_cycle = retained_hero.get("cycle")
9097
+ retained_cards = retained.get("accounts") or {}
9098
+ for card in accounts_wire:
9099
+ operands = retained_cards.get(str(card["accountKey"]))
9100
+ if isinstance(operands, Mapping):
9101
+ card.update(
9102
+ {name: operands[name] for name in _HERO_CARD_OPERANDS
9103
+ if name in operands}
9104
+ )
6951
9105
  return SourceDashboardState(
6952
9106
  source="codex",
6953
9107
  availability=availability,
@@ -6999,7 +9153,27 @@ def _build_codex_source_state(
6999
9153
  "start_at": cycle.start_at.astimezone(UTC).isoformat(),
7000
9154
  "resets_at": cycle.resets_at.astimezone(UTC).isoformat(),
7001
9155
  }
7002
- if cycle is not None and not hero_failure else None
9156
+ if cycle is not None and not hero_failure
9157
+ # #769 S6 / #753: during a gap the retained cohort's own
9158
+ # cycle travels with the operands it bounds. Publishing
9159
+ # the current build's cycle instead would describe the
9160
+ # retained spend with a boundary from a generation that
9161
+ # did not produce it.
9162
+ else (
9163
+ dict(retained_cycle)
9164
+ if hero_update_state == "updating"
9165
+ and isinstance(retained_cycle, Mapping)
9166
+ else None
9167
+ )
9168
+ ),
9169
+ # #769 S6 / #753 (D4, D5): additive, omitted on the healthy
9170
+ # path, so a coherent generation's payload is byte-identical.
9171
+ # `updating` means the figures beside it are the last coherent
9172
+ # ones and reconciliation is in flight; `pending` means this
9173
+ # process has never published a coherent hero.
9174
+ **(
9175
+ {"update_state": hero_update_state}
9176
+ if hero_update_state is not None else {}
7003
9177
  ),
7004
9178
  # #350 (spec §3.4): additive, hero-local staleness disclosure.
7005
9179
  # OMITTED when the cycle is fresh — never emitted as "fresh" —
@@ -7081,5 +9255,24 @@ def _build_codex_source_state(
7081
9255
  ),
7082
9256
  },
7083
9257
  private_session_labels=private_session_labels,
9258
+ # #819: the SAME reading that gated decoration above. Attaching it here
9259
+ # is what makes the two facts one: a caller that composes the All
9260
+ # source can no longer pair a decorated Codex payload with a scope some
9261
+ # second read withheld, and the share path — which never reaches the
9262
+ # TUI's own attachment — now carries the count at all.
9263
+ account_scope=account_registry.account_scope,
7084
9264
  combined_accounting=combined_accounting,
9265
+ hero_cohort=published_hero_cohort,
9266
+ # #834 S2 (#828, #829): the detail-horizon probe's typed result,
9267
+ # published from here and read back by both detail routes. A qualified
9268
+ # accounting read that FAILED overrides it, because a build whose own
9269
+ # read raised has established nothing about the rows however healthy
9270
+ # the probe found them. This constructor lists every field explicitly,
9271
+ # so an omission would publish a generation that reports its metadata
9272
+ # health as unknown on a store whose probe read fine.
9273
+ metadata_health=(
9274
+ build_metadata_health("transient_read_failure")
9275
+ if metadata_transient
9276
+ else detail_metadata_health
9277
+ ),
7085
9278
  )