cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -12,6 +12,17 @@ hooks never run therefore holds its certificate while its sources grow, so a
12
12
  certificate also expires purely on elapsed time after
13
13
  ``FRONTIER_CERTIFICATE_MAX_AGE_SECONDS``.
14
14
 
15
+ Identity is not continuity, though. The journal is rotated by replacement and
16
+ an inode number is reusable, so #769 S6 gives the ledger a durable generation
17
+ and a globally monotonic, gap-free sequence: every ticket carries both, each
18
+ consumer records the ``(epoch, sequence)`` it acknowledged, and seven named
19
+ discontinuities force an exhaustive walk instead of a targeted one. Six are
20
+ properties of one consumer's own slice — a changed generation, an
21
+ acknowledgement past the tail, a missing, duplicated or regressed sequence,
22
+ and a pre-protocol ticket — and the seventh is the cross-consumer one: two
23
+ consumers whose acknowledgements name different generations cannot tell which
24
+ tickets the other already consumed, so neither may plan a targeted pass.
25
+
15
26
  The activity journal is private runtime state. Diagnostic surfaces publish
16
27
  only the bounded ``mode``/``reason`` enums and counts, never its paths.
17
28
  """
@@ -20,14 +31,25 @@ from __future__ import annotations
20
31
  import json
21
32
  import fcntl
22
33
  import os
34
+ import secrets
35
+ import sqlite3
36
+ import threading
23
37
  import pathlib
24
38
  import shlex
25
39
  import time
26
40
  from dataclasses import dataclass
41
+ from typing import NamedTuple
27
42
  from _lib_retained_size import retained_size_bytes
28
43
 
44
+ #: The default for ``plan_provider``'s ``configuration_generation``, meaning
45
+ #: "derive it from the guard paths". An explicit ``None`` is a different
46
+ #: statement — the caller looked and found no configuration evidence — and the
47
+ #: two must stay distinguishable, so the default cannot itself be ``None``.
48
+ _DERIVE_CONFIGURATION_GENERATION = object()
49
+
29
50
  _MARKER_NAME = "dashboard-ingest-activity.jsonl"
30
51
  _MARKER_LOCK_NAME = "dashboard-ingest-activity.lock"
52
+ _LEDGER_STATE_NAME = "dashboard-ingest-activity.state.json"
31
53
  _MARKER_ROTATE_BYTES = 4 * 1024 * 1024
32
54
  _PROVIDERS = frozenset({"claude", "codex"})
33
55
  FRONTIER_MAX_BYTES = 16 * 1024 * 1024
@@ -91,6 +113,11 @@ def _now() -> float:
91
113
 
92
114
  Certificates are process-local, so a monotonic source keeps the bound
93
115
  correct across system clock steps, suspend/resume and timezone changes.
116
+
117
+ This is NOT the clock the recency window is measured on. See
118
+ ``_recency_floor_iso``, which has to read the wall clock because the
119
+ evidence it compares against is a wall-clock timestamp somebody else
120
+ wrote.
94
121
  """
95
122
  return time.monotonic()
96
123
 
@@ -99,6 +126,736 @@ def activity_marker_path(app_dir: pathlib.Path) -> pathlib.Path:
99
126
  return pathlib.Path(app_dir) / _MARKER_NAME
100
127
 
101
128
 
129
+ def ledger_state_path(app_dir: pathlib.Path) -> pathlib.Path:
130
+ """The durable ledger generation and counter beside the activity file.
131
+
132
+ The tickets themselves cannot hold this state. Rotation replaces the file
133
+ with one containing only the latest payload, so a counter recoverable only
134
+ from the file's contents would restart at every rotation, and a consumer
135
+ with no new tickets could not learn the current generation at all. Keeping
136
+ it in a small sidecar written under the SAME writer lock gives both
137
+ consumers one fact to compare against, at the cost of one tiny read per
138
+ plan.
139
+ """
140
+ return pathlib.Path(app_dir) / _LEDGER_STATE_NAME
141
+
142
+
143
+ def _mint_ledger_epoch() -> str:
144
+ """A generation token that can never collide with a previous one.
145
+
146
+ An incrementing integer would be wrong here. The counter it would have to
147
+ increment lives in the very file whose loss is one of the conditions this
148
+ token exists to detect, so a fresh ledger would restart at the same epoch
149
+ a consumer already acknowledged and the discontinuity would be invisible.
150
+ """
151
+ return secrets.token_hex(16)
152
+
153
+
154
+ class LedgerState(NamedTuple):
155
+ """The ledger's durable generation, counter and last Codex configuration.
156
+
157
+ A tuple rather than a record because both existing readers index it, and
158
+ the third field is optional evidence rather than part of the continuity
159
+ contract: it names the Codex hook configuration the MOST RECENT Codex
160
+ ticket was written under, which is what makes "observed to have executed
161
+ for this configuration generation" a claim with a source.
162
+ """
163
+
164
+ epoch: str
165
+ sequence: int
166
+ configuration_generation: "str | None" = None
167
+
168
+
169
+ def read_ledger_state(app_dir: pathlib.Path):
170
+ """The ledger's state, or ``None`` when unknown.
171
+
172
+ Absent, unreadable and malformed all report ``None``, which no
173
+ acknowledged epoch matches, so every one of them forces recovery rather
174
+ than being mistaken for continuity.
175
+ """
176
+ try:
177
+ raw = ledger_state_path(app_dir).read_bytes()
178
+ except OSError:
179
+ return None
180
+ try:
181
+ value = json.loads(raw.decode("utf-8"))
182
+ except (ValueError, UnicodeDecodeError):
183
+ return None
184
+ if not isinstance(value, dict):
185
+ return None
186
+ epoch = value.get("epoch")
187
+ sequence = value.get("sequence")
188
+ if not isinstance(epoch, str) or not epoch:
189
+ return None
190
+ if not isinstance(sequence, int) or isinstance(sequence, bool):
191
+ return None
192
+ if sequence < 0:
193
+ return None
194
+ generation = value.get("configuration_generation")
195
+ if generation is not None and not isinstance(generation, str):
196
+ generation = None
197
+ return LedgerState(epoch, sequence, generation or None)
198
+
199
+
200
+ def _write_ledger_state(
201
+ app_dir: pathlib.Path, epoch: str, sequence: int,
202
+ configuration_generation: "str | None" = None,
203
+ ) -> None:
204
+ """Publish the ledger generation atomically, under the caller's lock."""
205
+ target = ledger_state_path(app_dir)
206
+ staging = target.with_name(target.name + ".next")
207
+ body = {"epoch": epoch, "sequence": int(sequence)}
208
+ if configuration_generation:
209
+ body["configuration_generation"] = str(configuration_generation)
210
+ payload = json.dumps(
211
+ body, ensure_ascii=True, separators=(",", ":"), sort_keys=True,
212
+ ).encode("utf-8")
213
+ fd = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
214
+ try:
215
+ _write_all(fd, payload)
216
+ os.fsync(fd)
217
+ finally:
218
+ os.close(fd)
219
+ os.replace(staging, target)
220
+
221
+
222
+ def classify_ledger_continuity(
223
+ *, ack_epoch, ack_sequence: int, epoch, tail_sequence: int, sequences,
224
+ ) -> "str | None":
225
+ """Name the discontinuity in one consumed ledger slice, or return ``None``.
226
+
227
+ SIX of the seven discontinuities are decided here, and all six are
228
+ properties of one consumer's own slice. The seventh is cross-consumer and
229
+ cannot be: two consumers whose acknowledgements name different generations
230
+ are irreconcilable however self-consistent each one's slice is, and
231
+ :func:`reconcile_acknowledgements` is what decides that one.
232
+
233
+ Pure. ``sequences`` is the sequence number of every ticket the consumer is
234
+ about to consume, in file order, with ``None`` standing for a ticket
235
+ written before this protocol existed.
236
+
237
+ The order of the checks is the order of decreasing scope: a generation
238
+ that moved invalidates every sequence comparison inside it, and an
239
+ acknowledgement past the ledger's own tail means the consumer's memory
240
+ describes a ledger that no longer exists.
241
+ """
242
+ if epoch != ack_epoch:
243
+ # `None == None` is the one continuous case here: a store where no
244
+ # ticket has ever been written has no ledger, and the consumer
245
+ # acknowledged exactly that. Any pre-protocol tickets it does hold are
246
+ # caught below as legacy records rather than mistaken for continuity.
247
+ return "ledger_epoch_changed"
248
+ if ack_sequence > tail_sequence:
249
+ return "ledger_cursor_beyond_tail"
250
+ previous = int(ack_sequence)
251
+ for raw in sequences:
252
+ if raw is None:
253
+ return "ledger_legacy_record"
254
+ current = int(raw)
255
+ if current == previous:
256
+ return "ledger_sequence_duplicated"
257
+ if current < previous:
258
+ return "ledger_sequence_regressed"
259
+ if current > previous + 1:
260
+ return "ledger_sequence_gap"
261
+ previous = current
262
+ return None
263
+
264
+
265
+ class ObservedFile(NamedTuple):
266
+ """One restat of a tracked rollout: what is at that pathname right now."""
267
+
268
+ size: int
269
+ device: int
270
+ inode: int
271
+
272
+
273
+ def source_identity_replaced(
274
+ committed_device, committed_inode, observed_device, observed_inode,
275
+ ) -> bool:
276
+ """Does a DIFFERENT file now occupy the pathname this cursor describes?
277
+
278
+ The single point of truth for that verdict. `classify_recent_active_path`
279
+ below and both Codex walks (`_cctally_cache.sync_codex_cache` and
280
+ `sync_codex_conversations`) call it, because three copies of one
281
+ comparison is exactly the shape in which one of them silently stops
282
+ matching the others.
283
+
284
+ THE INODE DECIDES; THE DEVICE ONLY CORROBORATES. `st_dev` is assigned when
285
+ a volume is mounted rather than when a file is created, so a `$CODEX_HOME`
286
+ on an external or network volume presents every retained rollout at a new
287
+ device number after an ordinary remount, with no file changed at all.
288
+ Deciding replacement on the device would therefore classify the WHOLE
289
+ estate as replaced at once: every cursor resets to byte zero, every
290
+ incarnation bumps, no stored account range survives the bump, and every
291
+ rollout takes a fresh attribution from whichever account is logged in now.
292
+ That is #416's re-derive-attribution-per-sync failure, reached by a
293
+ filesystem event that changed no file. The device stays stored and stays
294
+ available as corroborating evidence and as a diagnostic; it is never a
295
+ decision input on its own.
296
+
297
+ NO EVIDENCE degrades to False, which returns the caller to its size
298
+ comparison — precisely the pre-#769 behaviour. It covers a missing INODE
299
+ on either side, which is what every row written before #769 S6 carries and
300
+ what a fresh column keeps until its path is next ingested, and it covers a
301
+ stored value that is not an integer. The second case matters because a
302
+ raise here escapes the per-file loop and takes every LATER file in the
303
+ estate with it, so an unreadable identity must degrade rather than fail.
304
+
305
+ NO-EVIDENCE IS THE INODE ALONE, because the DECISION is the inode alone.
306
+ An earlier form also required both device columns, so a row carrying a
307
+ real inode beside a NULL device answered "no evidence" and returned the
308
+ caller to the size comparison, which lets a same-size replacement pass as
309
+ unchanged. Both write sites stamp the pair from one `stat`, so no row in
310
+ the shipped tree splits them; the guard is written against the decision
311
+ rather than against the current shape of the rows.
312
+ """
313
+ if committed_inode is None or observed_inode is None:
314
+ return False
315
+ try:
316
+ return int(committed_inode) != int(observed_inode)
317
+ except (TypeError, ValueError):
318
+ return False
319
+
320
+
321
+ def classify_recent_active_path(
322
+ *, committed_size, committed_offset, committed_complete,
323
+ committed_device, committed_inode, observed,
324
+ ) -> str:
325
+ """Classify one recently-active rollout against its committed cursor.
326
+
327
+ Pure. Returns one of ``unchanged``, ``append``, ``resume``, ``replaced``,
328
+ ``cursor_gap`` or ``vanished``.
329
+
330
+ IDENTITY OUTRANKS SIZE. A larger file at a different inode is a
331
+ replacement, not an append: its first byte is not the continuation of the
332
+ committed offset, so resuming there would attribute bytes from one file to
333
+ the cursor of another. `source_identity_replaced` above owns that verdict
334
+ and states why the inode alone decides it.
335
+
336
+ ``resume`` is what makes a budget-stopped generation recoverable. Its
337
+ committed size is the whole observed size, so no size comparison can see
338
+ the unread suffix; only the completion flag can.
339
+ """
340
+ if observed is None:
341
+ return "vanished"
342
+ if source_identity_replaced(
343
+ committed_device, committed_inode, observed.device, observed.inode,
344
+ ):
345
+ return "replaced"
346
+ floor = max(int(committed_size or 0), int(committed_offset or 0))
347
+ if int(observed.size) < floor:
348
+ return "cursor_gap"
349
+ if int(observed.size) > int(committed_size or 0):
350
+ return "append"
351
+ if not committed_complete:
352
+ return "resume"
353
+ return "unchanged"
354
+
355
+
356
+ #: How long a Codex rollout stays in the recently-active set after cctally
357
+ #: last committed a read of it. Deliberately the SAME number as the
358
+ #: certificate age bound rather than a second knob, so that under a steady
359
+ #: clock the recency window ends exactly when the exhaustive backstop falls
360
+ #: due and no path is unwatched in between.
361
+ #:
362
+ #: THE TWO ARE MEASURED ON DIFFERENT CLOCKS, and an earlier version of this
363
+ #: comment claimed they "introduce no new time bound at all", which is false.
364
+ #: The certificate ages on ``time.monotonic()``; the recency window is a
365
+ #: comparison against ``codex_session_files.last_ingested_at``, which is a
366
+ #: wall-clock ISO timestamp the ingester wrote, so it can only be measured on
367
+ #: the wall clock. A forward clock step or a suspend and resume therefore
368
+ #: empties the recency set while the certificate is still young, and for the
369
+ #: remainder of that certificate's life a mid-turn append is invisible again.
370
+ #: That is a loss of freshness, not of correctness — it degrades to exactly
371
+ #: the pre-#724 behaviour, which the expiry backstop then answers — and a
372
+ #: BACKWARD step is safe in the other direction, because it only widens the
373
+ #: window and costs extra stats. The ``ingest_complete = 0`` leg of the
374
+ #: predicate is clock-independent, so pending work is never dropped by either.
375
+ FRONTIER_RECENT_ACTIVITY_SECONDS = FRONTIER_CERTIFICATE_MAX_AGE_SECONDS
376
+
377
+
378
+ def _recency_floor_iso(now=None) -> str:
379
+ """The lower bound of the recency window, in the writers' own spelling.
380
+
381
+ ``codex_session_files.last_ingested_at`` is written as
382
+ ``datetime.now(timezone.utc).isoformat()`` at COMMIT time, which is
383
+ cctally's own observation of when it read the file. Building the bound the
384
+ same way keeps the comparison a plain string comparison over one format.
385
+
386
+ WALL CLOCK, unavoidably. The stored value is a wall-clock timestamp
387
+ written by a process that may no longer exist, so there is no monotonic
388
+ reading of it to compare against. The certificate's own age bound uses
389
+ ``_now()`` instead, and ``FRONTIER_RECENT_ACTIVITY_SECONDS`` records what
390
+ the divergence between the two costs.
391
+ """
392
+ import datetime as _dt # noqa: PLC0415 — stdlib, one call per plan
393
+
394
+ moment = now or _dt.datetime.now(_dt.timezone.utc)
395
+ return (
396
+ moment - _dt.timedelta(seconds=FRONTIER_RECENT_ACTIVITY_SECONDS)
397
+ ).isoformat()
398
+
399
+
400
+ #: Host parameters per `IN (...)` query. SQLite's compiled default limit is
401
+ #: 999 on the builds this ships against; 400 is the figure
402
+ #: `_load_codex_session_files_rows` already chunks at, restated rather than
403
+ #: imported so the two modules stay independent.
404
+ _CURSOR_QUERY_CHUNK = 400
405
+
406
+
407
+ def select_recently_active_codex_paths(
408
+ conn, *, table: str = "codex_session_files", extra_paths=(),
409
+ ) -> dict:
410
+ """The bounded set of Codex rollouts one plan is entitled to restat.
411
+
412
+ Returns ``{path: (size_bytes, last_byte_offset, ingest_complete,
413
+ device_id, inode)}`` for the union of three sources: rollouts cctally
414
+ itself committed a read of inside the recency window, paths an interrupted
415
+ or budget-stopped generation left incomplete, and any ``extra_paths`` the
416
+ caller names — the valid Codex tickets in the slice being consumed.
417
+
418
+ RECENCY IS CCTALLY'S OWN COMMIT TIME. Not the rollout's event timestamps,
419
+ which the observed process writes; not the file's mtime, which any writer
420
+ moves; and not the time of a verification that found nothing changed,
421
+ which would keep a quiet file in this set for the life of the process
422
+ simply because it kept being looked at. Only a committed read advances
423
+ ``last_ingested_at``, and only a committed read renews recency.
424
+
425
+ ONE QUERY OVER ONE SMALL TABLE. Both cursor tables hold one row per
426
+ retained rollout — about 2,854 on the operator's store — and this
427
+ predicate is evaluated once per plan. It deliberately never touches
428
+ ``codex_session_entries``, which holds ~197,000 rows and whose cost is the
429
+ thing the whole frontier exists to avoid.
430
+
431
+ ``table`` selects the consumer's own cursor table. Only
432
+ ``codex_session_files`` carries ``ingest_complete``: the transcript
433
+ ingester has no budgeted mid-file stop, so it has no incomplete state to
434
+ resume and every one of its rows reports complete.
435
+ """
436
+ if table not in _CODEX_CURSOR_TABLES:
437
+ raise ValueError("unknown codex cursor table")
438
+ has_completion = _CODEX_CURSOR_TABLES[table]
439
+ completion = "ingest_complete" if has_completion else "1"
440
+ columns = (
441
+ f"path, size_bytes, last_byte_offset, {completion}, device_id, inode")
442
+ predicate = "last_ingested_at >= ?"
443
+ if has_completion:
444
+ predicate += " OR ingest_complete = 0"
445
+ floor = _recency_floor_iso()
446
+ selected: dict = {}
447
+
448
+ def _absorb(rows):
449
+ for path, size, offset, complete, device, inode in rows:
450
+ if not path:
451
+ continue
452
+ selected[str(path)] = (
453
+ int(size or 0), int(offset or 0), bool(complete),
454
+ None if device is None else int(device),
455
+ None if inode is None else int(inode),
456
+ )
457
+
458
+ _absorb(conn.execute(
459
+ f"SELECT {columns} FROM {table} WHERE {predicate}", (floor,)))
460
+ wanted = {str(item) for item in extra_paths if item}
461
+ missing = sorted(wanted - set(selected))
462
+ # CHUNKED, at the same 400 `_load_codex_session_files_rows` uses. One
463
+ # `IN (...)` over the whole missing set raises `OperationalError` past
464
+ # SQLite's host-parameter limit, and the caller catches that as
465
+ # `codex_cursor_unavailable` — a reason that describes store STATE, so a
466
+ # code defect would have been reported as an unmigrated store forever.
467
+ for start in range(0, len(missing), _CURSOR_QUERY_CHUNK):
468
+ chunk = missing[start:start + _CURSOR_QUERY_CHUNK]
469
+ placeholders = ",".join("?" for _ in chunk)
470
+ _absorb(conn.execute(
471
+ f"SELECT {columns} FROM {table} WHERE path IN ({placeholders})",
472
+ chunk))
473
+ return selected
474
+
475
+
476
+ #: Deterministic work counters, by cursor table. `retained_file_visits` counts
477
+ #: the stats the RESTAT performs — one per member of the bounded recently-active
478
+ #: set — which is the quantity #786 bounds and the one an unbounded restat
479
+ #: would blow up. It is not the total number of stats a plan performs:
480
+ #: `_target_has_cursor_gap` and `_target_in_roots` each stat a ticketed path
481
+ #: as well and are deliberately not counted here, so on a ticketed append the
482
+ #: same path is statted at least twice against one recorded visit. The counter
483
+ #: is exact for the case it exists to protect — an idle caught-up tick names
484
+ #: no path at all, so it stats nothing and records nothing — and every extra
485
+ #: stat it omits is proportional to ticketed activity rather than to the
486
+ #: retained estate. Plain integers under the coordinator's own lock discipline
487
+ #: is enough: they are read by the benchmark harness and by nothing that makes
488
+ #: a decision.
489
+ _FRONTIER_COUNTERS: "dict[str, int]" = {}
490
+ _FRONTIER_COUNTERS_LOCK = threading.Lock()
491
+
492
+
493
+ def frontier_counters() -> dict:
494
+ """A snapshot of the deterministic frontier work counters."""
495
+ with _FRONTIER_COUNTERS_LOCK:
496
+ return dict(_FRONTIER_COUNTERS)
497
+
498
+
499
+ def reset_frontier_counters() -> None:
500
+ with _FRONTIER_COUNTERS_LOCK:
501
+ _FRONTIER_COUNTERS.clear()
502
+
503
+
504
+ def _bump_counter(name: str, amount: int = 1) -> None:
505
+ with _FRONTIER_COUNTERS_LOCK:
506
+ _FRONTIER_COUNTERS[name] = _FRONTIER_COUNTERS.get(name, 0) + amount
507
+
508
+
509
+ def _observe_file(path: str, table: str) -> "ObservedFile | None":
510
+ _bump_counter(f"retained_file_visits.{table}")
511
+ try:
512
+ st = pathlib.Path(path).stat()
513
+ except OSError:
514
+ return None
515
+ return ObservedFile(st.st_size, st.st_dev, st.st_ino)
516
+
517
+
518
+ #: The two Codex cursor tables the restat may read, and whether each carries
519
+ #: an ``ingest_complete`` column. The accounting ingester has a budgeted
520
+ #: mid-file stop and therefore an incomplete state; the transcript ingester
521
+ #: does not.
522
+ _CODEX_CURSOR_TABLES = {
523
+ "codex_session_files": True,
524
+ "codex_conversation_source_files": False,
525
+ }
526
+
527
+
528
+ def _ingest_source_table(provider: str) -> str:
529
+ """The ACCOUNTING ingester's cursor table for one provider.
530
+
531
+ The twin of ``_conversation_source_table`` below, which answers the same
532
+ question for the transcript ingester. Both exist so that no planner spells
533
+ a table name out at its own call site: the accounting planner did, and one
534
+ literal beside one derived sibling is exactly the shape in which the two
535
+ stop matching.
536
+
537
+ Only the Codex answer is ever handed to ``plan_recent_active_restat``.
538
+ ``session_files`` is deliberately absent from ``_CODEX_CURSOR_TABLES``
539
+ above, so reaching the restat with the Claude answer raises rather than
540
+ reading a Codex table under a Claude plan. That refusal is a backstop, not
541
+ the mechanism: ``_restat_and_return`` gates the restat on
542
+ ``provider == "codex"`` and is what keeps the Claude answer away from it.
543
+ """
544
+ if provider == "claude":
545
+ return "session_files"
546
+ if provider == "codex":
547
+ return "codex_session_files"
548
+ raise ValueError("unknown provider")
549
+
550
+
551
+ def plan_recent_active_restat(
552
+ conn, *, table: str = "codex_session_files", extra_paths=(),
553
+ ):
554
+ """Restat the bounded recent set and return ``(targets, escalation)``.
555
+
556
+ ``targets`` are the paths whose suffix this tick should ingest;
557
+ ``escalation`` is a refusal reason when one of them cannot be handled by
558
+ targeted work at all, in which case the caller must walk exhaustively.
559
+
560
+ An empty recent set performs ZERO stats, which is the whole shape of an
561
+ idle caught-up tick on the production store.
562
+ """
563
+ try:
564
+ candidates = select_recently_active_codex_paths(
565
+ conn, table=table, extra_paths=extra_paths)
566
+ except sqlite3.DatabaseError:
567
+ # A store whose cursor columns this binary expects are absent is store
568
+ # STATE, not a code defect: an opener that has not yet run the
569
+ # migration, or a fixture standing in for the table. Escalating is the
570
+ # conservative answer, and it must be an answer rather than a raise,
571
+ # because the planner's own `except` clause does not cover
572
+ # `sqlite3.DatabaseError` and an escape here would end the tick.
573
+ return frozenset(), "codex_cursor_unavailable"
574
+ targets: set[str] = set()
575
+ for path, (size, offset, complete, device, inode) in sorted(
576
+ candidates.items()
577
+ ):
578
+ verdict = classify_recent_active_path(
579
+ committed_size=size,
580
+ committed_offset=offset,
581
+ committed_complete=complete,
582
+ committed_device=device,
583
+ committed_inode=inode,
584
+ observed=_observe_file(path, table),
585
+ )
586
+ if verdict in {"append", "resume"}:
587
+ targets.add(path)
588
+ elif verdict == "replaced":
589
+ return frozenset(), "source_replaced"
590
+ elif verdict == "cursor_gap":
591
+ return frozenset(), "cursor_gap"
592
+ elif verdict == "vanished":
593
+ return frozenset(), "filesystem_changed"
594
+ return frozenset(targets), None
595
+
596
+
597
+ def _ledger_slice_decision(owner, provider: str, state, records):
598
+ """The ledger verdict for one consumed slice, for EITHER consumer.
599
+
600
+ Returns ``(reason, ledger_epoch, ledger_sequence)``. ``reason`` names one
601
+ of the seven discontinuities, or is ``None`` when the slice is continuous
602
+ and reconcilable, in which case the other two values are what a clean plan
603
+ acknowledges.
604
+
605
+ Single-sourced because the ledger is ONE file and the discontinuities are
606
+ properties of it, not of whoever is reading it. The two consumers ran
607
+ byte-identical copies of this block, comment text included, and a copy is
608
+ exactly the shape in which one of them silently stops matching the other.
609
+ """
610
+ ledger = read_ledger_state(owner.app_dir)
611
+ reason = classify_ledger_continuity(
612
+ ack_epoch=state.ack_epoch,
613
+ ack_sequence=state.ack_sequence,
614
+ epoch=None if ledger is None else ledger.epoch,
615
+ tail_sequence=0 if ledger is None else ledger.sequence,
616
+ sequences=tuple(record[3] for record in records),
617
+ )
618
+ if reason is not None:
619
+ return reason, None, 0
620
+ # GUARDED, three lines after the same value was read defensively above.
621
+ # `ledger` is `None` exactly when no ledger has ever been written AND this
622
+ # consumer acknowledged that, which the classifier reports as continuous.
623
+ # Production does not reach it, because `capture_cutoff` materializes the
624
+ # sidecar before any seed; a caller that builds its own `MarkerCutoff`
625
+ # does, and a bare `ledger[0]` there raises a `TypeError` that the
626
+ # planner's `except (OSError, ValueError)` would not catch.
627
+ ledger_epoch = None if ledger is None else ledger.epoch
628
+ ledger_sequence = records[-1][3] if records else state.ack_sequence
629
+ # The seventh discontinuity. Reached only when this consumer's own slice
630
+ # is self-consistent, which is the whole reason it needs a separate check:
631
+ # each consumer compares its acknowledgement against the SAME live ledger,
632
+ # so a peer that has not planned since a rotation is the one state in
633
+ # which two acknowledgements can name different generations while every
634
+ # other guard is satisfied.
635
+ mine = (
636
+ None if not state.ack_epoch
637
+ else (state.ack_epoch, state.ack_sequence)
638
+ )
639
+ for peer in owner._generations.peer_ledger_acknowledgements(
640
+ provider, owner.CONSUMER,
641
+ ):
642
+ conflict = reconcile_acknowledgements(mine, peer)
643
+ if conflict is not None:
644
+ return conflict, None, 0
645
+ return None, ledger_epoch, ledger_sequence
646
+
647
+
648
+ def _configuration_generation_reason(
649
+ owner, provider: str, records, guard_paths, configuration_generation,
650
+ ):
651
+ """Escalate when a ticket's configuration is not the one on disk now.
652
+
653
+ A ticket written under a configuration that has since changed is not
654
+ evidence about the configuration on disk now: the handler that wrote it
655
+ may no longer be the managed one, so naming its path as the only target
656
+ would rest a precise claim on a stale observation. The ``guard_paths``
657
+ mtime comparison in each planner is the coarse form of the same check;
658
+ this one compares the bytes. It is a no-op when the caller supplies no
659
+ configuration evidence AND no ticket carries any, because Claude tickets
660
+ and the benchmarks legitimately carry none.
661
+ """
662
+ ticket_generations = {
663
+ record[4] for record in records if record[0] == provider}
664
+ stamped_generations = {
665
+ item for item in ticket_generations if item is not None}
666
+ current_generation = _resolve_configuration_generation(
667
+ provider, guard_paths, configuration_generation)
668
+ if current_generation is None and not stamped_generations:
669
+ return None
670
+ if all(
671
+ classify_codex_execution_observed(
672
+ observed_generation=stamped,
673
+ current_generation=current_generation,
674
+ )
675
+ for stamped in ticket_generations
676
+ ):
677
+ return None
678
+ if current_generation is not None and not codex_execution_observed(
679
+ owner.app_dir, current_generation,
680
+ ):
681
+ # #769 S6: the divergence diagnostic. The hook derives its digest from
682
+ # `_codex_lifecycle_roots()` and this module recomputes one from the
683
+ # guard paths; they agree today because both funnel through
684
+ # `codex_hook_roots(_codex_home_roots())`. A dashboard running under a
685
+ # different `$CODEX_HOME` than the hooks would break that, every Codex
686
+ # ticket would escalate here forever, and #724's fix would silently
687
+ # stop applying. This counter is what names that state: it rises on
688
+ # every plan while the ledger's own retained generation — written by
689
+ # whichever hook last ran — never matches the one recomputed here.
690
+ _bump_counter(f"configuration_unobserved.{provider}")
691
+ return "configuration_generation_changed"
692
+
693
+
694
+ def _ticket_target_paths(provider: str, records, roots):
695
+ """The ticketed paths this plan may target, or a refusal reason."""
696
+ provider_paths = tuple(
697
+ path for kind, path, _e, _s, _c in records if kind == provider)
698
+ if any(not path for path in provider_paths):
699
+ return None, "ambiguous_activity"
700
+ paths = frozenset(provider_paths)
701
+ if any(not _target_in_roots(path, roots) for path in paths):
702
+ return None, "target_outside_scope"
703
+ return paths, None
704
+
705
+
706
+ def _restat_and_return(
707
+ provider: str, conn, *, cursor_table: str, paths, marker_end: int,
708
+ ledger_epoch, ledger_sequence: int,
709
+ ):
710
+ """#724's bounded restat, then the caught-up or targeted plan it implies.
711
+
712
+ A mid-turn append lands before the post-turn hook writes anything, so it
713
+ moves no ticket, no directory mtime and no cursor, and the certificate's
714
+ age bound would otherwise be the only evidence that ever finds it.
715
+ Restatting the paths recent activity NAMES finds it on the next ordinary
716
+ tick at a cost proportional to that activity, and an idle caught-up estate
717
+ names nothing, so it stats nothing.
718
+
719
+ Scoped to Codex: the measured blind spot and the cursor-table evidence
720
+ this reads are Codex-specific, and Claude's behaviour is deliberately
721
+ unchanged this session. ``cursor_table`` is the CALLER's own table — the
722
+ two consumers hold independent offsets into the same files, so a path
723
+ caught up in one store can be behind in the other.
724
+ """
725
+ restat_paths = frozenset()
726
+ if provider == "codex":
727
+ restat_paths, escalation = plan_recent_active_restat(
728
+ conn, table=cursor_table, extra_paths=paths)
729
+ if escalation is not None:
730
+ return FrontierPlan(provider, "full", reason=escalation)
731
+ combined = paths | restat_paths
732
+ if not combined:
733
+ return FrontierPlan(
734
+ provider, "caught_up", marker_end=marker_end, reason="unchanged",
735
+ ledger_epoch=ledger_epoch, ledger_sequence=ledger_sequence,
736
+ )
737
+ return FrontierPlan(
738
+ provider,
739
+ "targeted",
740
+ paths=combined,
741
+ marker_end=marker_end,
742
+ # A path the restat found is not a path a ticket named, and a
743
+ # diagnostic that called both "activity" would hide exactly the
744
+ # discovery this session added.
745
+ reason="activity" if paths else "recent_activity",
746
+ ledger_epoch=ledger_epoch,
747
+ ledger_sequence=ledger_sequence,
748
+ )
749
+
750
+
751
+ def _plan_accounting(owner, provider, conn, **kwargs):
752
+ return DashboardIngestFrontier._plan_uncounted(
753
+ owner, provider, conn, **kwargs)
754
+
755
+
756
+ def _plan_conversation(owner, provider, conn, **kwargs):
757
+ return ConversationSyncFrontier._plan_uncounted(
758
+ owner, provider, conn, **kwargs)
759
+
760
+
761
+ def _counted_plan(owner, body, provider, conn, **kwargs):
762
+ """Record one plan's deterministic shape, by provider, consumer and mode.
763
+
764
+ The counters exist so the benchmark harness can assert the SHAPE of a tick
765
+ rather than only its wall clock: an idle caught-up tick that quietly starts
766
+ visiting every retained rollout is a regression no timing threshold would
767
+ reliably catch on a fast machine.
768
+ """
769
+ plan = body(owner, provider, conn, **kwargs)
770
+ consumer = owner.CONSUMER
771
+ _bump_counter(f"plans.{consumer}.{provider}.{plan.mode}")
772
+ if plan.mode == "full":
773
+ _bump_counter(f"full_walks.{consumer}.{provider}")
774
+ if plan.paths:
775
+ _bump_counter(
776
+ f"targeted_paths.{consumer}.{provider}", len(plan.paths))
777
+ return plan
778
+
779
+
780
+ def _resolve_configuration_generation(provider: str, guard_paths, supplied):
781
+ """The Codex configuration digest for one plan, or ``None``.
782
+
783
+ The frontier recomputes it from the current files rather than being told,
784
+ which keeps this protocol inside the two modules that own it: the guard
785
+ set the planner already holds names the hook roots, and
786
+ ``_lib_codex_hooks`` owns both directions of that relationship. A caller
787
+ may still pass a value explicitly, which the tests and any future consumer
788
+ with a cheaper source can do.
789
+
790
+ Claude has no Codex hook configuration, and this protocol is scoped to
791
+ Codex, so Claude always resolves to ``None`` and the check that reads it
792
+ becomes a no-op.
793
+ """
794
+ if supplied is not _DERIVE_CONFIGURATION_GENERATION:
795
+ return supplied
796
+ if provider != "codex":
797
+ return None
798
+ try:
799
+ import _lib_codex_hooks # noqa: PLC0415 — optional, resolved per plan
800
+
801
+ return _lib_codex_hooks.codex_configuration_generation_for_guard_paths(
802
+ guard_paths)
803
+ except Exception:
804
+ # An unreadable configuration is evidence of nothing, and `None` never
805
+ # matches a stamped generation, so a stamped ticket still forces the
806
+ # exhaustive walk rather than being trusted by default.
807
+ return None
808
+
809
+
810
+ def classify_codex_execution_observed(
811
+ *, observed_generation, current_generation,
812
+ ) -> bool:
813
+ """Whether a Codex hook is observed to have run for THIS configuration.
814
+
815
+ Pure. ``observed_generation`` is the configuration digest the most recent
816
+ Codex ticket was written under; ``current_generation`` is the digest
817
+ recomputed from the files on disk now.
818
+
819
+ This is a strictly stronger claim than
820
+ ``CodexHookObservation.observed_enabled``, which says only that the
821
+ current slots classify as installed and enabled, and which is deliberately
822
+ NOT relabelled: a root can be installed, enabled and trusted and still
823
+ never have fired a hook. Both ``None`` cases are false — a configuration
824
+ that cannot be read proves nothing, and a root that has written no ticket
825
+ has produced no execution evidence at all.
826
+ """
827
+ if not observed_generation or not current_generation:
828
+ return False
829
+ return observed_generation == current_generation
830
+
831
+
832
+ def codex_execution_observed(app_dir, current_generation) -> bool:
833
+ """The stored form of :func:`classify_codex_execution_observed`."""
834
+ state = read_ledger_state(pathlib.Path(app_dir))
835
+ return classify_codex_execution_observed(
836
+ observed_generation=None if state is None else (
837
+ state.configuration_generation),
838
+ current_generation=current_generation,
839
+ )
840
+
841
+
842
+ def reconcile_acknowledgements(first, second) -> "str | None":
843
+ """Whether two consumers' acknowledgements describe one ledger.
844
+
845
+ Pure. Each argument is an ``(epoch, sequence)`` pair or ``None`` for a
846
+ consumer that has never acknowledged anything. Two consumers legitimately
847
+ sit at different sequences within one generation, because they plan and
848
+ commit independently. Two DIFFERENT generations are the irreconcilable
849
+ case: neither consumer can tell which tickets the other one already
850
+ consumed, so only an exhaustive walk restores a shared frontier.
851
+ """
852
+ if first is None or second is None:
853
+ return None
854
+ if first[0] != second[0]:
855
+ return "ledger_acknowledgements_irreconcilable"
856
+ return None
857
+
858
+
102
859
  def invalidate_activity_marker(app_dir: pathlib.Path) -> bool:
103
860
  """Remove the activity certificate after a writer cannot append a ticket.
104
861
 
@@ -139,25 +896,72 @@ def invalidate_activity_marker(app_dir: pathlib.Path) -> bool:
139
896
  os.close(lock_fd)
140
897
 
141
898
 
142
- def record_activity(app_dir: pathlib.Path, provider: str, source_path: str) -> bool:
143
- """Append one hook-owned activity ticket before background work starts."""
899
+ def record_activity(
900
+ app_dir: pathlib.Path, provider: str, source_path: str,
901
+ *, configuration_generation: "str | None" = None,
902
+ ) -> bool:
903
+ """Append one hook-owned activity ticket before background work starts.
904
+
905
+ Every ticket carries the ledger generation it was written under and the
906
+ next durable sequence in that ledger. Both are assigned here, under the
907
+ writer flock, so two concurrent hooks can never mint the same sequence.
908
+
909
+ The sidecar is published BEFORE the ticket, and the ordering is
910
+ deliberate. A crash between the two leaves a sequence number that no
911
+ ticket ever claims, which the next consumer reads as a gap and answers
912
+ with an exhaustive walk. The opposite order would leave two tickets
913
+ sharing one sequence, which is the same recovery at the cost of a
914
+ duplicate the ledger can never explain.
915
+ """
144
916
  if provider not in _PROVIDERS:
145
917
  return False
146
918
  marker = activity_marker_path(pathlib.Path(app_dir))
147
919
  try:
148
920
  marker.parent.mkdir(parents=True, exist_ok=True)
149
- payload = json.dumps(
150
- {"provider": provider, "path": str(source_path or "")},
151
- ensure_ascii=True, separators=(",", ":"), sort_keys=True,
152
- ).encode("utf-8") + b"\n"
153
921
  lock_path = marker.with_name(_MARKER_LOCK_NAME)
154
922
  lock_fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600)
155
923
  try:
156
924
  fcntl.flock(lock_fd, fcntl.LOCK_EX)
157
925
  try:
158
926
  rotate = marker.stat().st_size >= _MARKER_ROTATE_BYTES
927
+ present = True
159
928
  except OSError:
160
929
  rotate = False
930
+ present = False
931
+ state = read_ledger_state(pathlib.Path(app_dir))
932
+ if state is None:
933
+ # No usable ledger state. Minting a fresh generation is what
934
+ # stops a restarted counter from looking like a continuation
935
+ # of the one every consumer already acknowledged.
936
+ state = LedgerState(_mint_ledger_epoch(), 0)
937
+ retained_generation = None
938
+ else:
939
+ retained_generation = state.configuration_generation
940
+ epoch, sequence = state.epoch, state.sequence
941
+ if rotate or not present:
942
+ # Both are replacements of the pathname, and a replacement is
943
+ # exactly what a consumer holding a byte cursor cannot follow.
944
+ epoch = _mint_ledger_epoch()
945
+ sequence += 1
946
+ # A writer that supplies no configuration evidence must not erase
947
+ # the evidence another writer left. One ledger serves both
948
+ # providers, so a Claude ticket would otherwise wipe the only
949
+ # durable record that the Codex hook ever ran.
950
+ generation = configuration_generation or retained_generation
951
+ _write_ledger_state(
952
+ pathlib.Path(app_dir), epoch, sequence, generation)
953
+ body = {
954
+ "provider": provider,
955
+ "path": str(source_path or ""),
956
+ "epoch": epoch,
957
+ "seq": sequence,
958
+ }
959
+ if configuration_generation:
960
+ body["cfg"] = str(configuration_generation)
961
+ payload = json.dumps(
962
+ body,
963
+ ensure_ascii=True, separators=(",", ":"), sort_keys=True,
964
+ ).encode("utf-8") + b"\n"
161
965
  if rotate:
162
966
  replacement = marker.with_name(marker.name + ".next")
163
967
  fd = os.open(
@@ -178,6 +982,7 @@ def record_activity(app_dir: pathlib.Path, provider: str, source_path: str) -> b
178
982
  try:
179
983
  os.chmod(marker, 0o600)
180
984
  os.chmod(lock_path, 0o600)
985
+ os.chmod(ledger_state_path(pathlib.Path(app_dir)), 0o600)
181
986
  except OSError:
182
987
  pass
183
988
  finally:
@@ -215,12 +1020,25 @@ class FrontierPlan:
215
1020
  paths: frozenset[str] = frozenset()
216
1021
  marker_end: int = 0
217
1022
  reason: str = "none"
1023
+ # The ledger generation this plan consumed and the last sequence in the
1024
+ # slice it consumed. Committing a non-full plan advances the consumer's
1025
+ # acknowledgement to exactly these, so the NEXT plan can prove it saw
1026
+ # every ticket in between.
1027
+ ledger_epoch: "str | None" = None
1028
+ ledger_sequence: int = 0
218
1029
 
219
1030
 
220
1031
  @dataclass(frozen=True)
221
1032
  class MarkerCutoff:
222
1033
  identity: tuple[int, int]
223
1034
  end: int
1035
+ # The ledger generation and sequence as at this boundary, read under the
1036
+ # SAME writer lock that produced `end`. Reading them afterwards would race
1037
+ # a ticket into the gap: the walk would leave that ticket pending on the
1038
+ # byte cursor while the consumer had already acknowledged its sequence,
1039
+ # and the next plan would report a duplicate that never happened.
1040
+ ledger_epoch: "str | None" = None
1041
+ ledger_sequence: int = 0
224
1042
 
225
1043
 
226
1044
  @dataclass(frozen=True)
@@ -244,6 +1062,15 @@ class _ProviderState:
244
1062
  # The default is deliberately far in the past so a state built without one
245
1063
  # is already expired rather than trusted forever.
246
1064
  seeded_at: float = 0.0
1065
+ # The ledger generation and the last sequence this consumer acknowledged.
1066
+ # `None` means no generation has ever been acknowledged, which no live
1067
+ # generation matches, so an unstamped state recovers rather than trusting.
1068
+ ack_epoch: "str | None" = None
1069
+ ack_sequence: int = 0
1070
+ # The shared provider evidence generation this certificate rests on, and
1071
+ # the normalized configured root set it was minted over.
1072
+ evidence_generation: "str | None" = None
1073
+ root_membership: tuple[str, ...] = ()
247
1074
 
248
1075
 
249
1076
  def _frontier_memory_stats(owner):
@@ -271,6 +1098,316 @@ def _admit_frontier_state(owner, provider: str, state: _ProviderState) -> bool:
271
1098
  return True
272
1099
 
273
1100
 
1101
+ def _issue_generation_for(owner, provider: str, conn) -> "str | None":
1102
+ """Join or mint this provider's evidence generation, or fail closed.
1103
+
1104
+ Returns ``None`` when this consumer's store contradicts another
1105
+ consumer's about which exhaustive walk it rests on. That is the
1106
+ cross-restart disagreement: the in-process generation is gone, the two
1107
+ stores' stamps are all that remain, and two different non-empty answers
1108
+ cannot both be true.
1109
+
1110
+ THE REFUSAL IS ONE-SHOT PER CONSUMER PER PROCESS, and it is not "self-
1111
+ correcting on the next walk" — commit ``2143bdfb8`` quoted that earlier
1112
+ wording as untrue of any walk, and this caller kept it. The refusal
1113
+ returns ``None``, so this seed FAILS and stamps nothing; the correction
1114
+ comes from the SECOND seed, which ``observe_stamp`` no longer refuses and
1115
+ which joins the live generation and writes it into this store. See
1116
+ ``FrontierGenerations.observe_stamp`` for why refusing more than once was
1117
+ an outage rather than a safeguard.
1118
+ """
1119
+ stamped = _read_generation_stamp(conn, provider)
1120
+ if not owner._generations.observe_stamp(
1121
+ provider, owner.CONSUMER, stamped
1122
+ ):
1123
+ return None
1124
+ generation = owner._generations.issue(provider, owner.CONSUMER)
1125
+ _write_generation_stamp(conn, provider, generation)
1126
+ return generation
1127
+
1128
+
1129
+ class FrontierGenerations:
1130
+ """One provider evidence generation, shared by every registered consumer.
1131
+
1132
+ #769 S6, #716 Task A. The two frontiers each held their own certificate
1133
+ state and their own byte cursor into one shared ticket ledger, and nothing
1134
+ tied them together: both could report themselves caught up while resting
1135
+ on different bodies of evidence, and neither could say whether the other
1136
+ had consumed the walk it was trusting.
1137
+
1138
+ A generation is minted by the first exhaustive walk that has no live
1139
+ generation to join. A later walk by another consumer JOINS it rather than
1140
+ replacing it, which is what makes "both consumers are on one generation" a
1141
+ statement with content. It RETIRES once every registered consumer has
1142
+ acknowledged it, and only then does the next walk mint a new one.
1143
+
1144
+ REGISTRATION IS BY PARTICIPATION, and that is the explicit form rather
1145
+ than an assumed one. A consumer registers the first time it actually seeds
1146
+ or plans a provider, so a process that never runs a transcript pass — the
1147
+ standalone TUI builds only an accounting frontier — has no conversation
1148
+ consumer to wait for and its generations retire normally. Registering at
1149
+ CONSTRUCTION would be wrong for a specific reason:
1150
+ `ConversationSyncFrontier.capture_cutoff` constructs a throwaway
1151
+ `DashboardIngestFrontier` purely to reuse one method, and that object
1152
+ would otherwise register an accounting consumer that never acknowledges
1153
+ anything and hold every generation open forever.
1154
+
1155
+ Every method takes the lock. Two dashboard threads reach one of these.
1156
+ """
1157
+
1158
+ def __init__(self, app_dir):
1159
+ self.app_dir = pathlib.Path(app_dir)
1160
+ self._lock = threading.RLock()
1161
+ self._registered: dict[str, set[str]] = {}
1162
+ self._live: dict[str, str] = {}
1163
+ self._acknowledged: dict[str, dict[str, str]] = {}
1164
+ self._observed: dict[str, dict[str, "str | None"]] = {}
1165
+ # The `(epoch, sequence)` each consumer last acknowledged into the
1166
+ # ticket ledger. Separate from `_acknowledged`, which tracks the
1167
+ # EVIDENCE generation: two consumers may legitimately sit at different
1168
+ # sequences inside one ledger generation, and the reconciliation below
1169
+ # judges only the generation.
1170
+ self._ledger_acks: dict[str, dict[str, tuple]] = {}
1171
+
1172
+ def _register_locked(self, provider: str, consumer: str) -> None:
1173
+ """Registration is by PARTICIPATION — see this class's docstring.
1174
+
1175
+ Private, and called from every method a consumer reaches, because
1176
+ there is no separate moment at which a consumer announces itself: the
1177
+ first plan, seed or acknowledgement IS the announcement, and a public
1178
+ `register` nobody called was the alternative this replaced.
1179
+ """
1180
+ self._registered.setdefault(provider, set()).add(consumer)
1181
+
1182
+ def record_ledger_acknowledgement(
1183
+ self, provider: str, consumer: str, epoch, sequence: int,
1184
+ ) -> None:
1185
+ """Publish where this consumer's byte cursor stands in the ledger."""
1186
+ if not epoch:
1187
+ return
1188
+ with self._lock:
1189
+ self._register_locked(provider, consumer)
1190
+ self._ledger_acks.setdefault(provider, {})[consumer] = (
1191
+ str(epoch), int(sequence))
1192
+
1193
+ def peer_ledger_acknowledgements(self, provider: str, consumer: str):
1194
+ """Every OTHER consumer's ledger acknowledgement, as a tuple."""
1195
+ with self._lock:
1196
+ return tuple(
1197
+ value for other, value in
1198
+ self._ledger_acks.get(provider, {}).items()
1199
+ if other != consumer
1200
+ )
1201
+
1202
+ def observe_stamp(self, provider: str, consumer: str, stamp) -> bool:
1203
+ """Record what this consumer's store claimed when first examined.
1204
+
1205
+ Returns False ONCE per consumer when that claim contradicts another
1206
+ consumer's. This is the cross-restart check: the in-process generation
1207
+ is gone, so the two stores' own stamps are the only remaining
1208
+ statement about which exhaustive walk each of them rests on, and two
1209
+ different non-empty answers cannot both be true.
1210
+
1211
+ REGISTERS THE CONSUMER, like every other method a consumer reaches.
1212
+ Without that, a refused consumer is not registered, so its peer can
1213
+ commit a clean plan, become the only registered consumer, retire the
1214
+ generation on its own, and leave the second seed minting a FRESH
1215
+ generation instead of joining the live one — which stamps this store
1216
+ with a third value and leaves the two stores still disagreeing. The
1217
+ cost of registering is the one the participation deviation in the
1218
+ session spec already records: a consumer that refuses and then never
1219
+ seeds again holds the live generation open for the life of the
1220
+ process. Retirement is NOT purely diagnostic — ``_retired_locked`` is
1221
+ what ``issue`` consults to choose between minting a fresh generation
1222
+ and rejoining the live one, while the public ``generation_retired``
1223
+ accessor is read only by tests. A generation held open therefore means
1224
+ later walks rejoin it and stamp both stores with the SAME value, which
1225
+ is agreement rather than divergence, and this path writes nothing to
1226
+ ``_ledger_acks``, so it cannot reach the seventh discontinuity. The
1227
+ repair it buys is what makes the two stores converge, so the trade is
1228
+ deliberate.
1229
+
1230
+ FIRST OBSERVATION WINS, and it is never overwritten. What this dict
1231
+ holds is each consumer's PRE-PROCESS claim — what its store said
1232
+ before this process touched it — and NOT a live mirror of disk. The
1233
+ distinction is load-bearing rather than pedantic: this process writes
1234
+ a generation into the store of every consumer that seeds
1235
+ successfully, so refreshing the record afterwards would let the value
1236
+ THIS process just wrote contradict a peer that had agreed with the
1237
+ estate all along, and force a spurious exhaustive walk. With more than
1238
+ two consumers the retained pre-process claim can outlive the
1239
+ disagreement it described, and a consumer whose first observation
1240
+ lands after another consumer's refusal was lifted may then refuse once
1241
+ against it. The one-shot key is ``(provider, consumer)``, so that
1242
+ bound is ONE extra exhaustive walk per consumer and the worst case
1243
+ over N consumers is N of them, not one. It is the direction to be
1244
+ wrong in, and production has exactly two consumers.
1245
+
1246
+ ONE-SHOT, and that bound is the whole difference between a safe check
1247
+ and an outage. Two stores holding different stamps is not a rare
1248
+ corruption: a generation retires once every registered consumer
1249
+ acknowledges it, the next exhaustive walk mints a fresh one, and only
1250
+ the store of the consumer that walked is stamped with it — so two
1251
+ consumers whose walks land in different rounds leave two different
1252
+ stamps behind as an ordinary outcome. Measured on the sealed #786
1253
+ current corpus after one ordinary dashboard run, both providers'
1254
+ stamps differed. A refusal that never lifted therefore held the
1255
+ refusing consumer on a full plan on EVERY tick, and #769 S6's own
1256
+ measurement caught the estate walking all 3,012 retained rollouts once
1257
+ every five seconds. Refusing once costs one exhaustive walk; that walk
1258
+ stamps this consumer's store, which is what makes the two agree again
1259
+ and what the recorded reasoning always claimed happened.
1260
+
1261
+ An absent stamp is not a claim. Every store predating this protocol is
1262
+ in that state, and so is a fresh install.
1263
+ """
1264
+ with self._lock:
1265
+ self._register_locked(provider, consumer)
1266
+ seen = self._observed.setdefault(provider, {})
1267
+ first_observation = consumer not in seen
1268
+ contradicted = stamp is not None and any(
1269
+ other != consumer and value is not None and value != stamp
1270
+ for other, value in seen.items()
1271
+ )
1272
+ seen.setdefault(consumer, stamp)
1273
+ return not (contradicted and first_observation)
1274
+
1275
+ def issue(self, provider: str, consumer: str) -> str:
1276
+ """The generation this consumer's exhaustive walk belongs to."""
1277
+ with self._lock:
1278
+ self._register_locked(provider, consumer)
1279
+ live = self._live.get(provider)
1280
+ if live is None or self._retired_locked(provider, live):
1281
+ live = secrets.token_hex(16)
1282
+ self._live[provider] = live
1283
+ self._acknowledged[provider] = {}
1284
+ return live
1285
+
1286
+ def acknowledge(self, provider: str, consumer: str, generation) -> None:
1287
+ if not generation:
1288
+ return
1289
+ with self._lock:
1290
+ self._register_locked(provider, consumer)
1291
+ self._acknowledged.setdefault(provider, {})[consumer] = generation
1292
+
1293
+ def drop(self, provider: str, consumer: str) -> None:
1294
+ """Deregister one consumer, e.g. after `drop_provider` discards it.
1295
+
1296
+ A consumer that has abandoned its certificate can no longer
1297
+ acknowledge anything, so leaving it registered would hold this
1298
+ generation and every later one open for the life of the process.
1299
+ """
1300
+ with self._lock:
1301
+ self._registered.get(provider, set()).discard(consumer)
1302
+ self._acknowledged.get(provider, {}).pop(consumer, None)
1303
+ # The ledger acknowledgement goes with it. A consumer that has
1304
+ # abandoned its certificate is no longer describing any ledger
1305
+ # position, so reconciling a live consumer against the cursor it
1306
+ # left behind would force exhaustive recovery over a claim nobody
1307
+ # is making any more.
1308
+ self._ledger_acks.get(provider, {}).pop(consumer, None)
1309
+
1310
+ def retired(self, provider: str, generation) -> bool:
1311
+ with self._lock:
1312
+ return self._retired_locked(provider, generation)
1313
+
1314
+ def _retired_locked(self, provider: str, generation) -> bool:
1315
+ if not generation:
1316
+ return True
1317
+ acknowledged = self._acknowledged.get(provider, {})
1318
+ registered = self._registered.get(provider, set())
1319
+ if not registered:
1320
+ return True
1321
+ return all(
1322
+ acknowledged.get(consumer) == generation
1323
+ for consumer in registered
1324
+ )
1325
+
1326
+
1327
+ #: One coordinator per data directory, because the generation is a property of
1328
+ #: the estate rather than of any object that happens to be looking at it. Two
1329
+ #: frontiers in one process must reach the same one, and they are constructed
1330
+ #: independently in `_cctally_dashboard` and `_cctally_tui`, so the shared
1331
+ #: object cannot be passed between them without changing both callers.
1332
+ _FRONTIER_GENERATIONS: "dict[str, FrontierGenerations]" = {}
1333
+ _FRONTIER_GENERATIONS_LOCK = threading.Lock()
1334
+
1335
+
1336
+ def frontier_generations(app_dir) -> FrontierGenerations:
1337
+ key = str(pathlib.Path(app_dir))
1338
+ with _FRONTIER_GENERATIONS_LOCK:
1339
+ existing = _FRONTIER_GENERATIONS.get(key)
1340
+ if existing is None:
1341
+ existing = FrontierGenerations(app_dir)
1342
+ _FRONTIER_GENERATIONS[key] = existing
1343
+ return existing
1344
+
1345
+
1346
+ # NOTE (#769 S6 T1 review R9): there is deliberately NO
1347
+ # `reset_frontier_generations`. One shipped in T1 as a production test hook and
1348
+ # was removed here, because #769 S6's constraints forbid adding one and the
1349
+ # `reset_*` family cited to justify it was created by this same session: as of
1350
+ # T1 the only commit introducing that sibling helper was 6d479343a, inside T1
1351
+ # itself. (Search for the definition rather than quoting the literal search
1352
+ # string, which would make this comment a second hit and falsify the claim.)
1353
+ # `reset_frontier_counters` stays because `bin/cctally-bench` calls it;
1354
+ # this one had no caller outside `tests/`. Isolation is now the test estate's
1355
+ # own job: `tests/conftest.py` substitutes `_FRONTIER_GENERATIONS` for the
1356
+ # duration of every test through `monkeypatch`, which restores the real
1357
+ # registry afterwards and is the mechanism `tests/_pytest_isolation_plugin.py`
1358
+ # already reasons about for a module global.
1359
+
1360
+
1361
+ def frontier_generation_meta_key(provider: str) -> str:
1362
+ """The `cache_meta` key each store stamps its acknowledged generation in."""
1363
+ return f"dashboard_frontier_generation_{provider}"
1364
+
1365
+
1366
+ def _read_generation_stamp(conn, provider: str) -> "str | None":
1367
+ try:
1368
+ row = conn.execute(
1369
+ "SELECT value FROM cache_meta WHERE key=? LIMIT 1",
1370
+ (frontier_generation_meta_key(provider),),
1371
+ ).fetchone()
1372
+ except sqlite3.DatabaseError:
1373
+ return None
1374
+ if row is None or not row[0]:
1375
+ return None
1376
+ return str(row[0])
1377
+
1378
+
1379
+ def _write_generation_stamp(conn, provider: str, generation: str) -> None:
1380
+ """Best effort. A store that cannot be stamped simply carries no claim.
1381
+
1382
+ Refusing the seed instead would turn a read-only or contended connection
1383
+ into a permanent full-walk loop, which is a worse outcome than losing one
1384
+ cross-restart consistency check.
1385
+ """
1386
+ try:
1387
+ conn.execute(
1388
+ "INSERT INTO cache_meta(key,value) VALUES(?,?) "
1389
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
1390
+ (frontier_generation_meta_key(provider), generation),
1391
+ )
1392
+ conn.commit()
1393
+ except sqlite3.DatabaseError:
1394
+ pass
1395
+
1396
+
1397
+ def normalized_root_membership(roots) -> tuple[str, ...]:
1398
+ """The configured root SET, as certificate identity.
1399
+
1400
+ Sorted and de-duplicated, so reordering `$CODEX_HOME` is not a change
1401
+ while adding or removing an entry is. Without this a configured root could
1402
+ be added or removed with no filesystem change anywhere and an old
1403
+ certificate would still look caught up, because plan-time roots were used
1404
+ only to validate ticket targets.
1405
+ """
1406
+ return tuple(sorted({
1407
+ str(pathlib.Path(root).absolute()) for root in roots
1408
+ }))
1409
+
1410
+
274
1411
  def _stat_identity(path: pathlib.Path) -> tuple[int, int, int, int]:
275
1412
  st = path.stat()
276
1413
  return (st.st_dev, st.st_ino, st.st_mtime_ns, st.st_ctime_ns)
@@ -320,14 +1457,10 @@ def _provider_complete(conn, provider: str) -> bool:
320
1457
 
321
1458
 
322
1459
  def _source_paths(conn, provider: str) -> tuple[pathlib.Path, ...]:
323
- query = (
324
- "SELECT path FROM session_files"
325
- if provider == "claude"
326
- else "SELECT path FROM codex_session_files"
327
- )
1460
+ table = _ingest_source_table(provider)
328
1461
  return tuple(
329
1462
  pathlib.Path(str(row[0]))
330
- for row in conn.execute(query)
1463
+ for row in conn.execute(f"SELECT path FROM {table}")
331
1464
  if row[0] and os.path.isabs(str(row[0]))
332
1465
  )
333
1466
 
@@ -382,7 +1515,9 @@ def _read_marker(marker: pathlib.Path, offset: int):
382
1515
  st = os.fstat(fh.fileno())
383
1516
  identity = (st.st_dev, st.st_ino)
384
1517
  if st.st_size < offset:
385
- raise ValueError("truncated")
1518
+ # The consumer's byte cursor points past the end of the ledger, so
1519
+ # its memory describes a file this one is not.
1520
+ raise ValueError("ledger_cursor_beyond_tail")
386
1521
  fh.seek(offset)
387
1522
  raw = fh.read()
388
1523
  end = fh.tell()
@@ -393,6 +1528,15 @@ def _read_marker(marker: pathlib.Path, offset: int):
393
1528
 
394
1529
 
395
1530
  def _decode_marker_records(raw: bytes):
1531
+ """Decode a ledger slice into ``(provider, path, epoch, sequence, cfg)``.
1532
+
1533
+ A record written before the continuity protocol carries neither an epoch
1534
+ nor a sequence, and it decodes to ``(provider, path, None, None)`` rather
1535
+ than raising. That distinction matters: an upgraded install still holds
1536
+ pre-protocol tickets ahead of its cursor, and treating them as malformed
1537
+ would refuse to seed a certificate at all instead of forcing one recovery
1538
+ walk that then consumes them.
1539
+ """
396
1540
  records = []
397
1541
  for line in raw.splitlines():
398
1542
  try:
@@ -404,7 +1548,22 @@ def _decode_marker_records(raw: bytes):
404
1548
  path = value.get("path")
405
1549
  if not isinstance(path, str):
406
1550
  raise ValueError("malformed")
407
- records.append((value["provider"], path))
1551
+ epoch = value.get("epoch")
1552
+ sequence = value.get("seq")
1553
+ if epoch is not None and not isinstance(epoch, str):
1554
+ raise ValueError("malformed")
1555
+ if sequence is not None and (
1556
+ not isinstance(sequence, int) or isinstance(sequence, bool)
1557
+ ):
1558
+ raise ValueError("malformed")
1559
+ if (epoch is None) != (sequence is None):
1560
+ # Half a stamp is not a legacy record; it is a record whose
1561
+ # continuity claim cannot be evaluated at all.
1562
+ raise ValueError("malformed")
1563
+ generation = value.get("cfg")
1564
+ if generation is not None and not isinstance(generation, str):
1565
+ raise ValueError("malformed")
1566
+ records.append((value["provider"], path, epoch, sequence, generation))
408
1567
  return records
409
1568
 
410
1569
 
@@ -502,11 +1661,25 @@ def provider_sync_certifiable(mode: str, stats: object) -> bool:
502
1661
  class DashboardIngestFrontier:
503
1662
  """One process-local certificate per provider, rebuilt after full sync."""
504
1663
 
1664
+ #: This frontier's name in the shared generation coordinator.
1665
+ CONSUMER = "accounting"
1666
+
505
1667
  def __init__(self, app_dir: pathlib.Path):
506
1668
  self.app_dir = pathlib.Path(app_dir)
507
1669
  self._states: dict[str, _ProviderState] = {}
508
1670
  self.last_seed_failure: dict[str, str] = {}
509
1671
  self._fallback_count = 0
1672
+ self._generations = frontier_generations(self.app_dir)
1673
+
1674
+ def _issue_generation(self, provider: str, conn) -> "str | None":
1675
+ return _issue_generation_for(self, provider, conn)
1676
+
1677
+ def evidence_generation(self, provider: str) -> "str | None":
1678
+ state = self._states.get(provider)
1679
+ return None if state is None else state.evidence_generation
1680
+
1681
+ def generation_retired(self, provider: str, generation) -> bool:
1682
+ return self._generations.retired(provider, generation)
510
1683
 
511
1684
  def memory_stats(self):
512
1685
  return _frontier_memory_stats(self)
@@ -529,6 +1702,20 @@ class DashboardIngestFrontier:
529
1702
  fcntl.flock(lock_fd, fcntl.LOCK_EX)
530
1703
  marker_fd = os.open(marker, os.O_RDONLY | os.O_CREAT, 0o600)
531
1704
  st = os.fstat(marker_fd)
1705
+ # Materialize the ledger generation here for the same reason the
1706
+ # marker itself is materialized here: the first full walk needs a
1707
+ # real zero-sequence boundary to acknowledge. Without one the
1708
+ # consumer would acknowledge "no ledger", the first hook ticket
1709
+ # would mint a generation, and that first ticket would present as
1710
+ # a replacement rather than as the append it is.
1711
+ ledger = read_ledger_state(self.app_dir)
1712
+ if ledger is None:
1713
+ # A `LedgerState`, not a bare pair. Every other reader of this
1714
+ # value reaches it by attribute, and a 2-tuple standing in for
1715
+ # a 3-field record is exactly the kind of substitution that
1716
+ # works until somebody reads the third field.
1717
+ ledger = LedgerState(_mint_ledger_epoch(), 0)
1718
+ _write_ledger_state(self.app_dir, ledger.epoch, ledger.sequence)
532
1719
  except OSError:
533
1720
  return _MARKER_CUTOFF_FAILURE
534
1721
  finally:
@@ -539,7 +1726,9 @@ class DashboardIngestFrontier:
539
1726
  fcntl.flock(lock_fd, fcntl.LOCK_UN)
540
1727
  finally:
541
1728
  os.close(lock_fd)
542
- return MarkerCutoff((st.st_dev, st.st_ino), st.st_size)
1729
+ return MarkerCutoff(
1730
+ (st.st_dev, st.st_ino), st.st_size,
1731
+ ledger.epoch, ledger.sequence)
543
1732
 
544
1733
  def seed_provider(
545
1734
  self, provider: str, conn, *, roots, guard_paths=(), trusted=True,
@@ -574,6 +1763,11 @@ class DashboardIngestFrontier:
574
1763
  self.last_seed_failure[provider] = "incomplete_store"
575
1764
  self._states.pop(provider, None)
576
1765
  return False
1766
+ generation = self._issue_generation(provider, conn)
1767
+ if generation is None:
1768
+ self.last_seed_failure[provider] = "generation_disagreement"
1769
+ self._states.pop(provider, None)
1770
+ return False
577
1771
  candidate = _ProviderState(
578
1772
  marker_identity=marker_identity,
579
1773
  marker_offset=boundary.end,
@@ -583,9 +1777,23 @@ class DashboardIngestFrontier:
583
1777
  directory_identity=_directory_identity(conn, provider, roots),
584
1778
  guard_identity=_guard_identity(guard_paths),
585
1779
  seeded_at=_now(),
1780
+ # The exhaustive walk consumed everything up to the cutoff, so
1781
+ # the ledger position AT the cutoff is what it acknowledges.
1782
+ ack_epoch=boundary.ledger_epoch,
1783
+ ack_sequence=boundary.ledger_sequence,
1784
+ evidence_generation=generation,
1785
+ root_membership=normalized_root_membership(roots),
586
1786
  )
587
1787
  if not _admit_frontier_state(self, provider, candidate):
588
1788
  return False
1789
+ # Published so the OTHER consumer can reconcile against it. A
1790
+ # successful exhaustive walk consumed everything up to the cutoff,
1791
+ # so the cutoff's ledger position IS this consumer's new
1792
+ # acknowledgement, and recording it only at commit time would
1793
+ # leave a freshly reseeded consumer invisible to its peer.
1794
+ self._generations.record_ledger_acknowledgement(
1795
+ provider, self.CONSUMER,
1796
+ boundary.ledger_epoch, boundary.ledger_sequence)
589
1797
  self.last_seed_failure[provider] = ""
590
1798
  return True
591
1799
  except (OSError, ValueError) as exc:
@@ -593,7 +1801,18 @@ class DashboardIngestFrontier:
593
1801
  self._states.pop(provider, None)
594
1802
  return False
595
1803
 
596
- def plan_provider(self, provider: str, conn, *, roots, guard_paths=()) -> FrontierPlan:
1804
+ def plan_provider(
1805
+ self, provider: str, conn, *, roots, guard_paths=(),
1806
+ configuration_generation=_DERIVE_CONFIGURATION_GENERATION,
1807
+ ) -> FrontierPlan:
1808
+ return _counted_plan(self, _plan_accounting, provider, conn,
1809
+ roots=roots, guard_paths=guard_paths,
1810
+ configuration_generation=configuration_generation)
1811
+
1812
+ def _plan_uncounted(
1813
+ self, provider: str, conn, *, roots, guard_paths=(),
1814
+ configuration_generation=_DERIVE_CONFIGURATION_GENERATION,
1815
+ ) -> FrontierPlan:
597
1816
  state = self._states.get(provider)
598
1817
  if state is None:
599
1818
  return FrontierPlan(provider, "full", reason="unseeded")
@@ -614,6 +1833,12 @@ class DashboardIngestFrontier:
614
1833
  raise ValueError("maintenance_changed")
615
1834
  if _guard_identity(guard_paths) != state.guard_identity:
616
1835
  raise ValueError("hook_config_changed")
1836
+ # The configured root SET, before any filesystem comparison. A
1837
+ # root added or removed with no filesystem change anywhere moves
1838
+ # none of the directory identities below, because those describe
1839
+ # only the roots the certificate was minted over.
1840
+ if normalized_root_membership(roots) != state.root_membership:
1841
+ raise ValueError("root_membership_changed")
617
1842
  if _restat_directory_identity(
618
1843
  state.directory_identity
619
1844
  ) != state.directory_identity:
@@ -625,27 +1850,25 @@ class DashboardIngestFrontier:
625
1850
  except (OSError, ValueError) as exc:
626
1851
  reason = str(exc) if str(exc) else "ambiguous"
627
1852
  return FrontierPlan(provider, "full", reason=reason)
628
- provider_paths = tuple(
629
- path for kind, path in records if kind == provider)
630
- if any(not path for path in provider_paths):
1853
+ reason, ledger_epoch, ledger_sequence = _ledger_slice_decision(
1854
+ self, provider, state, records)
1855
+ if reason is not None:
1856
+ return FrontierPlan(provider, "full", reason=reason)
1857
+ reason = _configuration_generation_reason(
1858
+ self, provider, records, guard_paths, configuration_generation)
1859
+ if reason is not None:
631
1860
  return FrontierPlan(
632
- provider, "full", marker_end=marker_end,
633
- reason="ambiguous_activity",
634
- )
635
- paths = frozenset(provider_paths)
636
- if any(not _target_in_roots(path, roots) for path in paths):
1861
+ provider, "full", marker_end=marker_end, reason=reason)
1862
+ paths, reason = _ticket_target_paths(provider, records, roots)
1863
+ if reason is not None:
637
1864
  return FrontierPlan(
638
- provider, "full", marker_end=marker_end,
639
- reason="target_outside_scope",
640
- )
1865
+ provider, "full", marker_end=marker_end, reason=reason)
641
1866
  if any(_target_has_cursor_gap(conn, provider, path) for path in paths):
642
1867
  return FrontierPlan(provider, "full", reason="cursor_gap")
643
- return FrontierPlan(
644
- provider,
645
- "targeted" if paths else "caught_up",
646
- paths=paths,
647
- marker_end=marker_end,
648
- reason="activity" if paths else "unchanged",
1868
+ return _restat_and_return(
1869
+ provider, conn, cursor_table=_ingest_source_table(provider),
1870
+ paths=paths, marker_end=marker_end, ledger_epoch=ledger_epoch,
1871
+ ledger_sequence=ledger_sequence,
649
1872
  )
650
1873
 
651
1874
  def commit_provider(
@@ -662,6 +1885,21 @@ class DashboardIngestFrontier:
662
1885
  state = self._states.get(plan.provider)
663
1886
  if state is None:
664
1887
  return
1888
+ # The acknowledgement moves with the BYTE cursor, not with this
1889
+ # provider's own tickets: one ledger serves both providers, so a
1890
+ # Claude ticket the Codex consumer skipped over is still a sequence
1891
+ # the Codex consumer has now passed. Leaving it unacknowledged would
1892
+ # make the next plan read a hole that is not a hole.
1893
+ if plan.ledger_epoch is not None:
1894
+ state.ack_epoch = plan.ledger_epoch
1895
+ state.ack_sequence = plan.ledger_sequence
1896
+ self._generations.record_ledger_acknowledgement(
1897
+ plan.provider, self.CONSUMER,
1898
+ plan.ledger_epoch, plan.ledger_sequence)
1899
+ # Committing a clean plan is this consumer's acknowledgement that it
1900
+ # has consumed the evidence generation its certificate rests on.
1901
+ self._generations.acknowledge(
1902
+ plan.provider, self.CONSUMER, state.evidence_generation)
665
1903
  if plan.mode == "caught_up":
666
1904
  # The plan already compared every guard. Advancing an empty marker
667
1905
  # slice must remain O(1); recomputing the estate here would erase
@@ -796,11 +2034,25 @@ class ConversationSyncFrontier:
796
2034
  guards the transcript store's own schema/cursor tables.
797
2035
  """
798
2036
 
2037
+ #: This frontier's name in the shared generation coordinator.
2038
+ CONSUMER = "conversation"
2039
+
799
2040
  def __init__(self, app_dir: pathlib.Path):
800
2041
  self.app_dir = pathlib.Path(app_dir)
801
2042
  self._states: dict[str, _ProviderState] = {}
802
2043
  self.last_seed_failure: dict[str, str] = {}
803
2044
  self._fallback_count = 0
2045
+ self._generations = frontier_generations(self.app_dir)
2046
+
2047
+ def _issue_generation(self, provider: str, conn) -> "str | None":
2048
+ return _issue_generation_for(self, provider, conn)
2049
+
2050
+ def evidence_generation(self, provider: str) -> "str | None":
2051
+ state = self._states.get(provider)
2052
+ return None if state is None else state.evidence_generation
2053
+
2054
+ def generation_retired(self, provider: str, generation) -> bool:
2055
+ return self._generations.retired(provider, generation)
804
2056
 
805
2057
  def memory_stats(self):
806
2058
  return _frontier_memory_stats(self)
@@ -811,6 +2063,26 @@ class ConversationSyncFrontier:
811
2063
  # different meaning to the first ticket in a newly-created marker.
812
2064
  return DashboardIngestFrontier(self.app_dir).capture_cutoff()
813
2065
 
2066
+ def drop_provider(self, provider: str) -> bool:
2067
+ """Discard one provider's certificate, returning whether one existed.
2068
+
2069
+ For work that mutates the transcript store OUTSIDE a sync pass, where
2070
+ no stats object reaches `provider_sync_certifiable` (#729). The orphan
2071
+ prune is that case: its deletions commit on their own, and when the
2072
+ rollup re-derive that should follow them is refused, a certificate
2073
+ minted before the prune would let the next pass skip exactly the work
2074
+ the refusal left undone.
2075
+ """
2076
+ if provider not in _PROVIDERS:
2077
+ raise ValueError("unknown provider")
2078
+ # Deregister before discarding the certificate. A consumer with no
2079
+ # certificate cannot acknowledge anything, so leaving it registered
2080
+ # would hold this generation and every later one open for the life of
2081
+ # the process — and the accounting frontier would then never see one
2082
+ # retire, however cleanly it committed.
2083
+ self._generations.drop(provider, self.CONSUMER)
2084
+ return self._states.pop(provider, None) is not None
2085
+
814
2086
  def seed_provider(
815
2087
  self, provider: str, conn, *, roots, guard_paths=(), trusted=True,
816
2088
  cutoff: "MarkerCutoff | _MarkerCutoffFailure | None" = None,
@@ -834,6 +2106,14 @@ class ConversationSyncFrontier:
834
2106
  pending = _conversation_pending_identity(conn)
835
2107
  if pending:
836
2108
  raise ValueError("maintenance_pending")
2109
+ generation = self._issue_generation(provider, conn)
2110
+ if generation is None:
2111
+ # Reported verbatim rather than through the enclosing
2112
+ # `except`, whose handler prefixes the exception type: this is
2113
+ # a named refusal, not an unexpected error.
2114
+ self.last_seed_failure[provider] = "generation_disagreement"
2115
+ self._states.pop(provider, None)
2116
+ return False
837
2117
  candidate = _ProviderState(
838
2118
  marker_identity=marker_identity,
839
2119
  marker_offset=boundary.end,
@@ -844,9 +2124,21 @@ class ConversationSyncFrontier:
844
2124
  conn, provider, roots),
845
2125
  guard_identity=_guard_identity(guard_paths),
846
2126
  seeded_at=_now(),
2127
+ ack_epoch=boundary.ledger_epoch,
2128
+ ack_sequence=boundary.ledger_sequence,
2129
+ evidence_generation=generation,
2130
+ root_membership=normalized_root_membership(roots),
847
2131
  )
848
2132
  if not _admit_frontier_state(self, provider, candidate):
849
2133
  return False
2134
+ # Published so the OTHER consumer can reconcile against it. A
2135
+ # successful exhaustive walk consumed everything up to the cutoff,
2136
+ # so the cutoff's ledger position IS this consumer's new
2137
+ # acknowledgement, and recording it only at commit time would
2138
+ # leave a freshly reseeded consumer invisible to its peer.
2139
+ self._generations.record_ledger_acknowledgement(
2140
+ provider, self.CONSUMER,
2141
+ boundary.ledger_epoch, boundary.ledger_sequence)
850
2142
  self.last_seed_failure[provider] = ""
851
2143
  return True
852
2144
  except (OSError, ValueError) as exc:
@@ -854,7 +2146,18 @@ class ConversationSyncFrontier:
854
2146
  self._states.pop(provider, None)
855
2147
  return False
856
2148
 
857
- def plan_provider(self, provider: str, conn, *, roots, guard_paths=()):
2149
+ def plan_provider(
2150
+ self, provider: str, conn, *, roots, guard_paths=(),
2151
+ configuration_generation=_DERIVE_CONFIGURATION_GENERATION,
2152
+ ):
2153
+ return _counted_plan(self, _plan_conversation, provider, conn,
2154
+ roots=roots, guard_paths=guard_paths,
2155
+ configuration_generation=configuration_generation)
2156
+
2157
+ def _plan_uncounted(
2158
+ self, provider: str, conn, *, roots, guard_paths=(),
2159
+ configuration_generation=_DERIVE_CONFIGURATION_GENERATION,
2160
+ ):
858
2161
  state = self._states.get(provider)
859
2162
  if state is None:
860
2163
  return FrontierPlan(provider, "full", reason="unseeded")
@@ -873,6 +2176,12 @@ class ConversationSyncFrontier:
873
2176
  raise ValueError("maintenance_changed")
874
2177
  if _guard_identity(guard_paths) != state.guard_identity:
875
2178
  raise ValueError("hook_config_changed")
2179
+ # The configured root SET, before any filesystem comparison. A
2180
+ # root added or removed with no filesystem change anywhere moves
2181
+ # none of the directory identities below, because those describe
2182
+ # only the roots the certificate was minted over.
2183
+ if normalized_root_membership(roots) != state.root_membership:
2184
+ raise ValueError("root_membership_changed")
876
2185
  if _restat_directory_identity(
877
2186
  state.directory_identity
878
2187
  ) != state.directory_identity:
@@ -884,17 +2193,19 @@ class ConversationSyncFrontier:
884
2193
  except (OSError, ValueError) as exc:
885
2194
  return FrontierPlan(
886
2195
  provider, "full", reason=str(exc) or "ambiguous")
887
- provider_paths = tuple(
888
- path for kind, path in records if kind == provider)
889
- if any(not path for path in provider_paths):
2196
+ reason, ledger_epoch, ledger_sequence = _ledger_slice_decision(
2197
+ self, provider, state, records)
2198
+ if reason is not None:
2199
+ return FrontierPlan(provider, "full", reason=reason)
2200
+ reason = _configuration_generation_reason(
2201
+ self, provider, records, guard_paths, configuration_generation)
2202
+ if reason is not None:
890
2203
  return FrontierPlan(
891
- provider, "full", marker_end=marker_end,
892
- reason="ambiguous_activity")
893
- paths = frozenset(provider_paths)
894
- if any(not _target_in_roots(path, roots) for path in paths):
2204
+ provider, "full", marker_end=marker_end, reason=reason)
2205
+ paths, reason = _ticket_target_paths(provider, records, roots)
2206
+ if reason is not None:
895
2207
  return FrontierPlan(
896
- provider, "full", marker_end=marker_end,
897
- reason="target_outside_scope")
2208
+ provider, "full", marker_end=marker_end, reason=reason)
898
2209
  target_risks = {
899
2210
  _conversation_target_risk(conn, provider, path)
900
2211
  for path in paths
@@ -903,12 +2214,10 @@ class ConversationSyncFrontier:
903
2214
  return FrontierPlan(provider, "full", reason="source_replaced")
904
2215
  if "cursor_gap" in target_risks:
905
2216
  return FrontierPlan(provider, "full", reason="cursor_gap")
906
- return FrontierPlan(
907
- provider,
908
- "targeted" if paths else "caught_up",
909
- paths=paths,
910
- marker_end=marker_end,
911
- reason="activity" if paths else "unchanged",
2217
+ return _restat_and_return(
2218
+ provider, conn, cursor_table=_conversation_source_table(provider),
2219
+ paths=paths, marker_end=marker_end, ledger_epoch=ledger_epoch,
2220
+ ledger_sequence=ledger_sequence,
912
2221
  )
913
2222
 
914
2223
  def commit_provider(
@@ -923,6 +2232,14 @@ class ConversationSyncFrontier:
923
2232
  state = self._states.get(plan.provider)
924
2233
  if state is None:
925
2234
  return
2235
+ if plan.ledger_epoch is not None:
2236
+ state.ack_epoch = plan.ledger_epoch
2237
+ state.ack_sequence = plan.ledger_sequence
2238
+ self._generations.record_ledger_acknowledgement(
2239
+ plan.provider, self.CONSUMER,
2240
+ plan.ledger_epoch, plan.ledger_sequence)
2241
+ self._generations.acknowledge(
2242
+ plan.provider, self.CONSUMER, state.evidence_generation)
926
2243
  state.marker_offset = plan.marker_end
927
2244
  if plan.mode == "caught_up":
928
2245
  return