cctally 1.114.0 → 1.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/bin/_cctally_account.py +32 -8
  3. package/bin/_cctally_cache.py +4394 -279
  4. package/bin/_cctally_completion.py +1287 -0
  5. package/bin/_cctally_config.py +37 -0
  6. package/bin/_cctally_core.py +178 -2
  7. package/bin/_cctally_dashboard.py +2 -2
  8. package/bin/_cctally_dashboard_conversation.py +292 -23
  9. package/bin/_cctally_dashboard_envelope.py +16 -3
  10. package/bin/_cctally_dashboard_sources.py +18 -9
  11. package/bin/_cctally_db.py +316 -5
  12. package/bin/_cctally_doctor.py +436 -27
  13. package/bin/_cctally_five_hour.py +82 -31
  14. package/bin/_cctally_forecast.py +81 -9
  15. package/bin/_cctally_journal.py +3884 -112
  16. package/bin/_cctally_journal_repair.py +101 -12
  17. package/bin/_cctally_parser.py +42 -0
  18. package/bin/_cctally_project.py +8 -2
  19. package/bin/_cctally_quarantine.py +652 -0
  20. package/bin/_cctally_quota.py +26 -4
  21. package/bin/_cctally_quota_model.py +360 -83
  22. package/bin/_cctally_record.py +677 -149
  23. package/bin/_cctally_rederive.py +297 -32
  24. package/bin/_cctally_refresh.py +84 -13
  25. package/bin/_cctally_source_analytics.py +23 -2
  26. package/bin/_cctally_statusline.py +6137 -517
  27. package/bin/_cctally_store.py +287 -48
  28. package/bin/_cctally_transcript.py +8 -7
  29. package/bin/_cctally_tui.py +57 -53
  30. package/bin/_lib_account_binding.py +152 -0
  31. package/bin/_lib_account_protocol.py +386 -0
  32. package/bin/_lib_aggregators.py +4 -0
  33. package/bin/_lib_cache_coverage.py +13 -1
  34. package/bin/_lib_cache_writer_lock.py +19 -3
  35. package/bin/_lib_clone_copy.py +142 -0
  36. package/bin/_lib_codex_conversation_query.py +133 -19
  37. package/bin/_lib_codex_conversation_watch.py +59 -14
  38. package/bin/_lib_codex_drain_admission.py +214 -0
  39. package/bin/_lib_codex_metadata.py +18 -1
  40. package/bin/_lib_codex_publication_barrier.py +277 -0
  41. package/bin/_lib_codex_rollout_container.py +502 -0
  42. package/bin/_lib_conversation_dispatch.py +3 -0
  43. package/bin/_lib_conversation_query.py +11 -3
  44. package/bin/_lib_detector_access_registry.py +3817 -0
  45. package/bin/_lib_doctor.py +452 -1
  46. package/bin/_lib_five_hour.py +126 -0
  47. package/bin/_lib_ingest_frontier.py +414 -20
  48. package/bin/_lib_journal.py +141 -10
  49. package/bin/_lib_jsonl.py +5 -4
  50. package/bin/_lib_legacy_spool.py +126 -0
  51. package/bin/_lib_legacy_writer_evidence.py +136 -0
  52. package/bin/_lib_merge_gate.py +29 -302
  53. package/bin/_lib_meter_rate_change.py +10 -0
  54. package/bin/_lib_outcome_quarantine.py +690 -0
  55. package/bin/_lib_pricing.py +7 -2
  56. package/bin/_lib_quarantine.py +154 -0
  57. package/bin/_lib_quota_calibration.py +53 -8
  58. package/bin/_lib_quota_copy.py +2 -1
  59. package/bin/_lib_quota_model.py +288 -46
  60. package/bin/_lib_rederive.py +94 -5
  61. package/bin/_lib_segment_summary.py +35 -2
  62. package/bin/_lib_statusline_candidates.py +292 -23
  63. package/bin/cctally +23 -1
  64. package/package.json +14 -1
@@ -27,11 +27,13 @@ import json
27
27
  import os
28
28
  import re
29
29
  import sqlite3
30
+ import stat
30
31
  import threading
31
32
  from collections import deque
32
33
  from types import MappingProxyType
33
34
 
34
35
  import _lib_codex_conversation as kern
36
+ import _lib_codex_rollout_container as _rc
35
37
  import _lib_codex_landmarks as landmarks
36
38
  import _lib_codex_segments as segkern
37
39
  from _lib_codex_find_projection import (
@@ -1863,6 +1865,29 @@ def codex_conversation_exists(conn: sqlite3.Connection, conversation_key: str) -
1863
1865
  return row is not None
1864
1866
 
1865
1867
 
1868
+ def codex_publication_pending(
1869
+ conn: sqlite3.Connection, conversation_key: str,
1870
+ ) -> bool:
1871
+ """A barrier-population conversation with no publication yet (#968 5.11).
1872
+
1873
+ Such a conversation stays in the existing pending-state handling until it
1874
+ has an eligible publication: it answers ``normalization_pending``, never
1875
+ ``not_found``. Any store error answers False (the caller's ``not_found``).
1876
+ """
1877
+ try:
1878
+ if conn.execute(
1879
+ "SELECT 1 FROM cache_db.codex_barrier_population "
1880
+ "WHERE conversation_key=?", (conversation_key,)).fetchone() is None:
1881
+ return False
1882
+ row = conn.execute(
1883
+ "SELECT published_revision, legacy_uncertified "
1884
+ "FROM main.codex_conversation_publications WHERE conversation_key=?",
1885
+ (conversation_key,)).fetchone()
1886
+ except sqlite3.Error:
1887
+ return False
1888
+ return row is None or (int(row[0] or 0) == 0 and not row[1])
1889
+
1890
+
1866
1891
  def codex_conversation_source_paths(
1867
1892
  conn: sqlite3.Connection, conversation_key: str
1868
1893
  ) -> list[str]:
@@ -2345,6 +2370,9 @@ def get_codex_conversation(
2345
2370
  # ── Phase A: the narrow index pass ───────────────────────────────────────
2346
2371
  rows, detail_bytes = _load_conversation_index_rows(conn, conversation_key)
2347
2372
  if not rows:
2373
+ if codex_publication_pending(conn, conversation_key):
2374
+ return {"status": "normalization_pending",
2375
+ "conversation_key": conversation_key, "items": [], "children": []}
2348
2376
  return {"status": "not_found", "conversation_key": conversation_key}
2349
2377
  undecodable_keys = undecodable_codex_conversation_keys(conn)
2350
2378
  kept, _suppressed = kern.pair_mirrors(rows)
@@ -3035,6 +3063,9 @@ def _outline_envelope(
3035
3063
  # The outline is not the route F3 bounds, and it stays turn-granular.
3036
3064
  rows = _load_conversation_rows(conn, conversation_key)
3037
3065
  if not rows:
3066
+ if codex_publication_pending(conn, conversation_key):
3067
+ return {"status": "normalization_pending",
3068
+ "conversation_key": conversation_key, "turns": [], "files": [], "children": []}
3038
3069
  return {"status": "not_found", "conversation_key": conversation_key}
3039
3070
  detail_bytes = _detail_bytes_of(rows)
3040
3071
  kept, _suppressed = kern.pair_mirrors(rows)
@@ -3421,20 +3452,26 @@ def _stored_browse_page_queries(
3421
3452
  "(COALESCE(r.last_activity_utc, '') = ? AND r.conversation_key < ?))")
3422
3453
  page_params.extend((cursor_last, cursor_last, cursor_key))
3423
3454
 
3455
+ # Keep native/parent enrichment inside the page statement. A LEFT JOIN
3456
+ # against the compound last-good view materializes retained history;
3457
+ # correlated probes push these predicates into its indexed branches.
3458
+ # The scalar parent lookup follows the detail reader's same-root lookup.
3459
+ parent_key_sql = (
3460
+ "(SELECT pt.conversation_key FROM codex_conversation_threads pt "
3461
+ "WHERE pt.source_root_key = r.source_root_key "
3462
+ "AND pt.native_thread_id = r.parent_thread_id "
3463
+ "AND pt.conversation_key != r.conversation_key LIMIT 1)"
3464
+ )
3424
3465
  sql = (
3425
3466
  "SELECT r.conversation_key, r.item_count, r.started_utc, "
3426
3467
  "r.last_activity_utc, r.project_key, r.project_label, r.models_json, "
3427
- "r.title, r.parent_thread_id, r.source_root_key, t.native_thread_id, "
3428
- "pt.conversation_key, pr.title, pr.project_label, pt.native_thread_id "
3468
+ "r.title, r.parent_thread_id, r.source_root_key, "
3469
+ "(SELECT t.native_thread_id FROM codex_conversation_threads t "
3470
+ "WHERE t.conversation_key = r.conversation_key), "
3471
+ f"{parent_key_sql}, pr.title, pr.project_label, r.parent_thread_id "
3429
3472
  "FROM codex_conversation_rollups r "
3430
- "LEFT JOIN codex_conversation_threads t "
3431
- "ON t.conversation_key = r.conversation_key "
3432
- "LEFT JOIN codex_conversation_threads pt "
3433
- "ON pt.source_root_key = r.source_root_key "
3434
- "AND pt.native_thread_id = r.parent_thread_id "
3435
- "AND pt.conversation_key != r.conversation_key "
3436
3473
  "LEFT JOIN codex_conversation_rollups pr "
3437
- "ON pr.conversation_key = pt.conversation_key "
3474
+ f"ON pr.conversation_key = {parent_key_sql} "
3438
3475
  f"WHERE {' AND '.join(page_where)} "
3439
3476
  "ORDER BY COALESCE(r.last_activity_utc, '') DESC, r.conversation_key DESC"
3440
3477
  )
@@ -4962,6 +4999,8 @@ def find_in_codex_conversation(
4962
4999
  return {**base, "status": "normalization_pending"}
4963
5000
  rows = _load_conversation_rows(conn, conversation_key)
4964
5001
  if not rows:
5002
+ if codex_publication_pending(conn, conversation_key):
5003
+ return {**base, "status": "normalization_pending"}
4965
5004
  return {"status": "not_found", "conversation_key": conversation_key}
4966
5005
  q = (query or "").strip()
4967
5006
  if not q:
@@ -5021,6 +5060,9 @@ def codex_conversation_prompts(conn: sqlite3.Connection, conversation_key: str)
5021
5060
  "conversation_key": conversation_key, "prompts": []}
5022
5061
  rows = _load_conversation_rows(conn, conversation_key)
5023
5062
  if not rows:
5063
+ if codex_publication_pending(conn, conversation_key):
5064
+ return {"status": "normalization_pending",
5065
+ "conversation_key": conversation_key, "prompts": []}
5024
5066
  return {"status": "not_found", "conversation_key": conversation_key}
5025
5067
  kept, _suppressed = kern.pair_mirrors(rows)
5026
5068
  items = kern.canonical_items(kept)
@@ -5061,7 +5103,40 @@ def _within_root(source_path: str | None, root_path: str | None) -> bool:
5061
5103
  return False
5062
5104
 
5063
5105
 
5064
- def _reread_codex_full_content(conn: sqlite3.Connection, row):
5106
+ class CodexContainerReadError(RuntimeError):
5107
+ """A present rollout failed to read; the query envelope reports HTTP 500."""
5108
+
5109
+ def __init__(self, category: str, logical: str) -> None:
5110
+ super().__init__(f"Codex rollout could not be read ({category})")
5111
+ self.category = category
5112
+ self.logical = logical
5113
+
5114
+
5115
+ class _CodexPayloadContainmentError(RuntimeError):
5116
+ """Internal 404 signal: the selected physical container escaped its root."""
5117
+
5118
+
5119
+ def _select_codex_payload_container(logical):
5120
+ # The discovery leaf deliberately treats stat failures as absent. A payload
5121
+ # request must distinguish a missing container (410) from inaccessible data
5122
+ # (500), including an inaccessible plain file beside a readable zstd twin.
5123
+ for candidate in (logical, _rc.zstd_twin(logical)):
5124
+ try:
5125
+ info = os.stat(candidate)
5126
+ except (FileNotFoundError, NotADirectoryError):
5127
+ continue
5128
+ except OSError as exc:
5129
+ raise CodexContainerReadError("read-error", str(logical)) from exc
5130
+ if not stat.S_ISREG(info.st_mode):
5131
+ raise CodexContainerReadError("read-error", str(logical))
5132
+ selected = _rc.select_container(logical)
5133
+ if selected is not None:
5134
+ return selected
5135
+ # It disappeared between the strict stat and selection; try the twin.
5136
+ return None
5137
+
5138
+
5139
+ def _reread_codex_full_content(conn: sqlite3.Connection, row, *, streams: "dict | None" = None):
5065
5140
  """Re-read the physical line at ``(row.source_path, row.line_offset)``, validate
5066
5141
  it against the stored ``codex_conversation_events.payload_json`` for that exact
5067
5142
  position (§3.4 structural gone-check — the canonical FULL record, not
@@ -5073,19 +5148,49 @@ def _reread_codex_full_content(conn: sqlite3.Connection, row):
5073
5148
  The full pre-cap content is re-derived through the SAME ``_extract`` the
5074
5149
  normalizer uses — which is how payload serves content beyond the normalized
5075
5150
  ``CODEX_TEXT_CAP``. Truncation/``truncated`` is against ``_FULL_PAYLOAD_CEILING``
5076
- (1,000,000 Python characters), the same ceiling the Claude payload path uses."""
5151
+ (1,000,000 Python characters), the same ceiling the Claude payload path uses.
5152
+
5153
+ #968: select plain first, else zstd, and seek in decoded offsets. Streams are
5154
+ request-local and reusable; no decoded copies persist. Present unreadable
5155
+ containers raise CodexContainerReadError, never a false gone result."""
5077
5156
  stored = conn.execute(
5078
5157
  "SELECT payload_json FROM codex_conversation_events "
5079
5158
  "WHERE source_path = ? AND line_offset = ?",
5080
5159
  (row.source_path, row.line_offset)).fetchone()
5081
5160
  if stored is None:
5082
5161
  return None
5083
- try:
5084
- with open(row.source_path, "rb") as fh:
5085
- fh.seek(row.line_offset)
5086
- line = fh.readline()
5087
- except OSError:
5088
- return None
5162
+ for attempt in range(2):
5163
+ selected = _select_codex_payload_container(row.source_path)
5164
+ if selected is None:
5165
+ return None
5166
+ # Validate exactly the descriptor we open, including after a compression
5167
+ # race. The caller has already checked the logical path and known root.
5168
+ root_path = _codex_source_root_path(conn, row.source_root_key)
5169
+ if not _within_root(str(selected.path), root_path):
5170
+ raise _CodexPayloadContainmentError()
5171
+ try:
5172
+ fh = None if streams is None else streams.get(str(selected.path))
5173
+ if fh is None:
5174
+ fh = _rc.open_decoded(selected)
5175
+ if streams is not None:
5176
+ streams[str(selected.path)] = fh
5177
+ try:
5178
+ fh.seek(row.line_offset)
5179
+ line = fh.readline()
5180
+ finally:
5181
+ if streams is None:
5182
+ fh.close()
5183
+ break
5184
+ except OSError as exc:
5185
+ # ContainerError subclasses OSError; preserve its approved leaf
5186
+ # category instead of flattening decoder/integrity errors.
5187
+ replacement = _select_codex_payload_container(row.source_path)
5188
+ if replacement is None:
5189
+ return None
5190
+ if replacement.path != selected.path and attempt == 0:
5191
+ continue # plain vanished during Codex's compression window
5192
+ category = exc.category if isinstance(exc, _rc.ContainerError) else "read-error"
5193
+ raise CodexContainerReadError(category, str(row.source_path)) from exc
5089
5194
  try:
5090
5195
  obj = json.loads(line.decode("utf-8"))
5091
5196
  except (UnicodeDecodeError, ValueError, TypeError):
@@ -5172,7 +5277,9 @@ def read_codex_payload(
5172
5277
  full bounded ``card``, and ``truncated`` reflects ``_FULL_PAYLOAD_CEILING``.
5173
5278
  ``gone`` (→ HTTP 410) means the physical
5174
5279
  record moved/mutated; ``not_found`` (→ 404) means no such block, no output
5175
- partner, or a containment miss (a read is never attempted outside the root)."""
5280
+ partner, or a containment miss (a read is never attempted outside the root).
5281
+ A present container that cannot be read raises CodexContainerReadError through
5282
+ the existing query-error envelope, including plain-file read failures."""
5176
5283
  miss = {"status": "not_found", "block_key": block_key, "which": which}
5177
5284
  if which not in ("call", "output", "event"):
5178
5285
  return miss
@@ -5187,7 +5294,14 @@ def read_codex_payload(
5187
5294
  root_path = _codex_source_root_path(conn, target.source_root_key)
5188
5295
  if not _within_root(target.source_path, root_path):
5189
5296
  return miss
5190
- outcome = _reread_codex_full_content(conn, target)
5297
+ streams: dict = {}
5298
+ try:
5299
+ outcome = _reread_codex_full_content(conn, target, streams=streams)
5300
+ except _CodexPayloadContainmentError:
5301
+ return miss
5302
+ finally:
5303
+ for stream in streams.values():
5304
+ stream.close()
5191
5305
  if outcome is None:
5192
5306
  return {"status": "gone", "block_key": block_key, "which": which}
5193
5307
  content, truncated, card = outcome
@@ -43,15 +43,16 @@ All six §5.4 bullets are contract:
43
43
  its first ingest attempt (so a discovered file is never lost even if its
44
44
  first ingest is dirty/lock-contended and leaves no cursor row). A path is
45
45
  retried while unclassified: with no committed cursor it is surfaced every
46
- time it is visited; with a committed cursor it is surfaced only when its size
47
- exceeds that cursor (an incomplete-``session_meta`` tail completing without
48
- any directory-mtime change). A path leaves the set when the driver ``reap``s
46
+ time it is visited; with a committed cursor it is surfaced only when its
47
+ signature advances beyond that cursor (an incomplete-``session_meta`` tail
48
+ completing without any directory-mtime change). A path leaves the set when the driver ``reap``s
49
49
  it (it gained a thread row — child or non-child), when it vanishes, or on a
50
50
  bounded per-candidate expiry (handed to the next full sync).
51
51
  """
52
52
  from __future__ import annotations
53
53
 
54
54
  import os
55
+ import _lib_codex_rollout_container as _rc
55
56
 
56
57
 
57
58
  # ── production-default injected callables ─────────────────────────────────────
@@ -101,6 +102,48 @@ def default_file_size(filepath):
101
102
  return st.st_size
102
103
 
103
104
 
105
+ def default_file_sig(filepath):
106
+ """The selected container's live-tail signature (#968 spec 5.9).
107
+
108
+ Plain keeps today's size (as ``("plain", size)``); a compressed-only
109
+ rollout reports its container fingerprint instead of vanishing.
110
+ """
111
+ return _rc.observe_watch_sig(filepath)
112
+
113
+
114
+ def _as_sig(value):
115
+ if value is None or isinstance(value, tuple):
116
+ return value
117
+ return (_rc.PLAIN, int(value))
118
+
119
+
120
+ def codex_sig_advanced(observed, committed) -> bool:
121
+ """Whether an observed signature is ahead of the committed one.
122
+
123
+ Plain grows (append-only, size decides); any other change of a compressed
124
+ container, or of the container kind, is a change. A bare int on either
125
+ side is a legacy plain size.
126
+ """
127
+ obs, com = _as_sig(observed), _as_sig(committed)
128
+ if obs is None:
129
+ return False
130
+ if com is None:
131
+ return True
132
+ if obs[0] == com[0] == _rc.PLAIN:
133
+ return obs[1] > com[1]
134
+ return obs != com
135
+
136
+
137
+ def committed_watch_sig(kind, size_bytes, container_size_bytes, mtime_ns,
138
+ device, inode):
139
+ """The signature a cursor row's ACCEPTED container would observe: the
140
+ leaf's ``ContainerFingerprint.watch_sig`` shape, device included."""
141
+ if kind in (None, _rc.PLAIN):
142
+ return (_rc.PLAIN, int(size_bytes or 0))
143
+ return (_rc.ZSTD, int(container_size_bytes or 0), int(mtime_ns or 0),
144
+ int(device or 0), int(inode or 0))
145
+
146
+
104
147
  _SENTINEL = object()
105
148
 
106
149
 
@@ -118,7 +161,7 @@ class CodexChildFrontier:
118
161
  pending_expiry_cycles: int = 50,
119
162
  scandir_fn=default_scandir,
120
163
  dir_mtime_fn=default_dir_mtime,
121
- file_size_fn=default_file_size,
164
+ file_size_fn=default_file_sig,
122
165
  clock_fn=None,
123
166
  ):
124
167
  self._walk_root = str(walk_root)
@@ -175,7 +218,7 @@ class CodexChildFrontier:
175
218
 
176
219
  ``known_paths`` — every path already tracked in ``codex_session_files``,
177
220
  so a newly enumerated file already known to some table is not
178
- re-registered. ``committed_sizes`` — ``{path: size_bytes}`` committed
221
+ re-registered. ``committed_sizes`` — ``{path: signature}`` committed
179
222
  cursors, for the pending-growth check.
180
223
 
181
224
  Returns the set of candidate paths to targeted-ingest this cycle (new
@@ -251,8 +294,8 @@ class CodexChildFrontier:
251
294
  mtime as the dir's baseline (so a create landing during the enumeration
252
295
  is re-examined next rotation), skips symlinked descendants (the
253
296
  ``follow_symlinks=False`` boundary), adds real subdirectories to the
254
- frontier, and registers brand-new ``*.jsonl`` files in ``pending`` BEFORE
255
- their first ingest — surfacing each for that first ingest attempt in the
297
+ frontier, and registers brand-new logical rollout files in ``pending``
298
+ BEFORE their first ingest — surfacing each for that first ingest attempt in the
256
299
  SAME cycle it is discovered (the pending sweep then owns the retries).
257
300
  Returns True iff it consumed a filesystem op."""
258
301
  if not self._enum_queue:
@@ -267,10 +310,12 @@ class CodexChildFrontier:
267
310
  if path not in self._dir_set:
268
311
  self._dir_set.add(path)
269
312
  self._dirs.append(path) # new subdir joins frontier
270
- elif path.endswith(".jsonl"):
271
- if path not in self._known and path not in self._pending:
272
- self._pending[path] = 0 # register BEFORE first ingest
273
- ingest.add(path) # first ingest attempt now
313
+ elif _rc.is_container_name(name):
314
+ # Both spellings enumerate one logical identity.
315
+ logical = os.fspath(_rc.logical_path(path))
316
+ if logical not in self._known and logical not in self._pending:
317
+ self._pending[logical] = 0 # register BEFORE first ingest
318
+ ingest.add(logical) # first ingest attempt now
274
319
  return True
275
320
 
276
321
  def _op_pending(self, ingest: set) -> bool:
@@ -286,12 +331,12 @@ class CodexChildFrontier:
286
331
  p = keys[idx]
287
332
  if p not in self._pending: # reaped/dropped mid-cycle
288
333
  continue
289
- size = self._file_size(p) # 1 fs op
290
- if size is None:
334
+ sig = self._file_size(p) # 1 signature probe
335
+ if sig is None:
291
336
  self._pending.pop(p, None) # vanished → drop
292
337
  else:
293
338
  committed = self._committed.get(p)
294
- if committed is None or size > committed:
339
+ if codex_sig_advanced(sig, committed):
295
340
  ingest.add(p)
296
341
  return True
297
342
  return False
@@ -0,0 +1,214 @@
1
+ """Waiting requests and the drain-admission gate (#968 spec 5.7, 5.12).
2
+
3
+ A Codex hook never waits for a lock. When it meets cache contention it
4
+ publishes a uniquely named, immutable request record under the data
5
+ directory (outside every SQLite store, because a contended hook cannot write
6
+ cache.db). Drain workers — the prune executor and the container-verification
7
+ executor — admit one identity at a time through ONE global gate and yield
8
+ while any request is fresh:
9
+
10
+ 1. check for fresh requests;
11
+ 2. acquire the drain gate (non-blocking), then check again;
12
+ 3. try the store's complete lock domain without blocking, then check again;
13
+ 4. a fresh request at any check releases everything acquired and yields;
14
+ 5. otherwise the caller executes one identity, commits, releases every store
15
+ lock and the gate (``finish``), and starts again at step 1.
16
+
17
+ Requests end only by expiry: 60 s after publication on a cross-process
18
+ monotonic clock. A worker observing a request, or restarting, never renews it;
19
+ a wall-clock jump cannot extend it (the wall age only shortens freshness, and
20
+ a NEGATIVE wall age — an mtime ahead of the wall clock after a backward step —
21
+ counts as fresh, bounded by the monotonic lifetime); a malformed record or one
22
+ from an incompatible clock is expired and removed. Stdlib only; no cctally
23
+ imports.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import fcntl
28
+ import json
29
+ import os
30
+ import pathlib
31
+ import time
32
+ import uuid
33
+ from typing import Callable, NamedTuple, Optional
34
+
35
+ REQUEST_DIR_NAME = "codex-drain-requests"
36
+ REQUEST_TTL_NS = 60_000_000_000
37
+ WALL_GUARD_SECONDS = 120.0
38
+ GATE_FILE_NAME = "codex-drain-admission.lock"
39
+ POLL_SECONDS = 5.0
40
+ _RECORD_VERSION = 1
41
+
42
+ # Module attributes so tests can drive a deterministic schedule.
43
+ _monotonic_ns: Callable[[], int] = time.monotonic_ns
44
+ _sleep: Callable[[float], None] = time.sleep
45
+ #: Post-release seam: called by ``finish`` after the store locks AND the gate
46
+ #: are released. Production leaves it None; the fairness test synchronizes on
47
+ #: it instead of sleeping (review P2-7).
48
+ _after_finish: Optional[Callable[[], None]] = None
49
+
50
+
51
+ def _request_dir(app_dir) -> pathlib.Path:
52
+ return pathlib.Path(app_dir) / REQUEST_DIR_NAME
53
+
54
+
55
+ def publish_waiting_request(app_dir, *, root_keys) -> "str | None":
56
+ """Atomically publish one immutable request; return its id or None.
57
+
58
+ ``None`` is a protocol failure the hook records; it is never fairness
59
+ evidence and never raises.
60
+ """
61
+ try:
62
+ directory = _request_dir(app_dir)
63
+ directory.mkdir(parents=True, exist_ok=True)
64
+ mono = int(_monotonic_ns())
65
+ request_id = f"{mono}-{os.getpid()}-{uuid.uuid4().hex}"
66
+ final = directory / f"req-{request_id}.json"
67
+ tmp = directory / f".tmp-{request_id}"
68
+ payload = json.dumps({
69
+ "v": _RECORD_VERSION, "id": request_id,
70
+ "roots": sorted(str(k) for k in root_keys), "mono_ns": mono,
71
+ }, separators=(",", ":"))
72
+ with open(tmp, "x", encoding="utf-8") as fh:
73
+ fh.write(payload)
74
+ fh.flush()
75
+ os.fsync(fh.fileno())
76
+ os.rename(tmp, final)
77
+ return request_id
78
+ except OSError:
79
+ return None
80
+
81
+
82
+ def _is_fresh(path: pathlib.Path, now_mono: int, now_wall: float) -> bool:
83
+ try:
84
+ record = json.loads(path.read_text(encoding="utf-8"))
85
+ mono = int(record["mono_ns"])
86
+ if record.get("v") != _RECORD_VERSION:
87
+ return False
88
+ wall_age = now_wall - path.stat().st_mtime
89
+ except (OSError, ValueError, TypeError, KeyError):
90
+ return False
91
+ age = now_mono - mono
92
+ # A negative monotonic age is an incompatible clock: expired. A negative
93
+ # WALL age is only an mtime ahead of the wall clock, which is not evidence
94
+ # of staleness: it stays fresh, and the monotonic bound still ends it
95
+ # within the single 60 s lifetime (review P3-5).
96
+ return 0 <= age < REQUEST_TTL_NS and wall_age < WALL_GUARD_SECONDS
97
+
98
+
99
+ def fresh_requests(app_dir) -> "list[str]":
100
+ """Ids of fresh requests; expired or malformed records are removed one by one."""
101
+ directory = _request_dir(app_dir)
102
+ try:
103
+ entries = sorted(directory.iterdir())
104
+ except OSError:
105
+ return []
106
+ now_mono = int(_monotonic_ns())
107
+ now_wall = time.time()
108
+ fresh: list[str] = []
109
+ for path in entries:
110
+ name = path.name
111
+ if name.startswith(".tmp-"):
112
+ try:
113
+ if now_wall - path.stat().st_mtime >= WALL_GUARD_SECONDS:
114
+ path.unlink()
115
+ except OSError:
116
+ pass
117
+ continue
118
+ if not (name.startswith("req-") and name.endswith(".json")):
119
+ continue
120
+ if _is_fresh(path, now_mono, now_wall):
121
+ fresh.append(name[len("req-"):-len(".json")])
122
+ else:
123
+ try:
124
+ path.unlink()
125
+ except OSError:
126
+ pass
127
+ return fresh
128
+
129
+
130
+ class DrainGate:
131
+ """The one global, non-blocking drain-admission flock."""
132
+
133
+ def __init__(self, app_dir):
134
+ self._path = pathlib.Path(app_dir) / GATE_FILE_NAME
135
+ self._fd: "int | None" = None
136
+
137
+ def try_acquire(self) -> bool:
138
+ if self._fd is not None:
139
+ return True
140
+ self._path.parent.mkdir(parents=True, exist_ok=True)
141
+ fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
142
+ try:
143
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
144
+ except OSError:
145
+ os.close(fd)
146
+ return False
147
+ self._fd = fd
148
+ return True
149
+
150
+ def release(self) -> None:
151
+ if self._fd is None:
152
+ return
153
+ try:
154
+ fcntl.flock(self._fd, fcntl.LOCK_UN)
155
+ finally:
156
+ os.close(self._fd)
157
+ self._fd = None
158
+
159
+
160
+ class Admission(NamedTuple):
161
+ admitted: bool
162
+ store_locks: object
163
+ gate: "DrainGate | None"
164
+
165
+
166
+ def admit(app_dir, try_store_locks: Callable[[], object],
167
+ release_store_locks: Callable[[object], None], *,
168
+ on_check: Optional[Callable[[str], None]] = None) -> Admission:
169
+ """Steps 1-4 of the protocol. ``try_store_locks`` must never block and
170
+ returns a token, or None on contention."""
171
+ def check(stage: str) -> bool:
172
+ if on_check is not None:
173
+ on_check(stage)
174
+ return bool(fresh_requests(app_dir))
175
+
176
+ if check("before-gate"):
177
+ return Admission(False, None, None)
178
+ gate = DrainGate(app_dir)
179
+ if not gate.try_acquire():
180
+ return Admission(False, None, None)
181
+ if check("after-gate"):
182
+ gate.release()
183
+ return Admission(False, None, None)
184
+ token = try_store_locks()
185
+ if token is None:
186
+ gate.release()
187
+ return Admission(False, None, None)
188
+ if check("after-store-locks"):
189
+ release_store_locks(token)
190
+ gate.release()
191
+ return Admission(False, None, None)
192
+ return Admission(True, token, gate)
193
+
194
+
195
+ def finish(admission: Admission,
196
+ release_store_locks: Callable[[object], None]) -> None:
197
+ """Step 5: release every store lock, then the gate."""
198
+ try:
199
+ if admission.store_locks is not None:
200
+ release_store_locks(admission.store_locks)
201
+ finally:
202
+ if admission.gate is not None:
203
+ admission.gate.release()
204
+ if _after_finish is not None:
205
+ _after_finish()
206
+
207
+
208
+ def wait_until_no_fresh_requests(
209
+ app_dir, *, should_continue: Optional[Callable[[], bool]] = None) -> None:
210
+ """Sleep in bounded steps until no request is fresh (requests expire)."""
211
+ while fresh_requests(app_dir):
212
+ if should_continue is not None and not should_continue():
213
+ return
214
+ _sleep(POLL_SECONDS)
@@ -26,6 +26,8 @@ import sqlite3
26
26
  import threading
27
27
  from collections import OrderedDict
28
28
 
29
+ from _lib_codex_publication_barrier import LAST_GOOD_THREADS_VIEW
30
+
29
31
 
30
32
  #: The physical name of the raw threads table. ``resolve_codex_threads_table``
31
33
  #: returns it qualified by whichever schema actually holds it.
@@ -185,6 +187,8 @@ def _schema_holds_threads_table(conn: sqlite3.Connection, schema: str) -> bool:
185
187
  def resolve_codex_threads_table(conn: sqlite3.Connection) -> str:
186
188
  """The RAW threads table reference for this connection.
187
189
 
190
+ A conversations connection carrying the #968 last-good view resolves to it.
191
+
188
192
  A bare cache connection (``open_cache_db()``) holds the table in its own
189
193
  ``main`` schema; a conversations connection
190
194
  (``open_conversations_db``/``open_conversations_db_readonly``) reaches it
@@ -222,8 +226,21 @@ def resolve_codex_threads_table(conn: sqlite3.Connection) -> str:
222
226
  _threads_table_resolution_cache.move_to_end(conn)
223
227
  return cached
224
228
 
229
+ # One metadata SELECT preserves last-good priority without adding a
230
+ # fifth query to the bounded accounting/metadata-health read contracts.
231
+ schema = conn.execute(
232
+ "SELECT CASE "
233
+ "WHEN EXISTS (SELECT 1 FROM sqlite_temp_master "
234
+ "WHERE type='view' AND name=?) THEN 'temp' "
235
+ "WHEN EXISTS (SELECT 1 FROM main.sqlite_master "
236
+ "WHERE type='table' AND name=?) THEN 'main' END",
237
+ (LAST_GOOD_THREADS_VIEW, CODEX_THREADS_TABLE),
238
+ ).fetchone()[0]
225
239
  resolved = None
226
- if _schema_holds_threads_table(conn, "main"):
240
+ if schema == "temp":
241
+ # #968 spec 5.11: unpublished metadata never overrides last-good.
242
+ resolved = f"temp.{LAST_GOOD_THREADS_VIEW}"
243
+ elif schema == "main":
227
244
  resolved = f"main.{CODEX_THREADS_TABLE}"
228
245
  else:
229
246
  attached = tuple(conn.execute("PRAGMA database_list"))