cctally 1.114.0 → 1.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/bin/_cctally_account.py +32 -8
  3. package/bin/_cctally_cache.py +4491 -295
  4. package/bin/_cctally_completion.py +1355 -0
  5. package/bin/_cctally_config.py +37 -0
  6. package/bin/_cctally_core.py +282 -9
  7. package/bin/_cctally_dashboard.py +8 -11
  8. package/bin/_cctally_dashboard_conversation.py +292 -23
  9. package/bin/_cctally_dashboard_envelope.py +42 -35
  10. package/bin/_cctally_dashboard_sources.py +18 -9
  11. package/bin/_cctally_db.py +401 -9
  12. package/bin/_cctally_doctor.py +469 -27
  13. package/bin/_cctally_five_hour.py +166 -1049
  14. package/bin/_cctally_forecast.py +81 -9
  15. package/bin/_cctally_journal.py +5076 -177
  16. package/bin/_cctally_journal_repair.py +110 -19
  17. package/bin/_cctally_oauth_samples.py +280 -0
  18. package/bin/_cctally_parser.py +47 -0
  19. package/bin/_cctally_project.py +8 -2
  20. package/bin/_cctally_quarantine.py +662 -0
  21. package/bin/_cctally_quota.py +96 -12
  22. package/bin/_cctally_quota_model.py +391 -83
  23. package/bin/_cctally_record.py +1088 -269
  24. package/bin/_cctally_rederive.py +854 -192
  25. package/bin/_cctally_refresh.py +113 -13
  26. package/bin/_cctally_source_analytics.py +23 -2
  27. package/bin/_cctally_statusline.py +7062 -505
  28. package/bin/_cctally_store.py +1244 -91
  29. package/bin/_cctally_transcript.py +8 -7
  30. package/bin/_cctally_tui.py +119 -60
  31. package/bin/_cctally_update.py +25 -0
  32. package/bin/_lib_account_binding.py +152 -0
  33. package/bin/_lib_account_protocol.py +569 -0
  34. package/bin/_lib_aggregators.py +4 -0
  35. package/bin/_lib_cache_coverage.py +13 -1
  36. package/bin/_lib_cache_writer_lock.py +19 -3
  37. package/bin/_lib_clone_copy.py +177 -0
  38. package/bin/_lib_codex_conversation_query.py +133 -19
  39. package/bin/_lib_codex_conversation_watch.py +59 -14
  40. package/bin/_lib_codex_drain_admission.py +214 -0
  41. package/bin/_lib_codex_metadata.py +18 -1
  42. package/bin/_lib_codex_publication_barrier.py +277 -0
  43. package/bin/_lib_codex_rollout_container.py +502 -0
  44. package/bin/_lib_conversation_dispatch.py +3 -0
  45. package/bin/_lib_conversation_query.py +11 -3
  46. package/bin/_lib_dashboard_sources.py +57 -15
  47. package/bin/_lib_detector_access_registry.py +4038 -0
  48. package/bin/_lib_doctor.py +753 -1
  49. package/bin/_lib_five_hour.py +126 -0
  50. package/bin/_lib_ingest_frontier.py +414 -20
  51. package/bin/_lib_journal.py +261 -10
  52. package/bin/_lib_journal_router.py +166 -48
  53. package/bin/_lib_jsonl.py +5 -4
  54. package/bin/_lib_legacy_spool.py +126 -0
  55. package/bin/_lib_legacy_writer_evidence.py +136 -0
  56. package/bin/_lib_meter_rate_change.py +10 -0
  57. package/bin/_lib_oauth_samples.py +238 -0
  58. package/bin/_lib_outcome_quarantine.py +690 -0
  59. package/bin/_lib_pricing.py +7 -2
  60. package/bin/_lib_pricing_check.py +0 -12
  61. package/bin/_lib_provider_activity.py +770 -0
  62. package/bin/_lib_quarantine.py +155 -0
  63. package/bin/_lib_quota_calibration.py +53 -8
  64. package/bin/_lib_quota_copy.py +2 -1
  65. package/bin/_lib_quota_model.py +384 -52
  66. package/bin/_lib_rebuild_progress.py +146 -0
  67. package/bin/_lib_record.py +22 -0
  68. package/bin/_lib_rederive.py +393 -8
  69. package/bin/_lib_segment_summary.py +35 -2
  70. package/bin/_lib_snapshot_cache.py +43 -1
  71. package/bin/_lib_stale_proof.py +708 -0
  72. package/bin/_lib_statusline_candidates.py +356 -27
  73. package/bin/_lib_worker_launches.py +315 -0
  74. package/bin/cctally +29 -12
  75. package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
  76. package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
  77. package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
  78. package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
  79. package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
  80. package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
  81. package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
  82. package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
  83. package/dashboard/static/dashboard.html +1 -1
  84. package/package.json +20 -2
  85. package/bin/_lib_merge_gate.py +0 -725
  86. package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
@@ -0,0 +1,177 @@
1
+ """Clone-first file and tree copies (stdlib only; #940 D2, spec §6.7).
2
+
3
+ On APFS, ``fclonefileat(2)`` gives a copy that shares the source's blocks
4
+ until either side is written, so a copy of a large test scaffold writes no
5
+ data. This module takes that clone wherever it can and a real copy wherever
6
+ it cannot, and otherwise behaves exactly as the stdlib copier it stands in
7
+ for:
8
+
9
+ - :func:`copy2` is ``shutil.copy2`` for the same arguments: a directory
10
+ destination keeps the source's basename, ``follow_symlinks=False`` copies a
11
+ link as a link, the same file raises ``shutil.SameFileError``, and an
12
+ existing destination is overwritten in place (same inode), which a clone
13
+ cannot do, so that case is a real copy. Metadata is ``shutil.copystat``'s,
14
+ applied after the clone as after a copy.
15
+ - :func:`try_clone` is the clone alone: True when `dst` was created as a
16
+ clone, False (nothing created) when the platform or filesystem refuses, so
17
+ a caller can choose where its real copy goes (#997 D13).
18
+ - :func:`copytree` is ``shutil.copytree`` with :func:`copy2` as its file
19
+ copier, so symlink policy, ignore lists, dangling-link errors and
20
+ destination collisions are the stdlib's own code.
21
+
22
+ A clone is attempted only on Darwin. An unsupported filesystem, a
23
+ cross-volume destination, a missing ``fclonefileat`` and any non-Darwin
24
+ platform take the real copy; any other error (EIO, ENOSPC, EACCES, …)
25
+ propagates and is never retried as a copy. A clone is never a hardlink: the
26
+ copy is a new inode with a link count of one.
27
+
28
+ The optional ``counts`` mapping is incremented under ``"clone"`` or
29
+ ``"copy"`` per file, so a caller or a measurement can see which path ran.
30
+ This module is public and imports nothing outside the standard library.
31
+
32
+ It binds ``fclonefileat`` itself rather than importing the binding from the
33
+ retired private staging module (#940 D-2 F17): a private module is absent
34
+ from the public tree that this module and its public callers ship in, and a
35
+ public module may not import it.
36
+ """
37
+
38
+ from __future__ import annotations
39
+
40
+ import ctypes
41
+ import errno
42
+ import os
43
+ import shutil
44
+ import stat
45
+ import sys
46
+
47
+ # Errors that mean "this filesystem or this pair of volumes cannot clone",
48
+ # never "the data could not be written". EEXIST is a destination created
49
+ # after the existence check: `shutil.copy2` overwrites it, and so do we.
50
+ _FALLBACK_ERRNOS = frozenset(
51
+ code for code in (
52
+ getattr(errno, "ENOTSUP", None), getattr(errno, "EOPNOTSUPP", None),
53
+ errno.ENOSYS, errno.EXDEV, errno.EINVAL, errno.EPERM, errno.EEXIST)
54
+ if code is not None)
55
+
56
+ _AT_FDCWD = -2 # Darwin's AT_FDCWD
57
+ _CLONE_NOOWNERCOPY = 0x0002 # leave ownership to the caller, as a copy does
58
+
59
+ _LIBC_CLONE = [] # the resolved fclonefileat, once looked up
60
+ # The platform this module decides on; a test sets it here instead of
61
+ # patching the interpreter-wide ``sys.platform`` (#940 D-2 F16).
62
+ _PLATFORM = sys.platform
63
+
64
+
65
+ def _fclonefileat():
66
+ """The C ``fclonefileat`` on Darwin, else None."""
67
+ if _PLATFORM != "darwin":
68
+ return None
69
+ if not _LIBC_CLONE:
70
+ libc = ctypes.CDLL(None, use_errno=True)
71
+ clone = getattr(libc, "fclonefileat", None)
72
+ if clone is not None:
73
+ clone.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32)
74
+ clone.restype = ctypes.c_int
75
+ _LIBC_CLONE.append(clone)
76
+ return _LIBC_CLONE[0]
77
+
78
+
79
+ def _clone_fd(source_fd: int, destination: str) -> None:
80
+ """Clone the open regular file `source_fd` to the new path `destination`."""
81
+ clone = _fclonefileat()
82
+ if clone is None:
83
+ raise OSError(errno.ENOTSUP, "fclonefileat is unavailable", destination)
84
+ if clone(source_fd, _AT_FDCWD, os.fsencode(os.path.abspath(destination)),
85
+ _CLONE_NOOWNERCOPY) != 0:
86
+ code = ctypes.get_errno()
87
+ raise OSError(code, os.strerror(code), destination)
88
+
89
+
90
+ def _clone_open_file(fd: int, src, dst, *, follow_symlinks: bool = True) -> bool:
91
+ """Clone the open regular file `fd` (opened from `src`) to the new path
92
+ `dst` and apply `src`'s metadata. False, with nothing created, when the
93
+ filesystem or the pair of volumes refuses the clone (`_FALLBACK_ERRNOS`);
94
+ any other error propagates."""
95
+ try:
96
+ _clone_fd(fd, dst)
97
+ except OSError as exc:
98
+ if exc.errno not in _FALLBACK_ERRNOS:
99
+ raise
100
+ return False
101
+ shutil.copystat(src, dst, follow_symlinks=follow_symlinks)
102
+ return True
103
+
104
+
105
+ def try_clone(src, dst) -> bool:
106
+ """Clone the regular file `src` to the NEW path `dst` (#997 D13).
107
+
108
+ True: `dst` is a clone sharing `src`'s blocks, with ``shutil.copystat``
109
+ metadata applied. False, with `dst` not created: the platform is not
110
+ Darwin (or has no ``fclonefileat``), `src` is not a regular file, or the
111
+ filesystem refused with a fallback errno (an unsupported filesystem, a
112
+ cross-volume destination, an existing `dst`). Any other error, including
113
+ a `src` that cannot be opened, propagates. A caller that needs a copy
114
+ either way makes the real copy itself on False.
115
+ """
116
+ if _fclonefileat() is None:
117
+ return False
118
+ fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
119
+ try:
120
+ if not stat.S_ISREG(os.fstat(fd).st_mode):
121
+ return False
122
+ return _clone_open_file(fd, src, dst)
123
+ finally:
124
+ os.close(fd)
125
+
126
+
127
+ def _count(counts, method: str) -> None:
128
+ if counts is not None:
129
+ counts[method] = counts.get(method, 0) + 1
130
+
131
+
132
+ def copy2(src, dst, *, follow_symlinks: bool = True, counts=None):
133
+ """``shutil.copy2(src, dst)``, by clone where the filesystem allows it.
134
+
135
+ Returns the destination path, as ``shutil.copy2`` does.
136
+ """
137
+ if os.path.isdir(dst):
138
+ dst = os.path.join(dst, os.path.basename(src))
139
+ if (not follow_symlinks and os.path.islink(src)) or os.path.lexists(dst):
140
+ # A link copied as a link, or a destination to overwrite in place.
141
+ result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
142
+ _count(counts, "copy")
143
+ return result
144
+ try:
145
+ fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
146
+ except OSError:
147
+ # The stdlib's own error for this source (missing, unreadable, …).
148
+ return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
149
+ try:
150
+ if not stat.S_ISREG(os.fstat(fd).st_mode):
151
+ # A FIFO, a device or a directory: the stdlib decides.
152
+ os.close(fd)
153
+ fd = -1
154
+ return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
155
+ if not _clone_open_file(fd, src, dst,
156
+ follow_symlinks=follow_symlinks):
157
+ result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
158
+ _count(counts, "copy")
159
+ return result
160
+ _count(counts, "clone")
161
+ return dst
162
+ finally:
163
+ if fd >= 0:
164
+ os.close(fd)
165
+
166
+
167
+ def copytree(src, dst, symlinks: bool = False, ignore=None,
168
+ ignore_dangling_symlinks: bool = False, dirs_exist_ok: bool = False,
169
+ *, counts=None):
170
+ """``shutil.copytree`` with :func:`copy2` as its file copier."""
171
+ def copier(source, destination):
172
+ return copy2(source, destination, counts=counts)
173
+
174
+ return shutil.copytree(src, dst, symlinks=symlinks, ignore=ignore,
175
+ copy_function=copier,
176
+ ignore_dangling_symlinks=ignore_dangling_symlinks,
177
+ dirs_exist_ok=dirs_exist_ok)
@@ -27,11 +27,13 @@ import json
27
27
  import os
28
28
  import re
29
29
  import sqlite3
30
+ import stat
30
31
  import threading
31
32
  from collections import deque
32
33
  from types import MappingProxyType
33
34
 
34
35
  import _lib_codex_conversation as kern
36
+ import _lib_codex_rollout_container as _rc
35
37
  import _lib_codex_landmarks as landmarks
36
38
  import _lib_codex_segments as segkern
37
39
  from _lib_codex_find_projection import (
@@ -1863,6 +1865,29 @@ def codex_conversation_exists(conn: sqlite3.Connection, conversation_key: str) -
1863
1865
  return row is not None
1864
1866
 
1865
1867
 
1868
+ def codex_publication_pending(
1869
+ conn: sqlite3.Connection, conversation_key: str,
1870
+ ) -> bool:
1871
+ """A barrier-population conversation with no publication yet (#968 5.11).
1872
+
1873
+ Such a conversation stays in the existing pending-state handling until it
1874
+ has an eligible publication: it answers ``normalization_pending``, never
1875
+ ``not_found``. Any store error answers False (the caller's ``not_found``).
1876
+ """
1877
+ try:
1878
+ if conn.execute(
1879
+ "SELECT 1 FROM cache_db.codex_barrier_population "
1880
+ "WHERE conversation_key=?", (conversation_key,)).fetchone() is None:
1881
+ return False
1882
+ row = conn.execute(
1883
+ "SELECT published_revision, legacy_uncertified "
1884
+ "FROM main.codex_conversation_publications WHERE conversation_key=?",
1885
+ (conversation_key,)).fetchone()
1886
+ except sqlite3.Error:
1887
+ return False
1888
+ return row is None or (int(row[0] or 0) == 0 and not row[1])
1889
+
1890
+
1866
1891
  def codex_conversation_source_paths(
1867
1892
  conn: sqlite3.Connection, conversation_key: str
1868
1893
  ) -> list[str]:
@@ -2345,6 +2370,9 @@ def get_codex_conversation(
2345
2370
  # ── Phase A: the narrow index pass ───────────────────────────────────────
2346
2371
  rows, detail_bytes = _load_conversation_index_rows(conn, conversation_key)
2347
2372
  if not rows:
2373
+ if codex_publication_pending(conn, conversation_key):
2374
+ return {"status": "normalization_pending",
2375
+ "conversation_key": conversation_key, "items": [], "children": []}
2348
2376
  return {"status": "not_found", "conversation_key": conversation_key}
2349
2377
  undecodable_keys = undecodable_codex_conversation_keys(conn)
2350
2378
  kept, _suppressed = kern.pair_mirrors(rows)
@@ -3035,6 +3063,9 @@ def _outline_envelope(
3035
3063
  # The outline is not the route F3 bounds, and it stays turn-granular.
3036
3064
  rows = _load_conversation_rows(conn, conversation_key)
3037
3065
  if not rows:
3066
+ if codex_publication_pending(conn, conversation_key):
3067
+ return {"status": "normalization_pending",
3068
+ "conversation_key": conversation_key, "turns": [], "files": [], "children": []}
3038
3069
  return {"status": "not_found", "conversation_key": conversation_key}
3039
3070
  detail_bytes = _detail_bytes_of(rows)
3040
3071
  kept, _suppressed = kern.pair_mirrors(rows)
@@ -3421,20 +3452,26 @@ def _stored_browse_page_queries(
3421
3452
  "(COALESCE(r.last_activity_utc, '') = ? AND r.conversation_key < ?))")
3422
3453
  page_params.extend((cursor_last, cursor_last, cursor_key))
3423
3454
 
3455
+ # Keep native/parent enrichment inside the page statement. A LEFT JOIN
3456
+ # against the compound last-good view materializes retained history;
3457
+ # correlated probes push these predicates into its indexed branches.
3458
+ # The scalar parent lookup follows the detail reader's same-root lookup.
3459
+ parent_key_sql = (
3460
+ "(SELECT pt.conversation_key FROM codex_conversation_threads pt "
3461
+ "WHERE pt.source_root_key = r.source_root_key "
3462
+ "AND pt.native_thread_id = r.parent_thread_id "
3463
+ "AND pt.conversation_key != r.conversation_key LIMIT 1)"
3464
+ )
3424
3465
  sql = (
3425
3466
  "SELECT r.conversation_key, r.item_count, r.started_utc, "
3426
3467
  "r.last_activity_utc, r.project_key, r.project_label, r.models_json, "
3427
- "r.title, r.parent_thread_id, r.source_root_key, t.native_thread_id, "
3428
- "pt.conversation_key, pr.title, pr.project_label, pt.native_thread_id "
3468
+ "r.title, r.parent_thread_id, r.source_root_key, "
3469
+ "(SELECT t.native_thread_id FROM codex_conversation_threads t "
3470
+ "WHERE t.conversation_key = r.conversation_key), "
3471
+ f"{parent_key_sql}, pr.title, pr.project_label, r.parent_thread_id "
3429
3472
  "FROM codex_conversation_rollups r "
3430
- "LEFT JOIN codex_conversation_threads t "
3431
- "ON t.conversation_key = r.conversation_key "
3432
- "LEFT JOIN codex_conversation_threads pt "
3433
- "ON pt.source_root_key = r.source_root_key "
3434
- "AND pt.native_thread_id = r.parent_thread_id "
3435
- "AND pt.conversation_key != r.conversation_key "
3436
3473
  "LEFT JOIN codex_conversation_rollups pr "
3437
- "ON pr.conversation_key = pt.conversation_key "
3474
+ f"ON pr.conversation_key = {parent_key_sql} "
3438
3475
  f"WHERE {' AND '.join(page_where)} "
3439
3476
  "ORDER BY COALESCE(r.last_activity_utc, '') DESC, r.conversation_key DESC"
3440
3477
  )
@@ -4962,6 +4999,8 @@ def find_in_codex_conversation(
4962
4999
  return {**base, "status": "normalization_pending"}
4963
5000
  rows = _load_conversation_rows(conn, conversation_key)
4964
5001
  if not rows:
5002
+ if codex_publication_pending(conn, conversation_key):
5003
+ return {**base, "status": "normalization_pending"}
4965
5004
  return {"status": "not_found", "conversation_key": conversation_key}
4966
5005
  q = (query or "").strip()
4967
5006
  if not q:
@@ -5021,6 +5060,9 @@ def codex_conversation_prompts(conn: sqlite3.Connection, conversation_key: str)
5021
5060
  "conversation_key": conversation_key, "prompts": []}
5022
5061
  rows = _load_conversation_rows(conn, conversation_key)
5023
5062
  if not rows:
5063
+ if codex_publication_pending(conn, conversation_key):
5064
+ return {"status": "normalization_pending",
5065
+ "conversation_key": conversation_key, "prompts": []}
5024
5066
  return {"status": "not_found", "conversation_key": conversation_key}
5025
5067
  kept, _suppressed = kern.pair_mirrors(rows)
5026
5068
  items = kern.canonical_items(kept)
@@ -5061,7 +5103,40 @@ def _within_root(source_path: str | None, root_path: str | None) -> bool:
5061
5103
  return False
5062
5104
 
5063
5105
 
5064
- def _reread_codex_full_content(conn: sqlite3.Connection, row):
5106
+ class CodexContainerReadError(RuntimeError):
5107
+ """A present rollout failed to read; the query envelope reports HTTP 500."""
5108
+
5109
+ def __init__(self, category: str, logical: str) -> None:
5110
+ super().__init__(f"Codex rollout could not be read ({category})")
5111
+ self.category = category
5112
+ self.logical = logical
5113
+
5114
+
5115
+ class _CodexPayloadContainmentError(RuntimeError):
5116
+ """Internal 404 signal: the selected physical container escaped its root."""
5117
+
5118
+
5119
+ def _select_codex_payload_container(logical):
5120
+ # The discovery leaf deliberately treats stat failures as absent. A payload
5121
+ # request must distinguish a missing container (410) from inaccessible data
5122
+ # (500), including an inaccessible plain file beside a readable zstd twin.
5123
+ for candidate in (logical, _rc.zstd_twin(logical)):
5124
+ try:
5125
+ info = os.stat(candidate)
5126
+ except (FileNotFoundError, NotADirectoryError):
5127
+ continue
5128
+ except OSError as exc:
5129
+ raise CodexContainerReadError("read-error", str(logical)) from exc
5130
+ if not stat.S_ISREG(info.st_mode):
5131
+ raise CodexContainerReadError("read-error", str(logical))
5132
+ selected = _rc.select_container(logical)
5133
+ if selected is not None:
5134
+ return selected
5135
+ # It disappeared between the strict stat and selection; try the twin.
5136
+ return None
5137
+
5138
+
5139
+ def _reread_codex_full_content(conn: sqlite3.Connection, row, *, streams: "dict | None" = None):
5065
5140
  """Re-read the physical line at ``(row.source_path, row.line_offset)``, validate
5066
5141
  it against the stored ``codex_conversation_events.payload_json`` for that exact
5067
5142
  position (§3.4 structural gone-check — the canonical FULL record, not
@@ -5073,19 +5148,49 @@ def _reread_codex_full_content(conn: sqlite3.Connection, row):
5073
5148
  The full pre-cap content is re-derived through the SAME ``_extract`` the
5074
5149
  normalizer uses — which is how payload serves content beyond the normalized
5075
5150
  ``CODEX_TEXT_CAP``. Truncation/``truncated`` is against ``_FULL_PAYLOAD_CEILING``
5076
- (1,000,000 Python characters), the same ceiling the Claude payload path uses."""
5151
+ (1,000,000 Python characters), the same ceiling the Claude payload path uses.
5152
+
5153
+ #968: select plain first, else zstd, and seek in decoded offsets. Streams are
5154
+ request-local and reusable; no decoded copies persist. Present unreadable
5155
+ containers raise CodexContainerReadError, never a false gone result."""
5077
5156
  stored = conn.execute(
5078
5157
  "SELECT payload_json FROM codex_conversation_events "
5079
5158
  "WHERE source_path = ? AND line_offset = ?",
5080
5159
  (row.source_path, row.line_offset)).fetchone()
5081
5160
  if stored is None:
5082
5161
  return None
5083
- try:
5084
- with open(row.source_path, "rb") as fh:
5085
- fh.seek(row.line_offset)
5086
- line = fh.readline()
5087
- except OSError:
5088
- return None
5162
+ for attempt in range(2):
5163
+ selected = _select_codex_payload_container(row.source_path)
5164
+ if selected is None:
5165
+ return None
5166
+ # Validate exactly the descriptor we open, including after a compression
5167
+ # race. The caller has already checked the logical path and known root.
5168
+ root_path = _codex_source_root_path(conn, row.source_root_key)
5169
+ if not _within_root(str(selected.path), root_path):
5170
+ raise _CodexPayloadContainmentError()
5171
+ try:
5172
+ fh = None if streams is None else streams.get(str(selected.path))
5173
+ if fh is None:
5174
+ fh = _rc.open_decoded(selected)
5175
+ if streams is not None:
5176
+ streams[str(selected.path)] = fh
5177
+ try:
5178
+ fh.seek(row.line_offset)
5179
+ line = fh.readline()
5180
+ finally:
5181
+ if streams is None:
5182
+ fh.close()
5183
+ break
5184
+ except OSError as exc:
5185
+ # ContainerError subclasses OSError; preserve its approved leaf
5186
+ # category instead of flattening decoder/integrity errors.
5187
+ replacement = _select_codex_payload_container(row.source_path)
5188
+ if replacement is None:
5189
+ return None
5190
+ if replacement.path != selected.path and attempt == 0:
5191
+ continue # plain vanished during Codex's compression window
5192
+ category = exc.category if isinstance(exc, _rc.ContainerError) else "read-error"
5193
+ raise CodexContainerReadError(category, str(row.source_path)) from exc
5089
5194
  try:
5090
5195
  obj = json.loads(line.decode("utf-8"))
5091
5196
  except (UnicodeDecodeError, ValueError, TypeError):
@@ -5172,7 +5277,9 @@ def read_codex_payload(
5172
5277
  full bounded ``card``, and ``truncated`` reflects ``_FULL_PAYLOAD_CEILING``.
5173
5278
  ``gone`` (→ HTTP 410) means the physical
5174
5279
  record moved/mutated; ``not_found`` (→ 404) means no such block, no output
5175
- partner, or a containment miss (a read is never attempted outside the root)."""
5280
+ partner, or a containment miss (a read is never attempted outside the root).
5281
+ A present container that cannot be read raises CodexContainerReadError through
5282
+ the existing query-error envelope, including plain-file read failures."""
5176
5283
  miss = {"status": "not_found", "block_key": block_key, "which": which}
5177
5284
  if which not in ("call", "output", "event"):
5178
5285
  return miss
@@ -5187,7 +5294,14 @@ def read_codex_payload(
5187
5294
  root_path = _codex_source_root_path(conn, target.source_root_key)
5188
5295
  if not _within_root(target.source_path, root_path):
5189
5296
  return miss
5190
- outcome = _reread_codex_full_content(conn, target)
5297
+ streams: dict = {}
5298
+ try:
5299
+ outcome = _reread_codex_full_content(conn, target, streams=streams)
5300
+ except _CodexPayloadContainmentError:
5301
+ return miss
5302
+ finally:
5303
+ for stream in streams.values():
5304
+ stream.close()
5191
5305
  if outcome is None:
5192
5306
  return {"status": "gone", "block_key": block_key, "which": which}
5193
5307
  content, truncated, card = outcome
@@ -43,15 +43,16 @@ All six §5.4 bullets are contract:
43
43
  its first ingest attempt (so a discovered file is never lost even if its
44
44
  first ingest is dirty/lock-contended and leaves no cursor row). A path is
45
45
  retried while unclassified: with no committed cursor it is surfaced every
46
- time it is visited; with a committed cursor it is surfaced only when its size
47
- exceeds that cursor (an incomplete-``session_meta`` tail completing without
48
- any directory-mtime change). A path leaves the set when the driver ``reap``s
46
+ time it is visited; with a committed cursor it is surfaced only when its
47
+ signature advances beyond that cursor (an incomplete-``session_meta`` tail
48
+ completing without any directory-mtime change). A path leaves the set when the driver ``reap``s
49
49
  it (it gained a thread row — child or non-child), when it vanishes, or on a
50
50
  bounded per-candidate expiry (handed to the next full sync).
51
51
  """
52
52
  from __future__ import annotations
53
53
 
54
54
  import os
55
+ import _lib_codex_rollout_container as _rc
55
56
 
56
57
 
57
58
  # ── production-default injected callables ─────────────────────────────────────
@@ -101,6 +102,48 @@ def default_file_size(filepath):
101
102
  return st.st_size
102
103
 
103
104
 
105
+ def default_file_sig(filepath):
106
+ """The selected container's live-tail signature (#968 spec 5.9).
107
+
108
+ Plain keeps today's size (as ``("plain", size)``); a compressed-only
109
+ rollout reports its container fingerprint instead of vanishing.
110
+ """
111
+ return _rc.observe_watch_sig(filepath)
112
+
113
+
114
+ def _as_sig(value):
115
+ if value is None or isinstance(value, tuple):
116
+ return value
117
+ return (_rc.PLAIN, int(value))
118
+
119
+
120
+ def codex_sig_advanced(observed, committed) -> bool:
121
+ """Whether an observed signature is ahead of the committed one.
122
+
123
+ Plain grows (append-only, size decides); any other change of a compressed
124
+ container, or of the container kind, is a change. A bare int on either
125
+ side is a legacy plain size.
126
+ """
127
+ obs, com = _as_sig(observed), _as_sig(committed)
128
+ if obs is None:
129
+ return False
130
+ if com is None:
131
+ return True
132
+ if obs[0] == com[0] == _rc.PLAIN:
133
+ return obs[1] > com[1]
134
+ return obs != com
135
+
136
+
137
+ def committed_watch_sig(kind, size_bytes, container_size_bytes, mtime_ns,
138
+ device, inode):
139
+ """The signature a cursor row's ACCEPTED container would observe: the
140
+ leaf's ``ContainerFingerprint.watch_sig`` shape, device included."""
141
+ if kind in (None, _rc.PLAIN):
142
+ return (_rc.PLAIN, int(size_bytes or 0))
143
+ return (_rc.ZSTD, int(container_size_bytes or 0), int(mtime_ns or 0),
144
+ int(device or 0), int(inode or 0))
145
+
146
+
104
147
  _SENTINEL = object()
105
148
 
106
149
 
@@ -118,7 +161,7 @@ class CodexChildFrontier:
118
161
  pending_expiry_cycles: int = 50,
119
162
  scandir_fn=default_scandir,
120
163
  dir_mtime_fn=default_dir_mtime,
121
- file_size_fn=default_file_size,
164
+ file_size_fn=default_file_sig,
122
165
  clock_fn=None,
123
166
  ):
124
167
  self._walk_root = str(walk_root)
@@ -175,7 +218,7 @@ class CodexChildFrontier:
175
218
 
176
219
  ``known_paths`` — every path already tracked in ``codex_session_files``,
177
220
  so a newly enumerated file already known to some table is not
178
- re-registered. ``committed_sizes`` — ``{path: size_bytes}`` committed
221
+ re-registered. ``committed_sizes`` — ``{path: signature}`` committed
179
222
  cursors, for the pending-growth check.
180
223
 
181
224
  Returns the set of candidate paths to targeted-ingest this cycle (new
@@ -251,8 +294,8 @@ class CodexChildFrontier:
251
294
  mtime as the dir's baseline (so a create landing during the enumeration
252
295
  is re-examined next rotation), skips symlinked descendants (the
253
296
  ``follow_symlinks=False`` boundary), adds real subdirectories to the
254
- frontier, and registers brand-new ``*.jsonl`` files in ``pending`` BEFORE
255
- their first ingest — surfacing each for that first ingest attempt in the
297
+ frontier, and registers brand-new logical rollout files in ``pending``
298
+ BEFORE their first ingest — surfacing each for that first ingest attempt in the
256
299
  SAME cycle it is discovered (the pending sweep then owns the retries).
257
300
  Returns True iff it consumed a filesystem op."""
258
301
  if not self._enum_queue:
@@ -267,10 +310,12 @@ class CodexChildFrontier:
267
310
  if path not in self._dir_set:
268
311
  self._dir_set.add(path)
269
312
  self._dirs.append(path) # new subdir joins frontier
270
- elif path.endswith(".jsonl"):
271
- if path not in self._known and path not in self._pending:
272
- self._pending[path] = 0 # register BEFORE first ingest
273
- ingest.add(path) # first ingest attempt now
313
+ elif _rc.is_container_name(name):
314
+ # Both spellings enumerate one logical identity.
315
+ logical = os.fspath(_rc.logical_path(path))
316
+ if logical not in self._known and logical not in self._pending:
317
+ self._pending[logical] = 0 # register BEFORE first ingest
318
+ ingest.add(logical) # first ingest attempt now
274
319
  return True
275
320
 
276
321
  def _op_pending(self, ingest: set) -> bool:
@@ -286,12 +331,12 @@ class CodexChildFrontier:
286
331
  p = keys[idx]
287
332
  if p not in self._pending: # reaped/dropped mid-cycle
288
333
  continue
289
- size = self._file_size(p) # 1 fs op
290
- if size is None:
334
+ sig = self._file_size(p) # 1 signature probe
335
+ if sig is None:
291
336
  self._pending.pop(p, None) # vanished → drop
292
337
  else:
293
338
  committed = self._committed.get(p)
294
- if committed is None or size > committed:
339
+ if codex_sig_advanced(sig, committed):
295
340
  ingest.add(p)
296
341
  return True
297
342
  return False