cctally 1.113.0 → 1.115.0

package/bin/_lib_clone_copy.py

@@ -0,0 +1,142 @@
+"""Clone-first file and tree copies (stdlib only; #940 D2, spec §6.7).
+
+On APFS, ``fclonefileat(2)`` gives a copy that shares the source's blocks
+until either side is written, so a copy of a large test scaffold writes no
+data. This module takes that clone wherever it can and a real copy wherever
+it cannot, and otherwise behaves exactly as the stdlib copier it stands in
+for:
+
+- :func:`copy2` is ``shutil.copy2`` for the same arguments: a directory
+  destination keeps the source's basename, ``follow_symlinks=False`` copies a
+  link as a link, the same file raises ``shutil.SameFileError``, and an
+  existing destination is overwritten in place (same inode), which a clone
+  cannot do, so that case is a real copy. Metadata is ``shutil.copystat``'s,
+  applied after the clone as after a copy.
+- :func:`copytree` is ``shutil.copytree`` with :func:`copy2` as its file
+  copier, so symlink policy, ignore lists, dangling-link errors and
+  destination collisions are the stdlib's own code.
+
+A clone is attempted only on Darwin. An unsupported filesystem, a
+cross-volume destination, a missing ``fclonefileat`` and any non-Darwin
+platform take the real copy; any other error (EIO, ENOSPC, EACCES, …)
+propagates and is never retried as a copy. A clone is never a hardlink: the
+copy is a new inode with a link count of one.
+
+The optional ``counts`` mapping is incremented under ``"clone"`` or
+``"copy"`` per file, so a caller or a measurement can see which path ran.
+This module is public and imports nothing outside the standard library.
+
+It binds ``fclonefileat`` itself rather than importing the binding in
+``bin/_lib_remote_staging.py`` (#940 D-2 F17): that module is private, so it
+is absent from the public tree that this module and its public callers ship
+in, and a public module may not import it. The staging module may import
+from this one instead (spec §6.7).
+"""
+
+from __future__ import annotations
+
+import ctypes
+import errno
+import os
+import shutil
+import stat
+import sys
+
+# Errors that mean "this filesystem or this pair of volumes cannot clone",
+# never "the data could not be written". EEXIST is a destination created
+# after the existence check: `shutil.copy2` overwrites it, and so do we.
+_FALLBACK_ERRNOS = frozenset(
+    code for code in (
+        getattr(errno, "ENOTSUP", None), getattr(errno, "EOPNOTSUPP", None),
+        errno.ENOSYS, errno.EXDEV, errno.EINVAL, errno.EPERM, errno.EEXIST)
+    if code is not None)
+
+_AT_FDCWD = -2            # Darwin's AT_FDCWD
+_CLONE_NOOWNERCOPY = 0x0002  # leave ownership to the caller, as a copy does
+
+_LIBC_CLONE = []  # the resolved fclonefileat, once looked up
+# The platform this module decides on; a test sets it here instead of
+# patching the interpreter-wide ``sys.platform`` (#940 D-2 F16).
+_PLATFORM = sys.platform
+
+
+def _fclonefileat():
+    """The C ``fclonefileat`` on Darwin, else None."""
+    if _PLATFORM != "darwin":
+        return None
+    if not _LIBC_CLONE:
+        libc = ctypes.CDLL(None, use_errno=True)
+        clone = getattr(libc, "fclonefileat", None)
+        if clone is not None:
+            clone.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32)
+            clone.restype = ctypes.c_int
+        _LIBC_CLONE.append(clone)
+    return _LIBC_CLONE[0]
+
+
+def _clone_fd(source_fd: int, destination: str) -> None:
+    """Clone the open regular file `source_fd` to the new path `destination`."""
+    clone = _fclonefileat()
+    if clone is None:
+        raise OSError(errno.ENOTSUP, "fclonefileat is unavailable", destination)
+    if clone(source_fd, _AT_FDCWD, os.fsencode(os.path.abspath(destination)),
+             _CLONE_NOOWNERCOPY) != 0:
+        code = ctypes.get_errno()
+        raise OSError(code, os.strerror(code), destination)
+
+
+def _count(counts, method: str) -> None:
+    if counts is not None:
+        counts[method] = counts.get(method, 0) + 1
+
+
+def copy2(src, dst, *, follow_symlinks: bool = True, counts=None):
+    """``shutil.copy2(src, dst)``, by clone where the filesystem allows it.
+
+    Returns the destination path, as ``shutil.copy2`` does.
+    """
+    if os.path.isdir(dst):
+        dst = os.path.join(dst, os.path.basename(src))
+    if (not follow_symlinks and os.path.islink(src)) or os.path.lexists(dst):
+        # A link copied as a link, or a destination to overwrite in place.
+        result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
+        _count(counts, "copy")
+        return result
+    try:
+        fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
+    except OSError:
+        # The stdlib's own error for this source (missing, unreadable, …).
+        return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
+    try:
+        if not stat.S_ISREG(os.fstat(fd).st_mode):
+            # A FIFO, a device or a directory: the stdlib decides.
+            os.close(fd)
+            fd = -1
+            return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
+        try:
+            _clone_fd(fd, dst)
+        except OSError as exc:
+            if exc.errno not in _FALLBACK_ERRNOS:
+                raise
+            result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
+            _count(counts, "copy")
+            return result
+        shutil.copystat(src, dst, follow_symlinks=follow_symlinks)
+        _count(counts, "clone")
+        return dst
+    finally:
+        if fd >= 0:
+            os.close(fd)
+
+
+def copytree(src, dst, symlinks: bool = False, ignore=None,
+             ignore_dangling_symlinks: bool = False, dirs_exist_ok: bool = False,
+             *, counts=None):
+    """``shutil.copytree`` with :func:`copy2` as its file copier."""
+    def copier(source, destination):
+        return copy2(source, destination, counts=counts)
+
+    return shutil.copytree(src, dst, symlinks=symlinks, ignore=ignore,
+                           copy_function=copier,
+                           ignore_dangling_symlinks=ignore_dangling_symlinks,
+                           dirs_exist_ok=dirs_exist_ok)

package/bin/_lib_codex_conversation.py

@@ -2167,6 +2167,38 @@ def _is_late_turn_anchor(
     )
 
 
+def in_physical_order(rows: Iterable[Any], *, timestamp, position) -> list:
+    """Rows read in ``(timestamp_utc, id)`` index order, returned in physical
+    order: ``(timestamp_utc, source_path, line_offset)``.
+
+    #901 Amendment 19 T1 (spec I1). ``idx_codex_conv_msgs_conversation`` is
+    ``(conversation_key, timestamp_utc, id)``, so a read ordered by the
+    physical position past the timestamp left SQLite a "LAST 2 TERMS" sorter
+    over every run of rows tied on one timestamp, and one conversation's tie
+    run (a whole conversation, when its rows share or lack a timestamp) is not
+    a bound. The caller now reads in index order and this orders each run in
+    Python: the same rows the caller already holds, no temp structure.
+
+    Exact: equal timestamps (NULL included) are contiguous in index order and
+    compare equal here as they do in an ORDER BY, and ``UNIQUE(source_path,
+    line_offset)`` makes ``position`` a total order inside a run.
+    """
+    ordered: list = []
+    run: list = []
+    run_timestamp = None
+    for row in rows:
+        stamp = timestamp(row)
+        if run and stamp != run_timestamp:
+            run.sort(key=position)
+            ordered.extend(run)
+            run = []
+        run_timestamp = stamp
+        run.append(row)
+    run.sort(key=position)
+    ordered.extend(run)
+    return ordered
+
+
 def codex_event_is_late_turn_anchor(event: Any) -> bool:
     """Public predicate shared by delta persistence and turn inference."""
     try:

package/bin/_lib_codex_conversation_query.py

@@ -27,11 +27,13 @@ import json
 import os
 import re
 import sqlite3
+import stat
 import threading
 from collections import deque
 from types import MappingProxyType
 
 import _lib_codex_conversation as kern
+import _lib_codex_rollout_container as _rc
 import _lib_codex_landmarks as landmarks
 import _lib_codex_segments as segkern
 from _lib_codex_find_projection import (
@@ -353,13 +355,19 @@ def _load_conversation_index_rows(
     """
     rows: list = []
     detail_bytes: dict[tuple[str, int], int] = {}
+    # #901 Amendment 19 T1: index order, each timestamp tie run then put in
+    # physical order in Python (``kern.in_physical_order``), not a sorter.
     for (source_path, line_offset, timestamp_utc, turn_id, call_id, kind,
          event_type, record_family, model, content_digest, content_len,
-         detail_json, detail_len) in conn.execute(
-        "SELECT " + _NARROW_ROW_COLS + " FROM codex_conversation_messages "
-        "WHERE conversation_key = ? "
-        "ORDER BY timestamp_utc, source_path, line_offset",
-        (conversation_key,),
+         detail_json, detail_len) in kern.in_physical_order(
+        conn.execute(
+            "SELECT " + _NARROW_ROW_COLS + " FROM codex_conversation_messages "
+            "WHERE conversation_key = ? "
+            "ORDER BY timestamp_utc, id",
+            (conversation_key,),
+        ),
+        timestamp=lambda row: row[2],
+        position=lambda row: (row[0], row[1]),
     ):
         rows.append(kern.CodexNormalizedRow(
             conversation_key=conversation_key, source_root_key="",
@@ -1857,6 +1865,29 @@ def codex_conversation_exists(conn: sqlite3.Connection, conversation_key: str) -
     return row is not None
 
 
+def codex_publication_pending(
+    conn: sqlite3.Connection, conversation_key: str,
+) -> bool:
+    """A barrier-population conversation with no publication yet (#968 5.11).
+
+    Such a conversation stays in the existing pending-state handling until it
+    has an eligible publication: it answers ``normalization_pending``, never
+    ``not_found``. Any store error answers False (the caller's ``not_found``).
+    """
+    try:
+        if conn.execute(
+                "SELECT 1 FROM cache_db.codex_barrier_population "
+                "WHERE conversation_key=?", (conversation_key,)).fetchone() is None:
+            return False
+        row = conn.execute(
+            "SELECT published_revision, legacy_uncertified "
+            "FROM main.codex_conversation_publications WHERE conversation_key=?",
+            (conversation_key,)).fetchone()
+    except sqlite3.Error:
+        return False
+    return row is None or (int(row[0] or 0) == 0 and not row[1])
+
+
 def codex_conversation_source_paths(
     conn: sqlite3.Connection, conversation_key: str
 ) -> list[str]:
@@ -2339,6 +2370,9 @@ def get_codex_conversation(
     # ── Phase A: the narrow index pass ───────────────────────────────────────
     rows, detail_bytes = _load_conversation_index_rows(conn, conversation_key)
     if not rows:
+        if codex_publication_pending(conn, conversation_key):
+            return {"status": "normalization_pending",
+                    "conversation_key": conversation_key, "items": [], "children": []}
         return {"status": "not_found", "conversation_key": conversation_key}
     undecodable_keys = undecodable_codex_conversation_keys(conn)
     kept, _suppressed = kern.pair_mirrors(rows)
@@ -3029,6 +3063,9 @@ def _outline_envelope(
     # The outline is not the route F3 bounds, and it stays turn-granular.
     rows = _load_conversation_rows(conn, conversation_key)
     if not rows:
+        if codex_publication_pending(conn, conversation_key):
+            return {"status": "normalization_pending",
+                    "conversation_key": conversation_key, "turns": [], "files": [], "children": []}
         return {"status": "not_found", "conversation_key": conversation_key}
     detail_bytes = _detail_bytes_of(rows)
     kept, _suppressed = kern.pair_mirrors(rows)
@@ -3415,20 +3452,26 @@ def _stored_browse_page_queries(
             "(COALESCE(r.last_activity_utc, '') = ? AND r.conversation_key < ?))")
         page_params.extend((cursor_last, cursor_last, cursor_key))
 
+    # Keep native/parent enrichment inside the page statement. A LEFT JOIN
+    # against the compound last-good view materializes retained history;
+    # correlated probes push these predicates into its indexed branches.
+    # The scalar parent lookup follows the detail reader's same-root lookup.
+    parent_key_sql = (
+        "(SELECT pt.conversation_key FROM codex_conversation_threads pt "
+        "WHERE pt.source_root_key = r.source_root_key "
+        "AND pt.native_thread_id = r.parent_thread_id "
+        "AND pt.conversation_key != r.conversation_key LIMIT 1)"
+    )
     sql = (
         "SELECT r.conversation_key, r.item_count, r.started_utc, "
         "r.last_activity_utc, r.project_key, r.project_label, r.models_json, "
-        "r.title, r.parent_thread_id, r.source_root_key, t.native_thread_id, "
-        "pt.conversation_key, pr.title, pr.project_label, pt.native_thread_id "
+        "r.title, r.parent_thread_id, r.source_root_key, "
+        "(SELECT t.native_thread_id FROM codex_conversation_threads t "
+        "WHERE t.conversation_key = r.conversation_key), "
+        f"{parent_key_sql}, pr.title, pr.project_label, r.parent_thread_id "
         "FROM codex_conversation_rollups r "
-        "LEFT JOIN codex_conversation_threads t "
-        "ON t.conversation_key = r.conversation_key "
-        "LEFT JOIN codex_conversation_threads pt "
-        "ON pt.source_root_key = r.source_root_key "
-        "AND pt.native_thread_id = r.parent_thread_id "
-        "AND pt.conversation_key != r.conversation_key "
         "LEFT JOIN codex_conversation_rollups pr "
-        "ON pr.conversation_key = pt.conversation_key "
+        f"ON pr.conversation_key = {parent_key_sql} "
         f"WHERE {' AND '.join(page_where)} "
         "ORDER BY COALESCE(r.last_activity_utc, '') DESC, r.conversation_key DESC"
     )
@@ -3872,34 +3915,165 @@ def _project_find_row(row, *, payload: dict | None = None, block: dict | None =
     return project_plain((RenderLeaf("t0", text),))
 
 
+#: Every ``codex_find_projection`` column in one fixed order: the builder's row
+#: tuples, the stored-row read and the write statements below all use it.
+#: ``(message_id, surface)`` (positions 0 and 5) is the table's primary key;
+#: everything else is an attribute the difference compares (#901 §5.3b).
+_FIND_PROJECTION_COLUMNS = (
+    "message_id,conversation_key,item_key,block_key,container_block_key,"
+    "surface,render_order,projected_text,leaves_json,disclosure_json,"
+    "projection_version"
+)
+_FIND_PROJECTION_KEY = (0, 5)
+_FIND_PROJECTION_ORDER = 6
+
+_FIND_PROJECTION_INSERT_SQL = (
+    "INSERT INTO codex_find_projection "
+    "(" + _FIND_PROJECTION_COLUMNS + ") "
+    "VALUES (?,?,?,?,?,?,?,?,?,?,?)"
+)
+#: An attribute change that keeps the row's place: the ordering index
+#: ``(conversation_key, render_order, message_id, surface)`` is untouched.
+_FIND_PROJECTION_UPDATE_IN_PLACE_SQL = (
+    "UPDATE codex_find_projection SET item_key=?,block_key=?,"
+    "container_block_key=?,projected_text=?,leaves_json=?,disclosure_json=?,"
+    "projection_version=? WHERE message_id=? AND surface=?"
+)
+#: A moved row (its dense ``render_order`` shifted): the index entry moves too.
+_FIND_PROJECTION_UPDATE_MOVED_SQL = (
+    "UPDATE codex_find_projection SET conversation_key=?,item_key=?,"
+    "block_key=?,container_block_key=?,render_order=?,projected_text=?,"
+    "leaves_json=?,disclosure_json=?,projection_version=? "
+    "WHERE message_id=? AND surface=?"
+)
+_FIND_PROJECTION_DELETE_SQL = (
+    "DELETE FROM codex_find_projection WHERE message_id=? AND surface=?"
+)
+_FIND_PROJECTION_GENERATION_SQL = (
+    "INSERT INTO cache_meta(key,value) VALUES"
+    "('codex_find_projection_generation','1') "
+    "ON CONFLICT(key) DO UPDATE SET value=CAST(value AS INTEGER)+1"
+)
+
+
+def _write_codex_find_projection_difference(
+    conn: sqlite3.Connection,
+    conversation_key: str,
+    built: list[tuple],
+) -> int:
+    """Make the stored projection of one conversation equal ``built``.
+
+    ``built`` is the complete canonical projection in the builder's order. Both
+    sides are keyed by the primary key ``(message_id, surface)`` and compared
+    on every other column; only the difference is written. Returns the number
+    of rows deleted, inserted or updated (0 for an unchanged conversation).
+    """
+    built_by_key = {
+        (row[_FIND_PROJECTION_KEY[0]], row[_FIND_PROJECTION_KEY[1]]): row
+        for row in built
+    }
+    if len(built_by_key) != len(built):
+        # The whole-rewrite writer raised on a duplicate key at INSERT time;
+        # a difference must not silently keep only one of the two rows.
+        raise sqlite3.IntegrityError(
+            "UNIQUE constraint failed: codex_find_projection.message_id, "
+            "codex_find_projection.surface"
+        )
+    stored = {
+        (row[_FIND_PROJECTION_KEY[0]], row[_FIND_PROJECTION_KEY[1]]): row
+        for row in conn.execute(
+            "SELECT " + _FIND_PROJECTION_COLUMNS + " FROM codex_find_projection "
+            "WHERE conversation_key=?",
+            (conversation_key,),
+        )
+    }
+    deletes = sorted(key for key in stored if key not in built_by_key)
+    inserts: list[tuple] = []
+    in_place: list[tuple] = []
+    moved: list[tuple] = []
+    for key, row in built_by_key.items():
+        old = stored.get(key)
+        if old is None:
+            inserts.append(row)
+            continue
+        if tuple(old) == row:
+            continue
+        message_id, conversation, item, block, container, surface, order, *rest = row
+        if old[1] == conversation and old[_FIND_PROJECTION_ORDER] == order:
+            in_place.append((item, block, container, *rest, message_id, surface))
+        else:
+            moved.append(
+                (conversation, item, block, container, order, *rest,
+                 message_id, surface)
+            )
+    if deletes:
+        conn.executemany(_FIND_PROJECTION_DELETE_SQL, deletes)
+    if in_place:
+        conn.executemany(_FIND_PROJECTION_UPDATE_IN_PLACE_SQL, in_place)
+    if moved:
+        conn.executemany(_FIND_PROJECTION_UPDATE_MOVED_SQL, moved)
+    if inserts:
+        conn.executemany(_FIND_PROJECTION_INSERT_SQL, inserts)
+    return len(deletes) + len(inserts) + len(in_place) + len(moved)
+
+
+def _load_conversation_rows(
+    conn: sqlite3.Connection, conversation_key: str,
+) -> list:
+    """A conversation's WIDE normalized rows in physical order:
+    ``(timestamp_utc, source_path, line_offset)``.
+
+    #901 Amendment 19 T1: read in ``(timestamp_utc, id)`` index order and each
+    timestamp tie run put in physical order by ``kern.in_physical_order``; the
+    old ``ORDER BY`` left a partial sorter over every tie run, and one
+    conversation is not a bound."""
+    return kern.in_physical_order(
+        (
+            kern.CodexNormalizedRow(*row)
+            for row in conn.execute(
+                "SELECT " + _ROW_COLS + " FROM codex_conversation_messages "
+                "WHERE conversation_key=? "
+                "ORDER BY timestamp_utc,id",
+                (conversation_key,),
+            )
+        ),
+        timestamp=lambda row: row.timestamp_utc,
+        position=lambda row: (row.source_path, row.line_offset),
+    )
+
+
 def materialize_codex_find_projection(
     conn: sqlite3.Connection,
     conversation_keys,
+    *,
+    cascaded: bool = False,
 ) -> None:
-    """Replace #482 projection rows for the affected conversations.
+    """Bring #482 projection rows of the affected conversations up to date.
 
     The existing item/block builder is the only authority for native folds.
     Every searchable physical row keeps its own block key; a folded output or
     completion separately records the visual call block that owns it.
+
+    #901 §5.3b (W6): each conversation's complete canonical projection is still
+    built in memory, exactly as before, but only its difference against the
+    stored rows is written, so an unchanged conversation writes no row.
+    ``codex_find_projection_generation`` advances exactly once per call that
+    changed a row, or when the caller says ``cascaded=True``: it deleted
+    messages of these conversations in the same transaction, so the message
+    trigger already cascaded their projection rows away and the builder may
+    find nothing left to change. A true no-op leaves the generation alone.
+    Callers make one call per transaction, so this is once per transaction.
     """
     keys = sorted({key for key in conversation_keys if key})
     if not keys:
         return
+    changed = 0
     for conversation_key in keys:
-        conn.execute(
-            "DELETE FROM codex_find_projection WHERE conversation_key=?",
-            (conversation_key,),
-        )
-        rows = [
-            kern.CodexNormalizedRow(*row)
-            for row in conn.execute(
-                "SELECT " + _ROW_COLS + " FROM codex_conversation_messages "
-                "WHERE conversation_key=? "
-                "ORDER BY timestamp_utc,source_path,line_offset",
-                (conversation_key,),
-            )
-        ]
+        built: list[tuple] = []
+        rows = _load_conversation_rows(conn, conversation_key)
         if not rows:
+            changed += _write_codex_find_projection_difference(
+                conn, conversation_key, built)
             continue
         kept, _suppressed = kern.pair_mirrors(rows)
         items = kern.canonical_items(kept)
@@ -3941,12 +4115,9 @@ def materialize_codex_find_projection(
                 if row.kind in {"reasoning", "meta"} or surface != "body"
                 else []
             )
-            conn.execute(
-                "INSERT INTO codex_find_projection "
-                "(message_id,conversation_key,item_key,block_key,"
-                "container_block_key,surface,render_order,projected_text,"
-                "leaves_json,disclosure_json,projection_version) "
-                "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+            # The builder collects complete rows; the difference below writes
+            # them (column order: `_FIND_PROJECTION_COLUMNS`).
+            built.append(
                 (
                     message_id,
                     conversation_key,
@@ -4032,11 +4203,11 @@ def materialize_codex_find_projection(
                 container = completion_owner.get(physical_key, physical_key)
                 store(row, container_block_key=container)
 
-    conn.execute(
-        "INSERT INTO cache_meta(key,value) VALUES"
-        "('codex_find_projection_generation','1') "
-        "ON CONFLICT(key) DO UPDATE SET value=CAST(value AS INTEGER)+1"
-    )
+        changed += _write_codex_find_projection_difference(
+            conn, conversation_key, built)
+
+    if changed or cascaded:
+        conn.execute(_FIND_PROJECTION_GENERATION_SQL)
 
 
 def _fts_query(query: str, column: str | None) -> str:
@@ -4828,6 +4999,8 @@ def find_in_codex_conversation(
         return {**base, "status": "normalization_pending"}
     rows = _load_conversation_rows(conn, conversation_key)
     if not rows:
+        if codex_publication_pending(conn, conversation_key):
+            return {**base, "status": "normalization_pending"}
         return {"status": "not_found", "conversation_key": conversation_key}
     q = (query or "").strip()
     if not q:
@@ -4887,6 +5060,9 @@ def codex_conversation_prompts(conn: sqlite3.Connection, conversation_key: str)
                 "conversation_key": conversation_key, "prompts": []}
     rows = _load_conversation_rows(conn, conversation_key)
     if not rows:
+        if codex_publication_pending(conn, conversation_key):
+            return {"status": "normalization_pending",
+                    "conversation_key": conversation_key, "prompts": []}
         return {"status": "not_found", "conversation_key": conversation_key}
     kept, _suppressed = kern.pair_mirrors(rows)
     items = kern.canonical_items(kept)
@@ -4927,7 +5103,40 @@ def _within_root(source_path: str | None, root_path: str | None) -> bool:
         return False
 
 
-def _reread_codex_full_content(conn: sqlite3.Connection, row):
+class CodexContainerReadError(RuntimeError):
+    """A present rollout failed to read; the query envelope reports HTTP 500."""
+
+    def __init__(self, category: str, logical: str) -> None:
+        super().__init__(f"Codex rollout could not be read ({category})")
+        self.category = category
+        self.logical = logical
+
+
+class _CodexPayloadContainmentError(RuntimeError):
+    """Internal 404 signal: the selected physical container escaped its root."""
+
+
+def _select_codex_payload_container(logical):
+    # The discovery leaf deliberately treats stat failures as absent. A payload
+    # request must distinguish a missing container (410) from inaccessible data
+    # (500), including an inaccessible plain file beside a readable zstd twin.
+    for candidate in (logical, _rc.zstd_twin(logical)):
+        try:
+            info = os.stat(candidate)
+        except (FileNotFoundError, NotADirectoryError):
+            continue
+        except OSError as exc:
+            raise CodexContainerReadError("read-error", str(logical)) from exc
+        if not stat.S_ISREG(info.st_mode):
+            raise CodexContainerReadError("read-error", str(logical))
+        selected = _rc.select_container(logical)
+        if selected is not None:
+            return selected
+        # It disappeared between the strict stat and selection; try the twin.
+    return None
+
+
+def _reread_codex_full_content(conn: sqlite3.Connection, row, *, streams: "dict | None" = None):
     """Re-read the physical line at ``(row.source_path, row.line_offset)``, validate
     it against the stored ``codex_conversation_events.payload_json`` for that exact
     position (§3.4 structural gone-check — the canonical FULL record, not
@@ -4939,19 +5148,49 @@ def _reread_codex_full_content(conn: sqlite3.Connection, row):
     The full pre-cap content is re-derived through the SAME ``_extract`` the
     normalizer uses — which is how payload serves content beyond the normalized
     ``CODEX_TEXT_CAP``. Truncation/``truncated`` is against ``_FULL_PAYLOAD_CEILING``
-    (1,000,000 Python characters), the same ceiling the Claude payload path uses."""
+    (1,000,000 Python characters), the same ceiling the Claude payload path uses.
+
+    #968: select plain first, else zstd, and seek in decoded offsets. Streams are
+    request-local and reusable; no decoded copies persist. Present unreadable
+    containers raise CodexContainerReadError, never a false gone result."""
     stored = conn.execute(
         "SELECT payload_json FROM codex_conversation_events "
         "WHERE source_path = ? AND line_offset = ?",
         (row.source_path, row.line_offset)).fetchone()
     if stored is None:
         return None
-    try:
-        with open(row.source_path, "rb") as fh:
-            fh.seek(row.line_offset)
-            line = fh.readline()
-    except OSError:
-        return None
+    for attempt in range(2):
+        selected = _select_codex_payload_container(row.source_path)
+        if selected is None:
+            return None
+        # Validate exactly the descriptor we open, including after a compression
+        # race. The caller has already checked the logical path and known root.
+        root_path = _codex_source_root_path(conn, row.source_root_key)
+        if not _within_root(str(selected.path), root_path):
+            raise _CodexPayloadContainmentError()
+        try:
+            fh = None if streams is None else streams.get(str(selected.path))
+            if fh is None:
+                fh = _rc.open_decoded(selected)
+                if streams is not None:
+                    streams[str(selected.path)] = fh
+            try:
+                fh.seek(row.line_offset)
+                line = fh.readline()
+            finally:
+                if streams is None:
+                    fh.close()
+            break
+        except OSError as exc:
+            # ContainerError subclasses OSError; preserve its approved leaf
+            # category instead of flattening decoder/integrity errors.
+            replacement = _select_codex_payload_container(row.source_path)
+            if replacement is None:
+                return None
+            if replacement.path != selected.path and attempt == 0:
+                continue  # plain vanished during Codex's compression window
+            category = exc.category if isinstance(exc, _rc.ContainerError) else "read-error"
+            raise CodexContainerReadError(category, str(row.source_path)) from exc
     try:
         obj = json.loads(line.decode("utf-8"))
     except (UnicodeDecodeError, ValueError, TypeError):
@@ -5038,7 +5277,9 @@ def read_codex_payload(
     full bounded ``card``, and ``truncated`` reflects ``_FULL_PAYLOAD_CEILING``.
     ``gone`` (→ HTTP 410) means the physical
     record moved/mutated; ``not_found`` (→ 404) means no such block, no output
-    partner, or a containment miss (a read is never attempted outside the root)."""
+    partner, or a containment miss (a read is never attempted outside the root).
+    A present container that cannot be read raises CodexContainerReadError through
+    the existing query-error envelope, including plain-file read failures."""
     miss = {"status": "not_found", "block_key": block_key, "which": which}
     if which not in ("call", "output", "event"):
         return miss
@@ -5053,7 +5294,14 @@ def read_codex_payload(
     root_path = _codex_source_root_path(conn, target.source_root_key)
     if not _within_root(target.source_path, root_path):
         return miss
-    outcome = _reread_codex_full_content(conn, target)
+    streams: dict = {}
+    try:
+        outcome = _reread_codex_full_content(conn, target, streams=streams)
+    except _CodexPayloadContainmentError:
+        return miss
+    finally:
+        for stream in streams.values():
+            stream.close()
     if outcome is None:
         return {"status": "gone", "block_key": block_key, "which": which}
     content, truncated, card = outcome