ccx-native 0.0.1 → 0.2.0

package/ccx/codex/history.py

@@ -0,0 +1,158 @@
+"""Reconstruct a native thread from source-attested paged Codex history."""
+from __future__ import annotations
+
+import time
+from typing import Any, Callable
+
+_PAGE_SIZE = 100
+_MAX_PAGES = 10_000
+
+
+def _remaining(deadline: float) -> float:
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise TimeoutError("Codex history paging timed out")
+    return remaining
+
+
+def _full_turn_items(request: Callable[..., dict], thread_id: str,
+                     turn_id: str, deadline: float) -> list[dict]:
+    items: list[dict] = []
+    cursor: str | None = None
+    seen: set[str] = set()
+    for _ in range(_MAX_PAGES):
+        params: dict[str, Any] = {
+            "threadId": thread_id, "turnId": turn_id,
+            "limit": _PAGE_SIZE, "sortDirection": "asc",
+        }
+        if cursor is not None:
+            params["cursor"] = cursor
+        page = request("thread/items/list", params, timeout=_remaining(deadline))
+        entries = page.get("data") if isinstance(page, dict) else None
+        if not isinstance(entries, list):
+            raise ValueError("Codex item-history page is malformed")
+        for wrapper in entries:
+            if (not isinstance(wrapper, dict) or wrapper.get("turnId") != turn_id
+                    or not isinstance(wrapper.get("item"), dict)):
+                raise ValueError("Codex item-history page belongs to another turn")
+            items.append(wrapper["item"])
+        next_cursor = page.get("nextCursor")
+        if next_cursor is None:
+            return items
+        if (not isinstance(next_cursor, str) or not next_cursor
+                or next_cursor in seen or not entries):
+            raise ValueError("Codex item-history cursor did not advance")
+        seen.add(next_cursor)
+        cursor = next_cursor
+    raise ValueError("Codex item history exceeded page budget")
+
+
+def fill_thread(
+    thread: dict, request: Callable[..., dict], *, deadline: float,
+    limit: int | None = None, newest_first: bool = False,
+    exact_turn_id: str | None = None,
+) -> dict:
+    """Return the thread/read shape, with actual turn statuses and full items."""
+    thread_id = thread.get("id") if isinstance(thread, dict) else None
+    if not isinstance(thread_id, str) or not thread_id:
+        raise ValueError("Codex source metadata has no original thread id")
+    if limit is not None and (type(limit) is not int or limit < 1):
+        raise ValueError("Codex history turn limit must be positive")
+    if exact_turn_id is not None and (not isinstance(exact_turn_id, str)
+                                       or not exact_turn_id):
+        raise ValueError("Codex exact turn id is invalid")
+    descending = newest_first or exact_turn_id is not None
+    turns: list[dict] = []
+    seen: set[str] = set()
+    seen_turns: set[str] = set()
+    cursor: str | None = None
+    for _ in range(_MAX_PAGES):
+        params: dict[str, Any] = {
+            "threadId": thread_id,
+            "limit": min(_PAGE_SIZE, limit - len(turns)) if limit else _PAGE_SIZE,
+            "sortDirection": "desc" if descending else "asc",
+            "itemsView": "notLoaded" if exact_turn_id else "full",
+        }
+        if cursor is not None:
+            params["cursor"] = cursor
+        page = request("thread/turns/list", params, timeout=_remaining(deadline))
+        data = page.get("data") if isinstance(page, dict) else None
+        if not isinstance(data, list):
+            raise ValueError("Codex turn-history page is malformed")
+        for turn in data:
+            if (not isinstance(turn, dict) or not isinstance(turn.get("id"), str)
+                    or not turn["id"] or not isinstance(turn.get("status"), str)
+                    or not isinstance(turn.get("items"), list)
+                    or turn["id"] in seen_turns):
+                raise ValueError("Codex native turn-history entry is malformed")
+            seen_turns.add(turn["id"])
+            if exact_turn_id is not None and turn["id"] != exact_turn_id:
+                continue
+            result = dict(turn)
+            if turn.get("itemsView", "full") != "full":
+                result["items"] = _full_turn_items(request, thread_id, turn["id"], deadline)
+                result["itemsView"] = "full"
+            turns.append(result)
+            if exact_turn_id is not None or (limit is not None and len(turns) >= limit):
+                return {**thread, "turns": list(reversed(turns)) if descending else turns,
+                        "historySource": "thread/turns/list"}
+        next_cursor = page.get("nextCursor")
+        if next_cursor is None:
+            return {**thread, "turns": list(reversed(turns)) if descending else turns,
+                    "historySource": "thread/turns/list"}
+        if (not isinstance(next_cursor, str) or not next_cursor
+                or next_cursor in seen or not data):
+            raise ValueError("Codex turn-history cursor did not advance")
+        seen.add(next_cursor)
+        cursor = next_cursor
+    raise ValueError("Codex turn history exceeded page budget")
+
+
+def read_original_thread(request: Callable[..., dict], thread_id: str, *,
+                         deadline: float, limit: int | None = None,
+                         newest_first: bool = False,
+                         exact_turn_id: str | None = None,
+                         expected_source: str | None = None) -> dict:
+    """One provider connection: metadata proof followed by bounded page reads."""
+    reply = request("thread/read", {"threadId": thread_id, "includeTurns": False},
+                    timeout=_remaining(deadline))
+    thread = reply.get("thread") if isinstance(reply, dict) else None
+    if not isinstance(thread, dict) or thread.get("id") != thread_id:
+        raise ValueError("Codex source metadata did not return the expected thread")
+    if expected_source is not None and thread.get("threadSource") != expected_source:
+        raise ValueError("Codex original thread source changed before history paging")
+    return fill_thread(thread, request, deadline=deadline, limit=limit,
+                       newest_first=newest_first, exact_turn_id=exact_turn_id)
+
+
+def read_fork_source_point(request: Callable[..., dict], thread_id: str, *,
+                           deadline: float) -> dict:
+    """Read only an unmanaged native source's metadata and newest turn.
+
+    This is NOT worker history authorization. It never starts, resumes,
+    unarchives or mutates the native source. Admission is a separate child
+    thread/fork with its own independently verified source tag.
+    """
+    from uuid import UUID
+    if (not isinstance(thread_id, str)
+            or str(UUID(thread_id)) != thread_id):
+        raise ValueError("import needs one exact native Codex thread UUID")
+    reply = request(
+        "thread/read", {"threadId": thread_id, "includeTurns": False},
+        timeout=_remaining(deadline),
+    )
+    thread = reply.get("thread") if isinstance(reply, dict) else None
+    if not isinstance(thread, dict) or thread.get("id") != thread_id:
+        raise ValueError("Codex native import source ID is not proven")
+    page = request("thread/turns/list", {
+        "threadId": thread_id, "limit": 1,
+        "sortDirection": "desc", "itemsView": "notLoaded",
+    }, timeout=_remaining(deadline))
+    items = page.get("data") if isinstance(page, dict) else None
+    if not isinstance(items, list) or len(items) > 1:
+        raise ValueError("Codex native import source turn page malformed")
+    last = items[0] if items else None
+    if last is not None and (not isinstance(last, dict)
+                             or not isinstance(last.get("id"), str)):
+        raise ValueError("Codex native import source turn ID unavailable")
+    return {"thread": thread, "lastTurn": last}

package/ccx/codex/history_store.py

@@ -0,0 +1,131 @@
+"""Read one Codex thread's history from Codex's local history database.
+
+Codex keeps a SQLite projection of every thread's rollout,
+``$CODEX_HOME/thread_history_1.sqlite``. Its ``thread_items.item_json`` holds
+the same ThreadItem JSON app-server returns, so a thread rebuilt from it feeds
+:func:`ccx.ops.conversation.build_codex` unchanged and yields the same turn
+sequence and item ordinals. Reading it takes milliseconds where app-server
+paging can take minutes, and loads nothing into codexd.
+
+It is Codex's private store, so every read is defensive: opened read-only
+(``mode=ro``, never ``immutable=1``, so SQLite still honours the WAL), gated on
+the tables and columns used here, short, and always closed. Any doubt returns
+``None`` and the caller falls back to app-server ``thread/read``.
+
+The thread id must come from the worker's verified original source
+(``codex.telemetry._original_source``), never from a record field read here.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from pathlib import Path
+from urllib.parse import quote
+
+HISTORY_DB_NAME = "thread_history_1.sqlite"
+
+# The schema this reader was checked against (2026-11, Codex `_sqlx_migrations`
+# version 8). A newer version is still read when the required columns exist;
+# the document then says so in a warning.
+VERIFIED_SCHEMA_VERSION = 8
+
+_REQUIRED = {
+    "thread_turns": {"thread_id", "turn_id", "rollout_ordinal", "status",
+                     "error_json", "started_at", "completed_at", "duration_ms"},
+    "thread_items": {"thread_id", "turn_id", "item_id", "rollout_ordinal",
+                     "item_json"},
+    "_sqlx_migrations": {"version", "success"},
+}
+_TURN_STATUSES = frozenset({"completed", "failed", "inProgress", "interrupted"})
+_BUSY_TIMEOUT_S = 2.0
+
+
+def history_db_path() -> Path:
+    # Like Codex itself, an empty CODEX_HOME means unset.
+    return Path(os.environ.get("CODEX_HOME") or "~/.codex").expanduser() / HISTORY_DB_NAME
+
+
+def _schema_ok(conn: sqlite3.Connection) -> bool:
+    for table, columns in _REQUIRED.items():
+        found = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
+        if not columns <= found:
+            return False
+    return True
+
+
+def read_thread_db(thread_id: str, *, path: Path | None = None) -> dict | None:
+    """Return the ``thread/read(includeTurns)`` shape for one thread, or ``None``.
+
+    Turns come in ``rollout_ordinal`` order, including turns with no items;
+    each turn's items come in their own ``rollout_ordinal`` order. The result
+    carries ``historySource: "codex-history-db"`` and ``historyDbVersion``.
+    """
+    if not isinstance(thread_id, str) or not thread_id:
+        return None
+    db = path or history_db_path()
+    if not db.is_file():
+        return None
+    try:
+        conn = sqlite3.connect(f"file:{quote(str(db))}?mode=ro", uri=True,
+                               timeout=_BUSY_TIMEOUT_S)
+    except sqlite3.Error:
+        return None
+    try:
+        if not _schema_ok(conn):
+            return None
+        row = conn.execute(
+            "SELECT max(version) FROM _sqlx_migrations WHERE success").fetchone()
+        version = row[0] if row else None
+        # One read transaction: turns and items come from the same snapshot.
+        conn.execute("BEGIN")
+        turn_rows = conn.execute(
+            "SELECT turn_id, status, error_json, started_at, completed_at, duration_ms"
+            " FROM thread_turns WHERE thread_id = ? ORDER BY rollout_ordinal",
+            (thread_id,)).fetchall()
+        item_rows = conn.execute(
+            "SELECT turn_id, item_json FROM thread_items"
+            " WHERE thread_id = ? ORDER BY rollout_ordinal",
+            (thread_id,)).fetchall()
+        conn.execute("COMMIT")
+    except sqlite3.Error:
+        return None
+    finally:
+        conn.close()
+
+    if not turn_rows:
+        return None
+    items: dict[str, list[dict]] = {}
+    for turn_id, item_json in item_rows:
+        try:
+            item = json.loads(item_json)
+        except (TypeError, ValueError):
+            return None
+        if not isinstance(item, dict):
+            return None
+        items.setdefault(turn_id, []).append(item)
+    known = {r[0] for r in turn_rows}
+    if set(items) - known:
+        return None  # items of a turn the projection doesn't list: not coherent
+
+    turns = []
+    for turn_id, status, error_json, started, completed, duration in turn_rows:
+        if status not in _TURN_STATUSES:
+            return None
+        error = None
+        if error_json:
+            try:
+                error = json.loads(error_json)
+            except ValueError:
+                return None
+        turns.append({
+            "id": turn_id, "status": status, "error": error,
+            "startedAt": started, "completedAt": completed, "durationMs": duration,
+            "items": items.get(turn_id, []), "itemsView": "full",
+        })
+    return {"id": thread_id, "turns": turns, "historySource": "codex-history-db",
+            "historyDbVersion": version}
+
+
+__all__ = ["HISTORY_DB_NAME", "VERIFIED_SCHEMA_VERSION", "history_db_path",
+           "read_thread_db"]

package/ccx/codex/notify_history.py

@@ -0,0 +1,140 @@
+"""Targeted Codex parent-notification delivery reconciliation.
+
+Notification delivery only needs to answer one question: did a native
+``worker_notification`` tool output with this delivery id already land?
+Hydrating an entire thread with ``thread/read(includeTurns=true)`` is both
+unnecessary and harmful for long-lived paginated Codex threads. Modern Codex
+exposes ``thread/items/list`` for incremental history access, so CCX pages
+that newest-first and never hydrates the full history for reconciliation.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Callable
+
+ITEM_PAGE_SIZE = 100
+MAX_ITEM_PAGES = 1000
+ITEM_PAGE_TIMEOUT_S = 10.0
+
+
+def _notification_payload(item: Any, *, name: str = "worker_notification") -> dict[str, Any] | None:
+    if not isinstance(item, dict):
+        return None
+    # ``thread/items/list`` wraps each native item with its containing turn id.
+    candidate = item.get("item") if isinstance(item.get("item"), dict) else item
+    if not isinstance(candidate, dict):
+        return None
+    if (
+        candidate.get("type") != "functionCallOutput"
+        or candidate.get("name") != name
+    ):
+        return None
+    raw = candidate.get("output")
+    if isinstance(raw, dict):
+        return raw
+    if not isinstance(raw, str):
+        return None
+    try:
+        value = json.loads(raw)
+    except json.JSONDecodeError:
+        return None
+    return value if isinstance(value, dict) else None
+
+
+def _item_delivery_match(
+    item: Any,
+    delivery_id: str,
+    *,
+    legacy_event_id: str | None = None,
+    name: str = "worker_notification",
+) -> dict[str, Any] | None:
+    value = _notification_payload(item, name=name)
+    if value is None:
+        return None
+    matches = (
+        value.get("deliveryId") == delivery_id
+        or bool(
+            legacy_event_id
+            and value.get("deliveryId") is None
+            and value.get("eventId") == legacy_event_id
+        )
+    )
+    if not matches:
+        return None
+    turn_id = item.get("turnId") if isinstance(item, dict) else None
+    return {
+        "deliveryId": delivery_id,
+        "eventId": value.get("eventId"),
+        "turnId": turn_id if isinstance(turn_id, str) and turn_id else None,
+    }
+
+
+def _paged_delivery_match(
+    request: Callable[..., dict[str, Any]],
+    thread_id: str,
+    delivery_id: str,
+    *,
+    legacy_event_id: str | None = None,
+    stop_at_turn_id: str | None = None,
+    name: str = "worker_notification",
+) -> dict[str, Any] | None:
+    cursor: str | None = None
+    seen_cursors: set[str] = set()
+    for _ in range(MAX_ITEM_PAGES):
+        params: dict[str, Any] = {
+            "threadId": thread_id,
+            "limit": ITEM_PAGE_SIZE,
+            "sortDirection": "desc",
+        }
+        if cursor is not None:
+            params["cursor"] = cursor
+        result = request("thread/items/list", params, timeout=ITEM_PAGE_TIMEOUT_S)
+        data = result.get("data") if isinstance(result, dict) else None
+        if not isinstance(data, list):
+            raise RuntimeError("thread/items/list returned malformed data")
+        for entry in data:
+            match = _item_delivery_match(
+                entry, delivery_id, legacy_event_id=legacy_event_id, name=name
+            )
+            if match is not None:
+                return match
+            if (
+                stop_at_turn_id
+                and isinstance(entry, dict)
+                and entry.get("turnId") == stop_at_turn_id
+            ):
+                # Newest-first reached a turn that had already completed
+                # before the first attempt; nothing older can hold it.
+                return None
+        next_cursor = result.get("nextCursor")
+        if not isinstance(next_cursor, str) or not next_cursor:
+            return None
+        if next_cursor in seen_cursors:
+            raise RuntimeError("thread/items/list repeated a pagination cursor")
+        seen_cursors.add(next_cursor)
+        cursor = next_cursor
+    raise RuntimeError(
+        f"thread/items/list exceeded {MAX_ITEM_PAGES} pages while reconciling notification"
+    )
+
+
+def notification_delivery_match(
+    request: Callable[..., dict[str, Any]],
+    thread_id: str,
+    delivery_id: str,
+    *,
+    legacy_event_id: str | None = None,
+    stop_at_turn_id: str | None = None,
+    name: str = "worker_notification",
+) -> dict[str, Any] | None:
+    """Find a landed notification, newest first.
+
+    ``stop_at_turn_id`` names a turn that had completed before the delivery's
+    first attempt: the paged walk ends there instead of at the thread's start.
+    """
+    return _paged_delivery_match(
+        request, thread_id, delivery_id,
+        legacy_event_id=legacy_event_id, stop_at_turn_id=stop_at_turn_id,
+        name=name,
+    )