ccx-native 0.1.0 → 0.2.0

package/ccx/ops/cancel.py

@@ -78,7 +78,13 @@ def cancel_turn(
                                 short=worker_id[:8], worker_id=worker_id)
     # In particular, an invented native id is never a cancel capability.
     if engine == "codex":
-        opening = codex_receipt_for_turn(handle, native_turn_id)
+        from ccx.codex.delivery import DeliveryReceiptError
+        try:
+            opening = codex_receipt_for_turn(handle, native_turn_id)
+        except DeliveryReceiptError:
+            # A damaged receipt somewhere proves nothing about this turn;
+            # the live exact turn below still decides, as with no receipt.
+            opening = None
     else:
         from ccx.claude.delivery import iter_receipts
         opening = next((row for row in iter_receipts(handle)
@@ -86,14 +92,11 @@ def cancel_turn(
                         and row.get("route") == "new_turn"), None)
     if prior is None and (not isinstance(opening, dict)
                           or opening.get("state") != "accepted"):
-        # Human input on the shared managed Claude PTY starts a genuine parent
-        # turn without a CCX delivery receipt. Do not fabricate one: the live
-        # authenticated host's exact activeTurnRef is sufficient to request a
-        # one-shot native Escape, under the same durable cancel intent. This
-        # exception never applies to Codex or an absent/invented/stale turn.
-        if engine != "claude":
-            raise DeliveryRejected("exact active turn has no original accepted delivery receipt",
-                                   short=worker_id[:8])
+        # Some genuine turns start without a CCX delivery receipt: human input
+        # on the shared managed Claude PTY, or a parent notification on Codex.
+        # Do not fabricate one: the live authenticated runtime's exact active
+        # turn is sufficient to request a one-shot native interrupt, under the
+        # same durable cancel intent. An absent/invented/stale turn is refused.
         known, active = _cancel_live_turn(handle, engine, worker_id)
         store.revalidate()
         if not known or active != turn_ref:
@@ -127,8 +130,12 @@ def cancel_turn(
             return None
 
         if engine == "codex":
-            seen = reconcile_turn(handle, turn_id=native_turn_id,
-                                  expected_generation=store.generation)
+            from ccx.codex.delivery import DeliveryReceiptError
+            try:
+                seen = reconcile_turn(handle, turn_id=native_turn_id,
+                                      expected_generation=store.generation)
+            except DeliveryReceiptError:
+                seen = {}  # no receipt evidence; the live turn below decides
             if seen.get("status") == "settled":
                 return public("confirmed", outcome="completed", owned=True,
                               receipt=receipt)

package/ccx/ops/conversation.py

@@ -1,4 +1,4 @@
-"Normalized conversation conversion for both CCX engines.\n\nOne versioned document shape is produced from two very different native\nsources:\n\n* Claude — the raw JSONL transcript ccx already records for every managed\n  worker. It is the authoritative source, so full mode reads it directly\n  instead of the cleaned renderer (whose 4 KB per-tool-result cap stays exactly\n  where it is, for ``logs()``/``peek``/dashboard previews).\n* Codex — the app-server ``thread/read(includeTurns=true)`` response. Rollout\n  files are never parsed, and the artifact is never called a raw transcript.\n\nThe document keys are camelCase because that is the shape the feature brief\nfroze and the shape ``transcript.render_transcript_json`` already writes;\n:class:`ccx.workers.WorkerClient` operation results stay snake_case. Only this\ndocument carries a ``schemaVersion``.\n\nEvery item carries an ``ordinal``: its position in the historical source this\nconverter was handed, per turn, dense and zero-based. The ``input``/``output``\nsplit is a classification, not an order — a Codex steer is user input that\nbelongs in the middle of a turn — so the ordinal is the only thing that\nreconstructs the source's own sequence.\n\nIt is not a claim about live chronology. Codex ``thread/read`` reconstructs a\nthread rather than replaying it: on 0.150.1 it has been observed to omit items\nthat certainly ran and to place user inputs differently from when they were\nactually delivered. The authoritative record of what CCX observed, in the order\nit observed it, is the evidence journal's ``sequence`` (:mod:`ccx.codex.evidence`).\nThe two orders are separate and must stay that way.\n\nMemory tradeoff (accepted for v1): both converters parse the whole native\nsource and materialize the whole normalized document, and\n:func:`write_export` serializes it in one ``json.dumps``. Peak memory is\nroughly 2-3x the raw native size, so a 50 MB transcript costs ~100-150 MB.\nTool results dominate large transcripts, so ``max_tool_result_bytes=N`` is the\nmitigation; the *native* copy is streamed to disk regardless, so full evidence\nsurvives even under a bound. Streaming conversion is deliberately deferred.\n"
+"Normalized conversation conversion for both CCX engines.\n\nOne versioned document shape is produced from two very different native\nsources:\n\n* Claude — the raw JSONL transcript ccx already records for every managed\n  worker. It is the authoritative source, so full mode reads it directly\n  instead of the cleaned renderer (whose 4 KB per-tool-result cap stays exactly\n  where it is, for ``logs()``/``peek``/dashboard previews).\n* Codex — the app-server ``thread/read(includeTurns=true)`` response, or the\n  same thread rebuilt from Codex's local history database\n  (``$CODEX_HOME/thread_history_1.sqlite``, read-only, schema-checked; see\n  :mod:`ccx.codex.history_store`), which ``ccx history`` prefers and which\n  falls back to app-server. Rollout files are never parsed, and the artifact is\n  never called a raw transcript.\n\nThe document keys are camelCase because that is the shape the feature brief\nfroze and the shape ``transcript.render_transcript`` already returns;\n:class:`ccx.workers.WorkerClient` operation results stay snake_case. Only this\ndocument carries a ``schemaVersion``.\n\nEvery item carries an ``ordinal``: its position in the historical source this\nconverter was handed, per turn, dense and zero-based. The ``input``/``output``\nsplit is a classification, not an order — a Codex steer is user input that\nbelongs in the middle of a turn — so the ordinal is the only thing that\nreconstructs the source's own sequence.\n\nIt is not a claim about live chronology. Codex ``thread/read`` reconstructs a\nthread rather than replaying it: on 0.150.1 it has been observed to omit items\nthat certainly ran and to place user inputs differently from when they were\nactually delivered. The authoritative record of what CCX observed, in the order\nit observed it, is the evidence journal's ``sequence`` (:mod:`ccx.codex.evidence`).\nThe two orders are separate and must stay that way.\n\nMemory tradeoff (accepted for v1): both converters parse the whole native\nsource and materialize the whole normalized document, and\n:func:`write_export` serializes it in one ``json.dumps``. Peak memory is\nroughly 2-3x the raw native size, so a 50 MB transcript costs ~100-150 MB.\nTool results dominate large transcripts, so ``max_tool_result_bytes=N`` is the\nmitigation; the *native* copy is streamed to disk regardless, so full evidence\nsurvives even under a bound. Streaming conversion is deliberately deferred.\n"
 
 from __future__ import annotations
 
@@ -533,14 +533,20 @@ _CLAUDE_BOOKKEEPING_TYPES = frozenset({
     "last-prompt", "custom-title", "ai-title", "agent-name", "mode",
     "permission-mode", "atis-latch", "file-history-snapshot",
     "file-history-delta", "api-request", "api-request-shape",
-    "api-request-blob",
+    "api-request-blob", "session-owner", "queue-operation", "tools-used",
+    "cost-state",
 })
 _CLAUDE_BOOKKEEPING_ATTACHMENTS = frozenset({
     "hook_success", "hook_non_blocking_error", "hook_permission_decision",
     "hook_additional_context", "environment", "model", "deferred_tools_delta",
     "agent_listing_delta", "mcp_instructions_delta", "skill_listing",
     "total_tokens_reminder", "session_context", "date", "credential_org",
-    "remote_session_change", "prompt_snapshot",
+    "remote_session_change", "prompt_snapshot", "deferred_tools_record",
+    "silent_turn_reminder", "instructions", "nested_memory", "auto_mode",
+    "auto_mode_exit", "plan_mode", "plan_mode_exit", "edited_text_file",
+    "compact_file_reference", "thinking_drop", "invoked_skills",
+    "skill_mention", "command_permissions", "bash_output_audience_note",
+    "batching_reminder_sent", "session_cron_carry",
 })
 _CLAUDE_BOOKKEEPING_SYSTEM = frozenset({"stop_hook_summary", "turn_duration"})
 
@@ -613,12 +619,19 @@ def _normalize_claude_events(
     events: list[dict],
     counters: _Counters,
     max_tool_result_bytes: int | None,
+    *,
+    fingerprints: bool = True,
 ) -> tuple[list[dict], str | None, list[dict]]:
-    """Normalize Claude events once for full documents and bounded pages."""
+    """Normalize Claude events once for full documents and bounded pages.
+
+    Only paging cursors need each source record's ``_ccx_fingerprint``, and it
+    costs most of a large transcript's parse, so full documents skip it.
+    """
     turns: list[dict] = []
     source_records: list[dict] = []
     observed_model: str | None = None
     current: dict | None = None
+    running: dict | None = None  # current, until Claude records the turn's end
     awaiting: dict[str, dict] = {}
     usage_record_by_message: dict[str, dict] = {}
     (
@@ -628,8 +641,8 @@ def _normalize_claude_events(
     ) = _claude_structural_links(events)
 
     def open_turn() -> dict:
-        nonlocal current
-        current = _turn(len(turns) + 1)
+        nonlocal current, running
+        current = running = _turn(len(turns) + 1)
         turns.append(current)
         return current
 
@@ -669,11 +682,14 @@ def _normalize_claude_events(
                     ensure_ascii=False,
                     default=str,
                 ).encode("utf-8")
-            ).hexdigest(),
+            ).hexdigest() if fingerprints else None,
         }
         source_records.append(record)
 
         def place(target: dict, bucket: str, item: dict) -> dict:
+            if isinstance(ts, int):  # the native row's time, named as Codex's
+                item["startedAtMs"] = ts
+                item["timingSource"] = "claude-transcript"
             placed = _place(target, bucket, item)
             if page_bearing:
                 record["items"].append(placed)
@@ -683,6 +699,8 @@ def _normalize_claude_events(
             # Never page-bearing, so bounded pages are unaffected; the source
             # record above still counts toward paging positions.
             counters.bookkeeping_excluded += 1
+            if etype == "system" and evt.get("subtype") == "turn_duration":
+                running = None  # Claude closed the current turn
             continue
 
         if etype == "user":
@@ -722,6 +740,9 @@ def _normalize_claude_events(
                     else:
                         place(opened, "input", _item(
                             "user-text", "user", evt.get("uuid"), text=text,
+                            # Claude Code's own injected rows (command caveats,
+                            # skill bodies, peer and scheduled prompts).
+                            **({"meta": True} if evt.get("isMeta") else {}),
                         ))
                     stamp(opened, ts)
                 elif btype == "tool_result":
@@ -734,6 +755,8 @@ def _normalize_claude_events(
                     if call is not None:
                         call["result"] = payload
                         call["status"] = "failed" if payload["isError"] else "completed"
+                        if isinstance(ts, int):
+                            call["completedAtMs"] = ts
                         continue
                     counters.warn(
                         f"tool_result {tid or '<no id>'} has no unpaired "
@@ -833,13 +856,42 @@ def _normalize_claude_events(
         if etype == "attachment":
             att = evt.get("attachment") or {}
             if att.get("type") == "queued_command":
-                text = (att.get("prompt") or "").strip()
-                if not text:
+                # A queued message is a string, or, when it carries an image
+                # or document, a content-block list like a user message's.
+                prompt = att.get("prompt")
+                media: list[dict] = []
+                others: list[dict] = []
+                if isinstance(prompt, list):
+                    parts = []
+                    for block in prompt:
+                        if isinstance(block, str):
+                            parts.append(block)
+                        elif not isinstance(block, dict):
+                            counters.omitted += 1
+                        elif block.get("type") == "text":
+                            parts.append(block.get("text") or "")
+                        elif block.get("type") in ("image", "document"):
+                            media.append(block)
+                        else:
+                            others.append(block)
+                    text = "\n".join(p for p in parts if p.strip()).strip()
+                else:
+                    text = prompt.strip() if isinstance(prompt, str) else ""
+                if not text and not media and not others:
                     continue
-                target = open_turn()
-                place(target, "input", _item(
-                    "user-text", "queued_command", evt.get("uuid"), text=text,
-                ))
+                # Claude queues a steer or task notification into the running
+                # turn, so it stays there; it opens a turn only when none runs.
+                target = running or open_turn()
+                if text:
+                    place(target, "input", _item(
+                        "user-text", "queued_command", evt.get("uuid"), text=text,
+                    ))
+                for block in media:
+                    place(target, "input", _claude_attachment(block))
+                for block in others:  # kept, as in a user message
+                    place(target, "output", _item(
+                        "other", block.get("type"), None, native=safe_native(block),
+                    ))
                 stamp(target, ts)
             else:
                 target = current or open_turn()
@@ -921,7 +973,7 @@ def build_claude(rec: dict, *, state: str | None, worker: str,
         )
 
     turns, observed_model, _records = _normalize_claude_events(
-        events, counters, max_tool_result_bytes,
+        events, counters, max_tool_result_bytes, fingerprints=False,
     )
 
     artifact = {
@@ -1203,6 +1255,7 @@ def _codex_item(entry: dict, counters: _Counters,
             "tool-result", itype, item_id,
             callId=entry.get("callId") or item_id,
             name=entry.get("name"),
+            namespace=entry.get("namespace"),
             result=_result_payload(entry.get("output"), False, limit, counters),
         )]
 

package/ccx/ops/conversation_paging.py

@@ -652,31 +652,32 @@ def _reverse_lines(
     """Yield (start offset, raw line) backwards from a certified source fence.
 
     The pinned reader uses pread, not the mutable native pathname or shared fd
-    seek position. The buffer also handles JSONL rows larger than a read block.
+    seek position. A row longer than a read block is collected piece by piece
+    and joined once, so a multi-MB row costs its size, not its size squared.
     """
     try:
         reader = (path.open_reader() if isinstance(path, PinnedNativeSource)
                   else open(path, "rb"))
         with reader as handle:
             position = before_offset
-            suffix = b""
+            pieces: list[bytes] = []  # the current row's tail, newest piece first
             while position > 0:
                 size = min(block_size, position)
                 position -= size
                 handle.seek(position)
-                data = handle.read(size) + suffix
-                parts = data.split(b"\n")
-                if position > 0:
-                    suffix = parts[0]
-                    parts = parts[1:]
-                else:
-                    suffix = b""
-                end = position + len(data)
-                for raw in reversed(parts):
-                    start = end - len(raw)
+                data = handle.read(size)
+                cut = len(data)
+                while (newline := data.rfind(b"\n", 0, cut)) >= 0:
+                    pieces.append(data[newline + 1:cut])
+                    raw = b"".join(reversed(pieces))
+                    pieces = []
                     if raw:
-                        yield start, raw
-                    end = start - 1
+                        yield position + newline + 1, raw
+                    cut = newline
+                pieces.append(data[:cut])
+            raw = b"".join(reversed(pieces))
+            if raw:
+                yield 0, raw
     except FileNotFoundError:
         return
 

package/ccx/ops/creation_steps.py

@@ -378,10 +378,6 @@ def prepare_worker(
         if engine == "claude":
             from ccx.store.paths import transcript_path_for
             record["transcript"] = str(transcript_path_for(directory, session_id))
-            record["transcriptJson"] = str(
-                store.pointer.generation_root / "workers" /
-                "claude" / chosen / "transcript.json"
-            )
         else:
             record.update(
                 nativeArchive={"state": "active", "attempts": 0},