ccx-native 0.1.0 → 0.2.0

package/ccx/claude/transcript.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import copy
+import difflib
+import hashlib
 from contextlib import ExitStack, contextmanager
 import io
 import json
@@ -12,7 +14,6 @@ import time
 
 from ccx.system import oslayer
 from ccx.claude.context import compaction_unknown_context, compute_context_usage
-from ccx.store.paths import transcript_json_path
 from ccx.store.record_cache import StampedCache
 from ccx.store.records import read_record
 
@@ -193,6 +194,7 @@ def queued_prompt_accepted(
     text: str,
     *,
     after_ms: int = 0,
+    after_offset: int = 0,
 ) -> bool:
     """Return True once Claude has natively accepted a queued prompt.
 
@@ -202,14 +204,16 @@ def queued_prompt_accepted(
     PostToolUse/SubagentStop steering paste was actually accepted, not merely
     written to the PTY.
 
-    Only the exact normalized prompt acknowledges this delivery.
+    Only the exact normalized prompt acknowledges this delivery. Callers pass
+    the transcript size from before the paste as ``after_offset``: this is
+    polled while a hook holds the turn, so it must not rescan a long history.
     """
     # Claude trims a submitted prompt; an inline paste has no envelope that
     # would keep the surrounding whitespace.
     text = normalize_semantic_prompt_text(text).strip()
     if not text:
         return False
-    for evt in _iter_transcript(transcript_path):
+    for evt in _iter_transcript(transcript_path, after_offset=after_offset):
         if evt.get("type") != "queue-operation":
             continue
         if evt.get("operation") != "enqueue":
@@ -357,7 +361,7 @@ _ESCAPED_PASTED_CONTENT_RE = re.compile(
 # Claude stores a typed `<pasted_content>` tag escaped as `<\\pasted_content>`.
 _ESCAPED_TAG_RE = re.compile(r"<\\(/?pasted_content)")
 # Claude turns a prompt that is only an image path into an `[Image #N]` row.
-_IMAGE_PATH_RE = re.compile(r"^[/~][^\n]*\.(?:png|jpe?g|gif|webp)$", re.IGNORECASE)
+BARE_IMAGE_PATH_RE = re.compile(r"^[/~][^\n]*\.(?:png|jpe?g|gif|webp)$", re.IGNORECASE)
 _IMAGE_PLACEHOLDER_RE = re.compile(r"^\[Image #\d+\]$")
 
 
@@ -379,7 +383,26 @@ def prompt_matches(expected: str, actual: str | None) -> bool:
     ``[Image #N]`` placeholder Claude stores in its place."""
     actual = normalize_semantic_prompt_text(actual).strip()
     return bool(expected) and (actual == expected or bool(
-        _IMAGE_PATH_RE.match(expected) and _IMAGE_PLACEHOLDER_RE.match(actual)))
+        BARE_IMAGE_PATH_RE.match(expected) and _IMAGE_PLACEHOLDER_RE.match(actual)))
+
+
+def prompt_mismatch_event(delivery_id: str, expected: str, actual: str | None) -> dict:
+    """A host event describing how native prompt text ``actual`` differs from
+    the sent prompt ``expected`` (already normalized and stripped), without
+    any of the text: sizes, hash prefixes, how many spans differ, and whether
+    it is the same characters in another order (a reordered paste)."""
+    actual = normalize_semantic_prompt_text(actual).strip()
+    spans = difflib.SequenceMatcher(None, expected, actual, autojunk=False).get_opcodes()
+    return {
+        "type": "semantic_prompt_mismatch",
+        "delivery_id": delivery_id,
+        "expected_chars": len(expected),
+        "actual_chars": len(actual),
+        "expected_sha256": hashlib.sha256(expected.encode()).hexdigest()[:16],
+        "actual_sha256": hashlib.sha256(actual.encode()).hexdigest()[:16],
+        "changed_spans": sum(op != "equal" for op, *_ in spans),
+        "same_characters": sorted(expected) == sorted(actual),
+    }
 
 
 def exact_user_prompt_event(
@@ -1248,15 +1271,33 @@ class BackgroundTaskReducer:
 
                 parsed_id = output_file = None
                 if isinstance(body, str):
-                    m = (
-                        _BG_LAUNCH_ID_RE.search(body)
-                        or _BG_AUTO_LAUNCH_ID_RE.search(body)
-                    )
-                    if m:
-                        parsed_id = m.group(1)
-                    mo = _BG_OUTPUT_FILE_RE.search(body)
-                    if mo:
-                        output_file = mo.group(1).rstrip(".")
+                    # Only this result's own launch counts. A foreground
+                    # command (e.g. `ccx peek other-worker`) may print another
+                    # worker's launch notice in its stdout; claiming that task
+                    # would keep it open here forever. Claude records the id
+                    # in toolUseResult.backgroundTaskId; without it, accept
+                    # only a notice that heads the result.
+                    if bg_id is None:
+                        notice = body.lstrip()
+                        if notice.startswith("Command running in background with ID:"):
+                            m = _BG_LAUNCH_ID_RE.search(notice)
+                        elif notice.startswith("Command did not complete within"):
+                            m = _BG_AUTO_LAUNCH_ID_RE.search(notice)
+                        else:
+                            m = None
+                        parsed_id = m.group(1) if m else None
+                    launched = bg_id or parsed_id
+                    if launched:
+                        # A command moved to the background after its timeout
+                        # prints its output so far before the notice, so match
+                        # the output path by task id rather than position.
+                        output_file = next((
+                            path
+                            for mo in _BG_OUTPUT_FILE_RE.finditer(body)
+                            if (path := mo.group(1).rstrip(".")).endswith(
+                                f"/{launched}.output"
+                            )
+                        ), None)
                 parsed_monitor_id = None
                 if isinstance(body, str) and body.startswith("Monitor started (task "):
                     mm = _MONITOR_LAUNCH_ID_RE.search(body)
@@ -1879,7 +1920,7 @@ def _iter_clean_turns(
             att = evt.get("attachment") or {}
             if att.get("type") != "queued_command":
                 continue
-            processed = _process_command_message(att.get("prompt") or "")
+            processed = _process_command_message(_block_text(att.get("prompt")))
             text = (processed or "").strip()
             if not text:
                 continue
@@ -2175,19 +2216,15 @@ def open_original_claude_source(store, owner, rec):
         yield pinned_source, source_stat, end_offset, source_path, current
 
 
-def render_transcript_json(short: str, *, store=None) -> dict | None:
-    """Refresh one V2 Claude-owned render from the original native JSONL.
 
-    The record's native session ID is not the CCX worker UUID. Bind source
-    derivation to the original immutable create intent, pin the provider file
-    before either read, and publish only through the ACTIVE generation's exact
-    full-UUID worker directory. An optional pinned store preserves the
-    caller's original ACTIVE generation. The published dict is also returned;
-    missing upstream JSONL returns None. Record transcriptJson is display metadata,
-    never a pathname from which to read or onto which to write.
+def render_transcript(short: str, *, store=None) -> dict | None:
+    """Render one Claude worker's cleaned transcript from its original JSONL.
+
+    Read-only and on demand (``logs``): parses the whole native transcript, so
+    nothing on a hot path may call it. Missing upstream JSONL returns None.
     """
     from ccx import constants
-    from ccx.store.root import StorageIntegrityError, _write_json, require_ccx_active
+    from ccx.store.root import require_ccx_active
     from ccx.store.identity import EngineMismatch
 
     if store is None:
@@ -2198,12 +2235,8 @@ def render_transcript_json(short: str, *, store=None) -> dict | None:
     rec = store.record(owner.worker)
     with open_original_claude_source(store, owner, rec) as source:
         if source is None:
-            # Preserve original render's missing-native-file no-output behavior.
             return None
-        pinned_source, source_stat, end_offset, source_path, current = source
-        native_sid = rec["native"]["sessionId"]
-        cwd = rec["cwd"]
-        fingerprint = rec["createSha256"]
+        pinned_source, _source_stat, end_offset, source_path, _current = source
         stats = {"malformedLines": 0, "truncatedToolResults": 0}
         turns = list(_iter_clean_turns(
             pinned_source, stats, before_offset=end_offset,
@@ -2212,58 +2245,282 @@ def render_transcript_json(short: str, *, store=None) -> dict | None:
         pending_compact = pending_compact_request(
             pinned_source, before_offset=end_offset,
         )
-        if pending_compact is not None:
+    if pending_compact is not None:
+        context = compaction_unknown_context(
+            "compacting", as_of_ms=pending_compact.get("as_of_ms"),
+        )
+    return {
+        "schemaVersion": TRANSCRIPT_SCHEMA_VERSION,
+        "generatedAt": int(time.time() * 1000),
+        "worker": owner.worker,
+        "sessionId": rec["native"]["sessionId"],
+        "name": rec.get("name", owner.short),
+        "cwd": rec["cwd"],
+        "transcriptJsonl": str(source_path),
+        "turns": turns,
+        "context": context,
+        "stats": stats,
+    }
+
+
+def _valid_observed_model(model) -> bool:
+    return (isinstance(model, str) and 0 < len(model) <= 256
+            and model != "<synthetic>")
+
+
+class _ObservationReducer:
+    """The context, model and pending-compact facts of a cleaned transcript,
+    fed one raw event at a time so the host can resume where it stopped.
+
+    It mirrors exactly the parts of ``_iter_clean_turns`` those facts depend
+    on: which events emit a turn, how one response's lines group by message
+    id, and which line's model and usage win. ``render_transcript`` stays the
+    full renderer; tests pin the two together at every line boundary. The
+    state is small and JSON-serializable.
+    """
+
+    def __init__(self, state: dict | None = None):
+        state = state or {}
+        self.any_turn = state.get("anyTurn", False)
+        # Latest compact boundary or usable assistant usage, as a minimal
+        # cleaned turn that compute_context_usage reads.
+        self.anchor = state.get("anchor")
+        self.model = state.get("model")
+        self.pending_compact = state.get("pendingCompact")
+        # The response still accumulating: {mid, ts, content, model, tokens}.
+        self.open = state.get("open")
+
+    @staticmethod
+    def valid_state(state) -> bool:
+        if not isinstance(state, dict) or not isinstance(state.get("anyTurn"), bool):
+            return False
+        anchor, model = state.get("anchor"), state.get("model")
+        pending, open_ = state.get("pendingCompact"), state.get("open")
+        return ((anchor is None or isinstance(anchor, dict))
+                and (model is None or _valid_observed_model(model))
+                and (pending is None or isinstance(pending, dict))
+                and (open_ is None or (isinstance(open_, dict)
+                                       and isinstance(open_.get("content"), bool))))
+
+    def state(self) -> dict:
+        return {"anyTurn": self.any_turn, "anchor": self.anchor, "model": self.model,
+                "pendingCompact": self.pending_compact, "open": self.open}
+
+    def _flush(self) -> None:
+        response, self.open = self.open, None
+        if response is None or not response["content"]:
+            return
+        self.any_turn = True
+        if _valid_observed_model(response.get("model")):
+            self.model = response["model"]
+        tokens = response.get("tokens")
+        if tokens is not None:
+            turn = {"role": "assistant", "ts": response.get("ts"), "tokens": tokens}
+            if (compute_context_usage([turn]) or {}).get("state") == "known":
+                self.anchor = turn
+
+    def _feed_compact(self, evt: dict, kind, ts) -> None:
+        # Same rules as pending_compact_request.
+        if kind == "system" and evt.get("subtype") == "compact_boundary":
+            self.pending_compact = None
+            return
+        content = None
+        if kind == "system" and evt.get("subtype") == "local_command":
+            content = evt.get("content")
+        elif kind == "user":
+            content = (evt.get("message") or {}).get("content")
+        if _is_compact_command_content(content):
+            self.pending_compact = {"as_of_ms": ts}
+        elif (self.pending_compact is not None and isinstance(content, str)
+                and ("<local-command-stdout>" in content
+                     or "<local-command-stderr>" in content)):
+            self.pending_compact = None
+
+    def feed(self, evt: dict) -> None:
+        kind = evt.get("type")
+        ts = _parse_event_ts_ms(evt)
+        self._feed_compact(evt, kind, ts)
+        if kind == "user":
+            content = (evt.get("message") or {}).get("content", "")
+            parts: list[str] = []
+            has_tool_results = False
+            if isinstance(content, str):
+                processed = _process_command_message(content)
+                if processed is None:
+                    return
+                parts.append(processed)
+            elif isinstance(content, list):
+                for block in content:
+                    if not isinstance(block, dict):
+                        continue
+                    if block.get("type") == "text":
+                        processed = _process_command_message(block.get("text", ""))
+                        if processed is not None:
+                            parts.append(processed)
+                    elif block.get("type") == "tool_result":
+                        has_tool_results = True
+            if has_tool_results and not parts:
+                return
+            self._flush()
+            if "\n".join(p for p in parts if p).strip():
+                self.any_turn = True
+        elif kind == "assistant":
+            msg = evt.get("message") or {}
+            mid = msg.get("id")
+            if self.open is not None and (not mid or self.open["mid"] != mid):
+                self._flush()
+            if self.open is None:
+                self.open = {"mid": mid, "ts": ts, "content": False,
+                             "model": None, "tokens": None}
+            content = msg.get("content", [])
+            for block in content if isinstance(content, list) else ():
+                if not isinstance(block, dict):
+                    continue
+                text = block.get("text", "")
+                if ((block.get("type") == "text" and isinstance(text, str) and text.strip())
+                        or block.get("type") == "tool_use"):
+                    self.open["content"] = True
+            if msg.get("model"):
+                self.open["model"] = msg["model"]
+            usage = msg.get("usage", {}) or {}
+            if usage:
+                tokens = _usage_tokens(evt, usage)
+                self.open["tokens"] = {
+                    key: tokens[key] for key in
+                    ("input", "cache_read_input", "cache_creation_input")
+                    if key in tokens
+                }
+        elif kind == "attachment":
+            attachment = evt.get("attachment") or {}
+            if attachment.get("type") != "queued_command":
+                return
+            processed = _process_command_message(_block_text(attachment.get("prompt")))
+            if (processed or "").strip():
+                self._flush()
+                self.any_turn = True
+        elif kind == "system":
+            self._flush()
+            data = _clean_system_event(evt.get("subtype"), evt)
+            if data is not None:
+                self.any_turn = True
+                if data[0] == "compact_boundary":
+                    self.anchor = {"role": "system", "kind": "compact_boundary", "ts": ts}
+
+    def observe(self) -> tuple[dict | None, str | None]:
+        """(context, observed model) as the full render would report them now:
+        the open response counts as finished, as it does at end of file."""
+        final = _ObservationReducer(copy.deepcopy(self.state()))
+        final._flush()
+        turns = [final.anchor or {"role": "user"}] if final.any_turn else []
+        context = compute_context_usage(turns)
+        if final.pending_compact is not None:
             context = compaction_unknown_context(
-                "compacting", as_of_ms=pending_compact.get("as_of_ms"),
+                "compacting", as_of_ms=final.pending_compact.get("as_of_ms"),
             )
-        observed_model = next((
-            turn["model"] for turn in reversed(turns)
-            if turn.get("role") == "assistant"
-            and isinstance(turn.get("model"), str)
-            and 0 < len(turn["model"]) <= 256
-            and turn["model"] != "<synthetic>"
-        ), None)
-        observed_at = int(time.time() * 1000)
+        return context, final.model
+
+
+OBSERVATION_CURSOR_VERSION = 1
+# Bytes just before the resume offset whose hash proves the file was only
+# appended to since the last refresh (not truncated and rewritten in place).
+_OBSERVATION_FENCE_BYTES = 4096
+
+
+def _observation_fence_sha(source: PinnedNativeSource, offset: int) -> str:
+    start = max(0, offset - _OBSERVATION_FENCE_BYTES)
+    return hashlib.sha256(os.pread(source.fd, offset - start, start)).hexdigest()
+
+
+def _resumable_observation_state(store, owner, native_sid, source, source_stat, end_offset):
+    """The prior observation's (offset, reducer state) when it still describes
+    a prefix of this exact source, else None (start again from the top)."""
+    try:
+        doc = store.read_worker_sidecar(owner.worker, "context-observation.json")
+    except Exception:
+        return None
+    if not isinstance(doc, dict):
+        return None
+    observed, cursor = doc.get("source"), doc.get("cursor")
+    if (doc.get("schemaVersion") != 1 or doc.get("workerId") != owner.worker_id
+            or doc.get("nativeSessionId") != native_sid
+            or doc.get("generation") != store.generation
+            or not isinstance(observed, dict) or not isinstance(cursor, dict)
+            or observed.get("dev") != source_stat.st_dev
+            or observed.get("ino") != source_stat.st_ino
+            or type(observed.get("endOffset")) is not int
+            or not 0 <= observed["endOffset"] <= end_offset
+            or cursor.get("version") != OBSERVATION_CURSOR_VERSION
+            or not _ObservationReducer.valid_state(cursor.get("state"))
+            or cursor.get("fenceSha256") != _observation_fence_sha(source, observed["endOffset"])):
+        return None
+    return observed["endOffset"], cursor["state"]
+
+
+def refresh_context_observation(short: str, *, store=None) -> dict | None:
+    """Update one Claude worker's context/model observation from its JSONL.
+
+    Incremental: parses only what Claude appended since the previous
+    observation of the same source file (same inode, same generation), so a
+    long-lived worker's refresh costs what its last turn wrote, with constant
+    memory. Anything else starts again from the top, streaming. Publishes only
+    through the ACTIVE generation's exact full-UUID worker directory; missing
+    upstream JSONL returns None.
+    """
+    from ccx import constants
+    from ccx.store.root import StorageIntegrityError, _write_json, require_ccx_active
+    from ccx.store.identity import EngineMismatch
+
+    if store is None:
+        store = require_ccx_active(constants.CCX_ROOT)
+    owner = store.resolve(short)
+    if owner.engine != "claude":
+        raise EngineMismatch("context observation requires a Claude full-UUID owner")
+    rec = store.record(owner.worker)
+    with open_original_claude_source(store, owner, rec) as source:
+        if source is None:
+            return None
+        pinned_source, source_stat, end_offset, source_path, current = source
+        native_sid = rec["native"]["sessionId"]
+        cwd = rec["cwd"]
+        fingerprint = rec["createSha256"]
+        prior = _resumable_observation_state(
+            store, owner, native_sid, pinned_source, source_stat, end_offset)
+        start, state = prior if prior is not None else (0, None)
+        reducer = _ObservationReducer(state)
+        for event in _iter_transcript(
+            pinned_source, after_offset=start, before_offset=end_offset,
+        ):
+            reducer.feed(event)
+        context, observed_model = reducer.observe()
         observation = {
             "schemaVersion": 1,
             "workerId": owner.worker_id, "nativeSessionId": native_sid,
             "generation": store.generation,
             "source": {"dev": source_stat.st_dev, "ino": source_stat.st_ino,
-                       "endOffset": end_offset, "observedAtMs": observed_at},
+                       "endOffset": end_offset,
+                       "observedAtMs": int(time.time() * 1000)},
             "context": context, "observedModel": observed_model,
+            "cursor": {"version": OBSERVATION_CURSOR_VERSION,
+                       "fenceSha256": _observation_fence_sha(pinned_source, end_offset),
+                       "state": reducer.state()},
         }
-        out = {
-            "schemaVersion": TRANSCRIPT_SCHEMA_VERSION,
-            "generatedAt": int(time.time() * 1000),
-            "worker": owner.worker,
-            "sessionId": native_sid,
-            "name": rec.get("name", owner.short),
-            "cwd": cwd,
-            "transcriptJsonl": str(source_path),
-            "turns": turns,
-            "context": context,
-            "stats": stats,
-        }
-        # If the upstream original pathname was replaced between the two
-        # parsing passes and publication, the pinned old inode cannot prove a
-        # current original provider source. Normal append preserves inode.
+        # If the upstream pathname was replaced during the read, the pinned
+        # old inode no longer proves the current provider source.
         try:
             current_source = os.stat(
                 native_sid + ".jsonl", dir_fd=current, follow_symlinks=False,
             )
         except OSError as exc:
-            raise StorageIntegrityError("original Claude native JSONL disappeared during render") from exc
+            raise StorageIntegrityError("original Claude native JSONL disappeared during refresh") from exc
         if (current_source.st_dev != source_stat.st_dev
                 or current_source.st_ino != source_stat.st_ino
                 or not stat.S_ISREG(current_source.st_mode)
                 or current_source.st_uid != os.getuid()
                 or current_source.st_mode & 0o022
                 or current_source.st_size < end_offset):
-            raise StorageIntegrityError("original Claude native JSONL changed identity during render")
-        # Pin and revalidate global ownership after the expensive native read;
-        # a generation rotation or changed original source must not publish
-        # into a new owner's directory. The existing atomic writer creates its
-        # temp inode 0600 before exposing it, then fsyncs and replaces via fd.
+            raise StorageIntegrityError("original Claude native JSONL changed identity during refresh")
+        # Revalidate global ownership after the read; a generation rotation or
+        # changed original source must not publish into a new owner's directory.
         with store.lease():
             latest_owner, latest = store._owned(owner.worker)
             latest_native = latest.get("native")
@@ -2272,10 +2529,7 @@ def render_transcript_json(short: str, *, store=None) -> dict | None:
                     or latest.get("cwd") != cwd
                     or latest.get("transcript") != str(source_path)
                     or latest.get("createSha256") != fingerprint):
-                raise StorageIntegrityError("Claude original JSONL ownership changed during render")
+                raise StorageIntegrityError("Claude original JSONL ownership changed during refresh")
             with store._gen("workers", "claude", owner.worker_id) as worker_dir_fd:
-                # Separate atomically published files are fine: readers consume
-                # this complete observation, never a transactional turn pair.
                 _write_json(worker_dir_fd, "context-observation.json", observation)
-                _write_json(worker_dir_fd, "transcript.json", out, rendered_transcript=True)
-        return out
+        return observation

package/ccx/claude/usage.py

@@ -4,20 +4,23 @@ import argparse
 from contextlib import contextmanager
 from email.utils import parsedate_to_datetime
 import fcntl
-import errno
 import math
 from pathlib import Path
 from datetime import datetime, timezone
 import json
 import os
 import stat
-import tempfile
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 
 from ccx.system import oslayer
+from ccx.system.private_json import (
+    _atomic_private_json,
+    _open_private_file,
+    _read_private_json,
+)
 from ccx.claude.oauth import oauth_headers, oauth_organization_uuid, oauth_token
 from ccx.constants import (
     CAP_METRICS,
@@ -56,22 +59,6 @@ _RETRY_HELD_INITIAL_S = 300.0
 _RETRY_HELD_MAX_S = 1800.0
 
 
-def _open_private_file(path: Path) -> int:
-    """Do not trust a symlink, another owner's file, or a readable private record."""
-    fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
-    st = os.fstat(fd)
-    if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or st.st_mode & 0o077:
-        os.close(fd)
-        raise PermissionError(f"not a private regular file owned by this user: {path}")
-    return fd
-
-
-def _read_private_json(path: Path) -> dict | None:
-    with os.fdopen(_open_private_file(path), "rb") as handle:
-        value = json.load(handle)
-    return value if isinstance(value, dict) else None
-
-
 def _open_usage_cache() -> int:
     return _open_private_file(usage_cache_path())
 
@@ -107,34 +94,6 @@ def _cache_age(data: dict | None = None) -> float | None:
     return max(0.0, time.time() - epoch) if epoch is not None else None
 
 
-def _atomic_private_json(path: Path, value: dict) -> None:
-    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
-    tmp = None
-    try:
-        fd, tmp = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
-        with os.fdopen(fd, "w", encoding="utf-8") as out:
-            json.dump(value, out, separators=(",", ":"), allow_nan=False)
-            out.flush()
-            os.fsync(out.fileno())
-        # Never overwrite a planted symlink/foreign file at the final path.
-        try:
-            existing = _open_private_file(path)
-        except FileNotFoundError:
-            pass
-        except OSError as exc:
-            # Replacing a planted *symlink itself* is safe: os.replace never
-            # follows the destination. Its target is neither read nor written.
-            if exc.errno != errno.ELOOP:
-                raise
-        else:
-            os.close(existing)
-        os.replace(tmp, path)
-        tmp = None
-    finally:
-        if tmp is not None:
-            os.unlink(tmp)
-
-
 def _write_usage_cache(data: dict) -> None:
     _atomic_private_json(usage_cache_path(), data)