ccx-native 0.0.0-stage → 0.1.0

package/ccx/store/paths.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+import errno
+import os
+import pathlib
+import re
+import socket
+import tempfile
+import unicodedata
+
+from ccx.system import oslayer
+from ccx import constants
+from ccx.constants import CLAUDE_HOME
+
+def private_mkdir(path: pathlib.Path | str) -> pathlib.Path:
+    """``mkdir -p`` where every directory it creates, and the leaf, is 0700.
+
+    ``Path.mkdir(parents=True, mode=...)`` applies the mode to the leaf only;
+    the parents it creates get the umask default, which would leave e.g.
+    ``operations/`` listable by other users.
+    """
+    path = pathlib.Path(path)
+    missing = []
+    probe = path
+    while not probe.exists() and probe != probe.parent:
+        missing.append(probe)
+        probe = probe.parent
+    for directory in reversed(missing):
+        try:
+            directory.mkdir(mode=0o700)
+        except FileExistsError:
+            continue
+        directory.chmod(0o700)
+    path.chmod(0o700)
+    return path
+
+
+def ensure_dirs():
+    """New runtime bootstrap is observational; never mkdir/chmod old short tree."""
+    from ccx import constants
+    from ccx.store.root import require_ccx_active
+    return require_ccx_active(constants.CCX_ROOT)
+
+
+
+def peer_is_same_user(conn: socket.socket) -> bool:
+    """Whether an accepted AF_UNIX peer runs as this user (Linux SO_PEERCRED).
+
+    The 0700 directory and 0600 socket are the primary guard; this is the
+    second one, so a socket reachable by mistake still only answers its
+    owner. Missing or unreadable kernel credentials always fail closed;
+    file permissions alone cannot authenticate a connected peer.
+    """
+    try:
+        credentials = oslayer.peer_credentials(conn)
+    except oslayer.PeerIdentityUnavailable:
+        return False
+    return credentials.uid == os.getuid()
+
+
+def accept_same_user(server: socket.socket) -> socket.socket | None:
+    """``server.accept()`` that closes and drops a peer of another user.
+
+    Returns None for a dropped peer; accept errors propagate as before.
+    """
+    conn, _ = server.accept()
+    if peer_is_same_user(conn):
+        return conn
+    try:
+        conn.close()
+    except OSError:
+        pass
+    return None
+
+
+def _require_unix_socket_budget(path: pathlib.Path) -> None:
+    """Check the final and fixed-width staged bind paths without writing."""
+    # mkdtemp(prefix=".") generates exactly eight characters after the dot.
+    for candidate, kind in (
+        (path, "final"),
+        (path.parent / ".XXXXXXXX" / "s", "staged"),
+    ):
+        if not oslayer.fits_unix_path(candidate):
+            raise OSError(errno.ENAMETOOLONG,
+                          f"{kind} Unix socket path exceeds AF_UNIX pathname budget",
+                          os.fspath(candidate))
+
+
+def bind_private_unix_socket(server: socket.socket, path: pathlib.Path | str) -> None:
+    """Bind ``server`` at ``path`` with mode 0600 and no permissive window.
+
+    Binding a socket creates the inode with umask permissions, so chmod alone
+    leaves an exposure window. Instead, bind inside a fresh private 0700
+    staging directory, chmod there, and atomically rename the 0600 socket.
+    This avoids changing the process-wide umask, which could race threads.
+    """
+    path = pathlib.Path(path)
+    _require_unix_socket_budget(path)  # before mkdtemp creates any directory
+    staging = pathlib.Path(tempfile.mkdtemp(prefix=".", dir=path.parent))
+    try:
+        tmp = staging / "s"
+        server.bind(str(tmp))
+        os.chmod(tmp, 0o600)
+        os.replace(tmp, path)
+    finally:
+        try:
+            (staging / "s").unlink()
+        except FileNotFoundError:
+            pass
+        try:
+            os.rmdir(staging)
+        except OSError:
+            pass
+
+def trust_ledger_path() -> pathlib.Path:
+    """Which ``.claude.json`` trust entries ccx itself wrote (see settings)."""
+    return constants.CCX_ROOT / "trust-ledger.json"
+
+def _owner_path(target: str, filename: str) -> pathlib.Path:
+    """Inert per-owned-worker pathname; actual writers must use pinned fds."""
+    from ccx import constants
+    from ccx.store.root import require_ccx_active
+    from ccx.store.identity import EngineMismatch
+    store = require_ccx_active(constants.CCX_ROOT)
+    owner = store.resolve(target)
+    if owner.engine != "claude":
+        raise EngineMismatch("Claude CCX sidecar requires Claude full worker owner")
+    # The path is NOT a second authority: the authoritative owner is checked
+    # on each call, and native internal sidecar I/O must use the pinned fd.
+    return (store.pointer.generation_root / "workers" /
+            owner.engine / owner.worker_id / filename)
+
+
+def sessions_dir() -> pathlib.Path:
+    from ccx import constants
+    from ccx.store.root import require_ccx_active
+    store = require_ccx_active(constants.CCX_ROOT)
+    return store.pointer.generation_root / "workers" / "claude"
+
+
+def record_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "record.json")
+
+
+def transcript_json_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "transcript.json")
+
+
+def statusline_json_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "statusline.json")
+
+
+def settings_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "settings.json")
+
+
+def events_ndjson_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "events.ndjson")
+
+
+def conversation_page_index_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "conversation-page-index.json")
+
+
+def compact_operations_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "compact.json")
+
+
+def steer_deliveries_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "deliveries.json")
+
+
+def steer_deliveries_lock_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "deliveries.lock")
+
+
+def sock_path(short: str) -> pathlib.Path:
+    from ccx import constants
+    from ccx.store.root import require_ccx_active
+    from ccx.rpc.transport import plan_transport
+    store = require_ccx_active(constants.CCX_ROOT)
+    owner = store.resolve(short)
+    if owner.engine != "claude":
+        from ccx.store.identity import EngineMismatch
+        raise EngineMismatch("Claude socket cannot use Codex worker")
+    return plan_transport(store, target=owner.worker).final
+
+
+def respawn_lock_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "respawn.lock")
+
+
+def host_pid_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "host.pid")
+
+
+def host_log_path(short: str) -> pathlib.Path:
+    return _owner_path(short, "host.log")
+
+
+def claude_project_slug(cwd: str) -> str:
+    """Mirror Claude 2.1.311's ``hL`` key (UTF-16 units / 200-unit cap)."""
+    # Claude's macOS cwd is NFC before hL; on Linux it retains on-disk NFD.
+    if oslayer.claude_project_cwd_is_nfc():
+        cwd = unicodedata.normalize("NFC", cwd)
+    encoded = cwd.encode("utf-16-le", "surrogatepass")
+    units = [encoded[i] | encoded[i + 1] << 8 for i in range(0, len(encoded), 2)]
+    # Claude's JS /[^a-zA-Z0-9]/g replaces each astral surrogate separately.
+    slug = "".join(
+        chr(unit) if (48 <= unit <= 57 or 65 <= unit <= 90 or 97 <= unit <= 122)
+        else "-" for unit in units
+    )
+    if len(slug) <= 200:
+        return slug
+    h = 0
+    for unit in units:
+        h = (31 * h + unit) & 0xffffffff
+    value = abs(h if h < 0x80000000 else h - 0x100000000)
+    digits = "0123456789abcdefghijklmnopqrstuvwxyz"
+    suffix = ""
+    while value:
+        value, remainder = divmod(value, 36)
+        suffix = digits[remainder] + suffix
+    return f"{slug[:200]}-{suffix or '0'}"
+
+
+def transcript_path_for(cwd: str, session_id: str) -> pathlib.Path:
+    """Mirror of Claude Code's path derivation for ~/.claude/projects/<slug>/<uuid>.jsonl."""
+    slug = claude_project_slug(cwd)
+    return CLAUDE_HOME / "projects" / slug / f"{session_id}.jsonl"
+
+def task_list_dir_for(session_id: str) -> pathlib.Path | None:
+    """Claude Code's structured task-list directory for one native session.
+
+    Claude currently keys ordinary task lists by the native session UUID under
+    ``~/.claude/tasks/<session-id>``. Keep that private upstream layout behind
+    CCX so consumers never need to derive it themselves. A malformed record id
+    degrades to ``None`` instead of letting a path separator escape the task root.
+    """
+    if not session_id or pathlib.PurePath(session_id).name != session_id:
+        return None
+    if session_id in {".", ".."}:
+        return None
+    return CLAUDE_HOME / "tasks" / session_id
+
+def subagent_dir_for(cwd: str, session_id: str) -> pathlib.Path:
+    slug = claude_project_slug(cwd)
+    return CLAUDE_HOME / "projects" / slug / session_id / "subagents"
+
+def iter_subagent_transcripts(cwd: str, session_id: str):
+    root = subagent_dir_for(cwd, session_id)
+    if not root.exists():
+        return
+    yield from sorted(root.rglob("*.jsonl"))

package/ccx/store/record_cache.py

@@ -0,0 +1,40 @@
+"""File-stamped caching for expensive derived reads."""
+from __future__ import annotations
+
+import threading
+import time
+from pathlib import Path
+
+# Avoid caching writes within coarse filesystem timestamp granularity.
+_MIN_AGE_NS = 50_000_000
+
+
+class StampedCache:
+    """Cache by mtime, size and inode; missing files delegate to the reader."""
+
+    def __init__(self) -> None:
+        self._entries: dict[str, tuple[tuple[int, int, int], object]] = {}
+        self._lock = threading.Lock()
+
+    def get(self, path, compute):
+        key = str(path)
+        try:
+            st = Path(path).stat()
+        except OSError:
+            with self._lock:
+                self._entries.pop(key, None)
+            return compute(path)
+        stamp = (st.st_mtime_ns, st.st_size, st.st_ino)
+        with self._lock:
+            hit = self._entries.get(key)
+        if hit is not None and hit[0] == stamp:
+            return hit[1]
+        value = compute(path)
+        if time.time_ns() - st.st_mtime_ns >= _MIN_AGE_NS:
+            with self._lock:
+                self._entries[key] = (stamp, value)
+        return value
+
+    def clear(self) -> None:
+        with self._lock:
+            self._entries.clear()

package/ccx/store/records.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import tempfile
+import time
+
+from ccx.constants import SELF_SHORT_ENV, S
+from ccx.store.paths import record_path, sessions_dir
+
+NAME_MAX_LEN = 80
+
+_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f]+")
+_WS_RE = re.compile(r"\s+")
+
+
+def sanitize_name(name: str | None) -> str:
+    """Normalize an operator-set display name to a safe single line.
+
+    Control characters and newlines collapse to a single space, surrounding
+    whitespace is stripped, and the result is capped at :data:`NAME_MAX_LEN`
+    characters. Returns ``""`` for ``None`` / empty / whitespace-only input — the
+    caller decides what default to substitute (the auto-name ``ccx-<short>``)."""
+    if not name:
+        return ""
+    collapsed = _WS_RE.sub(" ", _CONTROL_RE.sub(" ", name)).strip()
+    return collapsed[:NAME_MAX_LEN].strip()
+
+
+def iter_session_records():
+    """Read only global full-ID managed Claude owners for observational search."""
+    store = _uuid_claude_store()
+    for row in store.list_workers(engine="claude"):
+        yield store.record(row["worker"])
+
+
+def _uuid_claude_store():
+    """One operational ACTIVE-generation gate; never initialize a legacy root."""
+    from ccx.constants import CCX_ROOT
+    from ccx.store.root import require_ccx_active
+    return require_ccx_active(CCX_ROOT)
+
+
+def read_record(target: str) -> dict:
+    """Exact CCX-owned Claude full UUID only, no short/name/path fallback."""
+    from ccx.store.identity import EngineMismatch
+    store = _uuid_claude_store()
+    owner = store.resolve(target)
+    if owner.engine != "claude":
+        raise EngineMismatch("Claude record lookup requires Claude full worker UUID")
+    return store.record(owner.worker)
+
+
+def write_record(rec: dict) -> None:
+    """Explicit version-CAS only for existing CCX-owned Claude worker.
+
+    Historical host code that passes a stale whole-record snapshot will
+    fail RecordConflict. A missing revision, native session change, original
+    fork/owner change or short-keyed record fails closed before publication.
+    Prefer merge_lifecycle / GenerationStore.mutate_record for new writers.
+    """
+    from ccx.store.root import RecordConflict, StorageIntegrityError
+    if not isinstance(rec, dict) or rec.get("engine") != "claude":
+        raise StorageIntegrityError("Claude full-UUID record needs exact engine")
+    worker_id = rec.get("workerId")
+    revision = rec.get("recordRevision")
+    if type(revision) is not int or revision < 0:
+        raise RecordConflict(
+            "Claude whole-record update requires fresh original recordRevision"
+        )
+    store = _uuid_claude_store()
+    updated = store.replace_record_if_revision(
+        f"claude:{worker_id}", rec, expected_revision=revision,
+    )
+    rec.clear()
+    rec.update(updated)
+
+
+def merge_lifecycle(target: str, **patch) -> dict:
+    """One locked fresh-disk semantic patch; no stale whole-record write."""
+    store = _uuid_claude_store()
+    owner = store.resolve(target)
+    if owner.engine != "claude":
+        from ccx.store.identity import EngineMismatch
+        raise EngineMismatch("Claude lifecycle update requires Claude full worker UUID")
+    return store.patch_lifecycle(owner.worker, changes=patch)
+
+
+def default_claude_name(worker_id: str) -> str:
+    """Original v2 fallback display from a validated full UUID's short field."""
+    from ccx.store.identity import WorkerOwner
+    return "ccx-" + WorkerOwner(worker_id, "claude").short
+
+
+def take_host_boot_fields(target: str) -> tuple[dict | None, dict]:
+    """Atomically consume only host-owned run cap and stale waiting preview."""
+    from ccx.store.root import RecordConflict
+    store = _uuid_claude_store()
+    for _ in range(6):
+        current = store.record(target)
+        cap = current.get("activeCap")
+        if "activeCap" not in current and "waitingFor" not in current:
+            return None, current
+        try:
+            committed = store.mutate_record(
+                target, removals=("activeCap", "waitingFor"),
+                expect={"activeCap": cap, "waitingFor": current.get("waitingFor")},
+            )
+            return cap, committed
+        except RecordConflict:
+            continue  # changed host transient: re-read its actual value
+    raise RecordConflict("host boot transients changed during bounded consumption")
+
+
+def restore_host_turn_count(target: str, minimum: int) -> dict:
+    """Repair a sidecar-authoritative settled turn count, never decrease it."""
+    from ccx.store.root import RecordConflict, StorageIntegrityError
+    if type(minimum) is not int or minimum < 0:
+        raise StorageIntegrityError("invalid host settled turn floor")
+    store = _uuid_claude_store()
+    for _ in range(6):
+        current = store.record(target)
+        value = (current.get("stats") or {}).get("turns")
+        if type(value) is not int or value < 0:
+            raise StorageIntegrityError("host turn counter malformed")
+        if value >= minimum:
+            return current
+        try:
+            return store.mutate_record(
+                target, patches={"stats.turns": minimum},
+                expect={"stats.turns": value},
+            )
+        except RecordConflict:
+            continue
+    raise RecordConflict("host turn count changed during bounded recovery")
+
+
+def commit_host_turn_settlement(
+    target: str, *, turn_ref: str, prior_turns: int, outcome: str,
+    settled_at_ms: int, proposed_title: str | None = None,
+    context: dict | None = None, background_task_summary: dict | None = None,
+) -> dict:
+    """Atomic/idempotent settlement, rebased only over fields this host owns.
+
+    No lifecycle state is written here: an external terminal kill owns it.
+    Exact turn ref and counter/time fields are one CAS, including after a lost
+    acknowledgement. An independent CLI name/config patch is never lost.
+    """
+    from ccx.store.root import RecordConflict, StorageIntegrityError
+    if (not isinstance(turn_ref, str) or not turn_ref
+            or type(prior_turns) is not int or prior_turns < 0
+            or outcome not in ("completed", "interrupted", "unknown")
+            or type(settled_at_ms) is not int or settled_at_ms <= 0):
+        raise StorageIntegrityError("invalid exact host settlement")
+    store = _uuid_claude_store()
+    for _ in range(8):
+        current = store.record(target)
+        turns = (current.get("stats") or {}).get("turns")
+        prior = current.get("lastSettledTurn")
+        if (type(turns) is not int or turns < 0
+                or prior is not None and not isinstance(prior, dict)):
+            raise StorageIntegrityError("host settlement state malformed")
+        if prior and prior.get("turnRef") == turn_ref:
+            if (prior.get("outcome") != outcome
+                    or type(prior.get("turn")) is not int
+                    or turns < prior["turn"]):
+                raise RecordConflict("replayed host settlement contradicts original")
+            return current
+        if prior and type(prior.get("turn")) is int and prior["turn"] > prior_turns:
+            raise RecordConflict("a newer distinct host turn is already settled")
+        ordinal = max(turns, prior_turns) + 1
+        settled_time = max(settled_at_ms, int(current.get("lastSettledAt") or 0))
+        patches = {
+            "stats.turns": ordinal,
+            "lastSettledAt": settled_time,
+            "lastSettledTurn": {
+                "turn": ordinal, "turnRef": turn_ref, "outcome": outcome,
+                "settledAtMs": settled_time, "context": context,
+            },
+        }
+        if background_task_summary is not None:
+            patches["backgroundTaskSummary"] = background_task_summary
+        if outcome in ("completed", "interrupted"):
+            patches["lastTurnAt"] = max(settled_time, int(current.get("lastTurnAt") or 0))
+        expected = {"stats.turns": turns, "lastSettledTurn": prior}
+        fallback = default_claude_name(target)
+        if proposed_title and current.get("name") == fallback:
+            adopted = sanitize_name(proposed_title)
+            if adopted and adopted != fallback:
+                patches["name"] = adopted
+                expected["name"] = fallback
+        try:
+            return store.mutate_record(target, patches=patches, expect=expected)
+        except RecordConflict:
+            continue
+    raise RecordConflict("host settlement changed during bounded rebase")
+
+
+_HOST_TERMINAL_STATES = frozenset({S.DEAD, S.CRASHED, S.IDLE_KILLED, S.BUDGET_STOPPED})
+
+
+def host_live_transition(target: str, *, new_state: str, host_pid: int) -> dict | None:
+    """Only this host PID may publish a nonterminal state, never over a kill."""
+    from ccx.store.root import RecordConflict
+    if new_state in _HOST_TERMINAL_STATES:
+        raise ValueError("terminal host state must be published after socket teardown")
+    store = _uuid_claude_store()
+    for _ in range(6):
+        current = store.record(target)
+        lifecycle = current.get("lifecycle") or {}
+        if (lifecycle.get("state") in _HOST_TERMINAL_STATES
+                or lifecycle.get("hostPid") != host_pid):
+            return None
+        try:
+            return store.mutate_record(
+                target, patches={
+                    "lifecycle.state": new_state,
+                    "lifecycle.since": int(time.time() * 1000),
+                    "lifecycle.lastError": None,
+                    "lifecycle.apiError": None,
+                }, expect={"lifecycle.state": lifecycle.get("state"),
+                           "lifecycle.hostPid": host_pid},
+            )
+        except RecordConflict:
+            continue
+    raise RecordConflict("host live state changed during bounded transition")
+
+
+def host_terminal_transition(
+    target: str, *, state: str, host_pid: int, last_error: str | None,
+    clear_child: bool,
+) -> dict:
+    """Publish shutdown for this PID only; preserve another terminal verdict."""
+    from ccx.store.root import RecordConflict
+    if state not in _HOST_TERMINAL_STATES:
+        raise ValueError("host final state must be terminal")
+    store = _uuid_claude_store()
+    for _ in range(6):
+        current = store.record(target)
+        lifecycle = current.get("lifecycle") or {}
+        current_state = lifecycle.get("state")
+        if lifecycle.get("hostPid") != host_pid:
+            return current  # other/cleared host owner always wins
+        patches = {"lifecycle.hostPid": None, "lifecycle.hostStart": None}
+        if clear_child:
+            patches.update({"lifecycle.workerPid": None, "lifecycle.workerStart": None})
+        if current_state not in _HOST_TERMINAL_STATES:
+            patches.update({
+                "lifecycle.state": state,
+                "lifecycle.since": int(time.time() * 1000),
+                "lifecycle.lastError": last_error,
+            })
+        try:
+            return store.mutate_record(
+                target, patches=patches,
+                expect={"lifecycle.state": current_state,
+                        "lifecycle.hostPid": host_pid},
+            )
+        except RecordConflict:
+            continue
+    raise RecordConflict("terminal host state changed during bounded shutdown")
+
+
+def spawned_by_from_env() -> dict | None:
+    """Return attested full-UUID worker origin, or untrusted native metadata."""
+    from ccx.constants import SELF_ENGINE_ENV, SELF_WORKER_ID_ENV
+    from ccx.errors import SessionUnavailable
+    from ccx.store.identity import InvalidTarget, canonical_worker_id
+    old = os.environ.get(SELF_SHORT_ENV, "").strip()
+    if old:
+        raise SessionUnavailable(
+            "stale CCX_SELF_SHORT cannot identify a full-UUID parent"
+        )
+    full = os.environ.get(SELF_WORKER_ID_ENV, "").strip()
+    engine = os.environ.get(SELF_ENGINE_ENV, "").strip()
+    if full or engine:
+        if engine not in ("claude", "codex"):
+            raise SessionUnavailable("CCX_SELF_ENGINE must be explicit")
+        try:
+            wid = canonical_worker_id(full)
+        except InvalidTarget as exc:
+            raise SessionUnavailable("CCX_SELF_WORKER_ID is not full UUIDv4") from exc
+        try:
+            store = _uuid_claude_store()
+            owner = store.resolve(f"{engine}:{wid}")
+        except Exception as exc:
+            from ccx.store.root import GenerationError
+            from ccx.store.identity import IdentityError
+            if isinstance(exc, (GenerationError, IdentityError, OSError)):
+                raise SessionUnavailable(
+                    f"unproven full UUID caller identity/generation: {exc}"
+                ) from exc
+            raise
+        return {"kind": "ccx-worker", "workerId": owner.worker_id,
+                "engine": owner.engine}
+    # Native provider IDs are not managed CCX worker routing authority.
+    claude = os.environ.get("CLAUDE_CODE_SESSION_ID", "").strip()
+    codex = os.environ.get("CODEX_THREAD_ID", "").strip()
+    if claude and codex:
+        return {"kind": "provider-native-unknown",
+                "facts": {"claudeSessionId": claude, "codexThreadId": codex}}
+    if claude:
+        return {"kind": "provider-native-unknown", "engine": "claude",
+                "nativeId": claude}
+    if codex:
+        return {"kind": "provider-native-unknown", "engine": "codex",
+                "nativeId": codex}
+    return None