cdx-manager 0.25.1 → 0.25.2

package/src/run_observation.py

@@ -0,0 +1,93 @@
+"""Supervisor-owned, content-free snapshots of isolated provider observations."""
+
+import json
+import os
+import stat
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .history_models import _exact_transcript
+from .interactive_usage import MAX_TRANSCRIPT_BYTES
+from .run_usage import _usage_from_dict, extract_run_conversation_id, extract_run_usage
+
+
+def _open_rollout(root, path):
+    """Walk relative to the pinned attempt directory without following symlinks."""
+    parts = Path(path).relative_to(root).parts
+    directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        for part in parts[:-1]:
+            child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory)
+            os.close(directory)
+            directory = child
+        return os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
+    finally:
+        os.close(directory)
+
+
+def codex_attempt_evidence(auth_home, stdout_path, started_at):
+    """Read only the exact thread in a fresh attempt's volatile sessions mount.
+
+    Do not import claimed costs, requested models, messages, or arbitrary files
+    from the workspace. The exported object is never writable by the sandbox.
+    """
+    usage = extract_run_usage("codex", stdout_path)
+    identifier = extract_run_conversation_id("codex", stdout_path)
+    evidence = {"source": "isolated_codex_rollout", "stdout_path": str(stdout_path),
+                "thread_id": identifier, "started_at": started_at, "models": [],
+                "turns": [], "usage": usage}
+    if not identifier:
+        return evidence
+    path, error = _exact_transcript("codex", auth_home, identifier)
+    if error or not path:
+        return evidence
+    models = set()
+    turns = []
+    identity = False
+    complete_models = True
+    counters = None
+    ended_at = datetime.now(timezone.utc)
+    try:
+        descriptor = _open_rollout(auth_home, path)
+        with os.fdopen(descriptor, encoding="utf-8") as handle:
+            if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
+                return evidence
+            text = handle.read(MAX_TRANSCRIPT_BYTES + 1)
+        if len(text) > MAX_TRANSCRIPT_BYTES:
+            return evidence
+        for line in text.splitlines():
+            record = json.loads(line)
+            payload = record.get("payload") or {}
+            if record.get("type") == "session_meta":
+                if payload.get("id") != identifier:
+                    return evidence
+                identity = True
+            if record.get("type") == "event_msg" and payload.get("type") == "token_count":
+                counters = _usage_from_dict((payload.get("info") or {}).get("total_token_usage") or {}, provider="codex")
+            if record.get("type") != "turn_context":
+                continue
+            timestamp = record.get("timestamp")
+            if (not timestamp or datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
+                    < datetime.fromisoformat(started_at.replace("Z", "+00:00"))):
+                complete_models = False
+                continue
+            if datetime.fromisoformat(timestamp.replace("Z", "+00:00")) > ended_at:
+                complete_models = False
+                continue
+            model = payload.get("model")
+            if not isinstance(model, str) or not model:
+                complete_models = False
+                continue
+            models.add(model)
+            turn_id = payload.get("turn_id")
+            turns.append({"turn_id": turn_id if isinstance(turn_id, str) else None, "timestamp": timestamp, "model": model})
+    except (OSError, ValueError, TypeError, AttributeError):
+        return evidence
+    # Rollout counters must corroborate exec's measured counters. A free-form
+    # model/cost claim or a previous conversation's context is insufficient.
+    matches = counters is not None and all(value is None or counters.get(key) == value for key, value in usage.items())
+    if identity and matches:
+        evidence.update(models=sorted(models) if complete_models else [], turns=turns, ended_at=ended_at.isoformat())
+        if usage.get("total_tokens") is None:
+            evidence["measurement_usage"] = counters
+    return evidence

package/src/run_registry.py

@@ -244,7 +244,7 @@ class RunRegistry:
             _check_capacity(_read_registry(self.path), max_parallel)
 
     def start(self, run_id, *, kind, session, provider, model, cwd, artifacts=None, label=None, max_parallel=None,
-              parent_run_id=None, conversation_id=None, launch=None, reasoning_effort=None):
+              parent_run_id=None, conversation_id=None, launch=None, reasoning_effort=None, project=None, project_source="unknown"):
         with _registry_lock(self.path):
             data = _read_registry(self.path)
             data["runs"] = [run for run in data["runs"] if run.get("run_id") != run_id]
@@ -254,7 +254,8 @@ class RunRegistry:
                 _check_capacity(data, max_parallel)
             record = _base_record(run_id, kind=kind, session=session, provider=provider, model=model, cwd=cwd, artifacts=artifacts)
             record.update(parent_run_id=parent_run_id, conversation_id=conversation_id,
-                          launch=dict(launch or {}), reasoning_effort=reasoning_effort)
+                          launch=dict(launch or {}), reasoning_effort=reasoning_effort,
+                          project=project, project_source=project_source)
             if label:
                 record["label"] = label
             data["runs"].insert(0, record)
@@ -280,6 +281,18 @@ class RunRegistry:
                     return run
         return None
 
+    def record_attempt(self, run_id, artifacts):
+        """Keep live tail paths on this supervisor's current attempt."""
+        with _registry_lock(self.path):
+            data = _read_registry(self.path)
+            for run in data["runs"]:
+                if run.get("run_id") == run_id and run.get("status") == "running":
+                    run["attempt_id"] = artifacts.get("attempt_id")
+                    run["artifacts"] = {key: artifacts.get(key) for key in ("transcript_path", "stdout_path", "stderr_path")}
+                    _write_registry(self.path, data)
+                    return run
+        return None
+
     def record_background(self, run_id, *, path, provider_session_id=None, provider_agent_id=None):
         with _registry_lock(self.path):
             data = _read_registry(self.path)
@@ -376,6 +389,8 @@ class RunRegistry:
                 # used to store whichever shape the producing reader emitted.
                 run["usage"] = coerce_usage(final_payload.get("usage"), provider=run.get("provider"))
                 run["usage_model"] = final_payload.get("usage_model")
+                for key in ("usage_evidence", "usage_models", "attempt_id", "attempts", "cost_usd", "cost_status", "cost_reason", "budget_status"):
+                    run[key] = final_payload.get(key)
                 run["conversation_id"] = final_payload.get("conversation_id") or run.get("conversation_id")
                 run["parent_run_id"] = final_payload.get("parent_run_id") or run.get("parent_run_id")
                 if "reasoning_effort" in final_payload:

package/src/run_usage.py

@@ -363,6 +363,25 @@ def estimate_cost(usage, model, prices=None):
     return sum(present) / 1_000_000
 
 
+def cost_fields(usage, model, prices=None, *, models=(), attribution=None):
+    """Explain nullable estimates consistently on every JSON surface."""
+    cost = estimate_cost(usage, model, prices)
+    reason = None
+    if cost is None:
+        if attribution == "continuation_usage_unavailable":
+            reason = "attempt_usage_unavailable"
+        elif not _has_usage(usage):
+            reason = "usage_not_measured"
+        elif len(set(models)) > 1:
+            reason = "mixed_models_without_counters"
+        elif not model:
+            reason = "model_not_observed"
+        else:
+            reason = "model_price_unknown"
+    return {"cost_usd": cost, "cost_status": "estimated" if cost is not None else "unknown",
+            "cost_reason": reason}
+
+
 def _long_ttl_writes(usage):
     """One-hour cache writes, bounded by the writes they are a part of."""
     long_ttl = usage.get("cache_creation_1h_tokens")
@@ -475,7 +494,7 @@ def extract_run_usage(provider, stdout_path):
     return extract_run_observation(provider, stdout_path)[0]
 
 
-def extract_run_observation(provider, stdout_path, auth_home=None):
+def extract_run_observation(provider, stdout_path, auth_home=None, *, evidence=None, models_out=None):
     if not stdout_path or not provider:
         return empty_usage(), None
     if provider not in SUPPORTED_PROVIDERS:
@@ -493,6 +512,10 @@ def extract_run_observation(provider, stdout_path, auth_home=None):
         return empty_usage(), None
 
     usage = _extract_usage_from_records(records, provider=provider)
+    if (provider == "codex" and evidence is not None and evidence.get("stdout_path") == str(stdout_path)
+            and evidence.get("usage") == usage and evidence.get("thread_id") == extract_run_conversation_id(provider, stdout_path)
+            and not _has_usage(usage)):
+        usage = evidence.get("measurement_usage") or usage
     if not _has_usage(usage):
         return empty_usage(), None
     models = set()
@@ -506,8 +529,18 @@ def extract_run_observation(provider, stdout_path, auth_home=None):
         model_usage = record.get("modelUsage")
         if isinstance(model_usage, dict):
             models.update(model_usage)
-    if provider == "codex" and auth_home:
+    if provider == "codex" and evidence is not None:
+        # Only the supervisor supplies this snapshot, never a file in the checkout.
+        if (evidence.get("stdout_path") == str(stdout_path)
+                and evidence.get("thread_id") == extract_run_conversation_id(provider, stdout_path)
+                and (evidence.get("measurement_usage") or evidence.get("usage")) == usage):
+            models.update(evidence.get("models") or [])
+        else:
+            return usage, None
+    elif provider == "codex" and auth_home:
         models.update(_codex_run_models(records, auth_home))
+    if models_out is not None:
+        models_out.extend(sorted(models))
     return usage, next(iter(models)) if len(models) == 1 else None
 
 
@@ -539,8 +572,9 @@ def _codex_run_models(records, auth_home):
                     continue
                 payload = record.get("payload")
                 model = payload.get("model") if isinstance(payload, dict) else None
-                if isinstance(model, str) and model:
-                    models.add(model)
+                if not isinstance(model, str) or not model:
+                    return set()
+                models.add(model)
     except OSError:
         return set()
     return models

package/src/session_service.py

@@ -777,6 +777,14 @@ def backfill_history_models(store, base_dir, dry_run=True):
     )
 
 
+def backfill_history_projects(store, base_dir, mapping_path=None, dry_run=True):
+    from .history_models import read_run_reports
+    from .project_identity import load_mapping
+
+    mapping = load_mapping(mapping_path) if mapping_path else {}
+    return store["backfill_launch_projects"](mapping, read_run_reports(base_dir), dry_run=dry_run)
+
+
 def refresh_interactive_usage_models(store, dry_run=True):
     """Replace a placeholder model with the exact transcript's real model."""
     from .interactive_usage import MATCH_CONVERSATION_ID, extract_interactive_usage
@@ -813,6 +821,8 @@ def start_session_runtime(store, name, payload=None):
         "command": payload.get("command"),
         "label": payload.get("label"),
         "cwd": payload.get("cwd"),
+        "project": payload.get("project"),
+        "project_source": payload.get("project_source", "unknown"),
         "transcriptPath": payload.get("transcript_path") or payload.get("transcriptPath"),
     }
 
@@ -1010,6 +1020,7 @@ def create_session_service(options=None):
         "backfill_interactive_usage": partial(backfill_interactive_usage, store),
         "refresh_interactive_usage_models": partial(refresh_interactive_usage_models, store),
         "backfill_history_models": partial(backfill_history_models, store, base_dir),
+        "backfill_history_projects": partial(backfill_history_projects, store, base_dir),
         "update_auth_state": partial(update_auth_state, store),
         "get_status_row": partial(get_status_row, store, base_dir, env, fetch_codex_status),
         "get_status_rows": partial(get_status_rows, store, base_dir, env, fetch_codex_status),

package/src/session_store.py

@@ -395,6 +395,67 @@ def create_session_store(base_dir):
                 result["backfilled"] = recovered
             return result
 
+    def backfill_launch_projects(mapping, reports, dry_run=True):
+        """Plan and atomically apply evidence-only attribution under the history lock."""
+        import uuid
+
+        from .project_identity import recorded_project
+
+        result = {"dry_run": dry_run, "recovered": 0, "unchanged": 0, "unresolved": 0,
+                  "conflicting": 0, "applied": 0, "backup_path": None}
+        with _file_lock(lock_file):
+            try:
+                with open(launch_history_file, "rb") as handle:
+                    original = handle.read()
+            except FileNotFoundError:
+                original = b""
+            lines = original.splitlines(keepends=True)
+            entries = []
+            for line in lines:
+                try:
+                    entry = json.loads(line)
+                except (ValueError, UnicodeError):
+                    entry = None
+                entries.append(entry if isinstance(entry, dict) else None)
+            run_ids = {entry.get("run_id") for entry in entries if entry and isinstance(entry.get("run_id"), str)}
+            missing = set(mapping) - run_ids
+            if missing:
+                raise CdxError(f"Project mapping references unknown exact run ids: {', '.join(sorted(missing))}")
+            evidence = {}
+            for record in [*reports, *[entry for entry in entries if entry]]:
+                run_id = record.get("run_id")
+                project = recorded_project(record)
+                if isinstance(run_id, str) and project:
+                    evidence.setdefault(run_id, set()).add(project)
+            for run_id, project in mapping.items():
+                evidence.setdefault(run_id, set()).add(project)
+            rewritten = []
+            for line, entry in zip(lines, entries):
+                if entry is not None:
+                    run_id = entry.get("run_id")
+                    candidates = evidence.get(run_id, set()) if isinstance(run_id, str) else set()
+                    known = recorded_project(entry)
+                    if len(candidates) > 1:
+                        result["conflicting"] += 1
+                    elif known:
+                        result["unchanged"] += 1
+                    elif candidates:
+                        result["recovered"] += 1
+                        entry.update(project=next(iter(candidates)),
+                                     project_source="operator_mapping" if run_id in mapping else "run_report")
+                        line = (json.dumps(entry, separators=(",", ":")) + "\n").encode("utf-8")
+                    else:
+                        result["unresolved"] += 1
+                rewritten.append(line)
+            # A conflicting batch is entirely read-only, including otherwise valid rows.
+            if not dry_run and result["recovered"] and not result["conflicting"]:
+                backup = f"{launch_history_file}.projects-{uuid.uuid4().hex}.bak"
+                atomic_write(backup, original)
+                atomic_write(launch_history_file, b"".join(rewritten))
+                _fsync_directory(state_dir)
+                result.update(backup_path=backup, applied=result["recovered"])
+            return result
+
     def list_launch_history(session_name=None, limit=20):
         with _file_lock(lock_file):
             try:
@@ -433,5 +494,6 @@ def create_session_store(base_dir):
         "drop_unvouched_usage": drop_unvouched_usage,
         "backfill_launch_usage": backfill_launch_usage,
         "backfill_launch_models": backfill_launch_models,
+        "backfill_launch_projects": backfill_launch_projects,
         "list_launch_history": list_launch_history,
     }

package/src/workspace_sandbox.py

@@ -62,7 +62,7 @@ def clean_env(source: dict) -> dict:
 
 
 def sandbox_command(command: list[str], tree: Path, *, network: bool, writable: bool = True,
-                    source: dict | None = None, auth_home: str = "", provider: str = "") -> tuple[list[str], dict]:
+                    source: dict | None = None, auth_home: str = "", provider: str = "", observation_home: str = "") -> tuple[list[str], dict]:
     source = os.environ if source is None else source
     binary = shutil.which(command[0], path=source.get("PATH"))
     bwrap = shutil.which("bwrap", path=source.get("PATH"))
@@ -127,6 +127,9 @@ def sandbox_command(command: list[str], tree: Path, *, network: bool, writable:
     env = clean_env(source)
     if provider == "codex":
         env["CODEX_HOME"] = "/home/agent/.codex"
+        if observation_home:
+            # Fresh, volatile attempt storage; no host profile is exposed.
+            args += ["--bind", observation_home, "/home/agent/.codex/sessions"]
         args += ["--ro-bind", str(Path(auth_home) / "auth.json"), "/home/agent/.codex/auth.json"]
     elif provider == "claude":
         token = source.get("CLAUDE_CODE_OAUTH_TOKEN")