cdx-manager 0.14.0 → 0.15.1

package/src/run_registry.py

@@ -2,9 +2,12 @@ import json
 import os
 import sys
 import tempfile
+import time
 from contextlib import contextmanager
 from datetime import datetime, timezone
 
+from .errors import CdxError
+
 
 def utc_now_iso():
     return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
@@ -14,14 +17,49 @@ def registry_path(base_dir):
     return os.path.join(base_dir, "runs.json")
 
 
+# A waiter waits. `fcntl.flock` blocks until the lock is free, but Windows'
+# `LK_LOCK` retries ten times at one-second intervals and then raises
+# EDEADLOCK, so the same contention produced a wait on one platform and an
+# OSError on the other. Twenty parallel runs writing this registry is the load
+# cdx exists to carry, and on a physical Windows machine it exhausted that
+# budget - CI never showed it, being less contended than a real desktop.
+REGISTRY_LOCK_TIMEOUT_SECONDS = 120
+_LOCK_RETRY_SECONDS = 0.05
+
+
+def _acquire_windows_lock(handle, timeout_seconds):
+    """Block until the byte is locked, or give up with a nameable error.
+
+    Uses the non-blocking form and does the waiting here, so the wait is
+    bounded by a deadline cdx chooses rather than by `LK_LOCK`'s fixed ten
+    attempts. Unbounded would be worse than the original: a stuck holder would
+    hang every later command with nothing to report.
+    """
+    import msvcrt
+
+    deadline = time.monotonic() + timeout_seconds
+    while True:
+        handle.seek(0)
+        try:
+            msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+            return
+        except OSError:
+            if time.monotonic() >= deadline:
+                raise CdxError(
+                    "Timed out waiting for the run registry lock after "
+                    f"{timeout_seconds}s. Another cdx process may be stuck holding it.",
+                    75,
+                ) from None
+            time.sleep(_LOCK_RETRY_SECONDS)
+
+
 @contextmanager
-def _registry_lock(path):
+def _registry_lock(path, timeout_seconds=REGISTRY_LOCK_TIMEOUT_SECONDS):
     os.makedirs(os.path.dirname(path), exist_ok=True)
     with open(path + ".lock", "a") as handle:
         if sys.platform == "win32":
             import msvcrt
-            handle.seek(0)
-            msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
+            _acquire_windows_lock(handle, timeout_seconds)
             try:
                 yield
             finally:
@@ -101,6 +139,12 @@ def _refresh_stale_runs(data):
     for run in data["runs"]:
         if run.get("status") != "running":
             continue
+        if run.get("background_path") == "provider_native":
+            # The provider owns this agent's lifecycle and the pid it reports
+            # belongs to a daemon helper, not the run. Sweeping on pid liveness
+            # would mark a healthy delegated run stale; its terminal status
+            # comes from the provider instead.
+            continue
         pid = run.get("pid")
         if pid and _is_pid_alive(pid):
             continue
@@ -124,11 +168,31 @@ def _base_record(run_id, *, kind, session, provider, model, cwd, artifacts=None)
         # the liveness proxy until finish() records the provider child's pid.
         "pid": os.getpid(),
         "started_at": utc_now_iso(),
+        # A failover run occupies several sessions in turn while staying one
+        # run. `session` and `provider` above name the current occupant so every
+        # existing reader keeps working; this is the ordered history behind
+        # them, and it exists from the start so no run is ever written in a
+        # shape the reporting cannot express.
+        "occupancies": [{
+            "session": session,
+            "provider": provider,
+            "started_at": utc_now_iso(),
+            "ended_at": None,
+            "reason": "initial_selection",
+        }],
         "ended_at": None,
         "duration_seconds": None,
         "exit_code": None,
         "usage": None,
         "artifacts": dict(artifacts or {}),
+        # Which mechanism is carrying a detached run, and the provider's own
+        # handle when it is the provider's. Present on every record so a reader
+        # never has to infer it from absence.
+        "background_path": None,
+        # Two handles, because the provider uses two: the session id names the
+        # transcript file, the short agent id is what `claude stop` accepts.
+        "provider_session_id": None,
+        "provider_agent_id": None,
         "error": None,
         "task_report": None,
         "final_payload": None,
@@ -166,6 +230,50 @@ class RunRegistry:
                     return run
         return None
 
+    def record_background(self, run_id, *, path, provider_session_id=None, provider_agent_id=None):
+        with _registry_lock(self.path):
+            data = _read_registry(self.path)
+            for run in data["runs"]:
+                if run.get("run_id") != run_id:
+                    continue
+                run["background_path"] = path
+                run["provider_session_id"] = provider_session_id
+                run["provider_agent_id"] = provider_agent_id
+                _write_registry(self.path, data)
+                return run
+            return None
+
+    def migrate(self, run_id, *, session, provider, reason):
+        """Move a still-running run onto another session, keeping one run_id.
+
+        Closes the current occupancy with the reason it ended and opens the
+        next one. The top-level `session`/`provider` follow the new occupant so
+        readers that know nothing about occupancies stay correct.
+        """
+        with _registry_lock(self.path):
+            data = _read_registry(self.path)
+            now = utc_now_iso()
+            for run in data["runs"]:
+                if run.get("run_id") != run_id:
+                    continue
+                occupancies = run.get("occupancies") or []
+                if occupancies and occupancies[-1].get("ended_at") is None:
+                    occupancies[-1]["ended_at"] = now
+                    occupancies[-1]["reason"] = reason
+                occupancies.append({
+                    "session": session,
+                    "provider": provider,
+                    "started_at": now,
+                    "ended_at": None,
+                    "reason": "failover_target",
+                })
+                run["occupancies"] = occupancies
+                run["session"] = session
+                run["provider"] = provider
+                _write_registry(self.path, data)
+                return run
+            return None
+
     def finish(self, run_id, *, status, final_payload=None, run_info=None, error=None, task_report=None):
         with _registry_lock(self.path):
             return self._finish_locked(run_id, status=status, final_payload=final_payload, run_info=run_info, error=error, task_report=task_report)
@@ -178,6 +286,10 @@ class RunRegistry:
                 continue
             run["status"] = status
             run["ended_at"] = now
+            occupancies = run.get("occupancies") or []
+            if occupancies and occupancies[-1].get("ended_at") is None:
+                occupancies[-1]["ended_at"] = now
+                occupancies[-1]["reason"] = status
             if run.get("started_at"):
                 try:
                     start = datetime.fromisoformat(str(run["started_at"]).replace("Z", "+00:00"))

package/src/session_service.py

@@ -35,6 +35,8 @@ from functools import partial
 
 from .codex_usage import fetch_codex_rate_limits
 from .config import (
+    MAX_LAUNCH_BUDGET_USD,
+    MAX_LAUNCH_EXTRA_ARGS_LENGTH,
     PERMISSION_INPUT_VALUES,
     PERMISSION_VALUES,
     PROVIDER_CLAUDE,
@@ -42,6 +44,7 @@ from .config import (
     REASONING_EFFORT_VALUES,
     get_cdx_home,
     normalize_permission,
+    split_extra_args,
 )
 from .errors import CdxError
 from .fs_utils import atomic_write, remove_tree
@@ -107,6 +110,7 @@ from .session_status import (  # noqa: F401  (re-exported for callers and tests)
     record_status,
 )
 from .session_store import create_session_store
+from .status_source import find_latest_codex_conversation_id
 
 MAX_SESSION_LABEL_LENGTH = 64
 # Each live Codex probe spawns app-server, which can refresh+rotate the OAuth
@@ -214,6 +218,40 @@ def _normalize_launch_settings(settings, mark_fast_service_tier=True):
         if len(model) > MAX_LAUNCH_MODEL_LENGTH or any(ord(ch) < 32 or ord(ch) == 127 for ch in model):
             raise CdxError("Model contains unsupported characters.")
         normalized["model"] = model
+    if "fallback_model" in settings and settings["fallback_model"] is not None:
+        # Claude accepts a comma-separated list here and tries each in order, so
+        # this validates every element rather than the string as one model name.
+        raw = str(settings["fallback_model"]).strip()
+        if not raw:
+            raise CdxError("Fallback model cannot be empty.")
+        elements = [element.strip() for element in raw.split(",")]
+        if any(not element for element in elements):
+            raise CdxError("Fallback model cannot contain an empty entry.")
+        for element in elements:
+            if len(element) > MAX_LAUNCH_MODEL_LENGTH or any(ord(ch) < 32 or ord(ch) == 127 for ch in element):
+                raise CdxError("Fallback model contains unsupported characters.")
+        normalized["fallback_model"] = ",".join(elements)
+    if "extra_args" in settings and settings["extra_args"] is not None:
+        extra = str(settings["extra_args"]).strip()
+        if not extra:
+            raise CdxError("Extra args cannot be empty.")
+        if len(extra) > MAX_LAUNCH_EXTRA_ARGS_LENGTH or any(ord(ch) < 32 or ord(ch) == 127 for ch in extra):
+            raise CdxError("Extra args contain unsupported characters.")
+        # Parsed here, not at launch: a string that cannot be split into argv
+        # should be rejected when it is set, not when a run fails days later.
+        if split_extra_args(extra) is None:
+            raise CdxError("Extra args are not a valid argument list (unbalanced quotes).")
+        normalized["extra_args"] = extra
+    if "budget" in settings and settings["budget"] is not None:
+        try:
+            budget = float(settings["budget"])
+        except (TypeError, ValueError) as error:
+            raise CdxError(f"Unsupported budget: {settings['budget']}") from error
+        # NaN fails every comparison below, so it has to be rejected explicitly
+        # or it would sail through the range check.
+        if budget != budget or budget <= 0 or budget > MAX_LAUNCH_BUDGET_USD:
+            raise CdxError(f"Unsupported budget: {settings['budget']}")
+        normalized["budget"] = budget
     if "priority" in settings and settings["priority"] is not None:
         try:
             priority = int(settings["priority"])
@@ -310,7 +348,10 @@ def unset_launch_settings(store, name, keys):
         raise CdxError(f"Unknown session: {name}")
     if not keys:
         raise CdxError("At least one launch setting is required.")
-    allowed = {"power", "reasoning_effort", "reasoningEffort", "permission", "fast", "rtk", "logics", "model", "priority"}
+    allowed = {
+        "power", "reasoning_effort", "reasoningEffort", "permission", "fast", "rtk", "logics",
+        "model", "fallback_model", "budget", "extra_args", "priority",
+    }
     unknown = [key for key in keys if key not in allowed]
     if unknown:
         raise CdxError(f"Unsupported launch setting: {', '.join(unknown)}")
@@ -600,9 +641,39 @@ def record_launch_history(store, name, payload):
         **payload,
     }
     store["append_launch_history"](entry)
+    _observe_codex_conversation(store, session)
     return entry
 
 
+def _observe_codex_conversation(store, session):
+    """Read Codex's conversation id back after a run, if it wrote one.
+
+    Codex mints the id itself, so unlike Claude this can only be observed once
+    the rollout exists. Failing to find one is normal and leaves the session on
+    the recency resume path; it is never an error worth surfacing to a launch
+    that otherwise succeeded.
+    """
+    if session.get("provider") != PROVIDER_CODEX:
+        return
+    auth_home = session.get("authHome")
+    if not auth_home:
+        return
+    try:
+        identifier = find_latest_codex_conversation_id(auth_home)
+    except OSError:
+        return
+    if not identifier or identifier == (session.get("conversation") or {}).get("id"):
+        return
+    store["update_session"](session["name"], lambda s: {
+        **s,
+        "conversation": {
+            "id": identifier,
+            "provenance": "observed",
+            "recordedAt": _local_now_iso(),
+        },
+    })
+
+
 def get_launch_history(store, name=None, limit=20):
     if name and not store["get_session"](name):
         raise CdxError(f"Unknown session: {name}")
@@ -684,8 +755,60 @@ def launch_session(store, name):
         return {**state, "rehydratedAt": now}
 
     store["update_session_state"](name, state_updater)
+
+    def session_updater(s):
+        updated = {**s, "updatedAt": now, "lastLaunchedAt": now}
+        conversation = _conversation_identity_for_launch(s)
+        if conversation:
+            updated["conversation"] = conversation
+        return updated
+
+    return store["update_session"](name, session_updater)
+
+
+def _conversation_identity_for_launch(session):
+    """The conversation id this launch will carry, or None to leave it alone.
+
+    Claude accepts a caller-supplied `--session-id`, so cdx mints one here,
+    before the launch spec is built, and knows the conversation's identity
+    without having to find it afterwards. A fresh id per launch rather than one
+    reused forever: a launch starts a new conversation, and `cdx resume` should
+    mean the last one, so the stored value is always the newest. Reusing a
+    single id across launches would also depend on how the provider treats a
+    repeated `--session-id`, which is a behaviour cdx does not need to rely on.
+
+    Codex has no equivalent flag; its identity is read back from the rollout
+    after the run, so nothing is minted here.
+    """
+    if session.get("provider") != PROVIDER_CLAUDE:
+        return None
+    return {
+        "id": str(uuid.uuid4()),
+        "provenance": "imposed",
+        "recordedAt": _local_now_iso(),
+    }
+
+
+def begin_conversation(store, name):
+    """Mint the conversation id a Claude run is about to carry, and store it.
+
+    `launch_session` does this for interactive launches, but a headless run
+    never goes through it, so `cdx run` used to leave Claude sessions with no
+    identity at all - `cdx resume` then fell back to recency for Claude while
+    working by id for Codex, which records its own after the fact. Same shape
+    for both providers now: whatever ran last is what resume names.
+
+    Returns the updated session, or the unchanged one for providers that mint
+    nothing here.
+    """
+    session = store["get_session"](name)
+    if not session:
+        raise CdxError(f"Unknown session: {name}")
+    conversation = _conversation_identity_for_launch(session)
+    if not conversation:
+        return session
     return store["update_session"](name, lambda s: {
-        **s, "updatedAt": now, "lastLaunchedAt": now
+        **s, "conversation": conversation, "updatedAt": _local_now_iso(),
     })
 
 
@@ -726,6 +849,7 @@ def create_session_service(options=None):
         "copy_session": partial(copy_session, base_dir, store),
         "rename_session": partial(rename_session, base_dir, store),
         "launch_session": partial(launch_session, store),
+        "begin_conversation": partial(begin_conversation, store),
         "active_session_runtime": partial(_session_runtime, store),
         "start_session_runtime": partial(start_session_runtime, store),
         "finish_session_runtime": partial(finish_session_runtime, store),

package/src/session_status.py

@@ -79,6 +79,7 @@ def _normalize_status_payload(payload=None):
         "reset_5h_at": payload.get("reset_5h_at"),
         "reset_week_at": payload.get("reset_week_at"),
         "reset_at": payload.get("reset_at") or payload.get("reset_week_at") or payload.get("reset_5h_at"),
+        "rate_limit_reached": payload.get("rate_limit_reached"),
         "updated_at": _to_local_iso(payload.get("updated_at") or payload.get("captured_at") or now),
         "raw_status_text": payload.get("raw_status_text"),
         "source_ref": payload.get("source_ref"),
@@ -146,6 +147,18 @@ def _status_has_more_detail(candidate, current):
     ]
     return any(current.get(field) is None and candidate.get(field) is not None for field in fields)
 
+def _later_status_timestamp(current_at, candidate_at):
+    if not candidate_at:
+        return current_at
+    if not current_at:
+        return candidate_at
+    parsed_current = _parse_status_timestamp(current_at)
+    parsed_candidate = _parse_status_timestamp(candidate_at)
+    if not parsed_current or not parsed_candidate:
+        return current_at
+    return candidate_at if parsed_candidate > parsed_current else current_at
+
+
 def _merge_status_payload(current, candidate):
     if not current:
         return candidate
@@ -175,7 +188,11 @@ def _merge_status_payload(current, candidate):
     if merged.get("source_ref") is not None and merged["source_ref"] == candidate.get("source_ref"):
         merged["structured"] = bool(candidate.get("structured"))
 
-    merged["updated_at"] = candidate.get("updated_at") or current.get("updated_at")
+    # The merge is only reached when the candidate is not newer, so adopting its
+    # timestamp would date the record by its stalest contributor. That is not
+    # cosmetic: updated_at drives the freshness check, so a regressed stamp
+    # makes a just-resolved status read as expired and re-probes every call.
+    merged["updated_at"] = _later_status_timestamp(current.get("updated_at"), candidate.get("updated_at"))
     return merged
 
 def _compute_available_pct(status):

package/src/status_source.py

@@ -153,6 +153,13 @@ def _extract_structured_rate_limits(record):
         "reset_5h_at": _normalize_timestamp_value(_get_rate_limit_reset_at(five_hour)),
         "reset_week_at": _normalize_timestamp_value(_get_rate_limit_reset_at(weekly)),
         "reset_at": None,
+        # A typed reason the account is blocked, when the provider gives one:
+        # null in normal operation, and a value such as
+        # "workspace_owner_credits_depleted" when a limit is actually reached.
+        # Better evidence than any percentage - credits can run out while the
+        # 5-hour and weekly windows still read healthy - and better than
+        # matching prose, since it is an enum rather than a sentence.
+        "rate_limit_reached": _clean_reached_type(rate_limits.get("rate_limit_reached_type")),
         "raw_status_text": json.dumps(rate_limits, sort_keys=True),
     }
     if not five_hour and weekly:
@@ -172,6 +179,13 @@ def _extract_structured_rate_limits(record):
     return result
 
 
+def _clean_reached_type(value):
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    return text or None
+
+
 def _collect_structured_rate_limit_statuses(value, output=None):
     if output is None:
         output = []
@@ -597,6 +611,51 @@ def extract_named_statuses_from_text(text):
     }
 
 
+_ROLLOUT_NAME = re.compile(r"^rollout-.*-([0-9a-fA-F-]{36})\.jsonl$")
+
+
+def find_latest_codex_conversation_id(root_dir):
+    """The conversation id of the newest rollout under a Codex home, or None.
+
+    Codex exposes no flag that imposes a conversation id, so the only way to
+    learn one is to read it back after the run. Each rollout opens with a
+    session-meta record carrying `payload.session_id`, and the filename repeats
+    it; the record is preferred and the filename is the fallback for a file
+    whose first line is truncated.
+
+    Returns None rather than guessing whenever the newest rollout yields
+    nothing usable: a session left on the recency path behaves as it does
+    today, while a wrong id would resume somebody else's conversation.
+    """
+    sessions_dir = os.path.join(root_dir, "sessions")
+    if not os.path.isdir(sessions_dir):
+        return None
+    newest = None
+    for dirpath, _dirnames, filenames in os.walk(sessions_dir):
+        for fname in filenames:
+            if not _ROLLOUT_NAME.match(fname):
+                continue
+            fp = os.path.join(dirpath, fname)
+            stat = _safe_stat(fp)
+            if stat and (newest is None or stat.st_mtime > newest[0]):
+                newest = (stat.st_mtime, fp)
+    if not newest:
+        return None
+
+    path = newest[1]
+    try:
+        with open(path, encoding="utf-8", errors="replace") as handle:
+            record = json.loads(handle.readline())
+        identifier = (record.get("payload") or {}).get("session_id")
+        if isinstance(identifier, str) and identifier.strip():
+            return identifier.strip()
+    except (OSError, ValueError):
+        pass
+
+    match = _ROLLOUT_NAME.match(os.path.basename(path))
+    return match.group(1) if match else None
+
+
 def _collect_candidate_files(root_dir):
     priority_candidates = []
     history_candidates = []