cdx-manager 0.24.7 → 0.24.9

package/src/run_command.py

@@ -55,6 +55,14 @@ def run_cdx_error_code(error):
     return "cdx_error"
 
 
+def run_conversation_id(session, run_info=None):
+    if run_info is not None and "conversation_id" in run_info:
+        return run_info["conversation_id"]
+    if (session or {}).get("provider") == "codex" and not session.get("continue_run"):
+        return None
+    return ((session or {}).get("conversation") or {}).get("id")
+
+
 def run_payload_reasoning_effort(parsed, session):
     launch = (session.get("launch") or {}) if session else {}
     return (
@@ -149,7 +157,7 @@ def run_warnings(parsed, session, selection_row=None):
 
 
 def run_launch_payload(api_schema_version, parsed, session, artifacts, cwd=None,
-                       pid=None, launch_log_path=None, selection_row=None):
+                       pid=None, launch_log_path=None, selection_row=None, conversation_id=None):
     """Payload for a detached launch: identity and artifact paths, no outcome.
 
     Deliberately shaped like `run_result_payload` minus the fields only a
@@ -161,6 +169,8 @@ def run_launch_payload(api_schema_version, parsed, session, artifacts, cwd=None,
         "ok": True,
         "action": "run",
         "launcher": "cdx",
+        "parent_run_id": parsed.get("continue_run"),
+        "conversation_id": conversation_id or run_conversation_id(session),
         "detached": True,
         "session": session.get("name") if session else None,
         "provider": session.get("provider") if session else parsed.get("provider"),
@@ -208,9 +218,12 @@ def run_result_payload(api_schema_version, ok, parsed, session, run_info=None,
         "ok": bool(ok),
         "action": "run",
         "launcher": "cdx",
+        "parent_run_id": parsed.get("continue_run"),
+        "conversation_id": run_conversation_id(session, run_info),
         "session": session.get("name") if session else None,
         "provider": session.get("provider") if session else parsed.get("provider"),
-        "model": parsed.get("model") or ((session.get("launch") or {}).get("model") if session else None),
+        "model": run_info.get("effective_model", parsed.get("model") or ((session.get("launch") or {}).get("model") if session else None)),
+        "recovery_events": run_info.get("recovery_events", []),
         "reasoning_effort": run_payload_reasoning_effort(parsed, session),
         "power": parsed.get("power") or ((session.get("launch") or {}).get("power") if session else None),
         "cwd": os.path.abspath(parsed.get("cwd") or os.getcwd()),

package/src/run_failover.py

@@ -1,6 +1,6 @@
-"""Deciding whether a finished run stopped because its account ran out.
+"""Classify finished runs for account exhaustion or model capacity recovery.
 
-Two independent signals have to agree before cdx moves a task to another
+For account exhaustion, two independent signals must agree before cdx moves a task to another
 account, because the two ways of being wrong are not symmetric. Missing a rate
 limit costs nothing new: the run fails exactly as it does today. Inventing one
 migrates a healthy run off a working account, discards its progress, and
@@ -22,6 +22,7 @@ codebase reads provider output for this purpose.
 
 import json
 import os
+import re
 
 from .config import PROVIDER_CLAUDE, PROVIDER_CODEX
 
@@ -235,3 +236,32 @@ def failover_reason(provider, run_info, status_row):
 
 def should_fail_over(provider, run_info, status_row):
     return failover_reason(provider, run_info, status_row) is not None
+
+
+CAPACITY_RETRY_DELAYS = (1, 2)
+_CAPACITY_MESSAGE = re.compile(r"(?:selected model|(?:the )?model(?: [\w.-]+)?) is at capacity\b", re.IGNORECASE)
+
+
+def looks_model_at_capacity(provider, run_info):
+    """Recognize Codex failures, excluding successful output and assistant prose."""
+    if provider != PROVIDER_CODEX or run_info.get("timed_out") or run_info.get("returncode") in (None, 0):
+        return False
+    for key in ("stdout_path", "stderr_path"):
+        text = _read_tail(run_info.get(key))
+        for record in _structured_records(provider, text):
+            if any(_CAPACITY_MESSAGE.search(message) for message in _error_texts(provider, record)):
+                return True
+        # Some Codex versions print this diagnostic as a standalone plain line.
+        for line in text.splitlines():
+            line = line.strip()
+            if _CAPACITY_MESSAGE.match(re.sub(r"^error:\s*", "", line, flags=re.IGNORECASE)):
+                return True
+    return False
+
+
+def capacity_models(session):
+    """Configured alternatives in order, without retrying the current model."""
+    launch = session.get("launch") or {}
+    models = [launch.get("model")]
+    models.extend(str(launch.get("fallback_model") or "").split(","))
+    return list(dict.fromkeys(model.strip() if model else None for model in models if model is None or model.strip()))

package/src/run_registry.py

@@ -243,7 +243,8 @@ class RunRegistry:
         with _registry_lock(self.path):
             _check_capacity(_read_registry(self.path), max_parallel)
 
-    def start(self, run_id, *, kind, session, provider, model, cwd, artifacts=None, label=None, max_parallel=None):
+    def start(self, run_id, *, kind, session, provider, model, cwd, artifacts=None, label=None, max_parallel=None,
+              parent_run_id=None, conversation_id=None, launch=None, reasoning_effort=None):
         with _registry_lock(self.path):
             data = _read_registry(self.path)
             data["runs"] = [run for run in data["runs"] if run.get("run_id") != run_id]
@@ -252,6 +253,8 @@ class RunRegistry:
                 # detached child writes a second time under the same id.
                 _check_capacity(data, max_parallel)
             record = _base_record(run_id, kind=kind, session=session, provider=provider, model=model, cwd=cwd, artifacts=artifacts)
+            record.update(parent_run_id=parent_run_id, conversation_id=conversation_id,
+                          launch=dict(launch or {}), reasoning_effort=reasoning_effort)
             if label:
                 record["label"] = label
             data["runs"].insert(0, record)
@@ -285,6 +288,8 @@ class RunRegistry:
                     continue
                 run["background_path"] = path
                 run["provider_session_id"] = provider_session_id
+                if provider_session_id:
+                    run["conversation_id"] = provider_session_id
                 run["provider_agent_id"] = provider_agent_id
                 _write_registry(self.path, data)
                 return run
@@ -321,6 +326,16 @@ class RunRegistry:
                 return run
             return None
 
+    def record_recovery(self, run_id, event):
+        """Publish retries and fallbacks while a supervised run is still active."""
+        with _registry_lock(self.path):
+            data = _read_registry(self.path)
+            for run in data["runs"]:
+                if run.get("run_id") == run_id:
+                    run.setdefault("recovery_events", []).append({"at": utc_now_iso(), **event})
+                    _write_registry(self.path, data)
+                    return
+
     def finish(self, run_id, *, status, final_payload=None, run_info=None, error=None, task_report=None):
         with _registry_lock(self.path):
             return self._finish_locked(run_id, status=status, final_payload=final_payload, run_info=run_info, error=error, task_report=task_report)
@@ -346,6 +361,8 @@ class RunRegistry:
                     run["duration_seconds"] = None
             if run_info:
                 run["pid"] = run_info.get("pid")
+                if run_info.get("launch"):
+                    run["launch"] = run_info["launch"]
                 run["exit_code"] = run_info.get("returncode")
                 run["artifacts"] = {
                     **(run.get("artifacts") or {}),
@@ -357,8 +374,13 @@ class RunRegistry:
                 # Normalize on the way in, not on the way out: this record is
                 # what `cdx run-status` and `cdx run-report` publish, and it
                 # used to store whichever shape the producing reader emitted.
-                run["usage"] = coerce_usage(final_payload.get("usage"))
+                run["usage"] = coerce_usage(final_payload.get("usage"), provider=run.get("provider"))
                 run["usage_model"] = final_payload.get("usage_model")
+                run["conversation_id"] = final_payload.get("conversation_id") or run.get("conversation_id")
+                run["parent_run_id"] = final_payload.get("parent_run_id") or run.get("parent_run_id")
+                if "reasoning_effort" in final_payload:
+                    run["reasoning_effort"] = final_payload["reasoning_effort"]
+                run["model"] = final_payload.get("model")
                 run["final_payload"] = final_payload
             if error:
                 run["error"] = error

package/src/run_usage.py

@@ -24,6 +24,9 @@ The definitions below are therefore normative, not a suggestion:
                         published field of `cdx run --json`, and because the
                         stats table shows one CACHE column. It is a derived
                         sum, never a source.
+  cache_hit_ratio       cache_read / (input + cache_creation + cache_read).
+                        None when input, reads, or writes are unreported (except
+                        Codex writes), or total input is zero.
   output_tokens         Generated tokens.
   reasoning_tokens      Reasoning counted separately by the provider. Codex
                         reports one; Claude does not and never will -- it bills
@@ -40,7 +43,7 @@ differently -- 1.25x (2x for a one-hour TTL) and 0.1x of uncached input -- and
 a weighted figure cannot recover the split once they are summed.
 
 Absence stays absent: `None` means "not reported", which is not zero. Only
-`normalize_usage` may derive `cached_input_tokens` and `total_tokens`, so no
+`normalize_usage` derives the cache sum, total, and hit ratio, so no
 caller can invent a fourth arithmetic.
 """
 
@@ -50,7 +53,7 @@ import math
 from .errors import CdxError
 
 #: Order matters only for readability; every consumer addresses these by name.
-USAGE_KEYS = (
+TOKEN_USAGE_KEYS = (
     "input_tokens",
     "cached_input_tokens",
     "cache_creation_tokens",
@@ -60,17 +63,18 @@ USAGE_KEYS = (
     "reasoning_tokens",
     "total_tokens",
 )
-#: The two fields `normalize_usage` derives. Never read them from a provider.
-DERIVED_USAGE_KEYS = ("cached_input_tokens", "total_tokens")
+USAGE_KEYS = (*TOKEN_USAGE_KEYS, "cache_hit_ratio")
+#: Derived values are never read from a provider.
+DERIVED_USAGE_KEYS = ("cached_input_tokens", "total_tokens", "cache_hit_ratio")
 SUPPORTED_PROVIDERS = {"claude", "codex"}
 
 
 def empty_usage():
-    return {key: None for key in USAGE_KEYS}
+    return dict.fromkeys(USAGE_KEYS)
 
 
 def normalize_usage(input_tokens=None, cache_creation_tokens=None, cache_read_tokens=None,
-                    output_tokens=None, reasoning_tokens=None, cache_creation_1h_tokens=None):
+                    output_tokens=None, reasoning_tokens=None, cache_creation_1h_tokens=None, provider=None):
     """Build a usage record from the four measured quantities.
 
     The derived fields are computed here and only here. A record whose parts are
@@ -98,7 +102,50 @@ def normalize_usage(input_tokens=None, cache_creation_tokens=None, cache_read_to
         parts["cache_read_tokens"],
         parts["output_tokens"],
     )
-    return {**parts, "cached_input_tokens": cached, "total_tokens": total}
+    return {**parts, "cached_input_tokens": cached, "total_tokens": total,
+            "cache_hit_ratio": cache_hit_ratio(parts, provider=provider)}
+
+
+def cache_hit_ratio(usage, provider=None):
+    """Cache reads divided by all input, or None without a reported read split.
+
+    Missing writes leave Claude's denominator unknown. Only Codex can recover
+    the denominator from its cache-inclusive input total when writes are absent.
+    An empty input has no defined hit ratio.
+    """
+    read = _non_negative(usage.get("cache_read_tokens"))
+    uncached = _non_negative(usage.get("input_tokens"))
+    writes = _non_negative(usage.get("cache_creation_tokens"))
+    if read is None or uncached is None or (writes is None and provider != "codex"):
+        return None
+    total = uncached + read + (writes or 0)
+    return read / total if total else None
+
+
+def summarize_cache_usage(entries):
+    """Nullable measured sums and an input-weighted ratio over covered runs."""
+    keys = ("input_tokens", "cache_creation_tokens", "cache_read_tokens", "output_tokens")
+    sums = dict.fromkeys(keys)
+    reads = inputs = measured = 0
+    for entry in entries:
+        usage = entry.get("usage") or {}
+        if not is_vouched(usage):
+            continue
+        usage = coerce_usage(usage, provider=entry.get("provider"))
+        for key in keys:
+            value = usage.get(key)
+            if value is not None:
+                sums[key] = (sums[key] or 0) + value
+        if usage["cache_hit_ratio"] is not None:
+            measured += 1
+            reads += usage["cache_read_tokens"]
+            inputs += usage["input_tokens"] + usage["cache_read_tokens"] + (usage.get("cache_creation_tokens") or 0)
+    return {**sums, "cache_hit_ratio": reads / inputs if inputs else None,
+            "measured_runs": measured, "measured_input_tokens": inputs, "measured_cache_read_tokens": reads}
+
+
+def format_cache_ratio(ratio):
+    return "cache n/d" if ratio is None else f"cache {ratio * 100:.0f} %"
 
 
 #: What a cached token costs relative to one uncached input token.
@@ -370,7 +417,7 @@ def is_vouched(usage):
     return isinstance(usage, dict) and "cache_creation_tokens" in usage
 
 
-def coerce_usage(usage):
+def coerce_usage(usage, provider=None):
     """Bring an already-built usage record onto the canonical shape.
 
     For boundaries that receive a record rather than measure one -- the run
@@ -382,6 +429,7 @@ def coerce_usage(usage):
     if not isinstance(usage, dict):
         return None
     record = normalize_usage(
+        provider=provider,
         input_tokens=usage.get("input_tokens"),
         cache_creation_tokens=usage.get("cache_creation_tokens"),
         cache_read_tokens=usage.get("cache_read_tokens"),
@@ -409,6 +457,20 @@ def _sum_present(*values):
     return sum(present)
 
 
+def extract_run_conversation_id(provider, stdout_path):
+    """Read identity from this run's output, never from transcript recency."""
+    if not stdout_path:
+        return None
+    try:
+        with open(stdout_path, encoding="utf-8", errors="replace") as handle:
+            records = _parse_json_records(handle.read())
+    except OSError:
+        return None
+    key = "thread_id" if provider == "codex" else "session_id"
+    ids = {r[key] for r in records if isinstance(r, dict) and isinstance(r.get(key), str) and r[key]}
+    return next(iter(ids)) if len(ids) == 1 else None
+
+
 def extract_run_usage(provider, stdout_path):
     return extract_run_observation(provider, stdout_path)[0]
 
@@ -430,7 +492,7 @@ def extract_run_observation(provider, stdout_path, auth_home=None):
     if not records:
         return empty_usage(), None
 
-    usage = _extract_usage_from_records(records)
+    usage = _extract_usage_from_records(records, provider=provider)
     if not _has_usage(usage):
         return empty_usage(), None
     models = set()
@@ -439,7 +501,7 @@ def extract_run_observation(provider, stdout_path, auth_home=None):
             continue
         message = record.get("message")
         model = record.get("model") or (message.get("model") if isinstance(message, dict) else None)
-        if isinstance(model, str) and model and _has_usage(_find_usage(record)):
+        if isinstance(model, str) and model and _has_usage(_find_usage(record, provider=provider)):
             models.add(model)
         model_usage = record.get("modelUsage")
         if isinstance(model_usage, dict):
@@ -505,33 +567,33 @@ def _parse_json_records(text):
     return records
 
 
-def _extract_usage_from_records(records):
+def _extract_usage_from_records(records, provider=None):
     latest = None
     for record in records:
-        candidate = _find_usage(record)
+        candidate = _find_usage(record, provider=provider)
         if _has_usage(candidate):
             latest = candidate
     return latest or empty_usage()
 
 
-def _find_usage(value):
+def _find_usage(value, provider=None):
     if isinstance(value, dict):
-        direct = _usage_from_dict(value)
+        direct = _usage_from_dict(value, provider=provider)
         if _has_usage(direct):
             return direct
         for child in value.values():
-            found = _find_usage(child)
+            found = _find_usage(child, provider=provider)
             if _has_usage(found):
                 return found
     if isinstance(value, list):
         for child in value:
-            found = _find_usage(child)
+            found = _find_usage(child, provider=provider)
             if _has_usage(found):
                 return found
     return empty_usage()
 
 
-def _usage_from_dict(value):
+def _usage_from_dict(value, provider=None):
     """Map one provider usage object onto the canonical fields.
 
     This used to fold the cache counts into `input_tokens` *and* report them
@@ -544,16 +606,18 @@ def _usage_from_dict(value):
         return empty_usage()
 
     cache_creation = _first_int(usage.get("cache_creation_input_tokens"))
+    if cache_creation is None:
+        cache_creation = _first_int(usage.get("cache_write_input_tokens"))
     cache_read = _first_int(usage.get("cache_read_input_tokens"))
-    if cache_creation is None and cache_read is None:
-        # Codex reports one cached figure with no creation/read split; it is a
-        # read by construction, since nothing else could have written it.
+    if cache_read is None:
+        # Codex's cached input count measures reads, independently of writes.
         cache_read = _first_int(usage.get("cached_input_tokens"))
 
     input_tokens = _int_value(usage.get("input_tokens"), usage.get("prompt_tokens"))
     input_tokens = _uncached_input(input_tokens, cache_read, usage)
 
     return normalize_usage(
+        provider=provider,
         input_tokens=input_tokens,
         cache_creation_tokens=cache_creation,
         cache_creation_1h_tokens=_nested_int(usage, "cache_creation", "ephemeral_1h_input_tokens"),
@@ -583,17 +647,18 @@ def _uncached_input(input_tokens, cache_read, usage):
     Guessing wrong in that direction would silently erase real input tokens,
     which is the failure this whole definition exists to stop.
     """
-    if input_tokens is None or not cache_read:
+    if input_tokens is None:
         return input_tokens
     if usage.get("cache_creation_input_tokens") is not None:
         return input_tokens
     if usage.get("cache_read_input_tokens") is not None:
         return input_tokens
-    if usage.get("cached_input_tokens") is None:
+    if usage.get("cached_input_tokens") is None and usage.get("cache_write_input_tokens") is None:
         return input_tokens
-    if cache_read > input_tokens:
+    cached = (cache_read or 0) + (_first_int(usage.get("cache_write_input_tokens")) or 0)
+    if cached > input_tokens:
         return input_tokens
-    return input_tokens - cache_read
+    return input_tokens - cached
 
 
 def _nested_int(value, parent, child):