cdx-manager 0.15.0 → 0.16.0

package/src/run_failover.py

@@ -10,7 +10,9 @@ resolves to "not rate limited".
 Signal one is the provider's own structured output, which both headless paths
 already produce - `codex exec --json` emits JSONL events, and the claude spec
 passes `--print --output-format json`. Signal two is the account's rate-limit
-status, refreshed once and read through the existing status pipeline.
+status, refreshed once and read through the existing status pipeline - where
+`rate_limit_reached` is the strongest evidence available, being the provider's
+own typed reason rather than a threshold or a sentence.
 
 The matchers below are deliberately the only place that knows what exhaustion
 looks like on the wire. That shape is version-specific and is the part most
@@ -35,6 +37,14 @@ _MAX_TAIL_BYTES = 64 * 1024
 # "out of credits" was missing from the first version of this list, which is why
 # it never matched anything real. Credit exhaustion and a rate limit are the
 # same thing for this decision: the account cannot serve the task, try another.
+# Claude's own wording, captured from a real transcript on this machine:
+#   API error: 429 {"type":"error","error":{"type":"rate_limit_error",
+#     "message":"Usage credits are required for this model.", ...}}
+# `rate_limit_error` is already covered by "rate_limit" below. Worth knowing:
+# that same 429 is also how an org policy disabling a model is reported
+# (`disabled_reason: org_level_disabled`), which no amount of failing over
+# would fix - every account in the org hits it. The status corroboration in
+# `failover_reason` is what keeps that case from migrating a healthy run.
 _RATE_LIMIT_MARKERS = (
     "out of credits",
     "rate_limit",
@@ -161,6 +171,12 @@ def status_confirms_exhaustion(row):
     """
     if not row:
         return False
+    # The provider's own typed reason, when it gives one. Authoritative and
+    # language-free: `workspace_owner_credits_depleted` says the account cannot
+    # serve the task even when both percentage windows still read healthy,
+    # which is a case the thresholds below would miss entirely.
+    if row.get("rate_limit_reached"):
+        return True
     if row.get("blocking"):
         return True
     for key in ("remaining_5h_pct", "remaining_week_pct"):

package/src/run_registry.py

@@ -2,9 +2,12 @@ import json
 import os
 import sys
 import tempfile
+import time
 from contextlib import contextmanager
 from datetime import datetime, timezone
 
+from .errors import CdxError
+
 
 def utc_now_iso():
     return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
@@ -14,14 +17,49 @@ def registry_path(base_dir):
     return os.path.join(base_dir, "runs.json")
 
 
+# A waiter waits. `fcntl.flock` blocks until the lock is free, but Windows'
+# `LK_LOCK` retries ten times at one-second intervals and then raises
+# EDEADLOCK, so the same contention produced a wait on one platform and an
+# OSError on the other. Twenty parallel runs writing this registry is the load
+# cdx exists to carry, and on a physical Windows machine it exhausted that
+# budget - CI never showed it, being less contended than a real desktop.
+REGISTRY_LOCK_TIMEOUT_SECONDS = 120
+_LOCK_RETRY_SECONDS = 0.05
+
+
+def _acquire_windows_lock(handle, timeout_seconds):
+    """Block until the byte is locked, or give up with a nameable error.
+
+    Uses the non-blocking form and does the waiting here, so the wait is
+    bounded by a deadline cdx chooses rather than by `LK_LOCK`'s fixed ten
+    attempts. Unbounded would be worse than the original: a stuck holder would
+    hang every later command with nothing to report.
+    """
+    import msvcrt
+
+    deadline = time.monotonic() + timeout_seconds
+    while True:
+        handle.seek(0)
+        try:
+            msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+            return
+        except OSError:
+            if time.monotonic() >= deadline:
+                raise CdxError(
+                    "Timed out waiting for the run registry lock after "
+                    f"{timeout_seconds}s. Another cdx process may be stuck holding it.",
+                    75,
+                ) from None
+            time.sleep(_LOCK_RETRY_SECONDS)
+
+
 @contextmanager
-def _registry_lock(path):
+def _registry_lock(path, timeout_seconds=REGISTRY_LOCK_TIMEOUT_SECONDS):
     os.makedirs(os.path.dirname(path), exist_ok=True)
     with open(path + ".lock", "a") as handle:
         if sys.platform == "win32":
             import msvcrt
-            handle.seek(0)
-            msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
+            _acquire_windows_lock(handle, timeout_seconds)
             try:
                 yield
             finally:

package/src/session_service.py

@@ -211,6 +211,18 @@ def _normalize_launch_settings(settings, mark_fast_service_tier=True):
                 normalized["logics"] = False
             else:
                 raise CdxError(f"Unsupported logics value: {settings['logics']}")
+    if "notify" in settings and settings["notify"] is not None:
+        value = settings["notify"]
+        if isinstance(value, bool):
+            normalized["notify"] = value
+        else:
+            text = str(value).strip().lower()
+            if text in ("on", "true", "1", "yes"):
+                normalized["notify"] = True
+            elif text in ("off", "false", "0", "no"):
+                normalized["notify"] = False
+            else:
+                raise CdxError(f"Unsupported notify value: {settings['notify']}")
     if "model" in settings and settings["model"] is not None:
         model = str(settings["model"]).strip()
         if not model:
@@ -349,7 +361,7 @@ def unset_launch_settings(store, name, keys):
     if not keys:
         raise CdxError("At least one launch setting is required.")
     allowed = {
-        "power", "reasoning_effort", "reasoningEffort", "permission", "fast", "rtk", "logics",
+        "power", "reasoning_effort", "reasoningEffort", "permission", "fast", "rtk", "logics", "notify",
         "model", "fallback_model", "budget", "extra_args", "priority",
     }
     unknown = [key for key in keys if key not in allowed]

package/src/session_status.py

@@ -79,6 +79,7 @@ def _normalize_status_payload(payload=None):
         "reset_5h_at": payload.get("reset_5h_at"),
         "reset_week_at": payload.get("reset_week_at"),
         "reset_at": payload.get("reset_at") or payload.get("reset_week_at") or payload.get("reset_5h_at"),
+        "rate_limit_reached": payload.get("rate_limit_reached"),
         "updated_at": _to_local_iso(payload.get("updated_at") or payload.get("captured_at") or now),
         "raw_status_text": payload.get("raw_status_text"),
         "source_ref": payload.get("source_ref"),

package/src/status_source.py

@@ -153,6 +153,13 @@ def _extract_structured_rate_limits(record):
         "reset_5h_at": _normalize_timestamp_value(_get_rate_limit_reset_at(five_hour)),
         "reset_week_at": _normalize_timestamp_value(_get_rate_limit_reset_at(weekly)),
         "reset_at": None,
+        # A typed reason the account is blocked, when the provider gives one:
+        # null in normal operation, and a value such as
+        # "workspace_owner_credits_depleted" when a limit is actually reached.
+        # Better evidence than any percentage - credits can run out while the
+        # 5-hour and weekly windows still read healthy - and better than
+        # matching prose, since it is an enum rather than a sentence.
+        "rate_limit_reached": _clean_reached_type(rate_limits.get("rate_limit_reached_type")),
         "raw_status_text": json.dumps(rate_limits, sort_keys=True),
     }
     if not five_hour and weekly:
@@ -172,6 +179,13 @@ def _extract_structured_rate_limits(record):
     return result
 
 
+def _clean_reached_type(value):
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    return text or None
+
+
 def _collect_structured_rate_limit_statuses(value, output=None):
     if output is None:
         output = []