cdx-manager 0.14.0 → 0.15.1

package/src/provider_background.py

@@ -0,0 +1,263 @@
+"""Delegating detached runs to a provider's own background agents.
+
+cdx has always detached runs itself: fork the child into its own session, keep
+its pid, sweep the record when that pid dies. Claude Code now ships the same
+thing natively - `--bg` starts a background agent and `claude agents --json`
+lists them - so on that provider cdx can stop owning the mechanism and own the
+routing instead, which is the part no provider will do.
+
+Two deliberate limits:
+
+Claude only. codex-cli exposes `cloud` and `exec-server`, both marked
+EXPERIMENTAL in its own --help. Trading a working in-house path for a surface
+the provider may change without notice is a bad trade, so Codex keeps the
+in-house launcher until that surface settles.
+
+Capability, never version. The installed CLI is asked what it supports, the
+same way `cdx doctor --check-provider-flags` already does, because a version
+comparison goes stale the moment either side ships.
+"""
+
+import json
+import os
+import re
+import subprocess
+
+from .config import PROVIDER_CLAUDE
+
+BACKGROUND_PATH_PROVIDER = "provider_native"
+BACKGROUND_PATH_CDX = "cdx_detached"
+
+_CAPABILITY_PROBE_TIMEOUT_SECONDS = 10
+
+# Observed on Claude Code 2.1.226 by watching one agent from launch to finish:
+# completion is carried by `state`, not by `status`. `status` alternates
+# idle/busy - it describes activity, not outcome - and an agent waiting on
+# something sits at `state: "blocked"` with `status: "idle"`, which reads
+# exactly like a finished one if you look at `status`.
+_TERMINAL_STATES = {"done", "failed", "cancelled", "canceled", "error", "stopped"}
+_SUCCESS_STATE = "done"
+
+# `claude --bg` prints `backgrounded · <id>` on stdout. That is the authoritative
+# handle: the provider assigns the id itself and ignores --session-id, so this
+# line is the only thing that reliably names the agent just started.
+_BACKGROUNDED = re.compile(r"backgrounded\s*[^\w]?\s*([0-9a-fA-F]{6,})")
+
+
+def parse_backgrounded_id(output):
+    """The agent id `claude --bg` reported, or None."""
+    if not output:
+        return None
+    match = _BACKGROUNDED.search(output)
+    return match.group(1) if match else None
+
+
+EXPERIMENTAL_ENV = "CDX_EXPERIMENTAL_NATIVE_BG"
+
+
+def supports_native_background(provider, env=None, spawn_sync=None):
+    """Whether the installed provider CLI can run and report background agents.
+
+    Off unless `CDX_EXPERIMENTAL_NATIVE_BG=1`. Two live trials against Claude
+    Code 2.1.226 fixed the identification problems and exposed a deeper one.
+
+    Fixed: the provider assigns its own id and ignores `--session-id`, so the
+    agent is now named from the `backgrounded - <id>` line `--bg` prints, and
+    completion is read from `state` rather than `status` (which alternates
+    idle/busy and says nothing about outcome).
+
+    Not fixed, and the reason this stays off:
+
+    1. A delegated run cannot honour the `cdx run --json` contract. The
+       in-house path runs `claude --print --output-format json` and gets a
+       structured result, token usage, and a final payload. `--bg` runs the
+       interactive TUI; `claude logs <id>` returns an ANSI screen dump. So
+       usage accounting, the final payload, and the rate-limit classification
+       that `--failover` depends on all have no source.
+    2. Nothing yet drives a delegated run to a terminal status. `agent_terminal_status`
+       exists but no caller consults the provider, so the record relies on pid
+       liveness and a blocked agent reads as running indefinitely.
+    3. cdx's own launch spec makes the agent block. A hand-run
+       `claude --bg --permission-mode bypassPermissions "..."` reaches
+       `state: done`; the same task launched through cdx sits at
+       `state: blocked`, most likely because the spec passes a `--session-id`
+       that already exists.
+
+    Delegation therefore needs its own request, not a flag flip.
+    """
+    if provider != PROVIDER_CLAUDE:
+        return False
+    environment = env if env is not None else os.environ
+    if str(environment.get(EXPERIMENTAL_ENV, "")).strip() not in ("1", "true", "on", "yes"):
+        return False
+    runner = spawn_sync or _default_spawn_sync
+    try:
+        help_text = runner(["claude", "--help"], env=env)
+        agents_help = runner(["claude", "agents", "--help"], env=env)
+    except (OSError, subprocess.SubprocessError):
+        return False
+    if help_text is None or agents_help is None:
+        return False
+    return "--bg" in help_text and "--json" in agents_help
+
+
+def _default_spawn_sync(argv, env=None):
+    try:
+        completed = subprocess.run(
+            argv,
+            capture_output=True,
+            text=True,
+            timeout=_CAPABILITY_PROBE_TIMEOUT_SECONDS,
+            env=env,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return f"{completed.stdout or ''}{completed.stderr or ''}"
+
+
+def list_background_agents(env=None, cwd=None, spawn_sync=None):
+    """Every background agent the provider knows about, completed ones included.
+
+    Returns [] rather than raising for any failure: this feeds a status refresh
+    that must never take down the command that asked for it.
+    """
+    runner = spawn_sync or _default_spawn_sync
+    argv = ["claude", "agents", "--json", "--all"]
+    if cwd:
+        argv += ["--cwd", cwd]
+    output = runner(argv, env=env)
+    if not output:
+        return []
+    start = output.find("[")
+    if start < 0:
+        return []
+    try:
+        agents = json.loads(output[start:])
+    except ValueError:
+        return []
+    return agents if isinstance(agents, list) else []
+
+
+def find_agent(agents, identifier):
+    """The agent with this id, matching either the short id or the session id."""
+    for agent in agents:
+        if not isinstance(agent, dict):
+            continue
+        if identifier in (agent.get("id"), agent.get("sessionId")):
+            return agent
+    return None
+
+
+def stop_background_agent(identifier, env=None, spawn_sync=None):
+    """Release a parked session once its task is done.
+
+    `--bg` starts an interactive session that outlives the turn it was given,
+    so a one-shot run leaves it alive unless cdx stops it. Six were leaked
+    during development before this existed.
+    """
+    if not identifier:
+        return False
+    runner = spawn_sync or _default_spawn_sync
+    return runner(["claude", "stop", str(identifier)], env=env) is not None
+
+
+def turn_is_complete(outcome):
+    """Whether the provider recorded an answer for the prompt cdx submitted.
+
+    This, not the agent's state, is what says a one-shot delegated task
+    finished. A `--bg` session parks at `state: blocked, status: idle` after
+    answering and stays there indefinitely, waiting for a next prompt that a
+    one-shot run will never send - indistinguishable, by state alone, from an
+    agent still thinking.
+    """
+    return bool(outcome and outcome.get("result") is not None and outcome.get("usage"))
+
+
+def agent_terminal_status(agent):
+    """The cdx run status a provider agent maps to, or None while it runs.
+
+    An agent the provider no longer lists has finished in a way cdx cannot
+    name, so it reports `failed` rather than `succeeded`: claiming success for
+    a run whose outcome was never observed is the one answer that must not be
+    guessed.
+    """
+    if agent is None:
+        return "failed"
+    state = str(agent.get("state") or "").strip().lower()
+    if state not in _TERMINAL_STATES:
+        return None
+    return "succeeded" if state == _SUCCESS_STATE else "failed"
+
+
+def find_session_transcript(auth_home, session_id):
+    """The provider's own JSONL transcript for a background session, or None.
+
+    Claude Code writes one file per session under `.claude/projects/<cwd>/`,
+    named by session id. Located by walking rather than by reconstructing the
+    encoded cwd directory, because that encoding is the provider's and is not
+    part of any contract cdx should depend on.
+    """
+    if not auth_home or not session_id:
+        return None
+    root = os.path.join(auth_home, ".claude", "projects")
+    if not os.path.isdir(root):
+        return None
+    target = f"{session_id}.jsonl"
+    for dirpath, _dirnames, filenames in os.walk(root):
+        if target in filenames:
+            return os.path.join(dirpath, target)
+    return None
+
+
+def read_transcript_outcome(path):
+    """The final assistant text and summed token usage from a session transcript.
+
+    This is what makes a delegated run able to honour the `cdx run --json`
+    contract at all: `--bg` produces no structured stdout, but the provider
+    records the same facts here. Usage is summed across assistant messages
+    because a session can hold several turns.
+    """
+    outcome = {"result": None, "usage": None}
+    if not path:
+        return outcome
+    totals = {"input_tokens": 0, "output_tokens": 0, "reasoning_tokens": 0, "total_tokens": 0}
+    seen = False
+    try:
+        with open(path, encoding="utf-8", errors="replace") as handle:
+            for line in handle:
+                line = line.strip()
+                if not line.startswith("{"):
+                    continue
+                try:
+                    record = json.loads(line)
+                except ValueError:
+                    continue
+                if record.get("type") != "assistant":
+                    continue
+                message = record.get("message")
+                if not isinstance(message, dict):
+                    continue
+                content = message.get("content")
+                if isinstance(content, list):
+                    text = " ".join(
+                        block.get("text", "") for block in content
+                        if isinstance(block, dict) and block.get("type") == "text"
+                    ).strip()
+                    if text:
+                        outcome["result"] = text
+                usage = message.get("usage")
+                if isinstance(usage, dict):
+                    seen = True
+                    inp = (usage.get("input_tokens") or 0) + (usage.get("cache_creation_input_tokens") or 0) \
+                        + (usage.get("cache_read_input_tokens") or 0)
+                    out = usage.get("output_tokens") or 0
+                    totals["input_tokens"] += inp
+                    totals["output_tokens"] += out
+                    totals["total_tokens"] += inp + out
+    except OSError:
+        return outcome
+    if seen:
+        totals["reasoning_tokens"] = None
+        outcome["usage"] = totals
+    return outcome

package/src/provider_runtime.py

@@ -18,6 +18,7 @@ from .config import (
     PROVIDER_CODEX,
     PROVIDER_OLLAMA,
     REASONING_EFFORT_VALUES,
+    split_extra_args,
 )
 from .errors import CdxError
 
@@ -354,6 +355,45 @@ def _claude_cli_model(model):
     return raw
 
 
+def _conversation_id(session):
+    conversation = session.get("conversation") or {}
+    identifier = conversation.get("id")
+    return identifier if identifier else None
+
+
+def _claude_fallback_model(value):
+    """Normalize each element of the comma-separated fallback list.
+
+    Claude tries the entries in order, so the separator has to survive while
+    each name goes through the same aliasing a primary `--model` gets.
+    """
+    if not value:
+        return value
+    return ",".join(_claude_cli_model(element) for element in str(value).split(","))
+
+
+def _format_budget_arg(value):
+    """Render a budget without the float artefacts a bare str() would show.
+
+    A stored 5.0 must reach the CLI as `5`, not `5.0`, and 2.50 as `2.5`.
+    """
+    number = float(value)
+    if number == int(number):
+        return str(int(number))
+    return f"{number:g}"
+
+
+def _extra_args(session):
+    """Unvalidated provider arguments, appended after everything cdx maps.
+
+    Last wins in both provider CLIs, so a passthrough value overrides the
+    mapped setting it duplicates. That is the useful default - the escape hatch
+    would be pointless if `cdx set` always won - but it is why `cdx configs`
+    shows the raw string: the override has to be visible somewhere.
+    """
+    return split_extra_args((session.get("launch") or {}).get("extra_args")) or []
+
+
 def _launch_config_args(session):
     launch = session.get("launch") or {}
     args = []
@@ -482,9 +522,13 @@ def _build_launch_spec(session, cwd=None, env_override=None, initial_prompt=None
     if session["provider"] == PROVIDER_CLAUDE:
         launch = session.get("launch") or {}
         args = ["--name", session["name"]]
+        conversation_id = _conversation_id(session)
+        if conversation_id:
+            args += ["--session-id", conversation_id]
         if launch.get("model"):
             args += ["--model", _claude_cli_model(launch["model"])]
         args += _launch_config_args(session)
+        args += _extra_args(session)
         if initial_prompt:
             args.append(initial_prompt)
         auth_home = _get_auth_home(session)
@@ -555,20 +599,48 @@ def _redacted_resume_command_preview(session, cwd=None):
 def get_resume_capability(session, cwd=None):
     provider = session.get("provider")
     cwd = cwd or os.getcwd()
+    conversation = session.get("conversation") or {}
+    conversation_id = _conversation_id(session)
     if provider == PROVIDER_CODEX:
+        if conversation_id:
+            return {
+                "resumable": True,
+                "provider": provider,
+                "strategy": "provider_conversation_id",
+                "identity": conversation_id,
+                "provenance": conversation.get("provenance"),
+                "reason": "supported",
+                "command_preview": ["codex", "resume", conversation_id, "--cd", cwd],
+            }
         return {
             "resumable": True,
             "provider": provider,
             "strategy": "provider_last",
-            "reason": "supported",
+            "identity": None,
+            "provenance": None,
+            # Codex only reveals a conversation id once a run has written its
+            # rollout, so a session that has never run legitimately has none.
+            "reason": "no_recorded_conversation",
             "command_preview": ["codex", "resume", "--last", "--cd", cwd],
         }
     if provider == PROVIDER_CLAUDE:
+        if conversation_id:
+            return {
+                "resumable": True,
+                "provider": provider,
+                "strategy": "provider_conversation_id",
+                "identity": conversation_id,
+                "provenance": conversation.get("provenance"),
+                "reason": "supported",
+                "command_preview": ["claude", "--resume", conversation_id],
+            }
         return {
             "resumable": True,
             "provider": provider,
             "strategy": "provider_continue",
-            "reason": "supported",
+            "identity": None,
+            "provenance": None,
+            "reason": "no_recorded_conversation",
             "command_preview": ["claude", "--continue"],
         }
     return {
@@ -593,10 +665,16 @@ def _build_resume_spec(session, cwd=None, env_override=None, capture_transcript=
 
     if session["provider"] == PROVIDER_CLAUDE:
         launch = session.get("launch") or {}
-        args = ["--continue", "--name", session["name"]]
+        # Resuming by id targets the conversation this session actually last
+        # opened; --continue only means "the most recent one in this directory",
+        # which another session or another cwd can silently change.
+        conversation_id = _conversation_id(session)
+        args = ["--resume", conversation_id] if conversation_id else ["--continue"]
+        args += ["--name", session["name"]]
         if launch.get("model"):
             args += ["--model", _claude_cli_model(launch["model"])]
         args += _launch_config_args(session)
+        args += _extra_args(session)
         if resume_prompt:
             args.append(resume_prompt)
         auth_home = _get_auth_home(session)
@@ -615,10 +693,13 @@ def _build_resume_spec(session, cwd=None, env_override=None, capture_transcript=
         }, capture_transcript=capture_transcript, env=env)
 
     launch = session.get("launch") or {}
-    args = ["resume", "--last", "--cd", cwd]
+    conversation_id = _conversation_id(session)
+    args = ["resume", conversation_id] if conversation_id else ["resume", "--last"]
+    args += ["--cd", cwd]
     if launch.get("model"):
         args += ["--model", launch["model"]]
     args += _launch_config_args(session)
+    args += _extra_args(session)
     if resume_prompt:
         args.append(resume_prompt)
     return _wrap_launch_with_transcript(session, {
@@ -652,9 +733,20 @@ def _build_headless_launch_spec(session, cwd=None, env_override=None, initial_pr
 
     if session["provider"] == PROVIDER_CLAUDE:
         args = ["--print", "--output-format", "json", "--name", session["name"]]
+        conversation_id = _conversation_id(session)
+        if conversation_id:
+            args += ["--session-id", conversation_id]
         if model:
             args += ["--model", _claude_cli_model(model)]
+        # --max-budget-usd and --fallback-model are --print-only in the claude
+        # CLI, so they belong to this spec alone: the interactive and resume
+        # specs would be rejected by the provider for carrying them.
+        if launch.get("budget") is not None:
+            args += ["--max-budget-usd", _format_budget_arg(launch["budget"])]
+        if launch.get("fallback_model"):
+            args += ["--fallback-model", _claude_fallback_model(launch["fallback_model"])]
         args += _launch_config_args(session)
+        args += _extra_args(session)
         if initial_prompt:
             args.append(initial_prompt)
         auth_home = _get_auth_home(session)

package/src/run_failover.py

@@ -0,0 +1,237 @@
+"""Deciding whether a finished run stopped because its account ran out.
+
+Two independent signals have to agree before cdx moves a task to another
+account, because the two ways of being wrong are not symmetric. Missing a rate
+limit costs nothing new: the run fails exactly as it does today. Inventing one
+migrates a healthy run off a working account, discards its progress, and
+consumes a second account's quota to redo it. So every ambiguous case here
+resolves to "not rate limited".
+
+Signal one is the provider's own structured output, which both headless paths
+already produce - `codex exec --json` emits JSONL events, and the claude spec
+passes `--print --output-format json`. Signal two is the account's rate-limit
+status, refreshed once and read through the existing status pipeline - where
+`rate_limit_reached` is the strongest evidence available, being the provider's
+own typed reason rather than a threshold or a sentence.
+
+The matchers below are deliberately the only place that knows what exhaustion
+looks like on the wire. That shape is version-specific and is the part most
+likely to drift, so it is one function per provider and nothing else in the
+codebase reads provider output for this purpose.
+"""
+
+import json
+import os
+
+from .config import PROVIDER_CLAUDE, PROVIDER_CODEX
+
+MAX_FAILOVER_TRANSITIONS = 4
+
+# Read from the tail: a rate limit ends the run, so its marker is near the end,
+# and a long transcript should not be loaded whole to find it.
+_MAX_TAIL_BYTES = 64 * 1024
+
+# Observed against codex-cli 0.147.0 on an exhausted account:
+#   {"type":"error","message":"Your workspace is out of credits. Add credits to continue."}
+#   {"type":"turn.failed","error":{"message":"Your workspace is out of credits..."}}
+# "out of credits" was missing from the first version of this list, which is why
+# it never matched anything real. Credit exhaustion and a rate limit are the
+# same thing for this decision: the account cannot serve the task, try another.
+# Claude's own wording, captured from a real transcript on this machine:
+#   API error: 429 {"type":"error","error":{"type":"rate_limit_error",
+#     "message":"Usage credits are required for this model.", ...}}
+# `rate_limit_error` is already covered by "rate_limit" below. Worth knowing:
+# that same 429 is also how an org policy disabling a model is reported
+# (`disabled_reason: org_level_disabled`), which no amount of failing over
+# would fix - every account in the org hits it. The status corroboration in
+# `failover_reason` is what keeps that case from migrating a healthy run.
+_RATE_LIMIT_MARKERS = (
+    "out of credits",
+    "rate_limit",
+    "rate limit",
+    "usage limit",
+    "quota exceeded",
+    "insufficient_quota",
+)
+
+# Records the provider itself labelled as failures. Scoping the search to these
+# is what keeps assistant prose out of the decision - the model's own words are
+# never inside a record of this shape. The first version instead excluded the
+# `message` key everywhere, which also excluded the field codex actually puts
+# its error text in, so nothing could ever match.
+_CODEX_ERROR_TYPES = ("error", "turn.failed", "thread.failed")
+
+
+def _read_tail(path):
+    if not path or not os.path.isfile(path):
+        return ""
+    try:
+        size = os.path.getsize(path)
+        with open(path, encoding="utf-8", errors="replace") as handle:
+            if size > _MAX_TAIL_BYTES:
+                handle.seek(size - _MAX_TAIL_BYTES)
+            return handle.read()
+    except OSError:
+        return ""
+
+
+def _error_texts(provider, record):
+    """Message strings from a record the provider marked as a failure.
+
+    Returns nothing for any other record, which is what keeps the assistant's
+    own words out of the decision: prose lives in successful results, never in
+    an error record.
+    """
+    if not isinstance(record, dict):
+        return []
+    texts = []
+    if provider == PROVIDER_CODEX:
+        if str(record.get("type") or "") not in _CODEX_ERROR_TYPES:
+            return []
+        texts.append(record.get("message"))
+        error = record.get("error")
+        if isinstance(error, dict):
+            texts.append(error.get("message"))
+    elif provider == PROVIDER_CLAUDE:
+        # `--print --output-format json` returns one result object; its own
+        # is_error flag says whether `result` holds an error or an answer.
+        if record.get("is_error") is not True:
+            return []
+        texts.append(record.get("result"))
+        texts.append(record.get("subtype"))
+        error = record.get("error")
+        if isinstance(error, dict):
+            texts.append(error.get("message"))
+    return [text for text in texts if isinstance(text, str)]
+
+
+def _mentions_exhaustion(texts):
+    return any(
+        marker in text.lower()
+        for text in texts
+        for marker in _RATE_LIMIT_MARKERS
+    )
+
+
+def _structured_records(provider, text):
+    """The structured payloads a provider wrote, newest last.
+
+    Codex emits one JSON object per line; claude emits a single object. Lines
+    that do not decode are skipped rather than pattern-matched as raw text:
+    unparsed output is not a signal, it is an absence of one.
+    """
+    records = []
+    if not text:
+        return records
+    if provider == PROVIDER_CODEX:
+        for line in text.splitlines():
+            line = line.strip()
+            if not line.startswith("{"):
+                continue
+            try:
+                records.append(json.loads(line))
+            except ValueError:
+                continue
+        return records
+    if provider == PROVIDER_CLAUDE:
+        stripped = text.strip()
+        start = stripped.find("{")
+        if start >= 0:
+            try:
+                records.append(json.loads(stripped[start:]))
+            except ValueError:
+                return records
+    return records
+
+
+def looks_rate_limited(provider, run_info):
+    """Whether the provider's structured output says the account is exhausted.
+
+    This is the weaker of the two signals and is never acted on alone; see
+    `should_fail_over`.
+    """
+    if provider not in (PROVIDER_CODEX, PROVIDER_CLAUDE):
+        return False
+    if run_info.get("timed_out"):
+        # A timeout is cdx's own deadline, not the provider's verdict.
+        return False
+    if run_info.get("returncode") == 0:
+        return False
+    for record in _structured_records(provider, _read_tail(run_info.get("stdout_path"))):
+        if _mentions_exhaustion(_error_texts(provider, record)):
+            return True
+    return False
+
+
+def status_confirms_exhaustion(row):
+    """Whether a freshly refreshed status row agrees the account is spent.
+
+    An unreadable or empty row is not a confirmation. Treating "no status" as
+    exhaustion would make every probe failure look like a rate limit.
+    """
+    if not row:
+        return False
+    # The provider's own typed reason, when it gives one. Authoritative and
+    # language-free: `workspace_owner_credits_depleted` says the account cannot
+    # serve the task even when both percentage windows still read healthy,
+    # which is a case the thresholds below would miss entirely.
+    if row.get("rate_limit_reached"):
+        return True
+    if row.get("blocking"):
+        return True
+    for key in ("remaining_5h_pct", "remaining_week_pct"):
+        value = row.get(key)
+        if isinstance(value, (int, float)) and value <= 0:
+            return True
+    return False
+
+
+def provider_reported_failure(provider, run_info):
+    """A failure the provider itself emitted, whatever it says.
+
+    Deliberately wording-free. A timeout is cdx's own deadline, and a zero exit
+    is not a failure; everything else that produced a provider error record
+    counts.
+    """
+    if provider not in (PROVIDER_CODEX, PROVIDER_CLAUDE):
+        return False
+    if run_info.get("timed_out") or run_info.get("returncode") == 0:
+        return False
+    for record in _structured_records(provider, _read_tail(run_info.get("stdout_path"))):
+        if _error_texts(provider, record):
+            return True
+    return False
+
+
+def failover_reason(provider, run_info, status_row):
+    """Why this run should move, or None to stay put.
+
+    Two independent routes, because relying on either alone has been shown to
+    fail:
+
+    `provider_reported_exhaustion` - the provider said so in words cdx
+    recognises. Precise, but the wording is not one thing: an enterprise
+    workspace out of credits, a personal plan hitting its 5-hour window, and
+    Claude's own phrasing are all different sentences, and only the first has
+    ever been observed. Matching prose alone would silently miss the rest.
+
+    `account_status_exhausted` - the account's own rate-limit status says it has
+    nothing left, and the run failed. This is the authoritative route: it comes
+    from the provider's rate-limit API rather than from English, so it holds for
+    plans and providers whose wording nobody has seen. An account at zero cannot
+    serve the task no matter which sentence it used to say so.
+
+    Status confirmation is required either way, so a healthy account is never
+    abandoned on the strength of a matched phrase.
+    """
+    if not status_confirms_exhaustion(status_row):
+        return None
+    if looks_rate_limited(provider, run_info):
+        return "provider_reported_exhaustion"
+    if provider_reported_failure(provider, run_info):
+        return "account_status_exhausted"
+    return None
+
+
+def should_fail_over(provider, run_info, status_row):
+    return failover_reason(provider, run_info, status_row) is not None