cdx-manager 0.14.0 → 0.15.0

package/src/commands/runs.py

@@ -21,8 +21,27 @@ from ..cli_args import (
 )
 from ..cli_helpers import API_SCHEMA_VERSION, _json_failure, _json_success, _write_json
 from ..errors import CdxError
-from ..provider_runtime import _ensure_session_authentication, _headless_artifact_paths, _run_headless_provider_command
+from ..provider_background import (
+    BACKGROUND_PATH_CDX,
+    BACKGROUND_PATH_PROVIDER,
+    agent_terminal_status,
+    find_agent,
+    find_session_transcript,
+    list_background_agents,
+    parse_backgrounded_id,
+    read_transcript_outcome,
+    stop_background_agent,
+    supports_native_background,
+    turn_is_complete,
+)
+from ..provider_runtime import (
+    _build_launch_spec,
+    _ensure_session_authentication,
+    _headless_artifact_paths,
+    _run_headless_provider_command,
+)
 from ..run_command import read_run_prompt, run_cdx_error_code, run_launch_payload, run_result_payload
+from ..run_failover import MAX_FAILOVER_TRANSITIONS, failover_reason
 from ..run_registry import RunRegistry, build_code_review_report
 from ..run_usage import extract_run_usage
 from ..session_ranking import FACTOR_DESCRIPTIONS, rank_sessions, selection_policy
@@ -233,18 +252,144 @@ def _spawn_detached_run(parsed, prompt, artifacts, ctx, session_name, cwd):
         # registered run failed, instead of letting a raw OSError escape as an
         # unhandled traceback on a --json call.
         raise CdxError(f"Failed to start detached cdx run: {error}", 126) from error
-    return {"pid": getattr(child, "pid", None), "launch_log_path": launch_log_path}
+    return {
+        "pid": getattr(child, "pid", None),
+        "launch_log_path": launch_log_path,
+        "background_path": BACKGROUND_PATH_CDX,
+    }
+
+
+def _spawn_provider_background_run(run_session, prompt, artifacts, ctx, cwd):
+    """Start the run as the provider's own background agent.
+
+    Returns the same shape as the in-house launcher so the caller does not
+    branch on which path ran, plus the provider handle needed to observe it
+    later. Returns None when the provider declines to start one, which sends
+    the caller back to the in-house path rather than failing the run.
+    """
+    # Derived from the interactive spec, not the headless one. `--bg` starts a
+    # session; it does not pass `--print`, and the headless spec is built around
+    # flags the provider accepts only alongside `--print` - `--output-format`,
+    # `--max-budget-usd`, `--fallback-model`. Reusing it and stripping `--print`
+    # would hand the CLI print-only flags without `--print` and have the launch
+    # rejected.
+    spec = _build_launch_spec(
+        run_session, cwd=cwd, env_override=ctx.get("env"),
+        initial_prompt=prompt, capture_transcript=False,
+    )
+    argv = [spec["command"], "--bg"] + list(spec["args"])
+    env = (spec.get("options") or {}).get("env")
+    spawn = ctx.get("spawn_detached") or subprocess.Popen
+    prefix = os.path.splitext(artifacts["transcript_path"])[0]
+    launch_log_path = f"{prefix}.launch.log"
+    try:
+        with open(launch_log_path, "w", encoding="utf-8", errors="replace") as log_file:
+            child = spawn(
+                argv,
+                stdout=log_file,
+                stderr=subprocess.STDOUT,
+                stdin=subprocess.DEVNULL,
+                cwd=cwd,
+                env=env,
+                **_detached_spawn_options(),
+            )
+        # --bg returns immediately, so this waits for the launcher itself, not
+        # for the task: the id line is printed before it exits.
+        if hasattr(child, "wait"):
+            child.wait()
+    except OSError:
+        return None
+
+    with open(launch_log_path, encoding="utf-8", errors="replace") as handle:
+        identifier = parse_backgrounded_id(handle.read())
+    if not identifier:
+        # The agent may nonetheless be running. Falling back to the in-house
+        # launcher here would run the same task twice, once untracked, on the
+        # user's account - so this fails loudly instead.
+        raise CdxError(
+            "Started a provider background agent but it did not report an id; "
+            "refusing to launch a second run for the same task.",
+            126,
+        )
+    agent = find_agent(
+        list_background_agents(env=env, cwd=cwd, spawn_sync=ctx.get("spawn_text")),
+        identifier,
+    ) or {}
+    return {
+        "pid": agent.get("pid"),
+        "launch_log_path": launch_log_path,
+        "background_path": BACKGROUND_PATH_PROVIDER,
+        "provider_session_id": agent.get("sessionId") or identifier,
+        "provider_agent_id": identifier,
+    }
 
 def handle_schema(rest, ctx):
     _parse_schema_args(rest)
     _write_json(ctx, _json_success("schema", "Schema loaded.", **cdx_schema()))
     return 0
 
+def _reconcile_background_runs(ctx):
+    """Close out delegated runs the provider has finished.
+
+    A delegated run has no cdx child to report its own completion, so the
+    registry would leave it `running` forever. This asks the provider once, on
+    the read paths a caller actually uses, and closes what has finished -
+    including the usage and final result read back from the session transcript,
+    which is the only place a `--bg` run records them.
+
+    Never raises: a provider that is slow, missing, or unparseable must not
+    take down `cdx runs`.
+    """
+    registry = _run_registry(ctx)
+    try:
+        pending = [run for run in registry.list(limit=50)
+                   if run.get("status") == "running"
+                   and run.get("background_path") == BACKGROUND_PATH_PROVIDER]
+    except Exception:
+        return
+    if not pending:
+        return
+    for run in pending:
+        session = ctx["service"]["get_session"](run.get("session") or "")
+        if not session:
+            continue
+        auth_home = session.get("authHome")
+        env = {**os.environ, "HOME": auth_home} if auth_home else None
+        try:
+            outcome = read_transcript_outcome(
+                find_session_transcript(auth_home, run.get("provider_session_id"))
+            )
+            # The transcript decides, not the agent state: a --bg session parks
+            # after answering and never reports a terminal state on its own.
+            if turn_is_complete(outcome):
+                status = "succeeded"
+            else:
+                agents = list_background_agents(env=env, cwd=run.get("cwd"), spawn_sync=ctx.get("spawn_text"))
+                status = agent_terminal_status(find_agent(agents, run.get("provider_session_id")))
+            if not status:
+                continue
+            # `claude stop` accepts the short agent id only; the session id
+            # names the transcript file and is rejected here.
+            stop_background_agent(run.get("provider_agent_id"), env=env, spawn_sync=ctx.get("spawn_text"))
+            registry.finish(
+                run["run_id"],
+                status=status,
+                final_payload={
+                    "usage": outcome["usage"],
+                    "result": outcome["result"],
+                    "background_path": BACKGROUND_PATH_PROVIDER,
+                },
+            )
+        except Exception:
+            continue
+
+
 def handle_runs(rest, ctx):
     parsed = _parse_runs_args(rest)
     if not parsed["json"]:
         raise CdxError(RUNS_USAGE)
     since = parsed.get("since")
+    _reconcile_background_runs(ctx)
     runs = _run_registry(ctx).list(limit=parsed["limit"], since=since)
     warnings = []
     if since is not None and parsed.get("limit_given"):
@@ -315,6 +460,7 @@ def handle_run_tail(rest, ctx):
 
 def handle_run_status(rest, ctx):
     parsed = _parse_run_id_json_args(rest, RUN_STATUS_USAGE)
+    _reconcile_background_runs(ctx)
     run = _run_registry(ctx).get(parsed["run_id"])
     if not run:
         _write_json(ctx, _json_failure("run-status", "run_not_found", f"Unknown run: {parsed['run_id']}"))
@@ -324,6 +470,7 @@ def handle_run_status(rest, ctx):
 
 def handle_run_report(rest, ctx):
     parsed = _parse_run_id_json_args(rest, RUN_REPORT_USAGE)
+    _reconcile_background_runs(ctx)
     run = _run_registry(ctx).get(parsed["run_id"])
     if not run:
         _write_json(ctx, _json_failure("run-report", "run_not_found", f"Unknown run: {parsed['run_id']}"))
@@ -392,6 +539,11 @@ def handle_run(rest, ctx):
         if parsed.get("reasoning_effort"):
             launch_updates["reasoning_effort"] = parsed["reasoning_effort"]
             launch_updates["power"] = parsed["reasoning_effort"]
+        # Claude needs its conversation id before the spec is built; Codex
+        # records its own afterwards, from the rollout. Without this a headless
+        # Claude run left the session with no identity and `cdx resume` fell
+        # back to recency for Claude while working by id for Codex.
+        session = ctx["service"]["begin_conversation"](session["name"])
         run_session = {
             **session,
             "launch": {
@@ -433,7 +585,24 @@ def handle_run(rest, ctx):
             },
         )
         if parsed.get("detach"):
-            launch = _spawn_detached_run(parsed, prompt, artifacts, ctx, run_session["name"], cwd)
+            # Delegate where the installed CLI can both start a background
+            # agent and report on it; fall back to the in-house launcher
+            # otherwise, and whenever the provider declines to start one.
+            launch = None
+            if supports_native_background(
+                run_session.get("provider"),
+                env=ctx.get("env"),
+                spawn_sync=ctx.get("spawn_text"),
+            ):
+                launch = _spawn_provider_background_run(run_session, prompt, artifacts, ctx, cwd)
+            if launch is None:
+                launch = _spawn_detached_run(parsed, prompt, artifacts, ctx, run_session["name"], cwd)
+            registry.record_background(
+                run_id,
+                path=launch.get("background_path"),
+                provider_session_id=launch.get("provider_session_id"),
+                provider_agent_id=launch.get("provider_agent_id"),
+            )
             # Hand the record the child's pid: this process is about to exit,
             # and a run pointing at a dead pid gets swept up as stale.
             if launch.get("pid"):
@@ -450,17 +619,73 @@ def handle_run(rest, ctx):
             ))
             return 0
 
-        run_info = _run_headless_provider_command(
-            run_session,
-            cwd=cwd,
-            env_override=ctx.get("env"),
-            initial_prompt=prompt,
-            timeout_seconds=parsed.get("timeout_seconds"),
-            spawn=ctx.get("spawn_headless") or ctx.get("spawn"),
-            run_id=run_id,
-        )
-        usage = extract_run_usage(run_session.get("provider"), run_info.get("stdout_path"))
-        run_info = {**run_info, "usage": usage}
+        attempted = [run_session["name"]]
+        failover_error_code = None
+        while True:
+            run_info = _run_headless_provider_command(
+                run_session,
+                cwd=cwd,
+                env_override=ctx.get("env"),
+                initial_prompt=prompt,
+                timeout_seconds=parsed.get("timeout_seconds"),
+                spawn=ctx.get("spawn_headless") or ctx.get("spawn"),
+                run_id=run_id,
+            )
+            usage = extract_run_usage(run_session.get("provider"), run_info.get("stdout_path"))
+            run_info = {**run_info, "usage": usage}
+            if run_info.get("returncode") == 0 or not parsed.get("failover"):
+                break
+            reason = failover_reason(
+                run_session.get("provider"),
+                run_info,
+                _status_row_for(ctx, run_session["name"]),
+            )
+            if not reason:
+                break
+            if len(attempted) > MAX_FAILOVER_TRANSITIONS:
+                # A systematic misclassification must not be able to walk one
+                # task through every account the user owns.
+                failover_error_code = "failover_limit_reached"
+                break
+            successor = _select_headless_session(
+                ctx,
+                parsed["provider"] or run_session.get("provider"),
+                min_reasoning_effort=parsed["reasoning_effort"],
+                require_ready=True,
+                force_refresh=False,
+                exclude=attempted,
+            )
+            if not successor:
+                failover_error_code = "failover_exhausted"
+                break
+            selection_row = successor.get("row") or {}
+            run_session = {
+                **successor["session"],
+                "launch": {**(successor["session"].get("launch") or {}), **launch_updates},
+            }
+            attempted.append(run_session["name"])
+            # A different account means different credentials; an unauthenticated
+            # successor would fail for a reason unrelated to the task.
+            _ensure_session_authentication(
+                run_session,
+                ctx["service"],
+                spawn=ctx.get("spawn"),
+                spawn_sync=ctx.get("spawn_sync"),
+                env_override=ctx.get("env"),
+                stdin_is_tty=False,
+                behavior="launch",
+                signal_emitter=ctx.get("signal_emitter"),
+                trust_local_credentials=False,
+            )
+            registry.migrate(
+                run_id,
+                session=run_session["name"],
+                provider=run_session.get("provider"),
+                reason=reason,
+            )
+            # The successor is a different account with its own auth, so the
+            # context the task built so far travels with it as the prompt.
+            prompt = _failover_prompt(prompt, attempted)
         ok = run_info.get("returncode") == 0
         if ok:
             final_payload = run_result_payload(
@@ -483,7 +708,14 @@ def handle_run(rest, ctx):
             })
             _write_json(ctx, final_payload)
             return 0
-        message = "Provider process timed out." if run_info.get("timed_out") else "Provider process exited with a non-zero status."
+        if failover_error_code == "failover_exhausted":
+            message = f"Every eligible session was rate limited: {', '.join(attempted)}."
+        elif failover_error_code == "failover_limit_reached":
+            message = f"Failover stopped after {MAX_FAILOVER_TRANSITIONS} transitions: {', '.join(attempted)}."
+        elif run_info.get("timed_out"):
+            message = "Provider process timed out."
+        else:
+            message = "Provider process exited with a non-zero status."
         error = CdxError(message, run_info.get("returncode") or 1)
         final_payload = run_result_payload(
             API_SCHEMA_VERSION,
@@ -493,7 +725,11 @@ def handle_run(rest, ctx):
             run_info=run_info,
             error=error,
             error_source="provider",
-            error_code="provider_timeout" if run_info.get("timed_out") else "provider_failed",
+            # Running out of accounts is a different outcome from the task
+            # failing, and a caller has to be able to branch on which happened.
+            error_code=failover_error_code or (
+                "provider_timeout" if run_info.get("timed_out") else "provider_failed"
+            ),
             selection_row=selection_row,
         )
         registry.finish(
@@ -537,12 +773,17 @@ def handle_run(rest, ctx):
         _write_json(ctx, final_payload)
         return error.exit_code
 
-def _select_headless_session(ctx, provider, min_reasoning_effort=None, require_ready=False, force_refresh=False):
+def _select_headless_session(
+    ctx, provider, min_reasoning_effort=None, require_ready=False, force_refresh=False, exclude=None,
+):
     """The best session for a headless run, using the one shared ranking.
 
     Returns the winning session with its status row and why it won, or None if
-    no candidate survived the filters.
+    no candidate survived the filters. `exclude` drops sessions a failover has
+    already tried, which is the only selection input failover adds - the
+    ranking itself is untouched.
     """
+    excluded = set(exclude or ())
     sessions_by_name = {
         session["name"]: session for session in ctx["service"]["list_sessions"]()
     }
@@ -556,7 +797,53 @@ def _select_headless_session(ctx, provider, min_reasoning_effort=None, require_r
         min_reasoning_effort=min_reasoning_effort,
     )
     for row in ordered:
-        session = sessions_by_name.get(row.get("session_name"))
+        name = row.get("session_name")
+        if name in excluded:
+            continue
+        session = sessions_by_name.get(name)
         if session:
             return {"session": session, "row": row, "decision": decision}
     return None
+
+
+def _failover_prompt(prompt, attempted):
+    """The prompt the successor account receives.
+
+    What travels is the task, plus the fact that earlier accounts ran out
+    mid-way. What does NOT travel is the provider conversation: the successor
+    is a different account with its own auth and its own history, so it starts
+    fresh. Any work the previous attempt left on disk is still on disk, which
+    is why the note tells the assistant to check before redoing it.
+    """
+    previous = ", ".join(attempted[:-1])
+    return (
+        "This task was already started on another account "
+        f"({previous}) and stopped when that account hit its rate limit. "
+        "Work already written to the working tree is still there; check the "
+        "current state before redoing anything.\n\n"
+        f"{prompt}"
+    )
+
+
+def _status_row_for(ctx, name):
+    """The status to corroborate a rate-limited termination against.
+
+    Forced refresh on purpose: this runs once per rate-limited termination. The
+    Codex probe spawns app-server and can rotate the OAuth token, which is why
+    the cache exists, so this must stay rare and must never be called in a loop.
+
+    A refresh that comes back with no readable window falls back to the stored
+    status rather than being taken as "not exhausted": a failed probe is an
+    absence of evidence, and the account may well have a recorded 0% from the
+    moment it stopped answering.
+    """
+    def _read(force_refresh):
+        try:
+            return ctx["service"]["get_status_row"](name, force_refresh=force_refresh)
+        except CdxError:
+            return None
+
+    row = _read(True)
+    if row and any(row.get(key) is not None for key in ("remaining_5h_pct", "remaining_week_pct")):
+        return row
+    return _read(False) or row

package/src/commands/status.py

@@ -117,12 +117,25 @@ def _format_launch_configs(sessions, use_color=False):
             "",
             _dim(_format_launch_settings_hint(), use_color),
         ])
-    rows = [[_style(value, "1", use_color) for value in [
-        "SESSION", "PROVIDER", "POWER", "PERMISSION", "FAST", "RTK", "LOGICS", "MODEL", "PRIORITY"
-    ]]]
+    # The table is already nine columns wide, so the two headless-only settings
+    # follow the LABEL convention from the session list: shown once somebody
+    # uses them, absent otherwise.
+    has_headless = any(
+        (session.get("launch") or {}).get(key) is not None
+        for session in sessions
+        for key in ("fallback_model", "budget")
+    )
+    has_extra = any((session.get("launch") or {}).get("extra_args") for session in sessions)
+    headers = ["SESSION", "PROVIDER", "POWER", "PERMISSION", "FAST", "RTK", "LOGICS", "MODEL"]
+    if has_headless:
+        headers += ["FALLBACK", "BUDGET"]
+    if has_extra:
+        headers.append("EXTRA ARGS")
+    headers.append("PRIORITY")
+    rows = [[_style(value, "1", use_color) for value in headers]]
     for session in sessions:
         launch = session.get("launch") or {}
-        rows.append([
+        row = [
             _style(session["name"], "36", use_color),
             _dim(session.get("provider") or "-", use_color),
             _format_launch_setting_value(launch, "power", use_color),
@@ -131,14 +144,33 @@ def _format_launch_configs(sessions, use_color=False):
             _format_launch_setting_value(launch, "rtk", use_color),
             _format_launch_setting_value(launch, "logics", use_color),
             _format_launch_setting_value(launch, "model", use_color),
-            _format_launch_setting_value(launch, "priority", use_color),
-        ])
-    return "\n".join([
+        ]
+        if has_headless:
+            row.append(_format_launch_setting_value(launch, "fallback_model", use_color))
+            row.append(_format_launch_setting_value(launch, "budget", use_color))
+        if has_extra:
+            row.append(_format_launch_setting_value(launch, "extra_args", use_color))
+        row.append(_format_launch_setting_value(launch, "priority", use_color))
+        rows.append(row)
+    lines = [
         _style("Launch settings:", "1", use_color),
         _pad_table(rows),
         "",
-        _dim(_format_launch_settings_hint(), use_color),
-    ])
+    ]
+    if has_headless:
+        lines.append(_dim(
+            "FALLBACK and BUDGET apply to headless Claude runs only "
+            "(cdx run); the provider accepts them with --print alone.",
+            use_color,
+        ))
+    if has_extra:
+        lines.append(_dim(
+            "EXTRA ARGS are passed to the provider unchanged and unvalidated. "
+            "cdx does not check them, and cdx schema does not describe them.",
+            use_color,
+        ))
+    lines.append(_dim(_format_launch_settings_hint(), use_color))
+    return "\n".join(lines)
 
 def _format_duration_ms(value):
     if value is None:

package/src/config.py

@@ -34,6 +34,35 @@ PERMISSION_ALIASES = {
 # Everything accepted as input for a permission, canonical values first.
 PERMISSION_INPUT_VALUES = PERMISSION_VALUES + tuple(PERMISSION_ALIASES)
 
+# A budget is a ceiling, so the only bound worth enforcing is the one a caller
+# could cross by accident. Zero and negatives express nothing; the upper bound
+# catches a misplaced decimal rather than stating a policy. Defined here, not in
+# each parser, because `cdx set` and `cdx run` validating the same setting from
+# separate literals is how they came to disagree before.
+MAX_LAUNCH_BUDGET_USD = 10000
+
+MAX_LAUNCH_EXTRA_ARGS_LENGTH = 512
+
+
+def split_extra_args(value):
+    """Passthrough arguments as an argv list, or None if the string is malformed.
+
+    POSIX splitting on every platform, deliberately. The alternative - following
+    the host's own quoting rules - would make the same `cdx set` mean different
+    things on Windows and on macOS, and these strings travel between machines
+    through `cdx export`. One rule everywhere is worth more than matching the
+    local shell, especially since cdx never invokes a shell: the result is
+    passed as argv, so nothing here is ever re-interpreted.
+    """
+    import shlex
+
+    if value is None:
+        return None
+    try:
+        return shlex.split(str(value), posix=True)
+    except ValueError:
+        return None
+
 
 def normalize_permission(value):
     """Canonical form of a permission input, or None if it is not one.