cdx-manager 0.25.1 → 0.25.3

package/src/provider_runtime.py

@@ -1068,13 +1068,15 @@ def _build_headless_launch_spec(session, cwd=None, env_override=None, initial_pr
     )
 
 
-def _headless_artifact_paths(session, run_id=None):
+def _headless_artifact_paths(session, run_id=None, attempt=1):
     run_id = run_id or uuid.uuid4().hex
     log_dir = _get_launch_transcript_dir(session)
     os.makedirs(log_dir, exist_ok=True)
-    prefix = os.path.join(log_dir, f"cdx-run-{run_id}")
+    attempt_id = f"{run_id}:{attempt}"
+    prefix = os.path.join(log_dir, f"cdx-run-{run_id}-attempt-{attempt}")
     return {
         "run_id": run_id,
+        "attempt_id": attempt_id,
         "transcript_path": os.path.abspath(f"{prefix}.log"),
         "stdout_path": os.path.abspath(f"{prefix}.stdout.log"),
         "stderr_path": os.path.abspath(f"{prefix}.stderr.log"),
@@ -1205,7 +1207,33 @@ def _read_permission_denials(stdout_path):
 
 
 def _run_headless_provider_command(session, cwd=None, env_override=None, initial_prompt=None,
-                                   timeout_seconds=None, spawn=None, run_id=None, budget_check=None):
+                                   timeout_seconds=None, spawn=None, run_id=None, budget_check=None, attempt=1):
+    from .remote_run import remote_auth_lock, temporary_auth
+    from .workspace_sandbox import private_auth_home
+
+    isolated_codex = session.get("provider") == PROVIDER_CODEX and (session.get("launch") or {}).get("isolate_workspace")
+    if isolated_codex:
+        handlers = _route_termination_to_interrupt()
+        try:
+            with remote_auth_lock(_get_auth_home(session)):
+                auth = temporary_auth(_get_auth_home(session))
+            with private_auth_home(auth) as directory:
+                return _run_headless_provider_attempt(session, cwd, env_override, initial_prompt,
+                                                      timeout_seconds, spawn, run_id, budget_check, directory, attempt)
+        except KeyboardInterrupt as cause:
+            error = CdxError('Isolated Codex run was cancelled.', 130)
+            error.cancelled = True
+            raise error from cause
+        except (OSError, RuntimeError) as error:
+            raise CdxError(str(error)) from error
+        finally:
+            _restore_signal_handlers(handlers)
+    return _run_headless_provider_attempt(session, cwd, env_override, initial_prompt,
+                                          timeout_seconds, spawn, run_id, budget_check, attempt=attempt)
+
+
+def _run_headless_provider_attempt(session, cwd=None, env_override=None, initial_prompt=None,
+                                   timeout_seconds=None, spawn=None, run_id=None, budget_check=None, observation_home=None, attempt=1):
     spawn = spawn or subprocess.Popen
     spec = _build_headless_launch_spec(
         session,
@@ -1248,14 +1276,23 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
                 [spec["command"], *spec["args"]], Path(cwd),
                 network=launch.get("network_access", False),
                 writable=launch.get("permission") != "review",
-                source=spec["options"]["env"], auth_home=_get_auth_home(session),
-                provider=session.get("provider"))
+                source=spec["options"]["env"], auth_home=observation_home or _get_auth_home(session),
+                provider=session.get("provider"),
+                **({"observation_home": os.path.join(observation_home, "sessions")} if observation_home else {}))
         except (OSError, RuntimeError) as error:
             raise CdxError(str(error)) from error
         spec.update(command=argv[0], args=argv[1:])
         spec["options"]["env"] = child_env
-    paths = _headless_artifact_paths(session, run_id=run_id)
+    paths = _headless_artifact_paths(session, run_id=run_id, attempt=attempt)
     start_time = datetime.now(timezone.utc)
+
+    def snapshot():
+        if observation_home:
+            from .run_observation import codex_attempt_evidence
+            paths["usage_evidence"] = codex_attempt_evidence(
+                observation_home, paths["stdout_path"], start_time.isoformat())
+            paths["usage_evidence"].update(run_id=paths["run_id"], attempt_id=paths.get("attempt_id"))
+
     command = spec["command"]
     if spawn is subprocess.Popen:
         command = _resolve_command(command, spec.get("options", {}).get("env"))
@@ -1264,6 +1301,12 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
     timed_out = False
     cancelled = False
     budget_exceeded = False
+    credential_expired = False
+    expires_at = None
+    if observation_home:
+        from .remote_run import token_expiry
+        with open(os.path.join(observation_home, 'auth.json')) as handle:
+            expires_at = token_expiry(json.load(handle)['tokens']['access_token'])
     with open(paths["stdout_path"], "w", encoding="utf-8", errors="replace") as stdout_file, \
             open(paths["stderr_path"], "w", encoding="utf-8", errors="replace") as stderr_file:
         options = {k: v for k, v in spec.get("options", {}).items() if k not in ("stdio", "stdout", "stderr")}
@@ -1292,10 +1335,15 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
         # this supervisor and keep its run "running" forever.
         previous_handlers = _route_termination_to_interrupt()
         try:
-            if budget_check is not None:
+            if budget_check is not None or expires_at is not None:
                 deadline = time.monotonic() + timeout_seconds if timeout_seconds is not None else None
                 while True:
-                    if budget_check(paths):
+                    snapshot()
+                    if expires_at is not None and time.time() >= expires_at:
+                        credential_expired = True
+                        _terminate_child_tree(child)
+                        break
+                    if budget_check and budget_check(paths):
                         budget_exceeded = True
                         _terminate_child_tree(child)
                         break
@@ -1304,7 +1352,8 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
                         raise subprocess.TimeoutExpired(command, timeout_seconds)
                     try:
                         child.wait(timeout=min(0.25, remaining) if remaining is not None else 0.25)
-                        budget_exceeded = budget_check(paths)
+                        snapshot()
+                        budget_exceeded = bool(budget_check and budget_check(paths))
                         break
                     except subprocess.TimeoutExpired:
                         continue
@@ -1325,8 +1374,14 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
             cancelled = True
             _terminate_child_tree(child)
         finally:
+            snapshot()
             _restore_signal_handlers(previous_handlers)
 
+    if credential_expired or (expires_at is not None and time.time() >= expires_at):
+        _combine_headless_transcript(paths)
+        error = CdxError('Isolated temporary credential expired; start a new run from the controller.', code='remote_credential_expired')
+        error.run_info = _headless_run_info(paths, spec, start_time, child.returncode)
+        raise error
     if cancelled:
         _combine_headless_transcript(paths)
         returncode = getattr(child, "returncode", None)

package/src/remote_run.py

@@ -0,0 +1,230 @@
+"""Controller-side temporary Codex credentials and SSH transport."""
+
+import base64
+import json
+import math
+import os
+import pathlib
+import re
+import subprocess
+import time
+from contextlib import contextmanager
+from datetime import datetime, timezone
+
+from .codex_usage import _request_codex_app_server, codex_auth_lock
+from .errors import CdxError
+
+
+def token_expiry(token):
+    try:
+        part = token.split('.')[1]
+        expiry = float(json.loads(base64.urlsafe_b64decode(part + '=' * (-len(part) % 4)))['exp'])
+        if not math.isfinite(expiry) or expiry <= 0:
+            raise ValueError('Invalid expiration')
+        return expiry
+    except (ValueError, KeyError, IndexError, TypeError) as error:
+        raise CdxError('Cannot determine remote access token expiration; log in on the controller.') from error
+
+
+def needs_refresh(auth, now):
+    try:
+        refreshed = datetime.fromisoformat(auth['last_refresh'].replace('Z', '+00:00')).timestamp()
+    except (ValueError, KeyError, TypeError):
+        return True
+    return now - refreshed > 7 * 86400 or token_expiry(auth['tokens']['access_token']) - now < 86400
+
+
+def refresh_auth(home):
+    result = _request_codex_app_server(home, 'account/read', {'refreshToken': True}, 'refresh_failed', timeout=30)
+    if not result.get('ok'):
+        raise CdxError('Controller Codex credential refresh failed; log in on the controller.')
+
+
+def temporary_auth(home, now=None):
+    now = time.time() if now is None else now
+    try:
+        auth = json.loads((pathlib.Path(home) / 'auth.json').read_text())
+        if needs_refresh(auth, now):
+            refresh_auth(home)
+            auth = json.loads((pathlib.Path(home) / 'auth.json').read_text())
+            if needs_refresh(auth, now):
+                raise CdxError('Controller refresh did not produce a fresh remote credential.')
+        tokens = {key: auth['tokens'][key] for key in ('access_token', 'id_token', 'account_id')}
+        tokens['refresh_token'] = ''
+        result = {'tokens': tokens, 'last_refresh': auth['last_refresh']}
+        rotating = auth['tokens'].get('refresh_token')
+        if result['tokens']['refresh_token'] != '' or (rotating and rotating in json.dumps(result)):
+            raise CdxError('Refusing to send a refresh token to a worker.')
+        return result
+    except (OSError, ValueError, KeyError, TypeError) as error:
+        raise CdxError('Cannot read controller Codex OAuth credentials; log in on the controller.') from error
+
+
+def nonsecret_config(home):
+    # Copy only known scalar preferences. Provider URLs, hooks, env and arbitrary
+    # config entries can contain secrets or refer to controller-only programs.
+    path = pathlib.Path(home) / 'config.toml'
+    if not path.exists():
+        return ''
+    allowed = {'model', 'model_reasoning_effort', 'service_tier', 'personality',
+               'model_context_window', 'model_auto_compact_token_limit'}
+    lines = []
+    for line in path.read_text().splitlines():
+        if line.lstrip().startswith('['):
+            break
+        key, sep, value = line.partition('=')
+        if sep and key.strip() in allowed and re.fullmatch(r'\s*("[a-zA-Z0-9_. /-]+"|[0-9]+)\s*', value):
+            lines.append(line)
+    return '\n'.join(lines) + '\n'
+
+
+def worker_input():
+    # One JSON line frames the source, leaving the next line for the payload.
+    directory = pathlib.Path(__file__).parent
+    source = 'SANDBOX_SOURCE = ' + repr((directory / 'workspace_sandbox.py').read_text()) + '\n'
+    source += (directory / 'remote_worker.py').read_text()
+    return json.dumps(source).encode() + b'\n'
+
+
+def ssh_command(target, mode, remote_prefix=None):
+    if not re.fullmatch(r'[a-zA-Z0-9_@.:[\]-]+', target) or target.startswith('-'):
+        raise CdxError('Invalid --remote SSH target; use a host alias or user@host.')
+    prefix = (remote_prefix or '').strip()
+    if prefix and not re.fullmatch(r'[a-zA-Z0-9_./:=+ -]+', prefix):
+        raise CdxError('Invalid --remote-prefix; use a plain command prefix such as wsl.exe -d Ubuntu --.')
+    if mode not in ('probe', 'run'):
+        raise CdxError('Invalid remote worker mode.')
+    # Double quotes work in both POSIX sh and cmd.exe. The bootstrap contains
+    # no embedded double quotes, shell expansions, or worker source in argv.
+    command = (prefix + ' ' if prefix else '') + 'python3 -u -c "import sys,json;exec(json.loads(sys.stdin.buffer.readline()))" ' + mode
+    if len(command) > 8000:
+        raise CdxError('Remote command prefix exceeds the Windows command-line limit.')
+    return ['ssh', '-T', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', '-o', 'ConnectionAttempts=1',
+            '-o', 'ServerAliveInterval=5', '-o', 'ServerAliveCountMax=1', target, command]
+
+
+def preflight(target, env=None, *, remote_prefix=None, isolate_workspace=False, cwd=None, network=False, writable=True):
+    payload = {'isolate_workspace': isolate_workspace, 'cwd': cwd, 'network': network, 'writable': writable}
+    try:
+        local = subprocess.check_output(['codex', '--version'], env=env, stderr=subprocess.DEVNULL, timeout=10).decode().strip()
+        result = subprocess.run(ssh_command(target, 'probe', remote_prefix), env=env,
+                                input=worker_input() + json.dumps(payload).encode() + b'\n',
+                                capture_output=True, timeout=30)
+    except (OSError, subprocess.TimeoutExpired) as error:
+        raise CdxError('Remote worker unreachable or Codex CLI unavailable.', code='remote_unavailable') from error
+    if result.returncode:
+        message = ('Remote workspace isolation unavailable; requires bubblewrap, pasta, nftables, working namespaces and a valid checkout.'
+                   if isolate_workspace else 'Remote worker unreachable or missing Python 3, Codex or memory-backed /dev/shm.')
+        raise CdxError(message, code='remote_unavailable')
+    if local != result.stdout.decode().strip():
+        raise CdxError('Remote Codex CLI version mismatch; install the controller version on the worker.', code='remote_version_mismatch')
+    return local
+
+
+@contextmanager
+def remote_auth_lock(home):
+    try:
+        with codex_auth_lock(home, strict=True) as acquired:
+            if not acquired:
+                raise CdxError('Codex profile is busy on the controller or a worker.', code='session_busy')
+            yield
+    except OSError as error:
+        raise CdxError('Unable to hold the controller authentication lock.', code='auth_lock_failed') from error
+
+
+def run_remote(session, target, version, cwd=None, env_override=None, initial_prompt=None,
+               timeout_seconds=None, run_id=None, attempt=1, budget_check=None, remote_prefix=None, **_):
+    from .provider_runtime import (
+        _build_headless_launch_spec,
+        _combine_headless_transcript,
+        _headless_artifact_paths,
+        _headless_run_info,
+        _restore_signal_handlers,
+        _route_termination_to_interrupt,
+    )
+
+    home = session['authHome']
+    with remote_auth_lock(home):
+        auth = temporary_auth(home)
+        if auth['tokens'].get('refresh_token') != '':
+            raise CdxError('Refusing to send a refresh token to a worker.')
+        spec = _build_headless_launch_spec(session, cwd, env_override, initial_prompt)
+        if (session.get('launch') or {}).get('network_access'):
+            index = spec['args'].index('--') if '--' in spec['args'] else len(spec['args'])
+            spec['args'][index:index] = ['-c', 'sandbox_workspace_write.network_access=true']
+        config = nonsecret_config(home)
+        launch_text = json.dumps(spec['args']) + config
+        if any(value and value in launch_text for value in auth['tokens'].values()):
+            raise CdxError('Refusing a credential embedded in the remote prompt or configuration.')
+        payload = {'auth': auth, 'config': config, 'args': spec['args'], 'cwd': cwd,
+                   'version': version, 'expires_at': token_expiry(auth['tokens']['access_token']),
+                   'isolate_workspace': bool((session.get('launch') or {}).get('isolate_workspace')),
+                   'network': bool((session.get('launch') or {}).get('network_access')),
+                   'writable': (session.get('launch') or {}).get('permission') != 'review'}
+        rotating = json.loads((pathlib.Path(home) / 'auth.json').read_text())['tokens'].get('refresh_token')
+        if rotating and rotating in json.dumps(payload):
+            raise CdxError('Refusing to send a refresh token to a worker.')
+    paths = _headless_artifact_paths(session, run_id, attempt)
+    paths['remote'] = target
+    start = datetime.now(timezone.utc)
+    child = None
+    cancelled = timed_out = budget_exceeded = credential_expired = False
+    handlers = _route_termination_to_interrupt()
+    try:
+        with open(paths['stdout_path'], 'w') as out, open(paths['stderr_path'], 'w') as err:
+            child = subprocess.Popen(ssh_command(target, 'run', remote_prefix), env={**os.environ, **(env_override or {})},
+                                     stdin=subprocess.PIPE, stdout=out, stderr=err)
+            child.stdin.write(worker_input() + json.dumps(payload).encode() + b'\n')
+            child.stdin.flush()
+            deadline = time.monotonic() + timeout_seconds if timeout_seconds else None
+            while child.poll() is None:
+                if time.time() >= payload['expires_at']:
+                    credential_expired = True
+                    break
+                if budget_check and budget_check(paths):
+                    budget_exceeded = True
+                    break
+                if deadline and time.monotonic() >= deadline:
+                    timed_out = True
+                    break
+                time.sleep(0.1)
+    except KeyboardInterrupt:
+        cancelled = True
+    except (OSError, ValueError) as error:
+        raise CdxError('Remote SSH transport failed.', code='remote_unavailable') from error
+    finally:
+        if child:
+            # EOF makes the worker stop its process group and remove the home.
+            try:
+                child.stdin.close()
+            except (OSError, BrokenPipeError):
+                pass
+            try:
+                child.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                child.terminate()
+                try:
+                    child.wait(timeout=2)
+                except subprocess.TimeoutExpired:
+                    child.kill()
+                    child.wait()
+        _restore_signal_handlers(handlers)
+        _combine_headless_transcript(paths)
+    info = {**_headless_run_info(paths, spec, start, child.returncode), 'remote': target,
+            'timed_out': timed_out, 'budget_exceeded': budget_exceeded}
+    if cancelled:
+        error = CdxError('Remote run was cancelled.', 130)
+        error.cancelled = True
+        error.run_info = {**info, 'returncode': 130}
+        raise error
+    if credential_expired or time.time() >= payload['expires_at'] or child.returncode == 123:
+        error = CdxError('Remote temporary credential expired; start a new run from the controller.', code='remote_credential_expired')
+        error.run_info = info
+        raise error
+    if child.returncode == 255 and not (timed_out or budget_exceeded):
+        error = CdxError('Remote SSH connection failed or was lost.', code='remote_unavailable')
+        error.run_info = info
+        raise error
+    if timed_out or budget_exceeded:
+        info['returncode'] = 125 if budget_exceeded else 124
+    return info

package/src/remote_worker.py

@@ -0,0 +1,233 @@
+"""Self-contained Linux worker, framed over SSH stdin before the run payload."""
+
+import fcntl
+import json
+import os
+import pathlib
+import shutil
+import signal
+import stat
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+# Supplied by the controller before this module is executed.
+SANDBOX_SOURCE = globals().get('SANDBOX_SOURCE', '')
+
+
+def version():
+    return subprocess.check_output(["codex", "--version"], stderr=subprocess.DEVNULL, timeout=10).decode().strip()
+
+
+def storage():
+    mounts = [row.split() for row in pathlib.Path('/proc/mounts').read_text().splitlines()]
+    location = pathlib.Path('/dev/shm').resolve()
+    enclosing = [row for row in mounts if pathlib.Path(row[1]) == location or pathlib.Path(row[1]) in location.parents]
+    if not enclosing or max(enclosing, key=lambda row: len(row[1]))[2] != 'tmpfs':
+        raise RuntimeError("Remote execution requires memory-backed /dev/shm")
+    root = pathlib.Path('/dev/shm') / f'cdx-remote-{os.getuid()}'
+    root.mkdir(mode=0o700, exist_ok=True)
+    info = root.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
+        raise RuntimeError("Unsafe remote credential directory")
+    return root
+
+
+def allocate(root):
+    # Serialize sweep and allocation so another launcher cannot sweep a new home.
+    with open(root / '.sweep', 'a') as sweep:
+        fcntl.flock(sweep, fcntl.LOCK_EX)
+        for path in root.glob('run-*'):
+            if path.is_symlink() or not path.is_dir():
+                continue
+            try:
+                with open(path / '.live', 'a') as live:
+                    fcntl.flock(live, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    shutil.rmtree(path)
+            except (BlockingIOError, FileNotFoundError):
+                continue
+        home = pathlib.Path(tempfile.mkdtemp(prefix='run-', dir=root))
+        live = open(home / '.live', 'a')
+        fcntl.flock(live, fcntl.LOCK_EX)
+    return home, live
+
+
+def isolation_probe_command(payload, home, env):
+    # Execute exactly the local confinement implementation, shipped over stdin.
+    namespace = {'__name__': 'cdx_workspace_sandbox'}
+    exec(compile(SANDBOX_SOURCE, 'workspace_sandbox.py', 'exec'), namespace)
+    return namespace['sandbox_command'](
+        ['codex', '--version'], pathlib.Path(payload['cwd']), network=payload['network'],
+        writable=payload['writable'], source=env, auth_home=str(home), provider='codex',
+        observation_home=str(home / 'sessions'))
+
+
+def probe(root, payload):
+    if not payload.get('isolate_workspace'):
+        print(version())
+        return 0
+    # Exercise mounts, namespaces and the nft setup with a dummy home before
+    # the controller reads or sends a credential. Binary presence is insufficient.
+    with tempfile.TemporaryDirectory(prefix='probe-', dir=root) as directory:
+        home = pathlib.Path(directory)
+        (home / 'auth.json').write_text(json.dumps({'tokens': {'refresh_token': ''}}))
+        os.chmod(home / 'auth.json', 0o600)
+        (home / 'sessions').mkdir()
+        command, env = isolation_probe_command(payload, home, dict(os.environ))
+        result = subprocess.run(command, env=env, stdin=subprocess.DEVNULL,
+                                capture_output=True, timeout=20)
+        if result.returncode:
+            raise RuntimeError('Remote workspace isolation unavailable')
+        print(result.stdout.decode().strip())
+        return 0
+
+
+def main():
+    root = storage()
+    payload = json.loads(sys.stdin.buffer.readline())
+    if sys.argv[-1] == 'probe':
+        return probe(root, payload)
+    auth = payload['auth']
+    if auth['tokens'].get('refresh_token') != '':
+        raise RuntimeError('Refusing a rotating remote credential')
+    if version() != payload['version']:
+        raise RuntimeError('Remote Codex CLI version mismatch')
+    child = None
+    stopped = threading.Event()
+
+    def stop(*_):
+        stopped.set()
+        if child is not None and child.poll() is None:
+            try:
+                os.killpg(child.pid, signal.SIGTERM)
+                child.wait(timeout=3)
+            except subprocess.TimeoutExpired:
+                os.killpg(child.pid, signal.SIGKILL)
+                child.wait()
+            except ProcessLookupError:
+                pass
+
+    def disconnected():
+        while os.read(0, 1):
+            pass
+        stop()
+
+    signal.signal(signal.SIGTERM, stop)
+    signal.signal(signal.SIGHUP, stop)
+    signal.signal(signal.SIGINT, stop)
+    home, live = allocate(root)
+    try:
+        if stopped.is_set():
+            return 130
+        os.umask(0o077)
+        (home / 'auth.json').write_text(json.dumps(auth))
+        os.chmod(home / 'auth.json', 0o600)
+        (home / 'config.toml').write_text('cli_auth_credentials_store = "file"\n' + payload['config'])
+        env = {key: value for key, value in os.environ.items()
+               if key not in ('OPENAI_API_KEY', 'OPENAI_ACCESS_TOKEN', 'CODEX_ACCESS_TOKEN', 'CODEX_API_KEY', 'CODEX_HOME')}
+        env['CODEX_HOME'] = str(home)
+        command = ['codex', *payload['args']]
+        if payload.get('isolate_workspace'):
+            (home / 'sessions').mkdir(mode=0o700)
+            args = payload['args'][:]
+            while '-s' in args:
+                index = args.index('-s')
+                del args[index:index + 2]
+            if '--dangerously-bypass-approvals-and-sandbox' not in args:
+                index = args.index('--') if '--' in args else len(args)
+                args[index:index] = ['--dangerously-bypass-approvals-and-sandbox']
+            namespace = {'__name__': 'cdx_workspace_sandbox'}
+            exec(compile(SANDBOX_SOURCE, 'workspace_sandbox.py', 'exec'), namespace)
+            command, env = namespace['sandbox_command'](
+                ['codex', *args], pathlib.Path(payload['cwd']), network=payload['network'],
+                writable=payload['writable'], source=env, auth_home=str(home), provider='codex',
+                observation_home=str(home / 'sessions'))
+        child = subprocess.Popen(command, cwd=payload['cwd'], env=env,
+                                 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                 start_new_session=True)
+        if stopped.is_set():
+            stop()
+        threading.Thread(target=disconnected, daemon=True).start()
+        secrets = [value for value in auth['tokens'].values() if isinstance(value, str) and value]
+
+        output_lock = threading.Lock()
+        observed = {}
+
+        def snapshot():
+            # Only the fresh per-run home supplies model evidence. No transcript
+            # or authentication content leaves the worker.
+            with output_lock:
+                identifier = observed.get('thread_id')
+                usage = observed.get('usage')
+                if not identifier or not usage:
+                    return
+                for path in (home / 'sessions').rglob('*.jsonl'):
+                    if path.is_symlink() or path.stat().st_size > 16 * 1024 * 1024:
+                        continue
+                    try:
+                        records = [json.loads(line) for line in path.read_text().splitlines()]
+                    except (OSError, ValueError):
+                        continue
+                    if not any(row.get('type') == 'session_meta' and row.get('payload', {}).get('id') == identifier for row in records):
+                        continue
+                    contexts = [row.get('payload', {}) for row in records if row.get('type') == 'turn_context']
+                    models = {row.get('model') for row in contexts}
+                    counts = [row.get('payload', {}).get('info', {}).get('total_token_usage') for row in records
+                              if row.get('type') == 'event_msg' and row.get('payload', {}).get('type') == 'token_count']
+                    if (len(models) == 1 and None not in models and counts and counts[-1]
+                            and all(counts[-1].get(key) == usage.get(key) for key in ('input_tokens', 'output_tokens', 'cached_input_tokens'))):
+                        event = json.dumps({'type': 'cdx.remote.usage', 'usage': usage, 'model': next(iter(models))})
+                        for secret in secrets:
+                            event = event.replace(secret, '[REDACTED]')
+                        print(event, flush=True)
+                    return
+
+        def forward(source, target):
+            for line in iter(source.readline, b''):
+                for secret in secrets:
+                    line = line.replace(secret.encode(), b'[REDACTED]')
+                with output_lock:
+                    if target is sys.stdout:
+                        try:
+                            event = json.loads(line)
+                            if event.get('type') == 'thread.started':
+                                observed['thread_id'] = event.get('thread_id')
+                            if event.get('type') == 'turn.completed':
+                                observed['usage'] = event.get('usage')
+                        except (ValueError, AttributeError):
+                            pass
+                    target.buffer.write(line)
+                    target.buffer.flush()
+
+        threads = [threading.Thread(target=forward, args=pair) for pair in
+                   ((child.stdout, sys.stdout), (child.stderr, sys.stderr))]
+        for thread in threads:
+            thread.start()
+        while child.poll() is None:
+            try:
+                child.wait(timeout=0.25)
+            except subprocess.TimeoutExpired:
+                snapshot()
+        result = child.returncode
+        for thread in threads:
+            thread.join()
+        snapshot()
+        if time.time() >= payload['expires_at']:
+            print('Remote temporary credential expired; start a new run from the controller.', file=sys.stderr)
+            return 123
+        return 130 if stopped.is_set() else result
+    finally:
+        stop()
+        shutil.rmtree(home)
+        live.close()
+
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except Exception:
+        # Never render exception values: input and provider errors may contain credentials.
+        print('Remote worker setup or execution failed; check Python, Codex, cwd, /dev/shm and isolation dependencies.', file=sys.stderr)
+        sys.exit(126)

package/src/run_command.py

@@ -4,7 +4,7 @@ import sys
 from .cli_args import ARG_CODE_INVALID_VALUE
 from .errors import CdxArgumentError, CdxError
 from .provider_runtime import headless_permission_disables_network
-from .run_usage import empty_usage, estimate_cost, extract_run_observation
+from .run_usage import cost_fields, empty_usage, estimate_cost, extract_run_observation
 
 PROMPT_STDIN = "-"
 
@@ -169,6 +169,9 @@ def run_launch_payload(api_schema_version, parsed, session, artifacts, cwd=None,
         "ok": True,
         "action": "run",
         "launcher": "cdx",
+        **({"remote": parsed["remote"]} if parsed.get("remote") else {}),
+        "project": parsed.get("project"),
+        "project_source": parsed.get("project_source", "unknown"),
         "parent_run_id": parsed.get("continue_run"),
         "conversation_id": conversation_id or run_conversation_id(session),
         "detached": True,
@@ -219,6 +222,9 @@ def run_result_payload(api_schema_version, ok, parsed, session, run_info=None,
         "ok": bool(ok),
         "action": "run",
         "launcher": "cdx",
+        **({"remote": parsed["remote"]} if parsed.get("remote") else {}),
+        "project": parsed.get("project"),
+        "project_source": parsed.get("project_source", "unknown"),
         "parent_run_id": parsed.get("continue_run"),
         "conversation_id": run_conversation_id(session, run_info),
         "session": session.get("name") if session else None,
@@ -229,6 +235,8 @@ def run_result_payload(api_schema_version, ok, parsed, session, run_info=None,
         "power": parsed.get("power") or ((session.get("launch") or {}).get("power") if session else None),
         "cwd": os.path.abspath(parsed.get("cwd") or os.getcwd()),
         "run_id": run_info.get("run_id"),
+        "attempt_id": run_info.get("attempt_id"),
+        "attempts": run_info.get("attempts", []),
         "exit_code": run_info.get("returncode"),
         "duration_seconds": (run_info.get("duration_ms") / 1000.0) if run_info.get("duration_ms") is not None else None,
         "transcript_path": run_info.get("transcript_path"),
@@ -236,7 +244,14 @@ def run_result_payload(api_schema_version, ok, parsed, session, run_info=None,
         "stderr_path": run_info.get("stderr_path"),
         "usage": usage,
         "usage_model": usage_model,
+        "usage_models": run_info.get("usage_models", [usage_model] if usage_model else []),
+        **cost_fields(usage, usage_model, models=run_info.get("usage_models", (run_info.get("usage_evidence") or {}).get("models", [])), attribution=run_info.get("usage_attribution")),
         "cost_usd": run_info.get("cost_usd", estimate_cost(usage, usage_model)),
+        "usage_evidence": run_info.get("usage_evidence"),
+        "usage_attribution": run_info.get("usage_attribution"),
+        "budget_status": run_info.get("budget_status", "exceeded" if run_info.get("budget_exceeded") else
+                                      "usage_not_measured" if parsed.get("budget") is not None and usage.get("total_tokens") is None else
+                                      "estimated" if parsed.get("budget") is not None else "not_requested"),
         "budget_usd": parsed.get("budget"),
         "warnings": warnings,
         "error": None if ok else {