cdx-manager 0.25.1 → 0.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -1
- package/VERSION +1 -1
- package/changelogs/CHANGELOGS_0_25_2.md +15 -0
- package/changelogs/CHANGELOGS_0_25_3.md +20 -0
- package/changelogs/CHANGELOGS_UNRELEASED.md +28 -0
- package/checksums/release-archives.json +22 -0
- package/docs/cli.md +90 -4
- package/package.json +1 -1
- package/pyproject.toml +7 -4
- package/src/cli.py +7 -5
- package/src/cli_args.py +22 -2
- package/src/codex_usage.py +5 -1
- package/src/commands/launch.py +13 -5
- package/src/commands/runs.py +120 -37
- package/src/commands/status.py +32 -4
- package/src/history_models.py +10 -1
- package/src/project_identity.py +71 -0
- package/src/provider_runtime.py +64 -9
- package/src/remote_run.py +230 -0
- package/src/remote_worker.py +233 -0
- package/src/run_command.py +16 -1
- package/src/run_observation.py +93 -0
- package/src/run_registry.py +23 -2
- package/src/run_usage.py +38 -4
- package/src/session_service.py +11 -0
- package/src/session_store.py +62 -0
- package/src/workspace_sandbox.py +28 -1
package/src/commands/launch.py
CHANGED
|
@@ -36,7 +36,7 @@ from ..provider_runtime import (
|
|
|
36
36
|
from ..run_command import PROMPT_STDIN, read_run_prompt
|
|
37
37
|
from ..run_usage import estimate_cost, format_cache_ratio
|
|
38
38
|
|
|
39
|
-
LAUNCH_USAGE = "Usage: cdx <name> [--dir PATH] [--prompt TEXT|--prompt-file PATH|--task TEXT [--end-state TEXT]] [--resume] [--on-busy fail|ask] [--json]"
|
|
39
|
+
LAUNCH_USAGE = "Usage: cdx <name> [--dir PATH] [--project PATH] [--prompt TEXT|--prompt-file PATH|--task TEXT [--end-state TEXT]] [--resume] [--on-busy fail|ask] [--json]"
|
|
40
40
|
|
|
41
41
|
#: The first message of a session started with --task. The end state is what the
|
|
42
42
|
#: user's go-ahead covers: a bare "show me first" made sessions stop after a commit
|
|
@@ -206,6 +206,7 @@ def handle_can_resume(rest, ctx):
|
|
|
206
206
|
return 0
|
|
207
207
|
|
|
208
208
|
def handle_resume(rest, ctx):
|
|
209
|
+
project_path, rest = _pop_value_flag(rest, "--project", RESUME_USAGE)
|
|
209
210
|
conversation_id, rest = _pop_value_flag(rest, "--conversation", RESUME_USAGE)
|
|
210
211
|
directory, rest, prompt, on_busy = _parse_launch_args(rest)
|
|
211
212
|
if prompt["prompt"] is not None or prompt["prompt_file"] is not None:
|
|
@@ -214,7 +215,7 @@ def handle_resume(rest, ctx):
|
|
|
214
215
|
if len(args) != 1:
|
|
215
216
|
raise CdxError(RESUME_USAGE)
|
|
216
217
|
return handle_launch(args[0], ctx, resume=True, force_json=json_flag, directory=directory,
|
|
217
|
-
conversation_id=conversation_id, on_busy=on_busy)
|
|
218
|
+
conversation_id=conversation_id, on_busy=on_busy, project_path=project_path)
|
|
218
219
|
|
|
219
220
|
|
|
220
221
|
def _notification_provisioning_enabled(session, ctx, json_flag):
|
|
@@ -258,7 +259,7 @@ def _link_shared_memory(session, ctx, cwd, json_flag):
|
|
|
258
259
|
|
|
259
260
|
|
|
260
261
|
def handle_launch(command, ctx, initial_prompt=None, resume=False, force_json=None, directory=None,
|
|
261
|
-
conversation_id=None, on_busy="ask"):
|
|
262
|
+
conversation_id=None, on_busy="ask", project_path=None):
|
|
262
263
|
json_flag = "--json" in ctx.get("raw_args", ctx["options"].get("raw_args", []))
|
|
263
264
|
if force_json is not None:
|
|
264
265
|
json_flag = force_json
|
|
@@ -278,7 +279,10 @@ def handle_launch(command, ctx, initial_prompt=None, resume=False, force_json=No
|
|
|
278
279
|
raise CdxError(
|
|
279
280
|
f"Provider {session['provider']} does not support native resume through cdx."
|
|
280
281
|
)
|
|
282
|
+
from ..project_identity import resolve_project
|
|
283
|
+
|
|
281
284
|
cwd = _launch_directory(session, ctx, directory, json_flag)
|
|
285
|
+
project = resolve_project(cwd, project_path)
|
|
282
286
|
session = ctx["service"]["launch_session"](command, resume=resume)
|
|
283
287
|
if conversation_id:
|
|
284
288
|
session = with_requested_conversation(session, conversation_id)
|
|
@@ -330,7 +334,7 @@ def handle_launch(command, ctx, initial_prompt=None, resume=False, force_json=No
|
|
|
330
334
|
def runtime_lifecycle(event, info):
|
|
331
335
|
nonlocal runtime_run_id
|
|
332
336
|
if event == "started":
|
|
333
|
-
runtime = ctx["service"]["start_session_runtime"](session["name"], {**info, "cwd": cwd})
|
|
337
|
+
runtime = ctx["service"]["start_session_runtime"](session["name"], {**info, "cwd": cwd, **project})
|
|
334
338
|
runtime_run_id = runtime.get("runId")
|
|
335
339
|
elif event == "finished" and runtime_run_id:
|
|
336
340
|
ctx["service"]["finish_session_runtime"](
|
|
@@ -382,6 +386,8 @@ def handle_launch(command, ctx, initial_prompt=None, resume=False, force_json=No
|
|
|
382
386
|
"error": str(error),
|
|
383
387
|
"exit_code": error.exit_code,
|
|
384
388
|
**run_info,
|
|
389
|
+
**project,
|
|
390
|
+
"run_id": runtime_run_id,
|
|
385
391
|
})
|
|
386
392
|
raise
|
|
387
393
|
success_run_info = _attach_interactive_usage(
|
|
@@ -395,13 +401,15 @@ def handle_launch(command, ctx, initial_prompt=None, resume=False, force_json=No
|
|
|
395
401
|
"cwd": cwd,
|
|
396
402
|
"exit_code": 0,
|
|
397
403
|
**success_run_info,
|
|
404
|
+
**project,
|
|
405
|
+
"run_id": runtime_run_id,
|
|
398
406
|
})
|
|
399
407
|
if not json_flag:
|
|
400
408
|
ctx["out"](f"{_goodbye_line(session, success_run_info, ctx['use_color'])}\n")
|
|
401
409
|
if retro.start(session, cwd, success_run_info.get("provider_transcript_path"), env=ctx.get("env")) and not json_flag:
|
|
402
410
|
ctx["out"](f"{_dim('Session retro started; read it later with `cdx retro`', ctx['use_color'])}\n")
|
|
403
411
|
if json_flag:
|
|
404
|
-
extra = {"session": ctx["service"]["get_session"](session["name"]), "cwd": cwd}
|
|
412
|
+
extra = {"session": ctx["service"]["get_session"](session["name"]), "cwd": cwd, **project, "run_id": runtime_run_id}
|
|
405
413
|
if capability:
|
|
406
414
|
extra["resume"] = capability
|
|
407
415
|
_write_json(ctx, _json_success("resume" if resume else "launch", message, warnings=warnings, **extra))
|
package/src/commands/runs.py
CHANGED
|
@@ -5,7 +5,6 @@ Split out of cli_commands.py. Moved verbatim; re-exported by cli_commands.
|
|
|
5
5
|
|
|
6
6
|
import importlib.util
|
|
7
7
|
import os
|
|
8
|
-
import shutil
|
|
9
8
|
import subprocess
|
|
10
9
|
import sys
|
|
11
10
|
import time
|
|
@@ -49,6 +48,7 @@ from ..provider_runtime import (
|
|
|
49
48
|
_run_headless_provider_command,
|
|
50
49
|
with_requested_conversation,
|
|
51
50
|
)
|
|
51
|
+
from ..remote_run import preflight, run_remote
|
|
52
52
|
from ..run_command import read_run_prompt, run_cdx_error_code, run_launch_payload, run_result_payload
|
|
53
53
|
from ..run_failover import (
|
|
54
54
|
CAPACITY_RETRY_DELAYS,
|
|
@@ -58,7 +58,7 @@ from ..run_failover import (
|
|
|
58
58
|
looks_model_at_capacity,
|
|
59
59
|
)
|
|
60
60
|
from ..run_registry import RunRegistry, build_code_review_report, final_message
|
|
61
|
-
from ..run_usage import empty_usage, estimate_cost, extract_run_conversation_id, extract_run_observation, token_prices
|
|
61
|
+
from ..run_usage import cost_fields, empty_usage, estimate_cost, extract_run_conversation_id, extract_run_observation, token_prices
|
|
62
62
|
from ..session_ranking import FACTOR_DESCRIPTIONS, rank_sessions, selection_policy
|
|
63
63
|
from ..status_view import _now_timestamp, _priority_reset_timestamp
|
|
64
64
|
|
|
@@ -204,6 +204,9 @@ def _detached_child_argv(parsed, prompt_path, session_name, cwd):
|
|
|
204
204
|
# from the package root, so a relative "--cwd ." would resolve there.
|
|
205
205
|
argv = ["run", session_name, "--cwd", cwd, "--prompt-file", prompt_path]
|
|
206
206
|
for flag, key in (
|
|
207
|
+
("--remote", "remote"),
|
|
208
|
+
("--remote-prefix", "remote_prefix"),
|
|
209
|
+
("--project", "project"),
|
|
207
210
|
("--budget", "budget"),
|
|
208
211
|
("--continue-run", "continue_run"),
|
|
209
212
|
("--model", "model"),
|
|
@@ -423,6 +426,8 @@ def _reconcile_background_runs(ctx):
|
|
|
423
426
|
"run_id": run["run_id"],
|
|
424
427
|
"attempt": 1,
|
|
425
428
|
"cwd": run.get("cwd"),
|
|
429
|
+
"project": run.get("project"),
|
|
430
|
+
"project_source": run.get("project_source", "unknown"),
|
|
426
431
|
"usage": outcome["usage"],
|
|
427
432
|
"usage_model": outcome["usage_model"],
|
|
428
433
|
})
|
|
@@ -544,9 +549,19 @@ def _env_max_runs(ctx):
|
|
|
544
549
|
|
|
545
550
|
|
|
546
551
|
def _observe_headless_usage(session, run_info, baseline=None):
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
552
|
+
isolated = bool(run_info.get("remote")) or (session.get("launch") or {}).get("isolate_workspace")
|
|
553
|
+
evidence = run_info.get("usage_evidence")
|
|
554
|
+
observed_models = []
|
|
555
|
+
usage, model = extract_run_observation(session.get("provider"), run_info.get("stdout_path"),
|
|
556
|
+
None if isolated else session.get("authHome"), evidence=evidence, models_out=observed_models)
|
|
557
|
+
raw_measured = usage.get("total_tokens") is not None
|
|
558
|
+
observation = {"usage": usage, "usage_model": model, "usage_models": observed_models}
|
|
559
|
+
if evidence is not None:
|
|
560
|
+
observation["usage_evidence"] = evidence
|
|
561
|
+
if baseline is not None and isolated:
|
|
562
|
+
observation.update(usage=empty_usage(), usage_model=None, usage_attribution="continuation_usage_unavailable",
|
|
563
|
+
measurement_unavailable=raw_measured)
|
|
564
|
+
elif baseline is not None:
|
|
550
565
|
cumulative, path, match, transcript_model = extract_interactive_usage(
|
|
551
566
|
session["provider"], session.get("authHome"), conversation_id=(session.get("conversation") or {}).get("id"))
|
|
552
567
|
before, before_path, _before_match, _before_model = baseline
|
|
@@ -559,13 +574,18 @@ def _observe_headless_usage(session, run_info, baseline=None):
|
|
|
559
574
|
# establish this run's share; stdout cannot replace a missing boundary.
|
|
560
575
|
observation.update(usage=delta if delta is not None else empty_usage(), usage_cumulative=cumulative,
|
|
561
576
|
usage_attribution="launch_boundary" if delta is not None else "continuation_usage_unavailable",
|
|
577
|
+
measurement_unavailable=delta is None and bool(raw_measured or (cumulative and cumulative.get("total_tokens") is not None)),
|
|
562
578
|
provider_transcript_path=path, provider_transcript_match=match,
|
|
563
|
-
usage_model=model or transcript_model)
|
|
579
|
+
usage_model=model if session.get("provider") == "codex" else model or transcript_model)
|
|
580
|
+
observation.update(cost_fields(observation["usage"], observation["usage_model"],
|
|
581
|
+
models=observed_models, attribution=observation.get("usage_attribution")))
|
|
564
582
|
return observation
|
|
565
583
|
|
|
566
584
|
|
|
567
585
|
def _budget_observation(session, paths, baseline):
|
|
568
586
|
observation = _observe_headless_usage(session, paths, baseline if session["provider"] == "codex" else None)
|
|
587
|
+
if (session.get("launch") or {}).get("isolate_workspace"):
|
|
588
|
+
return observation
|
|
569
589
|
identifier = (extract_run_conversation_id(session["provider"], paths.get("stdout_path"))
|
|
570
590
|
or ((session.get("conversation") or {}).get("id")
|
|
571
591
|
if session["provider"] == "claude" or session.get("continue_run") else None))
|
|
@@ -580,8 +600,14 @@ def _budget_observation(session, paths, baseline):
|
|
|
580
600
|
comparable = (before and path == before_path
|
|
581
601
|
and all((usage.get(key) is None) == (before.get(key) is None) for key in MEASURED_KEYS))
|
|
582
602
|
usage = usage_delta(usage, before, provider=session["provider"]) if comparable else None
|
|
603
|
+
if usage is not None and session["provider"] == "codex":
|
|
604
|
+
from ..run_usage import _codex_run_models
|
|
605
|
+
models = _codex_run_models([{"type": "thread.started", "thread_id": identifier}], session.get("authHome"))
|
|
606
|
+
model = next(iter(models)) if len(models) == 1 else None
|
|
583
607
|
if usage is not None:
|
|
584
|
-
|
|
608
|
+
observed_models = sorted(models) if session["provider"] == "codex" else [model] if model else []
|
|
609
|
+
return {**observation, "usage": usage, "usage_model": model, "usage_models": observed_models,
|
|
610
|
+
**cost_fields(usage, model, models=observed_models), "provider_transcript_path": path,
|
|
585
611
|
"provider_transcript_match": match}
|
|
586
612
|
return observation
|
|
587
613
|
|
|
@@ -629,6 +655,12 @@ def _continue_run_session(parsed, ctx):
|
|
|
629
655
|
if arg.split("=", 1)[0] in pinned_options or arg.startswith(("-r", "-c")):
|
|
630
656
|
raise CdxError("--continue-run cannot use Claude extra_args that override model, effort, or conversation.",
|
|
631
657
|
code="continue_run_settings_mismatch")
|
|
658
|
+
from ..project_identity import recorded_project
|
|
659
|
+
|
|
660
|
+
project = recorded_project(previous)
|
|
661
|
+
if parsed.get("project") and parsed["project"] != project:
|
|
662
|
+
raise CdxError("--continue-run requires the previous run's recorded project.")
|
|
663
|
+
parsed["continued_project"] = {"project": project, "project_source": previous.get("project_source", "unknown")}
|
|
632
664
|
parsed.update(name=name, model=model, reasoning_effort=effort, power=effort)
|
|
633
665
|
launch = {**(session.get("launch") or {}), "model": model, "reasoning_effort": effort, "power": effort}
|
|
634
666
|
# A provider fallback would break the model pin even though cdx stayed on this profile.
|
|
@@ -681,9 +713,27 @@ def handle_run(rest, ctx):
|
|
|
681
713
|
session = selected["session"]
|
|
682
714
|
selection_row = selected.get("row") or {}
|
|
683
715
|
|
|
716
|
+
remote_version = None
|
|
717
|
+
if parsed.get("remote"):
|
|
718
|
+
if session["provider"] != "codex":
|
|
719
|
+
raise CdxError("--remote is only available for Codex profiles.", code="remote_provider_unsupported")
|
|
720
|
+
if parsed.get("continue_run"):
|
|
721
|
+
raise CdxError("--remote requires a fresh run without --continue-run.")
|
|
722
|
+
remote_version = preflight(
|
|
723
|
+
parsed["remote"], {**os.environ, **(ctx.get("env") or {})},
|
|
724
|
+
remote_prefix=parsed.get("remote_prefix"), isolate_workspace=parsed.get("isolate_workspace", False),
|
|
725
|
+
cwd=parsed["cwd"], network=parsed.get("network_access", False), writable=parsed.get("permission") != "review")
|
|
684
726
|
cwd = os.path.abspath(parsed["cwd"])
|
|
685
|
-
if not os.path.isdir(cwd):
|
|
727
|
+
if not parsed.get("remote") and not os.path.isdir(cwd):
|
|
686
728
|
raise CdxError(f"Invalid cwd: {parsed['cwd']}", code="invalid_cwd")
|
|
729
|
+
from ..project_identity import resolve_project
|
|
730
|
+
|
|
731
|
+
project = parsed.get("continued_project") or resolve_project(cwd, parsed.get("project"))
|
|
732
|
+
if detached_run_id:
|
|
733
|
+
previous = _run_registry(ctx).get(detached_run_id)
|
|
734
|
+
if previous:
|
|
735
|
+
project = {"project": previous.get("project"), "project_source": previous.get("project_source", "unknown")}
|
|
736
|
+
parsed.update(project)
|
|
687
737
|
prompt = read_run_prompt(parsed, stdin=ctx.get("prompt_stdin"))
|
|
688
738
|
_consume_detached_prompt_file(parsed, detached_run_id)
|
|
689
739
|
launch_updates = {key: parsed.get(key, False) for key in ("network_access", "isolate_workspace")}
|
|
@@ -716,17 +766,18 @@ def handle_run(rest, ctx):
|
|
|
716
766
|
**launch_updates,
|
|
717
767
|
},
|
|
718
768
|
}
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
769
|
+
if not parsed.get("remote"):
|
|
770
|
+
_ensure_session_authentication(
|
|
771
|
+
run_session,
|
|
772
|
+
ctx["service"],
|
|
773
|
+
spawn=ctx.get("spawn"),
|
|
774
|
+
spawn_sync=ctx.get("spawn_sync"),
|
|
775
|
+
env_override=ctx.get("env"),
|
|
776
|
+
stdin_is_tty=False,
|
|
777
|
+
behavior="launch",
|
|
778
|
+
signal_emitter=ctx.get("signal_emitter"),
|
|
779
|
+
trust_local_credentials=False,
|
|
780
|
+
)
|
|
730
781
|
registry = _run_registry(ctx)
|
|
731
782
|
# A detached child is handed its parent's run_id so both agree on the
|
|
732
783
|
# identity the launch payload already reported to the caller.
|
|
@@ -743,6 +794,7 @@ def handle_run(rest, ctx):
|
|
|
743
794
|
provider=run_session.get("provider"),
|
|
744
795
|
model=parsed.get("model") or ((run_session.get("launch") or {}).get("model")),
|
|
745
796
|
cwd=cwd,
|
|
797
|
+
**project,
|
|
746
798
|
artifacts={
|
|
747
799
|
"transcript_path": artifacts.get("transcript_path"),
|
|
748
800
|
"stdout_path": artifacts.get("stdout_path"),
|
|
@@ -750,6 +802,7 @@ def handle_run(rest, ctx):
|
|
|
750
802
|
},
|
|
751
803
|
label=parsed.get("label"),
|
|
752
804
|
max_parallel=max_parallel,
|
|
805
|
+
remote=parsed.get("remote"),
|
|
753
806
|
parent_run_id=parsed.get("continue_run"),
|
|
754
807
|
conversation_id=(run_session.get("conversation") or {}).get("id") if run_session["provider"] == "claude" or parsed.get("continue_run") else None,
|
|
755
808
|
launch=run_session.get("launch"),
|
|
@@ -797,30 +850,46 @@ def handle_run(rest, ctx):
|
|
|
797
850
|
failover_error_code = None
|
|
798
851
|
recovery_events = []
|
|
799
852
|
attempt = 0
|
|
853
|
+
attempt_observations = []
|
|
800
854
|
spent_usd = 0.0
|
|
801
855
|
models = capacity_models(run_session)
|
|
802
856
|
model_index = 0
|
|
803
857
|
capacity_retries = 0
|
|
804
858
|
while True:
|
|
805
859
|
attempt += 1
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
860
|
+
registry.record_attempt(run_id, _headless_artifact_paths(run_session, run_id=run_id, attempt=attempt))
|
|
861
|
+
baseline = None
|
|
862
|
+
if parsed.get("continue_run") and (run_session["provider"] == "codex" or parsed.get("budget")):
|
|
863
|
+
if (run_session.get("launch") or {}).get("isolate_workspace"):
|
|
864
|
+
baseline = (None, None, None, None)
|
|
865
|
+
else:
|
|
866
|
+
baseline = extract_interactive_usage(
|
|
867
|
+
run_session["provider"], run_session.get("authHome"),
|
|
868
|
+
conversation_id=(run_session.get("conversation") or {}).get("id"))
|
|
810
869
|
attempt_cost = None
|
|
811
870
|
budget_observation = None
|
|
871
|
+
budget_unverifiable = False
|
|
812
872
|
|
|
813
873
|
def check_budget(paths, session=run_session, before=baseline, already_spent=spent_usd):
|
|
814
|
-
nonlocal attempt_cost, budget_observation
|
|
874
|
+
nonlocal attempt_cost, budget_observation, budget_unverifiable
|
|
815
875
|
observation = _budget_observation(session, paths, before)
|
|
816
876
|
cost = estimate_cost(observation["usage"], observation["usage_model"], budget_prices)
|
|
877
|
+
measured = observation["usage"].get("total_tokens") is not None or observation.get("measurement_unavailable", False)
|
|
878
|
+
if cost is None and measured:
|
|
879
|
+
budget_unverifiable = True
|
|
880
|
+
budget_observation = observation
|
|
881
|
+
return True
|
|
817
882
|
if cost is not None:
|
|
818
883
|
if attempt_cost is None or cost >= attempt_cost:
|
|
819
884
|
attempt_cost = cost
|
|
820
885
|
budget_observation = observation
|
|
821
886
|
return attempt_cost is not None and already_spent + attempt_cost >= parsed["budget"]
|
|
822
887
|
|
|
823
|
-
|
|
888
|
+
runner = _run_headless_provider_command
|
|
889
|
+
if parsed.get("remote"):
|
|
890
|
+
def runner(*args, **kwargs):
|
|
891
|
+
return run_remote(*args, target=parsed["remote"], version=remote_version, remote_prefix=parsed.get("remote_prefix"), **kwargs)
|
|
892
|
+
run_info = runner(
|
|
824
893
|
run_session,
|
|
825
894
|
cwd=cwd,
|
|
826
895
|
env_override=ctx.get("env"),
|
|
@@ -828,6 +897,7 @@ def handle_run(rest, ctx):
|
|
|
828
897
|
timeout_seconds=parsed.get("timeout_seconds"),
|
|
829
898
|
spawn=ctx.get("spawn_headless") or ctx.get("spawn"),
|
|
830
899
|
run_id=run_id,
|
|
900
|
+
attempt=attempt,
|
|
831
901
|
**({"budget_check": check_budget} if parsed.get("budget") is not None else {}),
|
|
832
902
|
)
|
|
833
903
|
observation = _observe_headless_usage(run_session, run_info, baseline if run_session["provider"] == "codex" else None)
|
|
@@ -843,8 +913,20 @@ def handle_run(rest, ctx):
|
|
|
843
913
|
"recovery_events": recovery_events}
|
|
844
914
|
cost = estimate_cost(observation["usage"], observation["usage_model"], budget_prices)
|
|
845
915
|
spent_usd += max(attempt_cost or 0.0, cost or 0.0)
|
|
846
|
-
if parsed.get("budget") is not None and (
|
|
916
|
+
if parsed.get("budget") is not None and (budget_unverifiable or (
|
|
917
|
+
cost is None and (observation["usage"].get("total_tokens") is not None or observation.get("measurement_unavailable")))):
|
|
918
|
+
run_info.update(budget_unverifiable=True, budget_exceeded=False, returncode=125,
|
|
919
|
+
cost_usd=None, cost_status="unknown", budget_status="unverifiable")
|
|
920
|
+
elif parsed.get("budget") is not None and (run_info.get("budget_exceeded") or spent_usd >= parsed["budget"]):
|
|
847
921
|
run_info.update(budget_exceeded=True, returncode=125, cost_usd=spent_usd)
|
|
922
|
+
attempt_observations.append({
|
|
923
|
+
"attempt": attempt, "attempt_id": run_info.get("attempt_id"),
|
|
924
|
+
"session": run_session["name"], "provider": run_session["provider"],
|
|
925
|
+
**{key: run_info.get(key) for key in ("usage", "usage_model", "usage_models", "usage_evidence")},
|
|
926
|
+
**cost_fields(observation["usage"], observation["usage_model"], budget_prices,
|
|
927
|
+
models=observation.get("usage_models", []), attribution=observation.get("usage_attribution")),
|
|
928
|
+
})
|
|
929
|
+
run_info["attempts"] = list(attempt_observations)
|
|
848
930
|
ctx["service"]["record_launch_history"](run_session["name"], {
|
|
849
931
|
"status": "success" if run_info.get("returncode") == 0 else "failed",
|
|
850
932
|
"action": "run",
|
|
@@ -853,8 +935,9 @@ def handle_run(rest, ctx):
|
|
|
853
935
|
"cwd": cwd,
|
|
854
936
|
"exit_code": run_info.get("returncode"),
|
|
855
937
|
**run_info,
|
|
938
|
+
**project,
|
|
856
939
|
})
|
|
857
|
-
if run_info.get("budget_exceeded") or parsed.get("continue_run"):
|
|
940
|
+
if parsed.get("remote") or run_info.get("budget_exceeded") or run_info.get("budget_unverifiable") or parsed.get("continue_run"):
|
|
858
941
|
break
|
|
859
942
|
if looks_model_at_capacity(run_session.get("provider"), run_info):
|
|
860
943
|
event = {"reason": "model_at_capacity", "attempt": attempt,
|
|
@@ -979,7 +1062,10 @@ def handle_run(rest, ctx):
|
|
|
979
1062
|
)
|
|
980
1063
|
_write_json(ctx, final_payload)
|
|
981
1064
|
return 0
|
|
982
|
-
if run_info.get("
|
|
1065
|
+
if run_info.get("budget_unverifiable"):
|
|
1066
|
+
failover_error_code = "budget_unverifiable"
|
|
1067
|
+
message = "Measured usage cannot be priced; USD budget enforcement is unavailable. Check cost_reason and observed model evidence."
|
|
1068
|
+
elif run_info.get("budget_exceeded"):
|
|
983
1069
|
failover_error_code = "budget_exceeded"
|
|
984
1070
|
message = "Run exceeded its estimated USD budget."
|
|
985
1071
|
elif failover_error_code == "capacity_recovery_exhausted":
|
|
@@ -1000,7 +1086,7 @@ def handle_run(rest, ctx):
|
|
|
1000
1086
|
run_session,
|
|
1001
1087
|
run_info=run_info,
|
|
1002
1088
|
error=error,
|
|
1003
|
-
error_source="cdx" if run_info.get("budget_exceeded") else "provider",
|
|
1089
|
+
error_source="cdx" if run_info.get("budget_exceeded") or run_info.get("budget_unverifiable") else "provider",
|
|
1004
1090
|
# Running out of accounts is a different outcome from the task
|
|
1005
1091
|
# failing, and a caller has to be able to branch on which happened.
|
|
1006
1092
|
error_code=failover_error_code or (
|
|
@@ -1038,6 +1124,7 @@ def handle_run(rest, ctx):
|
|
|
1038
1124
|
"cwd": cwd,
|
|
1039
1125
|
"exit_code": error.exit_code,
|
|
1040
1126
|
**run_info,
|
|
1127
|
+
**project,
|
|
1041
1128
|
})
|
|
1042
1129
|
final_payload = run_result_payload(
|
|
1043
1130
|
API_SCHEMA_VERSION,
|
|
@@ -1065,14 +1152,10 @@ def handle_run(rest, ctx):
|
|
|
1065
1152
|
return error.exit_code
|
|
1066
1153
|
|
|
1067
1154
|
def _record_capacity_recovery(ctx, registry, run_id, event, run_info, events):
|
|
1068
|
-
#
|
|
1069
|
-
|
|
1070
|
-
for key in ("stdout_path", "stderr_path", "transcript_path")
|
|
1071
|
-
|
|
1072
|
-
if path and os.path.isfile(path):
|
|
1073
|
-
archived = f"{path}.attempt-{event['attempt']}"
|
|
1074
|
-
shutil.copyfile(path, archived)
|
|
1075
|
-
artifacts[key] = archived
|
|
1155
|
+
# Attempt paths are immutable; recovery events reference the same evidence
|
|
1156
|
+
# as the history row instead of copying output that a retry will overwrite.
|
|
1157
|
+
artifacts = {key: run_info[key] for key in ("stdout_path", "stderr_path", "transcript_path")
|
|
1158
|
+
if run_info.get(key) and os.path.isfile(run_info[key])}
|
|
1076
1159
|
event["artifacts"] = artifacts
|
|
1077
1160
|
events.append(event)
|
|
1078
1161
|
registry.record_recovery(run_id, event)
|
package/src/commands/status.py
CHANGED
|
@@ -55,6 +55,7 @@ from ..rtk_savings import rtk_saved
|
|
|
55
55
|
from ..run_usage import (
|
|
56
56
|
TOKEN_USAGE_KEYS,
|
|
57
57
|
cache_hit_ratio,
|
|
58
|
+
cost_fields,
|
|
58
59
|
estimate_cost,
|
|
59
60
|
format_cache_ratio,
|
|
60
61
|
is_vouched,
|
|
@@ -287,6 +288,7 @@ def _summarize_stats(entries):
|
|
|
287
288
|
"unweighted_runs": 0,
|
|
288
289
|
"priced_runs": 0,
|
|
289
290
|
"unpriced_models": set(),
|
|
291
|
+
"unpriced_reasons": {},
|
|
290
292
|
"unvouched_runs": 0,
|
|
291
293
|
"input_tokens": 0,
|
|
292
294
|
"cached_input_tokens": 0,
|
|
@@ -342,7 +344,10 @@ def _summarize_stats(entries):
|
|
|
342
344
|
if cost is not None:
|
|
343
345
|
row["cost_usd"] += cost
|
|
344
346
|
row["priced_runs"] += 1
|
|
345
|
-
|
|
347
|
+
else:
|
|
348
|
+
reason = cost_fields(usage, entry.get("usage_model"), models=entry.get("usage_models", (entry.get("usage_evidence") or {}).get("models", [])), attribution=entry.get("usage_attribution"))["cost_reason"]
|
|
349
|
+
row["unpriced_reasons"][reason] = row["unpriced_reasons"].get(reason, 0) + 1
|
|
350
|
+
if cost is None and entry.get("usage_model"):
|
|
346
351
|
# Named the model but could not price it. Recording which one
|
|
347
352
|
# turns an unexplained "-" into something actionable: that
|
|
348
353
|
# string is exactly what CDX_TOKEN_PRICES needs a key for.
|
|
@@ -524,9 +529,11 @@ def _format_rtk_saved(rtk):
|
|
|
524
529
|
|
|
525
530
|
|
|
526
531
|
def _stats_group(entry, by):
|
|
527
|
-
"""The
|
|
532
|
+
"""The recorded source repository or serving model a run is counted under."""
|
|
528
533
|
if by == "project":
|
|
529
|
-
|
|
534
|
+
from ..project_identity import recorded_project
|
|
535
|
+
|
|
536
|
+
return recorded_project(entry) or "-"
|
|
530
537
|
model = entry.get("usage_model") or "-"
|
|
531
538
|
return model.removesuffix("[1m]") if model.startswith("claude-") else model
|
|
532
539
|
|
|
@@ -640,6 +647,26 @@ def handle_configs(rest, ctx):
|
|
|
640
647
|
return 0
|
|
641
648
|
|
|
642
649
|
def handle_history(rest, ctx):
|
|
650
|
+
if rest and rest[0] == "backfill-projects":
|
|
651
|
+
usage = "Usage: cdx history backfill-projects [--mapping FILE] [--dry-run | --apply] [--json]"
|
|
652
|
+
parsed = _parse_flag_args(rest[1:], {
|
|
653
|
+
"--mapping": {"key": "mapping", "type": "str", "default": None},
|
|
654
|
+
"--dry-run": {"key": "dry_run", "type": "bool", "default": False},
|
|
655
|
+
"--apply": {"key": "apply", "type": "bool", "default": False},
|
|
656
|
+
"--json": {"key": "json", "type": "bool", "default": False},
|
|
657
|
+
}, usage)
|
|
658
|
+
if parsed["apply"] and parsed["dry_run"]:
|
|
659
|
+
raise CdxError(usage)
|
|
660
|
+
result = ctx["service"]["backfill_history_projects"](mapping_path=parsed["mapping"], dry_run=not parsed["apply"])
|
|
661
|
+
if parsed["json"]:
|
|
662
|
+
payload = (_json_failure("history", "project_attribution_conflict", "Conflicting project evidence; no changes applied.", **result)
|
|
663
|
+
if result["conflicting"] else _json_success("history", "Backfilled history projects", **result))
|
|
664
|
+
_write_json(ctx, payload)
|
|
665
|
+
else:
|
|
666
|
+
mode = "Dry run" if result["dry_run"] else "Not applied (conflicts)" if result["conflicting"] else "Applied"
|
|
667
|
+
ctx["out"](f"{mode}: {result['recovered']} recovered; {result['unchanged']} unchanged; "
|
|
668
|
+
f"{result['unresolved']} unresolved; {result['conflicting']} conflicting.\n")
|
|
669
|
+
return 1 if result["conflicting"] else 0
|
|
643
670
|
if rest and rest[0] == "backfill-models":
|
|
644
671
|
usage = "Usage: cdx history backfill-models [--dry-run | --apply] [--json]"
|
|
645
672
|
parsed = _parse_flag_args(rest[1:], {
|
|
@@ -672,7 +699,8 @@ def handle_history(rest, ctx):
|
|
|
672
699
|
entries = ctx["service"]["get_launch_history"](parsed["name"], limit=limit)
|
|
673
700
|
entries = _filter_history_period(entries, parsed["period"])
|
|
674
701
|
entries = [{**entry, "parent_run_id": entry.get("parent_run_id"),
|
|
675
|
-
|
|
702
|
+
**cost_fields(entry.get("usage") if is_vouched(entry.get("usage")) else None, entry.get("usage_model"),
|
|
703
|
+
models=entry.get("usage_models", (entry.get("usage_evidence") or {}).get("models", [])), attribution=entry.get("usage_attribution")),
|
|
676
704
|
"usage": {**entry["usage"], "cache_hit_ratio": cache_hit_ratio(entry["usage"], provider=entry.get("provider")) if is_vouched(entry["usage"]) else None}
|
|
677
705
|
if isinstance(entry.get("usage"), dict) else entry.get("usage")} for entry in entries]
|
|
678
706
|
if has_period and not parsed["summary"]:
|
package/src/history_models.py
CHANGED
|
@@ -7,7 +7,7 @@ import time
|
|
|
7
7
|
from .interactive_usage import MAX_TRANSCRIPT_BYTES, MAX_TRANSCRIPT_SCAN_SECONDS
|
|
8
8
|
from .run_registry import registry_path
|
|
9
9
|
|
|
10
|
-
EVIDENCE_KINDS = ("stdout", "run_report", "codex_rollout", "claude_transcript")
|
|
10
|
+
EVIDENCE_KINDS = ("isolated_observation", "stdout", "run_report", "codex_rollout", "claude_transcript")
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def missing_model(value):
|
|
@@ -108,6 +108,15 @@ def _exact_transcript(provider, auth_home, identifier):
|
|
|
108
108
|
def recover_model(entry, session=None, reports=()):
|
|
109
109
|
"""Return (model, evidence kind, reason); never consult live model settings."""
|
|
110
110
|
provider = entry.get("provider")
|
|
111
|
+
evidence = entry.get("usage_evidence")
|
|
112
|
+
if isinstance(evidence, dict):
|
|
113
|
+
if (evidence.get("stdout_path") != entry.get("stdout_path")
|
|
114
|
+
or evidence.get("attempt_id") != entry.get("attempt_id")
|
|
115
|
+
or evidence.get("run_id") != entry.get("run_id")
|
|
116
|
+
or (evidence.get("measurement_usage") or evidence.get("usage")) != entry.get("usage")):
|
|
117
|
+
return None, "isolated_observation", "observation_attempt_mismatch"
|
|
118
|
+
model, reason = _single_model(evidence.get("models") or [])
|
|
119
|
+
return model, "isolated_observation", reason or (None if model else "no_turn_models")
|
|
111
120
|
records, error = _read_records(entry.get("stdout_path"))
|
|
112
121
|
if error:
|
|
113
122
|
return None, None, error
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Repository attribution captured by the trusted launcher, separate from cwd."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import ntpath
|
|
5
|
+
import os
|
|
6
|
+
import posixpath
|
|
7
|
+
import subprocess
|
|
8
|
+
from pathlib import PureWindowsPath
|
|
9
|
+
|
|
10
|
+
from .errors import CdxError
|
|
11
|
+
from .tray_logics import _repository_root
|
|
12
|
+
|
|
13
|
+
PROJECT_SOURCES = {"git", "operator", "operator_mapping", "run_report"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def canonical_project(value):
|
|
17
|
+
"""Validate recorded identities without consulting a possibly deleted checkout."""
|
|
18
|
+
return (isinstance(value, str) and bool(value.strip()) and "\x00" not in value
|
|
19
|
+
and ((posixpath.isabs(value) and posixpath.normpath(value) == value)
|
|
20
|
+
or (PureWindowsPath(value).is_absolute() and ntpath.normpath(value) == value)))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def recorded_project(entry):
|
|
24
|
+
value = entry.get("project")
|
|
25
|
+
source = entry.get("project_source")
|
|
26
|
+
return value if canonical_project(value) and isinstance(source, str) and source in PROJECT_SOURCES else None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def resolve_project(cwd, explicit=None):
|
|
30
|
+
"""Resolve before isolation. Never infer clone provenance from remote URLs."""
|
|
31
|
+
if explicit is not None:
|
|
32
|
+
if not canonical_project(explicit):
|
|
33
|
+
raise CdxError("--project must be an absolute canonical source repository path.")
|
|
34
|
+
return {"project": explicit, "project_source": "operator"}
|
|
35
|
+
root = _repository_root(cwd)
|
|
36
|
+
if root:
|
|
37
|
+
if os.path.isdir(os.path.join(root, ".git")):
|
|
38
|
+
return {"project": root, "project_source": "git"}
|
|
39
|
+
try:
|
|
40
|
+
# Ignore ambient Git overrides: this lookup belongs to the launch cwd.
|
|
41
|
+
env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
|
|
42
|
+
common = subprocess.run(["git", "-C", root, "rev-parse", "--git-common-dir"],
|
|
43
|
+
env=env, capture_output=True, text=True, timeout=5, check=True).stdout.strip()
|
|
44
|
+
common = os.path.realpath(os.path.join(root, common))
|
|
45
|
+
if os.path.isdir(common):
|
|
46
|
+
return {"project": os.path.dirname(common) if os.path.basename(common) == ".git" else common,
|
|
47
|
+
"project_source": "git"}
|
|
48
|
+
except (OSError, subprocess.SubprocessError):
|
|
49
|
+
pass
|
|
50
|
+
return {"project": None, "project_source": "unknown"}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def load_mapping(path):
|
|
54
|
+
try:
|
|
55
|
+
with open(path, encoding="utf-8") as handle:
|
|
56
|
+
data = json.load(handle)
|
|
57
|
+
except (OSError, ValueError) as error:
|
|
58
|
+
raise CdxError(f"Cannot read project mapping: {error}") from error
|
|
59
|
+
if (not isinstance(data, dict) or set(data) != {"version", "projects"}
|
|
60
|
+
or type(data["version"]) is not int or data["version"] != 1 or not isinstance(data["projects"], list)):
|
|
61
|
+
raise CdxError("Invalid project mapping: expected version 1 and a projects array.")
|
|
62
|
+
projects = {}
|
|
63
|
+
for item in data["projects"]:
|
|
64
|
+
if (not isinstance(item, dict) or set(item) != {"run_id", "project"}
|
|
65
|
+
or not isinstance(item["run_id"], str) or not item["run_id"].strip()
|
|
66
|
+
or not canonical_project(item["project"])):
|
|
67
|
+
raise CdxError("Invalid project mapping: each entry needs an exact run_id and absolute canonical project path.")
|
|
68
|
+
if item["run_id"] in projects and projects[item["run_id"]] != item["project"]:
|
|
69
|
+
raise CdxError(f"Conflicting project mappings for run {item['run_id']}.")
|
|
70
|
+
projects[item["run_id"]] = item["project"]
|
|
71
|
+
return projects
|