cdx-manager 0.24.7 → 0.24.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1
- package/VERSION +1 -1
- package/changelogs/CHANGELOGS_0_24_7.md +7 -0
- package/changelogs/CHANGELOGS_0_24_8.md +24 -0
- package/changelogs/CHANGELOGS_0_24_9.md +18 -0
- package/checksums/release-archives.json +22 -0
- package/docs/cli.md +8 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/cli.py +1 -1
- package/src/cli_args.py +12 -2
- package/src/commands/launch.py +4 -4
- package/src/commands/runs.py +208 -21
- package/src/commands/status.py +48 -7
- package/src/interactive_usage.py +17 -6
- package/src/provider_background.py +8 -1
- package/src/provider_runtime.py +3 -1
- package/src/run_command.py +15 -2
- package/src/run_failover.py +32 -2
- package/src/run_registry.py +24 -2
- package/src/run_usage.py +89 -24
package/src/commands/runs.py
CHANGED
|
@@ -5,8 +5,10 @@ Split out of cli_commands.py. Moved verbatim; re-exported by cli_commands.
|
|
|
5
5
|
|
|
6
6
|
import importlib.util
|
|
7
7
|
import os
|
|
8
|
+
import shutil
|
|
8
9
|
import subprocess
|
|
9
10
|
import sys
|
|
11
|
+
import time
|
|
10
12
|
|
|
11
13
|
from ..cli_args import (
|
|
12
14
|
RUN_REPORT_USAGE,
|
|
@@ -21,7 +23,9 @@ from ..cli_args import (
|
|
|
21
23
|
cdx_schema,
|
|
22
24
|
)
|
|
23
25
|
from ..cli_helpers import API_SCHEMA_VERSION, _json_failure, _json_success, _write_json
|
|
26
|
+
from ..config import split_extra_args
|
|
24
27
|
from ..errors import CdxError
|
|
28
|
+
from ..interactive_usage import MEASURED_KEYS, extract_interactive_usage, usage_delta
|
|
25
29
|
from ..provider_background import (
|
|
26
30
|
BACKGROUND_PATH_CDX,
|
|
27
31
|
BACKGROUND_PATH_PROVIDER,
|
|
@@ -39,12 +43,22 @@ from ..provider_runtime import (
|
|
|
39
43
|
_build_launch_spec,
|
|
40
44
|
_ensure_session_authentication,
|
|
41
45
|
_headless_artifact_paths,
|
|
46
|
+
_launch_power,
|
|
47
|
+
_restore_signal_handlers,
|
|
48
|
+
_route_termination_to_interrupt,
|
|
42
49
|
_run_headless_provider_command,
|
|
50
|
+
with_requested_conversation,
|
|
43
51
|
)
|
|
44
52
|
from ..run_command import read_run_prompt, run_cdx_error_code, run_launch_payload, run_result_payload
|
|
45
|
-
from ..run_failover import
|
|
53
|
+
from ..run_failover import (
|
|
54
|
+
CAPACITY_RETRY_DELAYS,
|
|
55
|
+
MAX_FAILOVER_TRANSITIONS,
|
|
56
|
+
capacity_models,
|
|
57
|
+
failover_reason,
|
|
58
|
+
looks_model_at_capacity,
|
|
59
|
+
)
|
|
46
60
|
from ..run_registry import RunRegistry, build_code_review_report, final_message
|
|
47
|
-
from ..run_usage import extract_run_observation
|
|
61
|
+
from ..run_usage import empty_usage, extract_run_conversation_id, extract_run_observation
|
|
48
62
|
from ..session_ranking import FACTOR_DESCRIPTIONS, rank_sessions, selection_policy
|
|
49
63
|
from ..status_view import _now_timestamp, _priority_reset_timestamp
|
|
50
64
|
|
|
@@ -190,6 +204,7 @@ def _detached_child_argv(parsed, prompt_path, session_name, cwd):
|
|
|
190
204
|
# from the package root, so a relative "--cwd ." would resolve there.
|
|
191
205
|
argv = ["run", session_name, "--cwd", cwd, "--prompt-file", prompt_path]
|
|
192
206
|
for flag, key in (
|
|
207
|
+
("--continue-run", "continue_run"),
|
|
193
208
|
("--model", "model"),
|
|
194
209
|
("--kind", "kind"),
|
|
195
210
|
("--permission", "permission"),
|
|
@@ -527,6 +542,77 @@ def _env_max_runs(ctx):
|
|
|
527
542
|
return limit if limit > 0 else None
|
|
528
543
|
|
|
529
544
|
|
|
545
|
+
def _observe_headless_usage(session, run_info, baseline=None):
|
|
546
|
+
usage, model = extract_run_observation(session.get("provider"), run_info.get("stdout_path"), session.get("authHome"))
|
|
547
|
+
observation = {"usage": usage, "usage_model": model}
|
|
548
|
+
if baseline is not None:
|
|
549
|
+
cumulative, path, match, transcript_model = extract_interactive_usage(
|
|
550
|
+
session["provider"], session.get("authHome"), conversation_id=(session.get("conversation") or {}).get("id"))
|
|
551
|
+
before, before_path, _before_match, _before_model = baseline
|
|
552
|
+
comparable = (cumulative and before and path == before_path
|
|
553
|
+
and all((cumulative.get(key) is None) == (before.get(key) is None) for key in MEASURED_KEYS))
|
|
554
|
+
# A newly reported split includes unknown pre-launch usage; losing a
|
|
555
|
+
# split also prevents a comparable measurement across the boundary.
|
|
556
|
+
delta = usage_delta(cumulative, before, provider=session.get("provider")) if comparable else None
|
|
557
|
+
# Exec resume can emit conversation totals. Only exact launch boundaries
|
|
558
|
+
# establish this run's share; stdout cannot replace a missing boundary.
|
|
559
|
+
observation.update(usage=delta if delta is not None else empty_usage(), usage_cumulative=cumulative,
|
|
560
|
+
usage_attribution="launch_boundary" if delta is not None else "continuation_usage_unavailable",
|
|
561
|
+
provider_transcript_path=path, provider_transcript_match=match,
|
|
562
|
+
usage_model=model or transcript_model)
|
|
563
|
+
return observation
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _continue_run_session(parsed, ctx):
|
|
567
|
+
"""Resolve only the parent's profile and exact persisted conversation."""
|
|
568
|
+
previous = _run_registry(ctx).get(parsed["continue_run"])
|
|
569
|
+
if not previous:
|
|
570
|
+
raise CdxError(f"Cannot continue unknown run: {parsed['continue_run']}. Start a normal run instead.",
|
|
571
|
+
code="continue_run_not_found")
|
|
572
|
+
name, provider = previous.get("session"), previous.get("provider")
|
|
573
|
+
if (parsed.get("name") and parsed["name"] != name) or (parsed.get("provider") and parsed["provider"] != provider):
|
|
574
|
+
raise CdxError("Cannot continue a run from another profile or provider. Start a normal run instead.",
|
|
575
|
+
code="continue_run_mismatch")
|
|
576
|
+
session = ctx["service"]["get_session"](name) if name else None
|
|
577
|
+
if not session or session.get("provider") != provider:
|
|
578
|
+
raise CdxError("The previous run's profile or provider is no longer available. Start a normal run instead.",
|
|
579
|
+
code="continue_run_mismatch")
|
|
580
|
+
if provider not in ("claude", "codex") or previous.get("status") not in ("succeeded", "failed", "timed_out", "cancelled"):
|
|
581
|
+
raise CdxError("Only a finished Claude or Codex run can be continued. Start a normal run instead.",
|
|
582
|
+
code="continue_run_unavailable")
|
|
583
|
+
conversation = previous.get("conversation_id")
|
|
584
|
+
resumed = with_requested_conversation(session, conversation)
|
|
585
|
+
if not resumed:
|
|
586
|
+
raise CdxError("The previous run's provider session expired or its transcript is missing. Start a normal run instead.",
|
|
587
|
+
code="continue_run_unavailable")
|
|
588
|
+
model = previous.get("usage_model") or previous.get("model")
|
|
589
|
+
effort = previous.get("reasoning_effort")
|
|
590
|
+
if not isinstance(model, str) or not model or model.startswith("<") or not effort or not previous.get("launch"):
|
|
591
|
+
raise CdxError("The previous run's model and effort cannot be pinned. Start a normal run instead.",
|
|
592
|
+
code="continue_run_unavailable")
|
|
593
|
+
if ((parsed.get("model") and parsed["model"] != model)
|
|
594
|
+
or (parsed.get("reasoning_effort") and parsed["reasoning_effort"] != effort)):
|
|
595
|
+
raise CdxError("--continue-run requires the previous run's model and reasoning effort.",
|
|
596
|
+
code="continue_run_settings_mismatch")
|
|
597
|
+
if parsed.get("failover") or parsed.get("isolate_workspace"):
|
|
598
|
+
raise CdxError("--continue-run cannot be combined with --failover or --isolate-workspace.",
|
|
599
|
+
code="continue_run_settings_mismatch")
|
|
600
|
+
if provider == "claude":
|
|
601
|
+
# Passthrough options are appended last and can override the pinned settings.
|
|
602
|
+
pinned_options = {"--model", "--effort", "--fallback-model", "--resume", "--continue",
|
|
603
|
+
"--session-id", "--fork-session", "--resume-session-at", "--from-pr",
|
|
604
|
+
"--settings", "--agent", "--agents"}
|
|
605
|
+
for arg in split_extra_args((session.get("launch") or {}).get("extra_args")) or []:
|
|
606
|
+
if arg.split("=", 1)[0] in pinned_options or arg.startswith(("-r", "-c")):
|
|
607
|
+
raise CdxError("--continue-run cannot use Claude extra_args that override model, effort, or conversation.",
|
|
608
|
+
code="continue_run_settings_mismatch")
|
|
609
|
+
parsed.update(name=name, model=model, reasoning_effort=effort, power=effort)
|
|
610
|
+
launch = {**(session.get("launch") or {}), "model": model, "reasoning_effort": effort, "power": effort}
|
|
611
|
+
# A provider fallback would break the model pin even though cdx stayed on this profile.
|
|
612
|
+
launch.pop("fallback_model", None)
|
|
613
|
+
return {**resumed, "launch": launch, "continue_run": parsed["continue_run"]}
|
|
614
|
+
|
|
615
|
+
|
|
530
616
|
def handle_run(rest, ctx):
|
|
531
617
|
registry = None
|
|
532
618
|
run_id = None
|
|
@@ -538,8 +624,11 @@ def handle_run(rest, ctx):
|
|
|
538
624
|
detached_run_id = os.environ.pop(DETACHED_RUN_ID_ENV, None) or None
|
|
539
625
|
try:
|
|
540
626
|
parsed = _parse_run_args(rest)
|
|
541
|
-
session = None
|
|
542
|
-
if
|
|
627
|
+
session = _continue_run_session(parsed, ctx) if parsed.get("continue_run") else None
|
|
628
|
+
if session:
|
|
629
|
+
if session.get("enabled", True) is False:
|
|
630
|
+
raise CdxError(f"Session is disabled: {session['name']}", code="session_disabled")
|
|
631
|
+
elif parsed["name"]:
|
|
543
632
|
session = ctx["service"]["get_session"](parsed["name"])
|
|
544
633
|
if not session:
|
|
545
634
|
raise CdxError(f"Unknown session: {parsed['name']}")
|
|
@@ -594,7 +683,8 @@ def handle_run(rest, ctx):
|
|
|
594
683
|
# records its own afterwards, from the rollout. Without this a headless
|
|
595
684
|
# Claude run left the session with no identity and `cdx resume` fell
|
|
596
685
|
# back to recency for Claude while working by id for Codex.
|
|
597
|
-
|
|
686
|
+
if not parsed.get("continue_run"):
|
|
687
|
+
session = ctx["service"]["begin_conversation"](session["name"])
|
|
598
688
|
run_session = {
|
|
599
689
|
**session,
|
|
600
690
|
"launch": {
|
|
@@ -636,6 +726,10 @@ def handle_run(rest, ctx):
|
|
|
636
726
|
},
|
|
637
727
|
label=parsed.get("label"),
|
|
638
728
|
max_parallel=max_parallel,
|
|
729
|
+
parent_run_id=parsed.get("continue_run"),
|
|
730
|
+
conversation_id=(run_session.get("conversation") or {}).get("id") if run_session["provider"] == "claude" or parsed.get("continue_run") else None,
|
|
731
|
+
launch=run_session.get("launch"),
|
|
732
|
+
reasoning_effort=_launch_power(run_session),
|
|
639
733
|
)
|
|
640
734
|
if parsed.get("detach"):
|
|
641
735
|
# Delegate where the installed CLI can both start a background
|
|
@@ -644,7 +738,7 @@ def handle_run(rest, ctx):
|
|
|
644
738
|
launch = None
|
|
645
739
|
# A provider's own background agent has no cdx process watching it,
|
|
646
740
|
# so nothing could fail it over; --failover keeps the in-house child.
|
|
647
|
-
if not parsed.get("isolate_workspace") and not parsed.get("network_access") and not parsed.get("failover") and supports_native_background(
|
|
741
|
+
if not parsed.get("continue_run") and not parsed.get("isolate_workspace") and not parsed.get("network_access") and not parsed.get("failover") and supports_native_background(
|
|
648
742
|
run_session.get("provider"),
|
|
649
743
|
env=ctx.get("env"),
|
|
650
744
|
spawn_sync=ctx.get("spawn_text"),
|
|
@@ -670,13 +764,24 @@ def handle_run(rest, ctx):
|
|
|
670
764
|
cwd=cwd,
|
|
671
765
|
pid=launch.get("pid"),
|
|
672
766
|
launch_log_path=launch.get("launch_log_path"),
|
|
767
|
+
conversation_id=launch.get("provider_session_id"),
|
|
673
768
|
selection_row=selection_row,
|
|
674
769
|
))
|
|
675
770
|
return 0
|
|
676
771
|
|
|
677
772
|
attempted = [run_session["name"]]
|
|
678
773
|
failover_error_code = None
|
|
774
|
+
recovery_events = []
|
|
775
|
+
attempt = 0
|
|
776
|
+
models = capacity_models(run_session)
|
|
777
|
+
model_index = 0
|
|
778
|
+
capacity_retries = 0
|
|
679
779
|
while True:
|
|
780
|
+
attempt += 1
|
|
781
|
+
baseline = extract_interactive_usage(
|
|
782
|
+
run_session["provider"], run_session.get("authHome"),
|
|
783
|
+
conversation_id=(run_session.get("conversation") or {}).get("id"),
|
|
784
|
+
) if parsed.get("continue_run") and run_session["provider"] == "codex" else None
|
|
680
785
|
run_info = _run_headless_provider_command(
|
|
681
786
|
run_session,
|
|
682
787
|
cwd=cwd,
|
|
@@ -686,20 +791,78 @@ def handle_run(rest, ctx):
|
|
|
686
791
|
spawn=ctx.get("spawn_headless") or ctx.get("spawn"),
|
|
687
792
|
run_id=run_id,
|
|
688
793
|
)
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
794
|
+
observation = _observe_headless_usage(run_session, run_info, baseline)
|
|
795
|
+
run_info = {**run_info, **observation,
|
|
796
|
+
"parent_run_id": parsed.get("continue_run"),
|
|
797
|
+
"conversation_id": extract_run_conversation_id(run_session.get("provider"), run_info.get("stdout_path"))
|
|
798
|
+
or ((run_session.get("conversation") or {}).get("id") if run_session["provider"] == "claude" or parsed.get("continue_run") else None),
|
|
799
|
+
"launch": run_session.get("launch"),
|
|
800
|
+
"reasoning_effort": _launch_power(run_session),
|
|
801
|
+
"effective_model": (run_session.get("launch") or {}).get("model"),
|
|
802
|
+
"recovery_events": recovery_events}
|
|
694
803
|
ctx["service"]["record_launch_history"](run_session["name"], {
|
|
695
804
|
"status": "success" if run_info.get("returncode") == 0 else "failed",
|
|
696
805
|
"action": "run",
|
|
697
806
|
"run_id": run_id,
|
|
698
|
-
"attempt":
|
|
807
|
+
"attempt": attempt,
|
|
699
808
|
"cwd": cwd,
|
|
700
809
|
"exit_code": run_info.get("returncode"),
|
|
701
810
|
**run_info,
|
|
702
811
|
})
|
|
812
|
+
if parsed.get("continue_run"):
|
|
813
|
+
break
|
|
814
|
+
if looks_model_at_capacity(run_session.get("provider"), run_info):
|
|
815
|
+
event = {"reason": "model_at_capacity", "attempt": attempt,
|
|
816
|
+
"session": run_session["name"], "model": models[model_index]}
|
|
817
|
+
if capacity_retries < len(CAPACITY_RETRY_DELAYS):
|
|
818
|
+
delay = CAPACITY_RETRY_DELAYS[capacity_retries]
|
|
819
|
+
capacity_retries += 1
|
|
820
|
+
event.update(action="retry", retry=capacity_retries, delay_seconds=delay)
|
|
821
|
+
elif model_index + 1 < len(models):
|
|
822
|
+
model_index += 1
|
|
823
|
+
capacity_retries = 0
|
|
824
|
+
run_session["launch"]["model"] = models[model_index]
|
|
825
|
+
event.update(action="fallback_model", target_model=models[model_index])
|
|
826
|
+
else:
|
|
827
|
+
successor = None if len(attempted) > MAX_FAILOVER_TRANSITIONS else _select_headless_session(
|
|
828
|
+
ctx, "codex", min_reasoning_effort=parsed["reasoning_effort"],
|
|
829
|
+
require_ready=True, exclude=attempted,
|
|
830
|
+
)
|
|
831
|
+
if not successor:
|
|
832
|
+
failover_error_code = "capacity_recovery_exhausted"
|
|
833
|
+
event.update(action="exhausted")
|
|
834
|
+
else:
|
|
835
|
+
selection_row = successor.get("row") or {}
|
|
836
|
+
next_session = successor["session"]
|
|
837
|
+
run_session = {**next_session, "launch": {**(next_session.get("launch") or {}), **launch_updates}}
|
|
838
|
+
# Each new profile starts with the requested model, or its own default.
|
|
839
|
+
models = capacity_models(run_session)
|
|
840
|
+
model_index = capacity_retries = 0
|
|
841
|
+
attempted.append(run_session["name"])
|
|
842
|
+
event.update(action="fallback_profile", target_session=run_session["name"], target_model=models[0])
|
|
843
|
+
_record_capacity_recovery(ctx, registry, run_id, event, run_info, recovery_events)
|
|
844
|
+
if failover_error_code:
|
|
845
|
+
break
|
|
846
|
+
if event["action"] == "retry":
|
|
847
|
+
previous_handlers = _route_termination_to_interrupt()
|
|
848
|
+
try:
|
|
849
|
+
time.sleep(event["delay_seconds"])
|
|
850
|
+
except KeyboardInterrupt as interrupt:
|
|
851
|
+
error = CdxError("Codex capacity recovery was cancelled.", 130)
|
|
852
|
+
error.cancelled = True
|
|
853
|
+
error.history_recorded = True
|
|
854
|
+
error.run_info = {**run_info, "returncode": 130}
|
|
855
|
+
raise error from interrupt
|
|
856
|
+
finally:
|
|
857
|
+
_restore_signal_handlers(previous_handlers)
|
|
858
|
+
elif event["action"] == "fallback_profile":
|
|
859
|
+
_ensure_session_authentication(
|
|
860
|
+
run_session, ctx["service"], spawn=ctx.get("spawn"), spawn_sync=ctx.get("spawn_sync"),
|
|
861
|
+
env_override=ctx.get("env"), stdin_is_tty=False, behavior="launch",
|
|
862
|
+
signal_emitter=ctx.get("signal_emitter"), trust_local_credentials=False,
|
|
863
|
+
)
|
|
864
|
+
registry.migrate(run_id, session=run_session["name"], provider="codex", reason="model_at_capacity")
|
|
865
|
+
continue
|
|
703
866
|
if run_info.get("returncode") == 0 or not parsed.get("failover"):
|
|
704
867
|
break
|
|
705
868
|
reason = failover_reason(
|
|
@@ -731,6 +894,8 @@ def handle_run(rest, ctx):
|
|
|
731
894
|
"launch": {**(successor["session"].get("launch") or {}), **launch_updates},
|
|
732
895
|
}
|
|
733
896
|
attempted.append(run_session["name"])
|
|
897
|
+
models = capacity_models(run_session)
|
|
898
|
+
model_index = capacity_retries = 0
|
|
734
899
|
# A different account means different credentials; an unauthenticated
|
|
735
900
|
# successor would fail for a reason unrelated to the task.
|
|
736
901
|
_ensure_session_authentication(
|
|
@@ -769,7 +934,9 @@ def handle_run(rest, ctx):
|
|
|
769
934
|
)
|
|
770
935
|
_write_json(ctx, final_payload)
|
|
771
936
|
return 0
|
|
772
|
-
if failover_error_code == "
|
|
937
|
+
if failover_error_code == "capacity_recovery_exhausted":
|
|
938
|
+
message = "Codex model capacity recovery exhausted its retry, model, or profile budget."
|
|
939
|
+
elif failover_error_code == "failover_exhausted":
|
|
773
940
|
message = f"Every eligible session was rate limited: {', '.join(attempted)}."
|
|
774
941
|
elif failover_error_code == "failover_limit_reached":
|
|
775
942
|
message = f"Failover stopped after {MAX_FAILOVER_TRANSITIONS} transitions: {', '.join(attempted)}."
|
|
@@ -806,17 +973,20 @@ def handle_run(rest, ctx):
|
|
|
806
973
|
except CdxError as error:
|
|
807
974
|
run_info = getattr(error, "run_info", None)
|
|
808
975
|
cancelled = bool(getattr(error, "cancelled", False))
|
|
809
|
-
if run_info
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
976
|
+
if run_info is not None:
|
|
977
|
+
run_info = {**run_info, "recovery_events": locals().get("recovery_events", [])}
|
|
978
|
+
if run_info and run_id and not getattr(error, "history_recorded", False):
|
|
979
|
+
observation = _observe_headless_usage(run_session, run_info, locals().get("baseline"))
|
|
980
|
+
run_info = {**run_info, **observation,
|
|
981
|
+
"parent_run_id": parsed.get("continue_run"),
|
|
982
|
+
"conversation_id": extract_run_conversation_id(run_session.get("provider"), run_info.get("stdout_path"))
|
|
983
|
+
or ((run_session.get("conversation") or {}).get("id") if run_session["provider"] == "claude" or parsed.get("continue_run") else None),
|
|
984
|
+
"launch": run_session.get("launch"), "reasoning_effort": _launch_power(run_session)}
|
|
815
985
|
ctx["service"]["record_launch_history"](run_session["name"], {
|
|
816
986
|
"status": "failed",
|
|
817
987
|
"action": "run",
|
|
818
988
|
"run_id": run_id,
|
|
819
|
-
"attempt":
|
|
989
|
+
"attempt": locals().get("attempt", 1),
|
|
820
990
|
"cwd": cwd,
|
|
821
991
|
"exit_code": error.exit_code,
|
|
822
992
|
**run_info,
|
|
@@ -825,13 +995,14 @@ def handle_run(rest, ctx):
|
|
|
825
995
|
API_SCHEMA_VERSION,
|
|
826
996
|
False,
|
|
827
997
|
locals().get("parsed", {}) or {},
|
|
828
|
-
locals().get("session"),
|
|
998
|
+
locals().get("run_session", locals().get("session")),
|
|
829
999
|
run_info=run_info,
|
|
830
1000
|
error=error,
|
|
831
1001
|
error_source="provider" if cancelled else "cdx",
|
|
832
1002
|
error_code="run_cancelled" if cancelled else run_cdx_error_code(error),
|
|
833
1003
|
selection_row=selection_row,
|
|
834
1004
|
)
|
|
1005
|
+
final_payload["recovery_events"] = locals().get("recovery_events", [])
|
|
835
1006
|
if registry and run_id:
|
|
836
1007
|
registry.finish(
|
|
837
1008
|
run_id,
|
|
@@ -845,6 +1016,22 @@ def handle_run(rest, ctx):
|
|
|
845
1016
|
_write_json(ctx, final_payload)
|
|
846
1017
|
return error.exit_code
|
|
847
1018
|
|
|
1019
|
+
def _record_capacity_recovery(ctx, registry, run_id, event, run_info, events):
|
|
1020
|
+
# Subsequent attempts reuse the run's paths; retain the failed output first.
|
|
1021
|
+
artifacts = {}
|
|
1022
|
+
for key in ("stdout_path", "stderr_path", "transcript_path"):
|
|
1023
|
+
path = run_info.get(key)
|
|
1024
|
+
if path and os.path.isfile(path):
|
|
1025
|
+
archived = f"{path}.attempt-{event['attempt']}"
|
|
1026
|
+
shutil.copyfile(path, archived)
|
|
1027
|
+
artifacts[key] = archived
|
|
1028
|
+
event["artifacts"] = artifacts
|
|
1029
|
+
events.append(event)
|
|
1030
|
+
registry.record_recovery(run_id, event)
|
|
1031
|
+
details = ", ".join(f"{key}={value}" for key, value in event.items() if key != "artifacts")
|
|
1032
|
+
ctx["err"](f"cdx capacity recovery: {details}\n")
|
|
1033
|
+
|
|
1034
|
+
|
|
848
1035
|
def _select_headless_session(
|
|
849
1036
|
ctx, provider, min_reasoning_effort=None, require_ready=False, force_refresh=False, exclude=None,
|
|
850
1037
|
):
|
package/src/commands/status.py
CHANGED
|
@@ -53,10 +53,13 @@ from ..provider_runtime import (
|
|
|
53
53
|
)
|
|
54
54
|
from ..rtk_savings import rtk_saved
|
|
55
55
|
from ..run_usage import (
|
|
56
|
-
|
|
56
|
+
TOKEN_USAGE_KEYS,
|
|
57
|
+
cache_hit_ratio,
|
|
57
58
|
estimate_cost,
|
|
59
|
+
format_cache_ratio,
|
|
58
60
|
is_vouched,
|
|
59
61
|
normalize_usage,
|
|
62
|
+
summarize_cache_usage,
|
|
60
63
|
token_prices,
|
|
61
64
|
weighted_usage,
|
|
62
65
|
)
|
|
@@ -294,7 +297,9 @@ def _summarize_stats(entries):
|
|
|
294
297
|
"reasoning_tokens": 0,
|
|
295
298
|
"total_tokens": 0,
|
|
296
299
|
"last_started_at": None,
|
|
300
|
+
"_entries": [],
|
|
297
301
|
})
|
|
302
|
+
row["_entries"].append(entry)
|
|
298
303
|
row["launches"] += 1
|
|
299
304
|
if entry.get("provider"):
|
|
300
305
|
row["provider"].add(entry["provider"])
|
|
@@ -315,7 +320,7 @@ def _summarize_stats(entries):
|
|
|
315
320
|
usage = {}
|
|
316
321
|
parsed_usage = {
|
|
317
322
|
key: _token_value(usage, key)
|
|
318
|
-
for key in
|
|
323
|
+
for key in TOKEN_USAGE_KEYS
|
|
319
324
|
}
|
|
320
325
|
if any(value is not None for value in parsed_usage.values()):
|
|
321
326
|
row["usage_runs"] += 1
|
|
@@ -361,6 +366,16 @@ def _summarize_stats(entries):
|
|
|
361
366
|
# overwhelming majority of cached tokens are, and weighting the whole
|
|
362
367
|
# of it at the creation rate would overstate legacy sessions by more
|
|
363
368
|
# than reading it as reads understates them.
|
|
369
|
+
group_entries = row.pop("_entries")
|
|
370
|
+
row["cache_usage"] = summarize_cache_usage(group_entries)
|
|
371
|
+
row["cache_hit_ratio"] = row["cache_usage"]["cache_hit_ratio"]
|
|
372
|
+
row["run_types"] = {
|
|
373
|
+
kind: {"runs": len(subset), **summarize_cache_usage(subset)}
|
|
374
|
+
for kind, subset in (
|
|
375
|
+
("continued", [e for e in group_entries if e.get("parent_run_id")]),
|
|
376
|
+
("cold", [e for e in group_entries if not e.get("parent_run_id")]),
|
|
377
|
+
)
|
|
378
|
+
}
|
|
364
379
|
split = row["cache_creation_tokens"] + row["cache_read_tokens"]
|
|
365
380
|
legacy_cache = max(0, row["cached_input_tokens"] - split)
|
|
366
381
|
derived = normalize_usage(
|
|
@@ -381,15 +396,34 @@ def _summarize_stats(entries):
|
|
|
381
396
|
reverse=True,
|
|
382
397
|
)
|
|
383
398
|
|
|
399
|
+
def _merge_cache_usage(summaries):
|
|
400
|
+
totals = {}
|
|
401
|
+
for key in ("input_tokens", "cache_creation_tokens", "cache_read_tokens", "output_tokens",
|
|
402
|
+
"measured_runs", "measured_input_tokens", "measured_cache_read_tokens"):
|
|
403
|
+
values = [s[key] for s in summaries if s[key] is not None]
|
|
404
|
+
totals[key] = sum(values) if values else (0 if key.startswith("measured_") else None)
|
|
405
|
+
inputs = totals["measured_input_tokens"]
|
|
406
|
+
totals["cache_hit_ratio"] = totals["measured_cache_read_tokens"] / inputs if inputs else None
|
|
407
|
+
return totals
|
|
408
|
+
|
|
409
|
+
|
|
384
410
|
def _stats_totals(rows):
|
|
411
|
+
cache = _merge_cache_usage([row["cache_usage"] for row in rows])
|
|
385
412
|
return {
|
|
413
|
+
"cache_usage": cache,
|
|
414
|
+
"cache_hit_ratio": cache["cache_hit_ratio"],
|
|
415
|
+
"run_types": {
|
|
416
|
+
kind: {"runs": sum(row["run_types"][kind]["runs"] for row in rows),
|
|
417
|
+
**_merge_cache_usage([row["run_types"][kind] for row in rows])}
|
|
418
|
+
for kind in ("cold", "continued")
|
|
419
|
+
},
|
|
386
420
|
"sessions": len(rows),
|
|
387
421
|
"launches": sum(row["launches"] for row in rows),
|
|
388
422
|
"successes": sum(row["successes"] for row in rows),
|
|
389
423
|
"failures": sum(row["failures"] for row in rows),
|
|
390
424
|
"duration_ms": sum(row["duration_ms"] for row in rows),
|
|
391
425
|
"usage_runs": sum(row["usage_runs"] for row in rows),
|
|
392
|
-
**{key: sum(row[key] for row in rows) for key in
|
|
426
|
+
**{key: sum(row[key] for row in rows) for key in TOKEN_USAGE_KEYS},
|
|
393
427
|
"weighted_tokens": sum(row["weighted_tokens"] for row in rows),
|
|
394
428
|
"weighted_runs": sum(row["weighted_runs"] for row in rows),
|
|
395
429
|
"unweighted_runs": sum(row["unweighted_runs"] for row in rows),
|
|
@@ -448,7 +482,7 @@ def _format_history(entries, use_color=False, active_sessions=None):
|
|
|
448
482
|
active_sessions = active_sessions or set()
|
|
449
483
|
if not entries:
|
|
450
484
|
return "No launch history."
|
|
451
|
-
rows = [[_style(value, "1", use_color) for value in ["SESSION", "PROV.", "RESULT", "DURATION", "WHEN", "TRANSCRIPT"]]]
|
|
485
|
+
rows = [[_style(value, "1", use_color) for value in ["SESSION", "PROV.", "RESULT", "DURATION", "WHEN", "TOKENS", "TRANSCRIPT"]]]
|
|
452
486
|
for entry in entries:
|
|
453
487
|
transcript_path = entry.get("transcript_path")
|
|
454
488
|
session_name = entry.get("session_name") or "-"
|
|
@@ -461,6 +495,8 @@ def _format_history(entries, use_color=False, active_sessions=None):
|
|
|
461
495
|
_style(status, status_color, use_color),
|
|
462
496
|
_style(_format_duration_ms(entry.get("duration_ms")), "33" if entry.get("duration_ms") else "2", use_color),
|
|
463
497
|
_dim(_format_relative_age(entry.get("started_at")), use_color),
|
|
498
|
+
_dim(f"{_format_token_count((entry.get('usage') or {}).get('total_tokens'))} "
|
|
499
|
+
f"{format_cache_ratio((entry.get('usage') or {}).get('cache_hit_ratio'))}", use_color),
|
|
464
500
|
_dim(os.path.basename(transcript_path) if transcript_path else "-", use_color),
|
|
465
501
|
])
|
|
466
502
|
return "\n".join([
|
|
@@ -529,10 +565,12 @@ def _format_stats(rows, totals, period=None, use_color=False, active_sessions=No
|
|
|
529
565
|
_style(str(row["launches"]), "1", use_color),
|
|
530
566
|
_style(str(row["usage_runs"]), "32" if row["usage_runs"] else "2", use_color),
|
|
531
567
|
_style(_format_token_count(row["input_tokens"]), "96" if row["input_tokens"] else "2", use_color),
|
|
532
|
-
_style(_format_token_count(row["cached_input_tokens"])
|
|
568
|
+
_style(_format_token_count(row["cached_input_tokens"]) if row["cache_usage"]["cache_read_tokens"] is not None
|
|
569
|
+
or row["cache_usage"]["cache_creation_tokens"] is not None else "n/d",
|
|
570
|
+
"96" if row["cached_input_tokens"] else "2", use_color),
|
|
533
571
|
_style(_format_token_count(row["output_tokens"]), "96" if row["output_tokens"] else "2", use_color),
|
|
534
572
|
_style(_format_token_count(row["reasoning_tokens"]), "95" if row["reasoning_tokens"] else "2", use_color),
|
|
535
|
-
_style(_format_token_count(row[
|
|
573
|
+
_style(f"{_format_token_count(row['total_tokens'])} {format_cache_ratio(row.get('cache_hit_ratio'))}", "96" if row["total_tokens"] else "2", use_color),
|
|
536
574
|
_style(_format_token_count(row["weighted_tokens"]) if row["weighted_runs"] else "-",
|
|
537
575
|
"1;95" if row["weighted_runs"] else "2", use_color),
|
|
538
576
|
_style(_format_usd(row["cost_usd"]) if row["priced_runs"] else "-",
|
|
@@ -550,7 +588,7 @@ def _format_stats(rows, totals, period=None, use_color=False, active_sessions=No
|
|
|
550
588
|
_dim(
|
|
551
589
|
"Totals: "
|
|
552
590
|
f"{totals['launches']} runs, {totals['usage_runs']} with usage, "
|
|
553
|
-
f"{_format_token_count(totals['total_tokens'])} tokens "
|
|
591
|
+
f"{_format_token_count(totals['total_tokens'])} tokens {format_cache_ratio(totals.get('cache_hit_ratio'))} "
|
|
554
592
|
f"({_format_token_count(totals['weighted_tokens']) if totals['weighted_runs'] else '-'} cost-equivalent"
|
|
555
593
|
+ (f" on {totals['weighted_runs']}/{totals['usage_runs']} measured runs"
|
|
556
594
|
if totals["usage_runs"] else "")
|
|
@@ -633,6 +671,9 @@ def handle_history(rest, ctx):
|
|
|
633
671
|
limit = 0 if parsed["summary"] or has_period else parsed["limit"]
|
|
634
672
|
entries = ctx["service"]["get_launch_history"](parsed["name"], limit=limit)
|
|
635
673
|
entries = _filter_history_period(entries, parsed["period"])
|
|
674
|
+
entries = [{**entry, "parent_run_id": entry.get("parent_run_id"),
|
|
675
|
+
"usage": {**entry["usage"], "cache_hit_ratio": cache_hit_ratio(entry["usage"], provider=entry.get("provider")) if is_vouched(entry["usage"]) else None}
|
|
676
|
+
if isinstance(entry.get("usage"), dict) else entry.get("usage")} for entry in entries]
|
|
636
677
|
if has_period and not parsed["summary"]:
|
|
637
678
|
entries = entries[:parsed["limit"]]
|
|
638
679
|
message = (
|
package/src/interactive_usage.py
CHANGED
|
@@ -261,9 +261,13 @@ def _codex_usage(handle):
|
|
|
261
261
|
# larger than the input count is not a subset, so that record is
|
|
262
262
|
# left alone rather than forced to zero.
|
|
263
263
|
reported_input = _number(usage.get("input_tokens"))
|
|
264
|
-
cache_read = _number(usage.get("cached_input_tokens")
|
|
264
|
+
cache_read = _number(usage["cached_input_tokens"]) if usage.get("cached_input_tokens") is not None else None
|
|
265
|
+
cache_write = _number(usage["cache_write_input_tokens"]) if usage.get("cache_write_input_tokens") is not None else None
|
|
266
|
+
cached = (cache_read or 0) + (cache_write or 0)
|
|
265
267
|
latest = normalize_usage(
|
|
266
|
-
|
|
268
|
+
provider="codex",
|
|
269
|
+
input_tokens=reported_input - cached if cached <= reported_input else reported_input,
|
|
270
|
+
cache_creation_tokens=cache_write,
|
|
267
271
|
cache_read_tokens=cache_read,
|
|
268
272
|
output_tokens=_number(usage.get("output_tokens")),
|
|
269
273
|
reasoning_tokens=_number(usage.get("reasoning_output_tokens")) or None,
|
|
@@ -286,6 +290,7 @@ def _claude_usage(handle):
|
|
|
286
290
|
inside `output_tokens` -- so that field stays absent by design.
|
|
287
291
|
"""
|
|
288
292
|
totals = {"input_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 0, "output_tokens": 0}
|
|
293
|
+
missing = set(totals)
|
|
289
294
|
long_ttl = None
|
|
290
295
|
seen = set()
|
|
291
296
|
model = None
|
|
@@ -301,6 +306,11 @@ def _claude_usage(handle):
|
|
|
301
306
|
if identifier in seen:
|
|
302
307
|
continue
|
|
303
308
|
seen.add(identifier)
|
|
309
|
+
for source, target in (("input_tokens", "input_tokens"), ("output_tokens", "output_tokens"),
|
|
310
|
+
("cache_creation_input_tokens", "cache_creation_tokens"),
|
|
311
|
+
("cache_read_input_tokens", "cache_read_tokens")):
|
|
312
|
+
if usage.get(source) is not None:
|
|
313
|
+
missing.discard(target)
|
|
304
314
|
totals["input_tokens"] += _number(usage.get("input_tokens"))
|
|
305
315
|
totals["cache_creation_tokens"] += _number(usage.get("cache_creation_input_tokens"))
|
|
306
316
|
totals["cache_read_tokens"] += _number(usage.get("cache_read_input_tokens"))
|
|
@@ -316,7 +326,8 @@ def _claude_usage(handle):
|
|
|
316
326
|
continue
|
|
317
327
|
if not seen and not any(totals.values()):
|
|
318
328
|
return None, model
|
|
319
|
-
return normalize_usage(**
|
|
329
|
+
return normalize_usage(**{k: None if k in missing else v for k, v in totals.items()},
|
|
330
|
+
cache_creation_1h_tokens=long_ttl), model
|
|
320
331
|
|
|
321
332
|
|
|
322
333
|
def _number(value):
|
|
@@ -326,7 +337,7 @@ def _number(value):
|
|
|
326
337
|
return 0
|
|
327
338
|
|
|
328
339
|
|
|
329
|
-
#: The quantities a provider actually reports. The other
|
|
340
|
+
#: The quantities a provider actually reports. The other fields of a usage
|
|
330
341
|
#: record are derived by `normalize_usage` and must never be differenced.
|
|
331
342
|
MEASURED_KEYS = (
|
|
332
343
|
"input_tokens",
|
|
@@ -338,7 +349,7 @@ MEASURED_KEYS = (
|
|
|
338
349
|
)
|
|
339
350
|
|
|
340
351
|
|
|
341
|
-
def usage_delta(cumulative, previous):
|
|
352
|
+
def usage_delta(cumulative, previous, provider=None):
|
|
342
353
|
"""What this run added to a transcript that already held `previous`.
|
|
343
354
|
|
|
344
355
|
Both readers return a *cumulative* figure: Claude's is the sum over the
|
|
@@ -372,7 +383,7 @@ def usage_delta(cumulative, previous):
|
|
|
372
383
|
if difference < 0:
|
|
373
384
|
return None
|
|
374
385
|
parts[key] = difference
|
|
375
|
-
return normalize_usage(**parts)
|
|
386
|
+
return normalize_usage(**parts, provider=provider)
|
|
376
387
|
|
|
377
388
|
|
|
378
389
|
def transcript_predates_run(path, started_at):
|
|
@@ -224,6 +224,7 @@ def read_transcript_outcome(path):
|
|
|
224
224
|
if not path:
|
|
225
225
|
return outcome
|
|
226
226
|
totals = {"input_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 0, "output_tokens": 0}
|
|
227
|
+
missing = set(totals)
|
|
227
228
|
long_ttl = None
|
|
228
229
|
# This path had no de-duplication at all, so a transcript that records the
|
|
229
230
|
# same billed response more than once inflated the total outright.
|
|
@@ -266,6 +267,11 @@ def read_transcript_outcome(path):
|
|
|
266
267
|
seen = True
|
|
267
268
|
if isinstance(message.get("model"), str) and message["model"]:
|
|
268
269
|
models.add(message["model"])
|
|
270
|
+
for source, target in (("input_tokens", "input_tokens"), ("output_tokens", "output_tokens"),
|
|
271
|
+
("cache_creation_input_tokens", "cache_creation_tokens"),
|
|
272
|
+
("cache_read_input_tokens", "cache_read_tokens")):
|
|
273
|
+
if usage.get(source) is not None:
|
|
274
|
+
missing.discard(target)
|
|
269
275
|
totals["input_tokens"] += usage.get("input_tokens") or 0
|
|
270
276
|
totals["cache_creation_tokens"] += usage.get("cache_creation_input_tokens") or 0
|
|
271
277
|
totals["cache_read_tokens"] += usage.get("cache_read_input_tokens") or 0
|
|
@@ -276,7 +282,8 @@ def read_transcript_outcome(path):
|
|
|
276
282
|
except OSError:
|
|
277
283
|
return outcome
|
|
278
284
|
if seen:
|
|
279
|
-
outcome["usage"] = normalize_usage(**
|
|
285
|
+
outcome["usage"] = normalize_usage(**{k: None if k in missing else v for k, v in totals.items()},
|
|
286
|
+
cache_creation_1h_tokens=long_ttl)
|
|
280
287
|
if len(models) == 1:
|
|
281
288
|
outcome["usage_model"] = next(iter(models))
|
|
282
289
|
return outcome
|
package/src/provider_runtime.py
CHANGED
|
@@ -1006,7 +1006,7 @@ def _build_headless_launch_spec(session, cwd=None, env_override=None, initial_pr
|
|
|
1006
1006
|
args = ["--print", "--output-format", "json"]
|
|
1007
1007
|
conversation_id = _conversation_id(session)
|
|
1008
1008
|
if conversation_id:
|
|
1009
|
-
args += ["--session-id", conversation_id]
|
|
1009
|
+
args += ["--resume" if session.get("continue_run") else "--session-id", conversation_id]
|
|
1010
1010
|
if model:
|
|
1011
1011
|
args += ["--model", _claude_cli_model(model)]
|
|
1012
1012
|
# --max-budget-usd and --fallback-model are --print-only in the claude
|
|
@@ -1045,6 +1045,8 @@ def _build_headless_launch_spec(session, cwd=None, env_override=None, initial_pr
|
|
|
1045
1045
|
if permission:
|
|
1046
1046
|
args += HEADLESS_CODEX_PERMISSION_ARGS.get(permission, [])
|
|
1047
1047
|
args += preference_args
|
|
1048
|
+
if session.get("continue_run"):
|
|
1049
|
+
args += ["resume", _conversation_id(session)]
|
|
1048
1050
|
if initial_prompt:
|
|
1049
1051
|
# `--` keeps a prompt that starts with `-` from being read as an option.
|
|
1050
1052
|
args += ["--", initial_prompt]
|