cdx-manager 0.24.8 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -1
- package/VERSION +1 -1
- package/changelogs/CHANGELOGS_0_24_8.md +8 -0
- package/changelogs/CHANGELOGS_0_24_9.md +18 -0
- package/changelogs/CHANGELOGS_0_25_0.md +17 -0
- package/changelogs/CHANGELOGS_UNRELEASED.md +7 -0
- package/checksums/release-archives.json +22 -0
- package/docs/cli.md +20 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/cli.py +1 -1
- package/src/cli_args.py +26 -2
- package/src/commands/launch.py +4 -4
- package/src/commands/runs.py +160 -18
- package/src/commands/status.py +49 -7
- package/src/interactive_usage.py +17 -6
- package/src/provider_background.py +8 -1
- package/src/provider_runtime.py +24 -3
- package/src/run_command.py +18 -2
- package/src/run_registry.py +13 -2
- package/src/run_usage.py +89 -24
package/src/commands/runs.py
CHANGED
|
@@ -23,7 +23,9 @@ from ..cli_args import (
|
|
|
23
23
|
cdx_schema,
|
|
24
24
|
)
|
|
25
25
|
from ..cli_helpers import API_SCHEMA_VERSION, _json_failure, _json_success, _write_json
|
|
26
|
+
from ..config import split_extra_args
|
|
26
27
|
from ..errors import CdxError
|
|
28
|
+
from ..interactive_usage import MATCH_CONVERSATION_ID, MEASURED_KEYS, extract_interactive_usage, usage_delta
|
|
27
29
|
from ..provider_background import (
|
|
28
30
|
BACKGROUND_PATH_CDX,
|
|
29
31
|
BACKGROUND_PATH_PROVIDER,
|
|
@@ -41,9 +43,11 @@ from ..provider_runtime import (
|
|
|
41
43
|
_build_launch_spec,
|
|
42
44
|
_ensure_session_authentication,
|
|
43
45
|
_headless_artifact_paths,
|
|
46
|
+
_launch_power,
|
|
44
47
|
_restore_signal_handlers,
|
|
45
48
|
_route_termination_to_interrupt,
|
|
46
49
|
_run_headless_provider_command,
|
|
50
|
+
with_requested_conversation,
|
|
47
51
|
)
|
|
48
52
|
from ..run_command import read_run_prompt, run_cdx_error_code, run_launch_payload, run_result_payload
|
|
49
53
|
from ..run_failover import (
|
|
@@ -54,7 +58,7 @@ from ..run_failover import (
|
|
|
54
58
|
looks_model_at_capacity,
|
|
55
59
|
)
|
|
56
60
|
from ..run_registry import RunRegistry, build_code_review_report, final_message
|
|
57
|
-
from ..run_usage import extract_run_observation
|
|
61
|
+
from ..run_usage import empty_usage, estimate_cost, extract_run_conversation_id, extract_run_observation, token_prices
|
|
58
62
|
from ..session_ranking import FACTOR_DESCRIPTIONS, rank_sessions, selection_policy
|
|
59
63
|
from ..status_view import _now_timestamp, _priority_reset_timestamp
|
|
60
64
|
|
|
@@ -200,6 +204,8 @@ def _detached_child_argv(parsed, prompt_path, session_name, cwd):
|
|
|
200
204
|
# from the package root, so a relative "--cwd ." would resolve there.
|
|
201
205
|
argv = ["run", session_name, "--cwd", cwd, "--prompt-file", prompt_path]
|
|
202
206
|
for flag, key in (
|
|
207
|
+
("--budget", "budget"),
|
|
208
|
+
("--continue-run", "continue_run"),
|
|
203
209
|
("--model", "model"),
|
|
204
210
|
("--kind", "kind"),
|
|
205
211
|
("--permission", "permission"),
|
|
@@ -537,6 +543,99 @@ def _env_max_runs(ctx):
|
|
|
537
543
|
return limit if limit > 0 else None
|
|
538
544
|
|
|
539
545
|
|
|
546
|
+
def _observe_headless_usage(session, run_info, baseline=None):
|
|
547
|
+
usage, model = extract_run_observation(session.get("provider"), run_info.get("stdout_path"), session.get("authHome"))
|
|
548
|
+
observation = {"usage": usage, "usage_model": model}
|
|
549
|
+
if baseline is not None:
|
|
550
|
+
cumulative, path, match, transcript_model = extract_interactive_usage(
|
|
551
|
+
session["provider"], session.get("authHome"), conversation_id=(session.get("conversation") or {}).get("id"))
|
|
552
|
+
before, before_path, _before_match, _before_model = baseline
|
|
553
|
+
comparable = (cumulative and before and path == before_path
|
|
554
|
+
and all((cumulative.get(key) is None) == (before.get(key) is None) for key in MEASURED_KEYS))
|
|
555
|
+
# A newly reported split includes unknown pre-launch usage; losing a
|
|
556
|
+
# split also prevents a comparable measurement across the boundary.
|
|
557
|
+
delta = usage_delta(cumulative, before, provider=session.get("provider")) if comparable else None
|
|
558
|
+
# Exec resume can emit conversation totals. Only exact launch boundaries
|
|
559
|
+
# establish this run's share; stdout cannot replace a missing boundary.
|
|
560
|
+
observation.update(usage=delta if delta is not None else empty_usage(), usage_cumulative=cumulative,
|
|
561
|
+
usage_attribution="launch_boundary" if delta is not None else "continuation_usage_unavailable",
|
|
562
|
+
provider_transcript_path=path, provider_transcript_match=match,
|
|
563
|
+
usage_model=model or transcript_model)
|
|
564
|
+
return observation
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _budget_observation(session, paths, baseline):
|
|
568
|
+
observation = _observe_headless_usage(session, paths, baseline if session["provider"] == "codex" else None)
|
|
569
|
+
identifier = (extract_run_conversation_id(session["provider"], paths.get("stdout_path"))
|
|
570
|
+
or ((session.get("conversation") or {}).get("id")
|
|
571
|
+
if session["provider"] == "claude" or session.get("continue_run") else None))
|
|
572
|
+
if not identifier:
|
|
573
|
+
return observation
|
|
574
|
+
usage, path, match, model = extract_interactive_usage(
|
|
575
|
+
session["provider"], session.get("authHome"), conversation_id=identifier)
|
|
576
|
+
if usage is None or match != MATCH_CONVERSATION_ID:
|
|
577
|
+
return observation
|
|
578
|
+
if baseline is not None:
|
|
579
|
+
before, before_path, _match, _model = baseline
|
|
580
|
+
comparable = (before and path == before_path
|
|
581
|
+
and all((usage.get(key) is None) == (before.get(key) is None) for key in MEASURED_KEYS))
|
|
582
|
+
usage = usage_delta(usage, before, provider=session["provider"]) if comparable else None
|
|
583
|
+
if usage is not None:
|
|
584
|
+
return {**observation, "usage": usage, "usage_model": model, "provider_transcript_path": path,
|
|
585
|
+
"provider_transcript_match": match}
|
|
586
|
+
return observation
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _continue_run_session(parsed, ctx):
|
|
590
|
+
"""Resolve only the parent's profile and exact persisted conversation."""
|
|
591
|
+
previous = _run_registry(ctx).get(parsed["continue_run"])
|
|
592
|
+
if not previous:
|
|
593
|
+
raise CdxError(f"Cannot continue unknown run: {parsed['continue_run']}. Start a normal run instead.",
|
|
594
|
+
code="continue_run_not_found")
|
|
595
|
+
name, provider = previous.get("session"), previous.get("provider")
|
|
596
|
+
if (parsed.get("name") and parsed["name"] != name) or (parsed.get("provider") and parsed["provider"] != provider):
|
|
597
|
+
raise CdxError("Cannot continue a run from another profile or provider. Start a normal run instead.",
|
|
598
|
+
code="continue_run_mismatch")
|
|
599
|
+
session = ctx["service"]["get_session"](name) if name else None
|
|
600
|
+
if not session or session.get("provider") != provider:
|
|
601
|
+
raise CdxError("The previous run's profile or provider is no longer available. Start a normal run instead.",
|
|
602
|
+
code="continue_run_mismatch")
|
|
603
|
+
if provider not in ("claude", "codex") or previous.get("status") not in ("succeeded", "failed", "timed_out", "cancelled"):
|
|
604
|
+
raise CdxError("Only a finished Claude or Codex run can be continued. Start a normal run instead.",
|
|
605
|
+
code="continue_run_unavailable")
|
|
606
|
+
conversation = previous.get("conversation_id")
|
|
607
|
+
resumed = with_requested_conversation(session, conversation)
|
|
608
|
+
if not resumed:
|
|
609
|
+
raise CdxError("The previous run's provider session expired or its transcript is missing. Start a normal run instead.",
|
|
610
|
+
code="continue_run_unavailable")
|
|
611
|
+
model = previous.get("usage_model") or previous.get("model")
|
|
612
|
+
effort = previous.get("reasoning_effort")
|
|
613
|
+
if not isinstance(model, str) or not model or model.startswith("<") or not effort or not previous.get("launch"):
|
|
614
|
+
raise CdxError("The previous run's model and effort cannot be pinned. Start a normal run instead.",
|
|
615
|
+
code="continue_run_unavailable")
|
|
616
|
+
if ((parsed.get("model") and parsed["model"] != model)
|
|
617
|
+
or (parsed.get("reasoning_effort") and parsed["reasoning_effort"] != effort)):
|
|
618
|
+
raise CdxError("--continue-run requires the previous run's model and reasoning effort.",
|
|
619
|
+
code="continue_run_settings_mismatch")
|
|
620
|
+
if parsed.get("failover") or parsed.get("isolate_workspace"):
|
|
621
|
+
raise CdxError("--continue-run cannot be combined with --failover or --isolate-workspace.",
|
|
622
|
+
code="continue_run_settings_mismatch")
|
|
623
|
+
if provider == "claude":
|
|
624
|
+
# Passthrough options are appended last and can override the pinned settings.
|
|
625
|
+
pinned_options = {"--model", "--effort", "--fallback-model", "--resume", "--continue",
|
|
626
|
+
"--session-id", "--fork-session", "--resume-session-at", "--from-pr",
|
|
627
|
+
"--settings", "--agent", "--agents"}
|
|
628
|
+
for arg in split_extra_args((session.get("launch") or {}).get("extra_args")) or []:
|
|
629
|
+
if arg.split("=", 1)[0] in pinned_options or arg.startswith(("-r", "-c")):
|
|
630
|
+
raise CdxError("--continue-run cannot use Claude extra_args that override model, effort, or conversation.",
|
|
631
|
+
code="continue_run_settings_mismatch")
|
|
632
|
+
parsed.update(name=name, model=model, reasoning_effort=effort, power=effort)
|
|
633
|
+
launch = {**(session.get("launch") or {}), "model": model, "reasoning_effort": effort, "power": effort}
|
|
634
|
+
# A provider fallback would break the model pin even though cdx stayed on this profile.
|
|
635
|
+
launch.pop("fallback_model", None)
|
|
636
|
+
return {**resumed, "launch": launch, "continue_run": parsed["continue_run"]}
|
|
637
|
+
|
|
638
|
+
|
|
540
639
|
def handle_run(rest, ctx):
|
|
541
640
|
registry = None
|
|
542
641
|
run_id = None
|
|
@@ -548,8 +647,12 @@ def handle_run(rest, ctx):
|
|
|
548
647
|
detached_run_id = os.environ.pop(DETACHED_RUN_ID_ENV, None) or None
|
|
549
648
|
try:
|
|
550
649
|
parsed = _parse_run_args(rest)
|
|
551
|
-
|
|
552
|
-
if parsed
|
|
650
|
+
budget_prices = token_prices()[0] if parsed.get("budget") is not None else None
|
|
651
|
+
session = _continue_run_session(parsed, ctx) if parsed.get("continue_run") else None
|
|
652
|
+
if session:
|
|
653
|
+
if session.get("enabled", True) is False:
|
|
654
|
+
raise CdxError(f"Session is disabled: {session['name']}", code="session_disabled")
|
|
655
|
+
elif parsed["name"]:
|
|
553
656
|
session = ctx["service"]["get_session"](parsed["name"])
|
|
554
657
|
if not session:
|
|
555
658
|
raise CdxError(f"Unknown session: {parsed['name']}")
|
|
@@ -604,7 +707,8 @@ def handle_run(rest, ctx):
|
|
|
604
707
|
# records its own afterwards, from the rollout. Without this a headless
|
|
605
708
|
# Claude run left the session with no identity and `cdx resume` fell
|
|
606
709
|
# back to recency for Claude while working by id for Codex.
|
|
607
|
-
|
|
710
|
+
if not parsed.get("continue_run"):
|
|
711
|
+
session = ctx["service"]["begin_conversation"](session["name"])
|
|
608
712
|
run_session = {
|
|
609
713
|
**session,
|
|
610
714
|
"launch": {
|
|
@@ -646,6 +750,10 @@ def handle_run(rest, ctx):
|
|
|
646
750
|
},
|
|
647
751
|
label=parsed.get("label"),
|
|
648
752
|
max_parallel=max_parallel,
|
|
753
|
+
parent_run_id=parsed.get("continue_run"),
|
|
754
|
+
conversation_id=(run_session.get("conversation") or {}).get("id") if run_session["provider"] == "claude" or parsed.get("continue_run") else None,
|
|
755
|
+
launch=run_session.get("launch"),
|
|
756
|
+
reasoning_effort=_launch_power(run_session),
|
|
649
757
|
)
|
|
650
758
|
if parsed.get("detach"):
|
|
651
759
|
# Delegate where the installed CLI can both start a background
|
|
@@ -654,7 +762,7 @@ def handle_run(rest, ctx):
|
|
|
654
762
|
launch = None
|
|
655
763
|
# A provider's own background agent has no cdx process watching it,
|
|
656
764
|
# so nothing could fail it over; --failover keeps the in-house child.
|
|
657
|
-
if not parsed.get("isolate_workspace") and not parsed.get("network_access") and not parsed.get("failover") and supports_native_background(
|
|
765
|
+
if not parsed.get("budget") and not parsed.get("continue_run") and not parsed.get("isolate_workspace") and not parsed.get("network_access") and not parsed.get("failover") and supports_native_background(
|
|
658
766
|
run_session.get("provider"),
|
|
659
767
|
env=ctx.get("env"),
|
|
660
768
|
spawn_sync=ctx.get("spawn_text"),
|
|
@@ -680,6 +788,7 @@ def handle_run(rest, ctx):
|
|
|
680
788
|
cwd=cwd,
|
|
681
789
|
pid=launch.get("pid"),
|
|
682
790
|
launch_log_path=launch.get("launch_log_path"),
|
|
791
|
+
conversation_id=launch.get("provider_session_id"),
|
|
683
792
|
selection_row=selection_row,
|
|
684
793
|
))
|
|
685
794
|
return 0
|
|
@@ -688,11 +797,29 @@ def handle_run(rest, ctx):
|
|
|
688
797
|
failover_error_code = None
|
|
689
798
|
recovery_events = []
|
|
690
799
|
attempt = 0
|
|
800
|
+
spent_usd = 0.0
|
|
691
801
|
models = capacity_models(run_session)
|
|
692
802
|
model_index = 0
|
|
693
803
|
capacity_retries = 0
|
|
694
804
|
while True:
|
|
695
805
|
attempt += 1
|
|
806
|
+
baseline = extract_interactive_usage(
|
|
807
|
+
run_session["provider"], run_session.get("authHome"),
|
|
808
|
+
conversation_id=(run_session.get("conversation") or {}).get("id"),
|
|
809
|
+
) if parsed.get("continue_run") and (run_session["provider"] == "codex" or parsed.get("budget")) else None
|
|
810
|
+
attempt_cost = None
|
|
811
|
+
budget_observation = None
|
|
812
|
+
|
|
813
|
+
def check_budget(paths, session=run_session, before=baseline, already_spent=spent_usd):
|
|
814
|
+
nonlocal attempt_cost, budget_observation
|
|
815
|
+
observation = _budget_observation(session, paths, before)
|
|
816
|
+
cost = estimate_cost(observation["usage"], observation["usage_model"], budget_prices)
|
|
817
|
+
if cost is not None:
|
|
818
|
+
if attempt_cost is None or cost >= attempt_cost:
|
|
819
|
+
attempt_cost = cost
|
|
820
|
+
budget_observation = observation
|
|
821
|
+
return attempt_cost is not None and already_spent + attempt_cost >= parsed["budget"]
|
|
822
|
+
|
|
696
823
|
run_info = _run_headless_provider_command(
|
|
697
824
|
run_session,
|
|
698
825
|
cwd=cwd,
|
|
@@ -701,14 +828,23 @@ def handle_run(rest, ctx):
|
|
|
701
828
|
timeout_seconds=parsed.get("timeout_seconds"),
|
|
702
829
|
spawn=ctx.get("spawn_headless") or ctx.get("spawn"),
|
|
703
830
|
run_id=run_id,
|
|
831
|
+
**({"budget_check": check_budget} if parsed.get("budget") is not None else {}),
|
|
704
832
|
)
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
833
|
+
observation = _observe_headless_usage(run_session, run_info, baseline if run_session["provider"] == "codex" else None)
|
|
834
|
+
if run_info.get("budget_exceeded") and budget_observation is not None:
|
|
835
|
+
observation = budget_observation
|
|
836
|
+
run_info = {**run_info, **observation,
|
|
837
|
+
"parent_run_id": parsed.get("continue_run"),
|
|
838
|
+
"conversation_id": extract_run_conversation_id(run_session.get("provider"), run_info.get("stdout_path"))
|
|
839
|
+
or ((run_session.get("conversation") or {}).get("id") if run_session["provider"] == "claude" or parsed.get("continue_run") else None),
|
|
840
|
+
"launch": run_session.get("launch"),
|
|
841
|
+
"reasoning_effort": _launch_power(run_session),
|
|
710
842
|
"effective_model": (run_session.get("launch") or {}).get("model"),
|
|
711
843
|
"recovery_events": recovery_events}
|
|
844
|
+
cost = estimate_cost(observation["usage"], observation["usage_model"], budget_prices)
|
|
845
|
+
spent_usd += max(attempt_cost or 0.0, cost or 0.0)
|
|
846
|
+
if parsed.get("budget") is not None and (run_info.get("budget_exceeded") or spent_usd >= parsed["budget"]):
|
|
847
|
+
run_info.update(budget_exceeded=True, returncode=125, cost_usd=spent_usd)
|
|
712
848
|
ctx["service"]["record_launch_history"](run_session["name"], {
|
|
713
849
|
"status": "success" if run_info.get("returncode") == 0 else "failed",
|
|
714
850
|
"action": "run",
|
|
@@ -718,6 +854,8 @@ def handle_run(rest, ctx):
|
|
|
718
854
|
"exit_code": run_info.get("returncode"),
|
|
719
855
|
**run_info,
|
|
720
856
|
})
|
|
857
|
+
if run_info.get("budget_exceeded") or parsed.get("continue_run"):
|
|
858
|
+
break
|
|
721
859
|
if looks_model_at_capacity(run_session.get("provider"), run_info):
|
|
722
860
|
event = {"reason": "model_at_capacity", "attempt": attempt,
|
|
723
861
|
"session": run_session["name"], "model": models[model_index]}
|
|
@@ -841,7 +979,10 @@ def handle_run(rest, ctx):
|
|
|
841
979
|
)
|
|
842
980
|
_write_json(ctx, final_payload)
|
|
843
981
|
return 0
|
|
844
|
-
if
|
|
982
|
+
if run_info.get("budget_exceeded"):
|
|
983
|
+
failover_error_code = "budget_exceeded"
|
|
984
|
+
message = "Run exceeded its estimated USD budget."
|
|
985
|
+
elif failover_error_code == "capacity_recovery_exhausted":
|
|
845
986
|
message = "Codex model capacity recovery exhausted its retry, model, or profile budget."
|
|
846
987
|
elif failover_error_code == "failover_exhausted":
|
|
847
988
|
message = f"Every eligible session was rate limited: {', '.join(attempted)}."
|
|
@@ -859,7 +1000,7 @@ def handle_run(rest, ctx):
|
|
|
859
1000
|
run_session,
|
|
860
1001
|
run_info=run_info,
|
|
861
1002
|
error=error,
|
|
862
|
-
error_source="provider",
|
|
1003
|
+
error_source="cdx" if run_info.get("budget_exceeded") else "provider",
|
|
863
1004
|
# Running out of accounts is a different outcome from the task
|
|
864
1005
|
# failing, and a caller has to be able to branch on which happened.
|
|
865
1006
|
error_code=failover_error_code or (
|
|
@@ -869,7 +1010,7 @@ def handle_run(rest, ctx):
|
|
|
869
1010
|
)
|
|
870
1011
|
registry.finish(
|
|
871
1012
|
run_id,
|
|
872
|
-
status="timed_out" if run_info.get("timed_out") else "failed",
|
|
1013
|
+
status="timed_out" if run_info.get("timed_out") and not run_info.get("budget_exceeded") else "failed",
|
|
873
1014
|
final_payload=final_payload,
|
|
874
1015
|
run_info=run_info,
|
|
875
1016
|
error=final_payload.get("error"),
|
|
@@ -883,11 +1024,12 @@ def handle_run(rest, ctx):
|
|
|
883
1024
|
if run_info is not None:
|
|
884
1025
|
run_info = {**run_info, "recovery_events": locals().get("recovery_events", [])}
|
|
885
1026
|
if run_info and run_id and not getattr(error, "history_recorded", False):
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
1027
|
+
observation = _observe_headless_usage(run_session, run_info, locals().get("baseline"))
|
|
1028
|
+
run_info = {**run_info, **observation,
|
|
1029
|
+
"parent_run_id": parsed.get("continue_run"),
|
|
1030
|
+
"conversation_id": extract_run_conversation_id(run_session.get("provider"), run_info.get("stdout_path"))
|
|
1031
|
+
or ((run_session.get("conversation") or {}).get("id") if run_session["provider"] == "claude" or parsed.get("continue_run") else None),
|
|
1032
|
+
"launch": run_session.get("launch"), "reasoning_effort": _launch_power(run_session)}
|
|
891
1033
|
ctx["service"]["record_launch_history"](run_session["name"], {
|
|
892
1034
|
"status": "failed",
|
|
893
1035
|
"action": "run",
|
package/src/commands/status.py
CHANGED
|
@@ -53,10 +53,13 @@ from ..provider_runtime import (
|
|
|
53
53
|
)
|
|
54
54
|
from ..rtk_savings import rtk_saved
|
|
55
55
|
from ..run_usage import (
|
|
56
|
-
|
|
56
|
+
TOKEN_USAGE_KEYS,
|
|
57
|
+
cache_hit_ratio,
|
|
57
58
|
estimate_cost,
|
|
59
|
+
format_cache_ratio,
|
|
58
60
|
is_vouched,
|
|
59
61
|
normalize_usage,
|
|
62
|
+
summarize_cache_usage,
|
|
60
63
|
token_prices,
|
|
61
64
|
weighted_usage,
|
|
62
65
|
)
|
|
@@ -294,7 +297,9 @@ def _summarize_stats(entries):
|
|
|
294
297
|
"reasoning_tokens": 0,
|
|
295
298
|
"total_tokens": 0,
|
|
296
299
|
"last_started_at": None,
|
|
300
|
+
"_entries": [],
|
|
297
301
|
})
|
|
302
|
+
row["_entries"].append(entry)
|
|
298
303
|
row["launches"] += 1
|
|
299
304
|
if entry.get("provider"):
|
|
300
305
|
row["provider"].add(entry["provider"])
|
|
@@ -315,7 +320,7 @@ def _summarize_stats(entries):
|
|
|
315
320
|
usage = {}
|
|
316
321
|
parsed_usage = {
|
|
317
322
|
key: _token_value(usage, key)
|
|
318
|
-
for key in
|
|
323
|
+
for key in TOKEN_USAGE_KEYS
|
|
319
324
|
}
|
|
320
325
|
if any(value is not None for value in parsed_usage.values()):
|
|
321
326
|
row["usage_runs"] += 1
|
|
@@ -361,6 +366,16 @@ def _summarize_stats(entries):
|
|
|
361
366
|
# overwhelming majority of cached tokens are, and weighting the whole
|
|
362
367
|
# of it at the creation rate would overstate legacy sessions by more
|
|
363
368
|
# than reading it as reads understates them.
|
|
369
|
+
group_entries = row.pop("_entries")
|
|
370
|
+
row["cache_usage"] = summarize_cache_usage(group_entries)
|
|
371
|
+
row["cache_hit_ratio"] = row["cache_usage"]["cache_hit_ratio"]
|
|
372
|
+
row["run_types"] = {
|
|
373
|
+
kind: {"runs": len(subset), **summarize_cache_usage(subset)}
|
|
374
|
+
for kind, subset in (
|
|
375
|
+
("continued", [e for e in group_entries if e.get("parent_run_id")]),
|
|
376
|
+
("cold", [e for e in group_entries if not e.get("parent_run_id")]),
|
|
377
|
+
)
|
|
378
|
+
}
|
|
364
379
|
split = row["cache_creation_tokens"] + row["cache_read_tokens"]
|
|
365
380
|
legacy_cache = max(0, row["cached_input_tokens"] - split)
|
|
366
381
|
derived = normalize_usage(
|
|
@@ -381,15 +396,34 @@ def _summarize_stats(entries):
|
|
|
381
396
|
reverse=True,
|
|
382
397
|
)
|
|
383
398
|
|
|
399
|
+
def _merge_cache_usage(summaries):
|
|
400
|
+
totals = {}
|
|
401
|
+
for key in ("input_tokens", "cache_creation_tokens", "cache_read_tokens", "output_tokens",
|
|
402
|
+
"measured_runs", "measured_input_tokens", "measured_cache_read_tokens"):
|
|
403
|
+
values = [s[key] for s in summaries if s[key] is not None]
|
|
404
|
+
totals[key] = sum(values) if values else (0 if key.startswith("measured_") else None)
|
|
405
|
+
inputs = totals["measured_input_tokens"]
|
|
406
|
+
totals["cache_hit_ratio"] = totals["measured_cache_read_tokens"] / inputs if inputs else None
|
|
407
|
+
return totals
|
|
408
|
+
|
|
409
|
+
|
|
384
410
|
def _stats_totals(rows):
|
|
411
|
+
cache = _merge_cache_usage([row["cache_usage"] for row in rows])
|
|
385
412
|
return {
|
|
413
|
+
"cache_usage": cache,
|
|
414
|
+
"cache_hit_ratio": cache["cache_hit_ratio"],
|
|
415
|
+
"run_types": {
|
|
416
|
+
kind: {"runs": sum(row["run_types"][kind]["runs"] for row in rows),
|
|
417
|
+
**_merge_cache_usage([row["run_types"][kind] for row in rows])}
|
|
418
|
+
for kind in ("cold", "continued")
|
|
419
|
+
},
|
|
386
420
|
"sessions": len(rows),
|
|
387
421
|
"launches": sum(row["launches"] for row in rows),
|
|
388
422
|
"successes": sum(row["successes"] for row in rows),
|
|
389
423
|
"failures": sum(row["failures"] for row in rows),
|
|
390
424
|
"duration_ms": sum(row["duration_ms"] for row in rows),
|
|
391
425
|
"usage_runs": sum(row["usage_runs"] for row in rows),
|
|
392
|
-
**{key: sum(row[key] for row in rows) for key in
|
|
426
|
+
**{key: sum(row[key] for row in rows) for key in TOKEN_USAGE_KEYS},
|
|
393
427
|
"weighted_tokens": sum(row["weighted_tokens"] for row in rows),
|
|
394
428
|
"weighted_runs": sum(row["weighted_runs"] for row in rows),
|
|
395
429
|
"unweighted_runs": sum(row["unweighted_runs"] for row in rows),
|
|
@@ -448,7 +482,7 @@ def _format_history(entries, use_color=False, active_sessions=None):
|
|
|
448
482
|
active_sessions = active_sessions or set()
|
|
449
483
|
if not entries:
|
|
450
484
|
return "No launch history."
|
|
451
|
-
rows = [[_style(value, "1", use_color) for value in ["SESSION", "PROV.", "RESULT", "DURATION", "WHEN", "TRANSCRIPT"]]]
|
|
485
|
+
rows = [[_style(value, "1", use_color) for value in ["SESSION", "PROV.", "RESULT", "DURATION", "WHEN", "TOKENS", "TRANSCRIPT"]]]
|
|
452
486
|
for entry in entries:
|
|
453
487
|
transcript_path = entry.get("transcript_path")
|
|
454
488
|
session_name = entry.get("session_name") or "-"
|
|
@@ -461,6 +495,8 @@ def _format_history(entries, use_color=False, active_sessions=None):
|
|
|
461
495
|
_style(status, status_color, use_color),
|
|
462
496
|
_style(_format_duration_ms(entry.get("duration_ms")), "33" if entry.get("duration_ms") else "2", use_color),
|
|
463
497
|
_dim(_format_relative_age(entry.get("started_at")), use_color),
|
|
498
|
+
_dim(f"{_format_token_count((entry.get('usage') or {}).get('total_tokens'))} "
|
|
499
|
+
f"{format_cache_ratio((entry.get('usage') or {}).get('cache_hit_ratio'))}", use_color),
|
|
464
500
|
_dim(os.path.basename(transcript_path) if transcript_path else "-", use_color),
|
|
465
501
|
])
|
|
466
502
|
return "\n".join([
|
|
@@ -529,10 +565,12 @@ def _format_stats(rows, totals, period=None, use_color=False, active_sessions=No
|
|
|
529
565
|
_style(str(row["launches"]), "1", use_color),
|
|
530
566
|
_style(str(row["usage_runs"]), "32" if row["usage_runs"] else "2", use_color),
|
|
531
567
|
_style(_format_token_count(row["input_tokens"]), "96" if row["input_tokens"] else "2", use_color),
|
|
532
|
-
_style(_format_token_count(row["cached_input_tokens"])
|
|
568
|
+
_style(_format_token_count(row["cached_input_tokens"]) if row["cache_usage"]["cache_read_tokens"] is not None
|
|
569
|
+
or row["cache_usage"]["cache_creation_tokens"] is not None else "n/d",
|
|
570
|
+
"96" if row["cached_input_tokens"] else "2", use_color),
|
|
533
571
|
_style(_format_token_count(row["output_tokens"]), "96" if row["output_tokens"] else "2", use_color),
|
|
534
572
|
_style(_format_token_count(row["reasoning_tokens"]), "95" if row["reasoning_tokens"] else "2", use_color),
|
|
535
|
-
_style(_format_token_count(row[
|
|
573
|
+
_style(f"{_format_token_count(row['total_tokens'])} {format_cache_ratio(row.get('cache_hit_ratio'))}", "96" if row["total_tokens"] else "2", use_color),
|
|
536
574
|
_style(_format_token_count(row["weighted_tokens"]) if row["weighted_runs"] else "-",
|
|
537
575
|
"1;95" if row["weighted_runs"] else "2", use_color),
|
|
538
576
|
_style(_format_usd(row["cost_usd"]) if row["priced_runs"] else "-",
|
|
@@ -550,7 +588,7 @@ def _format_stats(rows, totals, period=None, use_color=False, active_sessions=No
|
|
|
550
588
|
_dim(
|
|
551
589
|
"Totals: "
|
|
552
590
|
f"{totals['launches']} runs, {totals['usage_runs']} with usage, "
|
|
553
|
-
f"{_format_token_count(totals['total_tokens'])} tokens "
|
|
591
|
+
f"{_format_token_count(totals['total_tokens'])} tokens {format_cache_ratio(totals.get('cache_hit_ratio'))} "
|
|
554
592
|
f"({_format_token_count(totals['weighted_tokens']) if totals['weighted_runs'] else '-'} cost-equivalent"
|
|
555
593
|
+ (f" on {totals['weighted_runs']}/{totals['usage_runs']} measured runs"
|
|
556
594
|
if totals["usage_runs"] else "")
|
|
@@ -633,6 +671,10 @@ def handle_history(rest, ctx):
|
|
|
633
671
|
limit = 0 if parsed["summary"] or has_period else parsed["limit"]
|
|
634
672
|
entries = ctx["service"]["get_launch_history"](parsed["name"], limit=limit)
|
|
635
673
|
entries = _filter_history_period(entries, parsed["period"])
|
|
674
|
+
entries = [{**entry, "parent_run_id": entry.get("parent_run_id"),
|
|
675
|
+
"cost_usd": estimate_cost(entry.get("usage"), entry.get("usage_model")) if is_vouched(entry.get("usage")) else None,
|
|
676
|
+
"usage": {**entry["usage"], "cache_hit_ratio": cache_hit_ratio(entry["usage"], provider=entry.get("provider")) if is_vouched(entry["usage"]) else None}
|
|
677
|
+
if isinstance(entry.get("usage"), dict) else entry.get("usage")} for entry in entries]
|
|
636
678
|
if has_period and not parsed["summary"]:
|
|
637
679
|
entries = entries[:parsed["limit"]]
|
|
638
680
|
message = (
|
package/src/interactive_usage.py
CHANGED
|
@@ -261,9 +261,13 @@ def _codex_usage(handle):
|
|
|
261
261
|
# larger than the input count is not a subset, so that record is
|
|
262
262
|
# left alone rather than forced to zero.
|
|
263
263
|
reported_input = _number(usage.get("input_tokens"))
|
|
264
|
-
cache_read = _number(usage.get("cached_input_tokens")
|
|
264
|
+
cache_read = _number(usage["cached_input_tokens"]) if usage.get("cached_input_tokens") is not None else None
|
|
265
|
+
cache_write = _number(usage["cache_write_input_tokens"]) if usage.get("cache_write_input_tokens") is not None else None
|
|
266
|
+
cached = (cache_read or 0) + (cache_write or 0)
|
|
265
267
|
latest = normalize_usage(
|
|
266
|
-
|
|
268
|
+
provider="codex",
|
|
269
|
+
input_tokens=reported_input - cached if cached <= reported_input else reported_input,
|
|
270
|
+
cache_creation_tokens=cache_write,
|
|
267
271
|
cache_read_tokens=cache_read,
|
|
268
272
|
output_tokens=_number(usage.get("output_tokens")),
|
|
269
273
|
reasoning_tokens=_number(usage.get("reasoning_output_tokens")) or None,
|
|
@@ -286,6 +290,7 @@ def _claude_usage(handle):
|
|
|
286
290
|
inside `output_tokens` -- so that field stays absent by design.
|
|
287
291
|
"""
|
|
288
292
|
totals = {"input_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 0, "output_tokens": 0}
|
|
293
|
+
missing = set(totals)
|
|
289
294
|
long_ttl = None
|
|
290
295
|
seen = set()
|
|
291
296
|
model = None
|
|
@@ -301,6 +306,11 @@ def _claude_usage(handle):
|
|
|
301
306
|
if identifier in seen:
|
|
302
307
|
continue
|
|
303
308
|
seen.add(identifier)
|
|
309
|
+
for source, target in (("input_tokens", "input_tokens"), ("output_tokens", "output_tokens"),
|
|
310
|
+
("cache_creation_input_tokens", "cache_creation_tokens"),
|
|
311
|
+
("cache_read_input_tokens", "cache_read_tokens")):
|
|
312
|
+
if usage.get(source) is not None:
|
|
313
|
+
missing.discard(target)
|
|
304
314
|
totals["input_tokens"] += _number(usage.get("input_tokens"))
|
|
305
315
|
totals["cache_creation_tokens"] += _number(usage.get("cache_creation_input_tokens"))
|
|
306
316
|
totals["cache_read_tokens"] += _number(usage.get("cache_read_input_tokens"))
|
|
@@ -316,7 +326,8 @@ def _claude_usage(handle):
|
|
|
316
326
|
continue
|
|
317
327
|
if not seen and not any(totals.values()):
|
|
318
328
|
return None, model
|
|
319
|
-
return normalize_usage(**
|
|
329
|
+
return normalize_usage(**{k: None if k in missing else v for k, v in totals.items()},
|
|
330
|
+
cache_creation_1h_tokens=long_ttl), model
|
|
320
331
|
|
|
321
332
|
|
|
322
333
|
def _number(value):
|
|
@@ -326,7 +337,7 @@ def _number(value):
|
|
|
326
337
|
return 0
|
|
327
338
|
|
|
328
339
|
|
|
329
|
-
#: The quantities a provider actually reports. The other
|
|
340
|
+
#: The quantities a provider actually reports. The other fields of a usage
|
|
330
341
|
#: record are derived by `normalize_usage` and must never be differenced.
|
|
331
342
|
MEASURED_KEYS = (
|
|
332
343
|
"input_tokens",
|
|
@@ -338,7 +349,7 @@ MEASURED_KEYS = (
|
|
|
338
349
|
)
|
|
339
350
|
|
|
340
351
|
|
|
341
|
-
def usage_delta(cumulative, previous):
|
|
352
|
+
def usage_delta(cumulative, previous, provider=None):
|
|
342
353
|
"""What this run added to a transcript that already held `previous`.
|
|
343
354
|
|
|
344
355
|
Both readers return a *cumulative* figure: Claude's is the sum over the
|
|
@@ -372,7 +383,7 @@ def usage_delta(cumulative, previous):
|
|
|
372
383
|
if difference < 0:
|
|
373
384
|
return None
|
|
374
385
|
parts[key] = difference
|
|
375
|
-
return normalize_usage(**parts)
|
|
386
|
+
return normalize_usage(**parts, provider=provider)
|
|
376
387
|
|
|
377
388
|
|
|
378
389
|
def transcript_predates_run(path, started_at):
|
|
@@ -224,6 +224,7 @@ def read_transcript_outcome(path):
|
|
|
224
224
|
if not path:
|
|
225
225
|
return outcome
|
|
226
226
|
totals = {"input_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 0, "output_tokens": 0}
|
|
227
|
+
missing = set(totals)
|
|
227
228
|
long_ttl = None
|
|
228
229
|
# This path had no de-duplication at all, so a transcript that records the
|
|
229
230
|
# same billed response more than once inflated the total outright.
|
|
@@ -266,6 +267,11 @@ def read_transcript_outcome(path):
|
|
|
266
267
|
seen = True
|
|
267
268
|
if isinstance(message.get("model"), str) and message["model"]:
|
|
268
269
|
models.add(message["model"])
|
|
270
|
+
for source, target in (("input_tokens", "input_tokens"), ("output_tokens", "output_tokens"),
|
|
271
|
+
("cache_creation_input_tokens", "cache_creation_tokens"),
|
|
272
|
+
("cache_read_input_tokens", "cache_read_tokens")):
|
|
273
|
+
if usage.get(source) is not None:
|
|
274
|
+
missing.discard(target)
|
|
269
275
|
totals["input_tokens"] += usage.get("input_tokens") or 0
|
|
270
276
|
totals["cache_creation_tokens"] += usage.get("cache_creation_input_tokens") or 0
|
|
271
277
|
totals["cache_read_tokens"] += usage.get("cache_read_input_tokens") or 0
|
|
@@ -276,7 +282,8 @@ def read_transcript_outcome(path):
|
|
|
276
282
|
except OSError:
|
|
277
283
|
return outcome
|
|
278
284
|
if seen:
|
|
279
|
-
outcome["usage"] = normalize_usage(**
|
|
285
|
+
outcome["usage"] = normalize_usage(**{k: None if k in missing else v for k, v in totals.items()},
|
|
286
|
+
cache_creation_1h_tokens=long_ttl)
|
|
280
287
|
if len(models) == 1:
|
|
281
288
|
outcome["usage_model"] = next(iter(models))
|
|
282
289
|
return outcome
|
package/src/provider_runtime.py
CHANGED
|
@@ -8,6 +8,7 @@ import signal
|
|
|
8
8
|
import subprocess
|
|
9
9
|
import sys
|
|
10
10
|
import threading
|
|
11
|
+
import time
|
|
11
12
|
import uuid
|
|
12
13
|
from datetime import datetime, timezone
|
|
13
14
|
|
|
@@ -1006,7 +1007,7 @@ def _build_headless_launch_spec(session, cwd=None, env_override=None, initial_pr
|
|
|
1006
1007
|
args = ["--print", "--output-format", "json"]
|
|
1007
1008
|
conversation_id = _conversation_id(session)
|
|
1008
1009
|
if conversation_id:
|
|
1009
|
-
args += ["--session-id", conversation_id]
|
|
1010
|
+
args += ["--resume" if session.get("continue_run") else "--session-id", conversation_id]
|
|
1010
1011
|
if model:
|
|
1011
1012
|
args += ["--model", _claude_cli_model(model)]
|
|
1012
1013
|
# --max-budget-usd and --fallback-model are --print-only in the claude
|
|
@@ -1045,6 +1046,8 @@ def _build_headless_launch_spec(session, cwd=None, env_override=None, initial_pr
|
|
|
1045
1046
|
if permission:
|
|
1046
1047
|
args += HEADLESS_CODEX_PERMISSION_ARGS.get(permission, [])
|
|
1047
1048
|
args += preference_args
|
|
1049
|
+
if session.get("continue_run"):
|
|
1050
|
+
args += ["resume", _conversation_id(session)]
|
|
1048
1051
|
if initial_prompt:
|
|
1049
1052
|
# `--` keeps a prompt that starts with `-` from being read as an option.
|
|
1050
1053
|
args += ["--", initial_prompt]
|
|
@@ -1202,7 +1205,7 @@ def _read_permission_denials(stdout_path):
|
|
|
1202
1205
|
|
|
1203
1206
|
|
|
1204
1207
|
def _run_headless_provider_command(session, cwd=None, env_override=None, initial_prompt=None,
|
|
1205
|
-
timeout_seconds=None, spawn=None, run_id=None):
|
|
1208
|
+
timeout_seconds=None, spawn=None, run_id=None, budget_check=None):
|
|
1206
1209
|
spawn = spawn or subprocess.Popen
|
|
1207
1210
|
spec = _build_headless_launch_spec(
|
|
1208
1211
|
session,
|
|
@@ -1260,6 +1263,7 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
|
|
|
1260
1263
|
child = None
|
|
1261
1264
|
timed_out = False
|
|
1262
1265
|
cancelled = False
|
|
1266
|
+
budget_exceeded = False
|
|
1263
1267
|
with open(paths["stdout_path"], "w", encoding="utf-8", errors="replace") as stdout_file, \
|
|
1264
1268
|
open(paths["stderr_path"], "w", encoding="utf-8", errors="replace") as stderr_file:
|
|
1265
1269
|
options = {k: v for k, v in spec.get("options", {}).items() if k not in ("stdio", "stdout", "stderr")}
|
|
@@ -1288,7 +1292,23 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
|
|
|
1288
1292
|
# this supervisor and keep its run "running" forever.
|
|
1289
1293
|
previous_handlers = _route_termination_to_interrupt()
|
|
1290
1294
|
try:
|
|
1291
|
-
if
|
|
1295
|
+
if budget_check is not None:
|
|
1296
|
+
deadline = time.monotonic() + timeout_seconds if timeout_seconds is not None else None
|
|
1297
|
+
while True:
|
|
1298
|
+
if budget_check(paths):
|
|
1299
|
+
budget_exceeded = True
|
|
1300
|
+
_terminate_child_tree(child)
|
|
1301
|
+
break
|
|
1302
|
+
remaining = deadline - time.monotonic() if deadline is not None else None
|
|
1303
|
+
if remaining is not None and remaining <= 0:
|
|
1304
|
+
raise subprocess.TimeoutExpired(command, timeout_seconds)
|
|
1305
|
+
try:
|
|
1306
|
+
child.wait(timeout=min(0.25, remaining) if remaining is not None else 0.25)
|
|
1307
|
+
budget_exceeded = budget_check(paths)
|
|
1308
|
+
break
|
|
1309
|
+
except subprocess.TimeoutExpired:
|
|
1310
|
+
continue
|
|
1311
|
+
elif timeout_seconds is None:
|
|
1292
1312
|
child.wait()
|
|
1293
1313
|
else:
|
|
1294
1314
|
child.wait(timeout=timeout_seconds)
|
|
@@ -1335,6 +1355,7 @@ def _run_headless_provider_command(session, cwd=None, env_override=None, initial
|
|
|
1335
1355
|
"pid": getattr(child, "pid", None),
|
|
1336
1356
|
"returncode": returncode,
|
|
1337
1357
|
"timed_out": timed_out,
|
|
1358
|
+
"budget_exceeded": budget_exceeded,
|
|
1338
1359
|
"permission_denials": (
|
|
1339
1360
|
_read_permission_denials(paths["stdout_path"])
|
|
1340
1361
|
if session.get("provider") == PROVIDER_CLAUDE else []
|