claude-dev-env 2.17.0 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/orchestrator/SKILL.md +6 -4
- package/.agents/skills/orchestrator-refresh/SKILL.md +9 -5
- package/.agents/skills/team-advisor/SKILL.md +5 -5
- package/_shared/advisor/AGENTS.md +5 -5
- package/_shared/advisor/advisor-protocol.md +54 -31
- package/_shared/advisor/reference/advisor-block.md +5 -1
- package/_shared/advisor/reference/consult-format.md +1 -1
- package/_shared/advisor/reference/identity.md +28 -0
- package/_shared/advisor/reference/lifecycle.md +8 -1
- package/_shared/advisor/reference/sol-rung.md +12 -7
- package/_shared/advisor/reference/spawn-walk-log.md +6 -5
- package/_shared/advisor/reference/third-party-bind.md +5 -6
- package/_shared/advisor/reference/warm-up.md +9 -2
- package/_shared/advisor/scripts/codex_sol_advisor.py +66 -11
- package/_shared/advisor/scripts/config/advisor_scripts_constants/advisor_route_constants.py +9 -0
- package/_shared/advisor/scripts/config/advisor_scripts_constants/model_tier_run_validator_constants.py +16 -4
- package/_shared/advisor/scripts/config/advisor_scripts_constants/sol_advisor_constants.py +11 -10
- package/_shared/advisor/scripts/model_tier_run_validator.py +50 -23
- package/_shared/advisor/scripts/tests/test_codex_sol_advisor.py +108 -3
- package/_shared/advisor/scripts/tests/test_model_tier_run_validator.py +171 -82
- package/_shared/advisor/scripts/tests/test_tier_model_ids.py +43 -0
- package/_shared/advisor/scripts/tier_model_ids.py +75 -7
- package/_shared/pr-loop/worker-spawn.md +4 -3
- package/docs/references/team-advisor-skill.md +2 -2
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Bind and consult a read-only Codex CLI session at Sol
|
|
1
|
+
"""Bind and consult a read-only Codex CLI session at Sol low effort."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -22,6 +22,8 @@ if _config_directory_text not in sys.path:
|
|
|
22
22
|
|
|
23
23
|
from advisor_scripts_constants.sol_advisor_constants import ( # noqa: E402
|
|
24
24
|
ADVISOR_CODEX_EXECUTABLE_ENV_VAR,
|
|
25
|
+
ALL_SOL_TRUTHY_VALUES,
|
|
26
|
+
CLAUDE_CONFIG_DIRECTORY_NAME,
|
|
25
27
|
CODEX_CONFIG_FLAG,
|
|
26
28
|
CODEX_EXECUTABLE,
|
|
27
29
|
CODEX_EXEC_SUBCOMMAND,
|
|
@@ -29,15 +31,15 @@ from advisor_scripts_constants.sol_advisor_constants import ( # noqa: E402
|
|
|
29
31
|
CODEX_MODEL_FLAG,
|
|
30
32
|
CODEX_PROMPT_FROM_STDIN,
|
|
31
33
|
CODEX_READ_ONLY_SANDBOX,
|
|
32
|
-
|
|
34
|
+
CODEX_REASONING_CONFIG_TEMPLATE,
|
|
33
35
|
CODEX_RESUME_SUBCOMMAND,
|
|
34
36
|
CODEX_SANDBOX_FLAG,
|
|
35
|
-
CLAUDE_CONFIG_DIRECTORY_NAME,
|
|
36
37
|
SOL_BIND_FAILURE_REASON,
|
|
37
38
|
SOL_CODEX_TIMEOUT_REASON,
|
|
38
39
|
SOL_CODEX_TIMEOUT_SECONDS,
|
|
39
|
-
|
|
40
|
+
SOL_EFFORT_FLAG,
|
|
40
41
|
SOL_ENABLE_FLAG,
|
|
42
|
+
SOL_ENV_VAR,
|
|
41
43
|
SOL_EXECUTABLE_NOT_FOUND_REASON,
|
|
42
44
|
SOL_FALLBACK_KIND_BROKEN,
|
|
43
45
|
SOL_FALLBACK_KIND_DECLINED,
|
|
@@ -48,14 +50,16 @@ from advisor_scripts_constants.sol_advisor_constants import ( # noqa: E402
|
|
|
48
50
|
SOL_PROBE_TIMEOUT_REASON,
|
|
49
51
|
SOL_REPLY_FAILURE_REASON,
|
|
50
52
|
SOL_SESSION_ID_METAVAR,
|
|
51
|
-
ALL_SOL_TRUTHY_VALUES,
|
|
52
53
|
SOL_USAGE_PROBE_TIMEOUT_SECONDS,
|
|
53
54
|
)
|
|
54
55
|
from advisor_scripts_constants.advisor_route_constants import ( # noqa: E402
|
|
55
56
|
ADVISOR_CODEX_MODEL_ID,
|
|
57
|
+
ADVISOR_EFFORT_DEFAULT,
|
|
58
|
+
ADVISOR_EFFORT_ENV_VAR,
|
|
56
59
|
ADVISOR_FALLBACK_RESULT,
|
|
57
60
|
ADVISOR_FALLBACK_TIER,
|
|
58
61
|
ADVISOR_MODEL_TIER,
|
|
62
|
+
ALL_ADVISOR_EFFORT_LEVELS,
|
|
59
63
|
ALL_ADVISOR_GUIDANCE_SIGNALS,
|
|
60
64
|
CODEX_BIND_SUCCESS_TOKEN,
|
|
61
65
|
SPAWN_OUTCOME_KEY,
|
|
@@ -143,7 +147,7 @@ def _resolved_setting_by_name(
|
|
|
143
147
|
def is_sol_advisor_enabled(
|
|
144
148
|
setting_by_name: Mapping[str, str] | None,
|
|
145
149
|
) -> bool:
|
|
146
|
-
"""Return whether the optional Sol
|
|
150
|
+
"""Return whether the optional Sol rung is enabled.
|
|
147
151
|
|
|
148
152
|
Args:
|
|
149
153
|
setting_by_name: Optional environment-like settings mapping.
|
|
@@ -158,6 +162,42 @@ def is_sol_advisor_enabled(
|
|
|
158
162
|
)
|
|
159
163
|
|
|
160
164
|
|
|
165
|
+
def resolve_advisor_effort(
|
|
166
|
+
setting_by_name: Mapping[str, str] | None,
|
|
167
|
+
) -> str:
|
|
168
|
+
"""Return the shared advisor effort from settings, or the default.
|
|
169
|
+
|
|
170
|
+
::
|
|
171
|
+
|
|
172
|
+
resolve_advisor_effort({"ADVISOR_EFFORT": "medium"})
|
|
173
|
+
# ok: "medium"
|
|
174
|
+
resolve_advisor_effort({"ADVISOR_EFFORT": "MAX"})
|
|
175
|
+
# ok: "max"
|
|
176
|
+
resolve_advisor_effort({})
|
|
177
|
+
# ok: default low
|
|
178
|
+
resolve_advisor_effort({"ADVISOR_EFFORT": "nope"})
|
|
179
|
+
# ok: default low
|
|
180
|
+
resolve_advisor_effort({"ADVISOR_SOL_EFFORT": "high"})
|
|
181
|
+
# ok: default low
|
|
182
|
+
|
|
183
|
+
Fable and Sol both read this value. Unset and unrecognized values use
|
|
184
|
+
low. The Sol-only name does not set effort.
|
|
185
|
+
|
|
186
|
+
Args:
|
|
187
|
+
setting_by_name: Optional environment-like settings mapping.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
One of low, medium, high, xhigh, or max.
|
|
191
|
+
"""
|
|
192
|
+
resolved_setting_by_name = _resolved_setting_by_name(setting_by_name)
|
|
193
|
+
requested_effort = (
|
|
194
|
+
resolved_setting_by_name.get(ADVISOR_EFFORT_ENV_VAR, "").strip().lower()
|
|
195
|
+
)
|
|
196
|
+
if requested_effort in ALL_ADVISOR_EFFORT_LEVELS:
|
|
197
|
+
return requested_effort
|
|
198
|
+
return ADVISOR_EFFORT_DEFAULT
|
|
199
|
+
|
|
200
|
+
|
|
161
201
|
def resolve_usage_probe_path(home_directory: Path) -> Path:
|
|
162
202
|
"""Return the installed Codex weekly usage probe path.
|
|
163
203
|
|
|
@@ -290,12 +330,14 @@ def resolve_codex_executable(
|
|
|
290
330
|
def build_codex_arguments(
|
|
291
331
|
codex_executable: str,
|
|
292
332
|
session_id: str | None = None,
|
|
333
|
+
reasoning_effort: str = ADVISOR_EFFORT_DEFAULT,
|
|
293
334
|
) -> list[str]:
|
|
294
335
|
"""Build the installed CLI's shell-free bind or resume argv.
|
|
295
336
|
|
|
296
337
|
Args:
|
|
297
338
|
codex_executable: Resolved executable name or path to invoke.
|
|
298
339
|
session_id: Optional existing session to resume.
|
|
340
|
+
reasoning_effort: Codex model_reasoning_effort token.
|
|
299
341
|
|
|
300
342
|
Returns:
|
|
301
343
|
The shell-free Codex command argument vector.
|
|
@@ -306,7 +348,7 @@ def build_codex_arguments(
|
|
|
306
348
|
CODEX_MODEL_FLAG,
|
|
307
349
|
ADVISOR_CODEX_MODEL_ID,
|
|
308
350
|
CODEX_CONFIG_FLAG,
|
|
309
|
-
|
|
351
|
+
CODEX_REASONING_CONFIG_TEMPLATE.format(effort=reasoning_effort),
|
|
310
352
|
CODEX_SANDBOX_FLAG,
|
|
311
353
|
CODEX_READ_ONLY_SANDBOX,
|
|
312
354
|
CODEX_JSON_FLAG,
|
|
@@ -435,7 +477,11 @@ def run_codex_sol_advisor(
|
|
|
435
477
|
)
|
|
436
478
|
try:
|
|
437
479
|
completed_process = process_runner(
|
|
438
|
-
build_codex_arguments(
|
|
480
|
+
build_codex_arguments(
|
|
481
|
+
codex_executable,
|
|
482
|
+
session_id=session_id,
|
|
483
|
+
reasoning_effort=resolve_advisor_effort(setting_by_name),
|
|
484
|
+
),
|
|
439
485
|
cwd=str(working_directory),
|
|
440
486
|
input=prompt,
|
|
441
487
|
capture_output=True,
|
|
@@ -467,7 +513,7 @@ def build_argument_parser() -> argparse.ArgumentParser:
|
|
|
467
513
|
The parser for the helper's bind and resume modes.
|
|
468
514
|
"""
|
|
469
515
|
argument_parser = argparse.ArgumentParser(
|
|
470
|
-
description="Bind or consult a read-only Codex Sol
|
|
516
|
+
description="Bind or consult a read-only Codex Sol advisor."
|
|
471
517
|
)
|
|
472
518
|
mode_group = argument_parser.add_mutually_exclusive_group(required=True)
|
|
473
519
|
mode_group.add_argument("--bind", action="store_true")
|
|
@@ -479,6 +525,13 @@ def build_argument_parser() -> argparse.ArgumentParser:
|
|
|
479
525
|
action="store_true",
|
|
480
526
|
help="Open the Sol rung for this invocation without an environment flag.",
|
|
481
527
|
)
|
|
528
|
+
argument_parser.add_argument(
|
|
529
|
+
SOL_EFFORT_FLAG,
|
|
530
|
+
dest="sol_effort",
|
|
531
|
+
choices=ALL_ADVISOR_EFFORT_LEVELS,
|
|
532
|
+
default=None,
|
|
533
|
+
help="Shared advisor effort for this invocation.",
|
|
534
|
+
)
|
|
482
535
|
return argument_parser
|
|
483
536
|
|
|
484
537
|
|
|
@@ -492,9 +545,11 @@ def main(all_cli_arguments: Sequence[str]) -> int:
|
|
|
492
545
|
Zero for a successful Sol response, or one for an explicit fallback.
|
|
493
546
|
"""
|
|
494
547
|
parsed_arguments = build_argument_parser().parse_args(list(all_cli_arguments))
|
|
495
|
-
setting_by_name:
|
|
548
|
+
setting_by_name: dict[str, str] = dict(os.environ)
|
|
496
549
|
if parsed_arguments.is_sol_requested:
|
|
497
|
-
setting_by_name =
|
|
550
|
+
setting_by_name[SOL_ENV_VAR] = "1"
|
|
551
|
+
if parsed_arguments.sol_effort is not None:
|
|
552
|
+
setting_by_name[ADVISOR_EFFORT_ENV_VAR] = parsed_arguments.sol_effort
|
|
498
553
|
advisor_reply = run_codex_sol_advisor(
|
|
499
554
|
prompt=sys.stdin.read(),
|
|
500
555
|
working_directory=parsed_arguments.cwd,
|
|
@@ -6,6 +6,15 @@ ADVISOR_MODEL_TIER: str = "Sol"
|
|
|
6
6
|
ADVISOR_CODEX_MODEL_ID: str = "gpt-5.6-sol"
|
|
7
7
|
ADVISOR_FALLBACK_TIER: str = "Fable"
|
|
8
8
|
ADVISOR_FALLBACK_RESULT: str = "fable"
|
|
9
|
+
ADVISOR_EFFORT_ENV_VAR: str = "ADVISOR_EFFORT"
|
|
10
|
+
ADVISOR_EFFORT_DEFAULT: str = "low"
|
|
11
|
+
ALL_ADVISOR_EFFORT_LEVELS: tuple[str, ...] = (
|
|
12
|
+
"low",
|
|
13
|
+
"medium",
|
|
14
|
+
"high",
|
|
15
|
+
"xhigh",
|
|
16
|
+
"max",
|
|
17
|
+
)
|
|
9
18
|
ALL_ADVISOR_GUIDANCE_SIGNALS: frozenset[str] = frozenset(
|
|
10
19
|
{"ENDORSE", "CORRECTION", "PLAN", "STOP"}
|
|
11
20
|
)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Constants for the model-tier-run validator and CLI / Agent alias map.
|
|
2
2
|
|
|
3
3
|
They name the parts of a spawn-walk log: the tier ladder (strongest
|
|
4
|
-
first), the
|
|
4
|
+
first), the three host profiles, and the validation messages. Shared log
|
|
5
5
|
keys and bind-result tokens live in ``advisor_route_constants``.
|
|
6
6
|
|
|
7
7
|
::
|
|
@@ -11,9 +11,15 @@ The alias map turns each tier into its short CLI / Agent name (``opus``,
|
|
|
11
11
|
|
|
12
12
|
Host-profile detection (see ``detect_host_profile``):
|
|
13
13
|
|
|
14
|
-
- ``ADVISOR_HOST_PROFILE=ThirdParty
|
|
14
|
+
- ``ADVISOR_HOST_PROFILE=ThirdParty``, ``=Claude``, or ``=Codex`` — explicit override
|
|
15
15
|
- ``THIRD_PARTY=1`` (or ``true`` / ``yes``) — a third-party (non-Claude) harness
|
|
16
16
|
- default when neither is set: Claude
|
|
17
|
+
|
|
18
|
+
Session identity (see ``resolve_session_identity``):
|
|
19
|
+
|
|
20
|
+
- a ``codex`` token maps to Codex
|
|
21
|
+
- a ``claude`` token maps to Claude
|
|
22
|
+
- any other identity maps to ThirdParty
|
|
17
23
|
"""
|
|
18
24
|
|
|
19
25
|
from __future__ import annotations
|
|
@@ -24,11 +30,17 @@ from advisor_scripts_constants.advisor_route_constants import (
|
|
|
24
30
|
)
|
|
25
31
|
|
|
26
32
|
HOST_PROFILE_CLAUDE: str = "Claude"
|
|
33
|
+
HOST_PROFILE_CODEX: str = "Codex"
|
|
27
34
|
HOST_PROFILE_THIRD_PARTY: str = "ThirdParty"
|
|
28
35
|
ALL_HOST_PROFILES: tuple[str, ...] = (
|
|
29
36
|
HOST_PROFILE_CLAUDE,
|
|
37
|
+
HOST_PROFILE_CODEX,
|
|
30
38
|
HOST_PROFILE_THIRD_PARTY,
|
|
31
39
|
)
|
|
40
|
+
SESSION_IDENTITY_CLAUDE_TOKEN: str = "claude"
|
|
41
|
+
SESSION_IDENTITY_CODEX_TOKEN: str = "codex"
|
|
42
|
+
SESSION_IDENTITY_WORD_PATTERN: str = r"[a-z0-9]+"
|
|
43
|
+
HOST_PROFILE_JSON_KEY: str = "host_profile"
|
|
32
44
|
|
|
33
45
|
ALL_MODEL_TIERS: tuple[str, ...] = (
|
|
34
46
|
ADVISOR_FALLBACK_TIER,
|
|
@@ -42,7 +54,6 @@ ALL_KNOWN_TIER_NAMES: tuple[str, ...] = (
|
|
|
42
54
|
*ALL_MODEL_TIERS,
|
|
43
55
|
THIRD_PARTY_MODEL_TIER,
|
|
44
56
|
)
|
|
45
|
-
THIRD_PARTY_CLI_ADVISOR_FLOOR_TIER: str = "Opus"
|
|
46
57
|
|
|
47
58
|
ADVISOR_SENDMESSAGE_REPLY_WAIT_SECONDS: int = 120
|
|
48
59
|
|
|
@@ -62,8 +73,9 @@ ALL_THIRD_PARTY_TRUTHY_VALUES: frozenset[str] = frozenset(
|
|
|
62
73
|
UNKNOWN_OWN_TIER_MESSAGE: str = "own_tier is not a known model tier"
|
|
63
74
|
UNKNOWN_LADDER_NAME_ERROR: str = "ladder name is not a known model tier: {!r}"
|
|
64
75
|
UNKNOWN_HOST_PROFILE_ERROR: str = "host profile is not a known profile: {!r}"
|
|
76
|
+
HOST_PROFILE_MUST_BE_STRING_MESSAGE: str = "host_profile must be a string"
|
|
65
77
|
CANDIDATE_TIERS_MISMATCH_MESSAGE: str = (
|
|
66
|
-
"candidate_tiers does not match the
|
|
78
|
+
"candidate_tiers does not match the advisor walk for this host"
|
|
67
79
|
)
|
|
68
80
|
ATTEMPT_TIER_OUT_OF_SLICE_MESSAGE: str = (
|
|
69
81
|
"a spawn try names a tier outside the candidate slice"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Constants for the optional Codex Sol
|
|
1
|
+
"""Constants for the optional Codex Sol advisor bind."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -8,7 +8,7 @@ CODEX_READ_ONLY_SANDBOX: str = "read-only"
|
|
|
8
8
|
CODEX_JSON_FLAG: str = "--json"
|
|
9
9
|
CODEX_MODEL_FLAG: str = "--model"
|
|
10
10
|
CODEX_CONFIG_FLAG: str = "--config"
|
|
11
|
-
|
|
11
|
+
CODEX_REASONING_CONFIG_TEMPLATE: str = 'model_reasoning_effort="{effort}"'
|
|
12
12
|
CODEX_PROMPT_FROM_STDIN: str = "-"
|
|
13
13
|
CODEX_EXEC_SUBCOMMAND: str = "exec"
|
|
14
14
|
CODEX_RESUME_SUBCOMMAND: str = "resume"
|
|
@@ -17,17 +17,18 @@ CLAUDE_CONFIG_DIRECTORY_NAME: str = ".claude"
|
|
|
17
17
|
SOL_SESSION_ID_METAVAR: str = "SESSION_ID"
|
|
18
18
|
SOL_CODEX_TIMEOUT_SECONDS: float = 120.0
|
|
19
19
|
SOL_USAGE_PROBE_TIMEOUT_SECONDS: float = 30.0
|
|
20
|
-
SOL_ENV_VAR: str = "
|
|
20
|
+
SOL_ENV_VAR: str = "ADVISOR_SOL"
|
|
21
21
|
ALL_SOL_TRUTHY_VALUES: frozenset[str] = frozenset({"1", "true", "yes", "on"})
|
|
22
22
|
SOL_PREFLIGHT_FAILURE_REASON: str = "sol preflight did not establish an eligible Codex meter"
|
|
23
|
-
SOL_BIND_FAILURE_REASON: str = "Codex Sol
|
|
24
|
-
SOL_REPLY_FAILURE_REASON: str = "Codex Sol
|
|
23
|
+
SOL_BIND_FAILURE_REASON: str = "Codex Sol bind failed"
|
|
24
|
+
SOL_REPLY_FAILURE_REASON: str = "Codex Sol returned no advisor guidance"
|
|
25
25
|
SOL_PROBE_TIMEOUT_REASON: str = "Codex Sol meter check timed out"
|
|
26
|
-
SOL_CODEX_TIMEOUT_REASON: str = "Codex Sol
|
|
27
|
-
SOL_MALFORMED_JSONL_REASON: str = "Codex Sol
|
|
28
|
-
SOL_MISSING_SESSION_REASON: str = "Codex Sol
|
|
29
|
-
SOL_INVALID_SIGNAL_REASON: str = "Codex Sol
|
|
30
|
-
SOL_EXECUTABLE_NOT_FOUND_REASON: str = "Codex Sol
|
|
26
|
+
SOL_CODEX_TIMEOUT_REASON: str = "Codex Sol request timed out"
|
|
27
|
+
SOL_MALFORMED_JSONL_REASON: str = "Codex Sol returned malformed JSONL"
|
|
28
|
+
SOL_MISSING_SESSION_REASON: str = "Codex Sol returned no session id"
|
|
29
|
+
SOL_INVALID_SIGNAL_REASON: str = "Codex Sol returned an invalid guidance signal"
|
|
30
|
+
SOL_EXECUTABLE_NOT_FOUND_REASON: str = "Codex Sol could not find the codex executable on PATH"
|
|
31
31
|
SOL_FALLBACK_KIND_DECLINED: str = "declined"
|
|
32
32
|
SOL_FALLBACK_KIND_BROKEN: str = "broken"
|
|
33
33
|
SOL_ENABLE_FLAG: str = "--enable-sol"
|
|
34
|
+
SOL_EFFORT_FLAG: str = "--effort"
|
|
@@ -8,23 +8,29 @@ from data, not inferred from a transcript.
|
|
|
8
8
|
|
|
9
9
|
ladder_walk = ModelTierRun(
|
|
10
10
|
own_tier="Opus",
|
|
11
|
-
candidate_tiers=["Fable"
|
|
12
|
-
attempts=[
|
|
13
|
-
|
|
14
|
-
{"tier": "Opus", "result": "spawned"},
|
|
15
|
-
],
|
|
16
|
-
selected_tier="Opus",
|
|
11
|
+
candidate_tiers=["Fable"],
|
|
12
|
+
attempts=[{"tier": "Fable", "result": "spawned"}],
|
|
13
|
+
selected_tier="Fable",
|
|
17
14
|
)
|
|
18
15
|
validate_model_tier_run(ladder_walk) # ok: returns None, raises nothing
|
|
19
16
|
|
|
20
17
|
cli_bind = ModelTierRun(
|
|
21
18
|
own_tier="Opus",
|
|
22
|
-
candidate_tiers=["Fable"
|
|
19
|
+
candidate_tiers=["Fable"],
|
|
23
20
|
attempts=[{"tier": "Fable", "result": "cli"}],
|
|
24
21
|
selected_tier="Fable",
|
|
25
22
|
)
|
|
26
23
|
validate_model_tier_run(cli_bind) # ok: third-party-host CLI Claude-chain bind
|
|
27
24
|
|
|
25
|
+
codex_bind = ModelTierRun(
|
|
26
|
+
own_tier="Opus",
|
|
27
|
+
candidate_tiers=["Sol"],
|
|
28
|
+
attempts=[{"tier": "Sol", "result": "spawned"}],
|
|
29
|
+
selected_tier="Sol",
|
|
30
|
+
host_profile="Codex",
|
|
31
|
+
)
|
|
32
|
+
validate_model_tier_run(codex_bind) # ok: Codex in-session Sol spawn
|
|
33
|
+
|
|
28
34
|
A run whose selected_tier is not the first successful bind fails.
|
|
29
35
|
On any broken invariant, validate_model_tier_run raises ModelTierRunError.
|
|
30
36
|
|
|
@@ -48,7 +54,6 @@ if _scripts_directory not in sys.path:
|
|
|
48
54
|
sys.path.insert(0, _scripts_directory)
|
|
49
55
|
|
|
50
56
|
from advisor_scripts_constants.model_tier_run_validator_constants import ( # noqa: E402
|
|
51
|
-
ALL_MODEL_TIERS,
|
|
52
57
|
ATTEMPT_ORDER_MISMATCH_MESSAGE,
|
|
53
58
|
ATTEMPT_TIER_OUT_OF_SLICE_MESSAGE,
|
|
54
59
|
CANDIDATE_TIERS_MISMATCH_MESSAGE,
|
|
@@ -57,15 +62,20 @@ from advisor_scripts_constants.model_tier_run_validator_constants import ( # no
|
|
|
57
62
|
CLI_SUCCESS_EXIT_CODE,
|
|
58
63
|
CLI_USAGE_MESSAGE,
|
|
59
64
|
CLI_VALIDATION_FAILURE_EXIT_CODE,
|
|
65
|
+
HOST_PROFILE_CLAUDE,
|
|
66
|
+
HOST_PROFILE_CODEX,
|
|
67
|
+
HOST_PROFILE_JSON_KEY,
|
|
68
|
+
HOST_PROFILE_MUST_BE_STRING_MESSAGE,
|
|
60
69
|
INCOMPLETE_FALLBACK_WALK_MESSAGE,
|
|
61
70
|
MISSING_FALLBACK_REASON_MESSAGE,
|
|
62
71
|
SELECTED_TIER_MISMATCH_MESSAGE,
|
|
63
72
|
SELECTED_TIER_NOT_NULL_MESSAGE,
|
|
64
|
-
THIRD_PARTY_CLI_ADVISOR_FLOOR_TIER,
|
|
65
73
|
THIRD_PARTY_MODEL_TIER,
|
|
74
|
+
UNKNOWN_HOST_PROFILE_ERROR,
|
|
66
75
|
UNKNOWN_OWN_TIER_MESSAGE,
|
|
67
76
|
)
|
|
68
77
|
from advisor_scripts_constants.advisor_route_constants import ( # noqa: E402
|
|
78
|
+
ADVISOR_FALLBACK_TIER,
|
|
69
79
|
ADVISOR_MODEL_TIER,
|
|
70
80
|
CODEX_BIND_SUCCESS_TOKEN,
|
|
71
81
|
CLI_BIND_SUCCESS_TOKEN,
|
|
@@ -73,7 +83,7 @@ from advisor_scripts_constants.advisor_route_constants import ( # noqa: E402
|
|
|
73
83
|
SPAWN_SUCCESS_TOKEN,
|
|
74
84
|
TIER_KEY,
|
|
75
85
|
)
|
|
76
|
-
from tier_model_ids import canonical_tier_name # noqa: E402
|
|
86
|
+
from tier_model_ids import canonical_host_profile, canonical_tier_name # noqa: E402
|
|
77
87
|
|
|
78
88
|
|
|
79
89
|
@dataclass(frozen=True)
|
|
@@ -84,6 +94,7 @@ class ModelTierRun:
|
|
|
84
94
|
selected_tier: str | None
|
|
85
95
|
fallback_reason: str | None = None
|
|
86
96
|
is_sol_enabled: bool = False
|
|
97
|
+
host_profile: str = HOST_PROFILE_CLAUDE
|
|
87
98
|
|
|
88
99
|
|
|
89
100
|
class ModelTierRunError(ValueError):
|
|
@@ -101,30 +112,39 @@ def _canonical_tier_list(all_tier_names: list[str]) -> list[str] | None:
|
|
|
101
112
|
|
|
102
113
|
|
|
103
114
|
def _expected_candidate_tiers(
|
|
104
|
-
own_tier: str,
|
|
115
|
+
own_tier: str,
|
|
116
|
+
is_sol_enabled: bool = False,
|
|
117
|
+
host_profile: str = HOST_PROFILE_CLAUDE,
|
|
105
118
|
) -> list[str]:
|
|
106
119
|
maybe_canonical_own_tier = canonical_tier_name(own_tier)
|
|
107
120
|
if maybe_canonical_own_tier is None:
|
|
108
121
|
raise ModelTierRunError(f"{UNKNOWN_OWN_TIER_MESSAGE}: {own_tier!r}")
|
|
109
122
|
if maybe_canonical_own_tier == ADVISOR_MODEL_TIER:
|
|
110
123
|
raise ModelTierRunError(f"{UNKNOWN_OWN_TIER_MESSAGE}: {own_tier!r}")
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
124
|
+
maybe_canonical_host = canonical_host_profile(host_profile)
|
|
125
|
+
if maybe_canonical_host is None:
|
|
126
|
+
raise ModelTierRunError(UNKNOWN_HOST_PROFILE_ERROR.format(host_profile))
|
|
127
|
+
if maybe_canonical_host == HOST_PROFILE_CODEX:
|
|
128
|
+
return [ADVISOR_MODEL_TIER]
|
|
129
|
+
all_expected_candidates = [ADVISOR_FALLBACK_TIER]
|
|
116
130
|
if is_sol_enabled:
|
|
117
|
-
all_expected_candidates.
|
|
131
|
+
all_expected_candidates.append(ADVISOR_MODEL_TIER)
|
|
118
132
|
return all_expected_candidates
|
|
119
133
|
|
|
120
134
|
|
|
121
135
|
def _is_successful_attempt_outcome(
|
|
122
136
|
canonical_tier: str,
|
|
123
137
|
outcome_token: str,
|
|
138
|
+
host_profile: str = HOST_PROFILE_CLAUDE,
|
|
124
139
|
) -> bool:
|
|
140
|
+
maybe_canonical_host = canonical_host_profile(host_profile)
|
|
125
141
|
if canonical_tier == THIRD_PARTY_MODEL_TIER:
|
|
126
142
|
return False
|
|
127
143
|
if canonical_tier == ADVISOR_MODEL_TIER:
|
|
144
|
+
if maybe_canonical_host == HOST_PROFILE_CODEX:
|
|
145
|
+
if outcome_token == SPAWN_SUCCESS_TOKEN:
|
|
146
|
+
return True
|
|
147
|
+
return outcome_token == CODEX_BIND_SUCCESS_TOKEN
|
|
128
148
|
return outcome_token == CODEX_BIND_SUCCESS_TOKEN
|
|
129
149
|
if outcome_token == CODEX_BIND_SUCCESS_TOKEN:
|
|
130
150
|
return False
|
|
@@ -144,12 +164,13 @@ def validate_model_tier_run(run: ModelTierRun) -> None:
|
|
|
144
164
|
validate_model_tier_run(cli_bind) # ok: CLI Claude-chain bind
|
|
145
165
|
validate_model_tier_run(broken_log) # flag: ModelTierRunError
|
|
146
166
|
|
|
147
|
-
Candidate tiers
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
167
|
+
Candidate tiers are Fable, plus Sol when ``is_sol_enabled`` is true, on
|
|
168
|
+
Claude and ThirdParty hosts. A Codex host walks Sol only. Consumer
|
|
169
|
+
``own_tier`` is recorded and must be a known tier; it does not add Opus
|
|
170
|
+
to the advisor walk. Tries walk that list in order. Early stop only
|
|
171
|
+
after ``spawned``, ``cli``, or Sol ``codex`` (and Sol ``spawned`` on a
|
|
172
|
+
Codex host). A null selected_tier requires a full walk plus
|
|
173
|
+
fallback_reason.
|
|
153
174
|
|
|
154
175
|
Args:
|
|
155
176
|
run: The structured spawn-walk log to check.
|
|
@@ -163,6 +184,7 @@ def validate_model_tier_run(run: ModelTierRun) -> None:
|
|
|
163
184
|
all_expected_candidates = _expected_candidate_tiers(
|
|
164
185
|
run.own_tier,
|
|
165
186
|
is_sol_enabled=run.is_sol_enabled,
|
|
187
|
+
host_profile=run.host_profile,
|
|
166
188
|
)
|
|
167
189
|
maybe_canonical_candidates = _canonical_tier_list(run.candidate_tiers)
|
|
168
190
|
if maybe_canonical_candidates != all_expected_candidates:
|
|
@@ -199,6 +221,7 @@ def _validate_selected_tier(
|
|
|
199
221
|
if _is_successful_attempt_outcome(
|
|
200
222
|
canonical_tier=each_tier,
|
|
201
223
|
outcome_token=each_attempt[SPAWN_OUTCOME_KEY],
|
|
224
|
+
host_profile=run.host_profile,
|
|
202
225
|
)
|
|
203
226
|
]
|
|
204
227
|
if all_bound_tiers:
|
|
@@ -237,6 +260,9 @@ def load_model_tier_run_from_json_path(from_path: Path) -> ModelTierRun:
|
|
|
237
260
|
raw_sol_enabled = parsed_payload.get("sol_enabled", False)
|
|
238
261
|
if not isinstance(raw_sol_enabled, bool):
|
|
239
262
|
raise TypeError("sol_enabled must be a boolean")
|
|
263
|
+
raw_host_profile = parsed_payload.get(HOST_PROFILE_JSON_KEY, HOST_PROFILE_CLAUDE)
|
|
264
|
+
if not isinstance(raw_host_profile, str):
|
|
265
|
+
raise TypeError(HOST_PROFILE_MUST_BE_STRING_MESSAGE)
|
|
240
266
|
return ModelTierRun(
|
|
241
267
|
own_tier=parsed_payload["own_tier"],
|
|
242
268
|
candidate_tiers=list(parsed_payload["candidate_tiers"]),
|
|
@@ -244,6 +270,7 @@ def load_model_tier_run_from_json_path(from_path: Path) -> ModelTierRun:
|
|
|
244
270
|
selected_tier=parsed_payload.get("selected_tier"),
|
|
245
271
|
fallback_reason=parsed_payload.get("fallback_reason"),
|
|
246
272
|
is_sol_enabled=raw_sol_enabled,
|
|
273
|
+
host_profile=raw_host_profile,
|
|
247
274
|
)
|
|
248
275
|
|
|
249
276
|
|
|
@@ -97,8 +97,35 @@ def _two_step_process_runner(calls: list[list[str]], guidance: str) -> _ProcessR
|
|
|
97
97
|
|
|
98
98
|
|
|
99
99
|
def test_sol_flag_accepts_documented_truthy_values() -> None:
|
|
100
|
-
assert sol_advisor.is_sol_advisor_enabled({"
|
|
101
|
-
assert not sol_advisor.is_sol_advisor_enabled({"
|
|
100
|
+
assert sol_advisor.is_sol_advisor_enabled({"ADVISOR_SOL": "yes"})
|
|
101
|
+
assert not sol_advisor.is_sol_advisor_enabled({"ADVISOR_SOL": "0"})
|
|
102
|
+
assert not sol_advisor.is_sol_advisor_enabled({"ADVISOR_SOL_XHIGH": "1"})
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_resolve_advisor_effort_defaults_and_accepts_documented_levels() -> None:
|
|
106
|
+
assert sol_advisor.resolve_advisor_effort({}) == "low"
|
|
107
|
+
assert sol_advisor.resolve_advisor_effort({"ADVISOR_EFFORT": "MEDIUM"}) == "medium"
|
|
108
|
+
assert sol_advisor.resolve_advisor_effort({"ADVISOR_EFFORT": "nope"}) == "low"
|
|
109
|
+
assert sol_advisor.resolve_advisor_effort({"ADVISOR_SOL_EFFORT": "high"}) == "low"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@pytest.mark.parametrize(
|
|
113
|
+
"reasoning_effort",
|
|
114
|
+
["low", "medium", "high", "xhigh", "max"],
|
|
115
|
+
)
|
|
116
|
+
def test_codex_arguments_use_selected_sol_reasoning_effort(
|
|
117
|
+
reasoning_effort: str,
|
|
118
|
+
) -> None:
|
|
119
|
+
command_arguments = sol_advisor.build_codex_arguments(
|
|
120
|
+
"codex",
|
|
121
|
+
reasoning_effort=reasoning_effort,
|
|
122
|
+
)
|
|
123
|
+
config_flag_index = command_arguments.index("--config")
|
|
124
|
+
|
|
125
|
+
assert (
|
|
126
|
+
command_arguments[config_flag_index + 1]
|
|
127
|
+
== f'model_reasoning_effort="{reasoning_effort}"'
|
|
128
|
+
)
|
|
102
129
|
|
|
103
130
|
|
|
104
131
|
def test_resolve_usage_probe_path_uses_supplied_home_directory(tmp_path: Path) -> None:
|
|
@@ -162,7 +189,7 @@ def test_bind_and_resume_arguments_match_installed_codex_interface() -> None:
|
|
|
162
189
|
"--model",
|
|
163
190
|
sol_advisor.ADVISOR_CODEX_MODEL_ID,
|
|
164
191
|
"--config",
|
|
165
|
-
'model_reasoning_effort="
|
|
192
|
+
'model_reasoning_effort="low"',
|
|
166
193
|
"--sandbox",
|
|
167
194
|
"read-only",
|
|
168
195
|
"--json",
|
|
@@ -326,6 +353,40 @@ def test_enable_sol_flag_opens_the_rung_without_an_environment_flag(
|
|
|
326
353
|
assert payload["fallback_kind"] == sol_advisor.SOL_FALLBACK_KIND_DECLINED
|
|
327
354
|
|
|
328
355
|
|
|
356
|
+
def test_effort_cli_flag_overrides_environment_effort(
|
|
357
|
+
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
358
|
+
) -> None:
|
|
359
|
+
captured_settings: dict[str, str] = {}
|
|
360
|
+
|
|
361
|
+
def fake_advisor(**kwargs: object) -> object:
|
|
362
|
+
captured_settings.update(dict(kwargs["setting_by_name"])) # type: ignore[arg-type]
|
|
363
|
+
return sol_advisor._reply_fallback(
|
|
364
|
+
"probe declined",
|
|
365
|
+
True,
|
|
366
|
+
fallback_kind=sol_advisor.SOL_FALLBACK_KIND_DECLINED,
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
monkeypatch.setattr(sol_advisor, "run_codex_sol_advisor", fake_advisor)
|
|
370
|
+
monkeypatch.setenv("ADVISOR_EFFORT", "high")
|
|
371
|
+
monkeypatch.setattr(sys, "stdin", io.StringIO("first consult"))
|
|
372
|
+
|
|
373
|
+
exit_code = sol_advisor.main(
|
|
374
|
+
[
|
|
375
|
+
"--bind",
|
|
376
|
+
"--cwd",
|
|
377
|
+
".",
|
|
378
|
+
sol_advisor.SOL_ENABLE_FLAG,
|
|
379
|
+
sol_advisor.SOL_EFFORT_FLAG,
|
|
380
|
+
"medium",
|
|
381
|
+
]
|
|
382
|
+
)
|
|
383
|
+
payload = json.loads(capsys.readouterr().out)
|
|
384
|
+
|
|
385
|
+
assert exit_code == 1
|
|
386
|
+
assert captured_settings["ADVISOR_EFFORT"] == "medium"
|
|
387
|
+
assert payload["fallback_kind"] == sol_advisor.SOL_FALLBACK_KIND_DECLINED
|
|
388
|
+
|
|
389
|
+
|
|
329
390
|
def test_successful_probe_requires_finite_meter_above_configured_gate() -> None:
|
|
330
391
|
def probe_runner(arguments: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
|
331
392
|
return _probe_process({"percent_left": 90})
|
|
@@ -474,6 +535,50 @@ def test_bind_runs_probe_then_codex_with_read_only_xhigh_settings() -> None:
|
|
|
474
535
|
assert calls[1][1]["timeout"]
|
|
475
536
|
|
|
476
537
|
|
|
538
|
+
def test_protocol_docs_name_shared_effort_and_fable_sol_ladder() -> None:
|
|
539
|
+
sol_rung_path = SCRIPTS_ROOT.parent / "reference" / "sol-rung.md"
|
|
540
|
+
protocol_path = SCRIPTS_ROOT.parent / "advisor-protocol.md"
|
|
541
|
+
third_party_bind_path = SCRIPTS_ROOT.parent / "reference" / "third-party-bind.md"
|
|
542
|
+
advisor_block_path = SCRIPTS_ROOT.parent / "reference" / "advisor-block.md"
|
|
543
|
+
sol_rung = sol_rung_path.read_text(encoding="utf-8")
|
|
544
|
+
protocol_text = protocol_path.read_text(encoding="utf-8")
|
|
545
|
+
third_party_bind_text = third_party_bind_path.read_text(encoding="utf-8")
|
|
546
|
+
advisor_block_text = advisor_block_path.read_text(encoding="utf-8")
|
|
547
|
+
|
|
548
|
+
assert 'model_reasoning_effort="low"' in sol_rung
|
|
549
|
+
assert "out of usage" in protocol_text
|
|
550
|
+
assert "ADVISOR_EFFORT" in protocol_text
|
|
551
|
+
assert "ADVISOR_EFFORT" in sol_rung
|
|
552
|
+
assert "ADVISOR_EFFORT" in third_party_bind_text
|
|
553
|
+
assert "ADVISOR_SOL=1" in protocol_text
|
|
554
|
+
assert "ADVISOR_SOL_EFFORT" not in protocol_text
|
|
555
|
+
assert "ADVISOR_SOL_EFFORT" not in sol_rung
|
|
556
|
+
assert "ADVISOR_SOL_XHIGH" not in protocol_text
|
|
557
|
+
assert "ADVISOR_SOL_XHIGH" not in sol_rung
|
|
558
|
+
assert "--effort xhigh" not in third_party_bind_text
|
|
559
|
+
assert "--effort medium" not in third_party_bind_text
|
|
560
|
+
assert "then Opus" not in protocol_text
|
|
561
|
+
assert "then Claude Opus" not in advisor_block_text
|
|
562
|
+
assert 'candidate_tiers = ["Fable", "Opus"]' not in protocol_text
|
|
563
|
+
assert 'candidate_tiers = ["Fable", "Sol", "Opus"]' not in protocol_text
|
|
564
|
+
team_advisor_text = (
|
|
565
|
+
SCRIPTS_ROOT.parents[2] / ".agents" / "skills" / "team-advisor" / "SKILL.md"
|
|
566
|
+
).read_text(encoding="utf-8")
|
|
567
|
+
orchestrator_text = (
|
|
568
|
+
SCRIPTS_ROOT.parents[2] / ".agents" / "skills" / "orchestrator" / "SKILL.md"
|
|
569
|
+
).read_text(encoding="utf-8")
|
|
570
|
+
refresh_text = (
|
|
571
|
+
SCRIPTS_ROOT.parents[2]
|
|
572
|
+
/ ".agents"
|
|
573
|
+
/ "skills"
|
|
574
|
+
/ "orchestrator-refresh"
|
|
575
|
+
/ "SKILL.md"
|
|
576
|
+
).read_text(encoding="utf-8")
|
|
577
|
+
assert "then Opus" not in team_advisor_text
|
|
578
|
+
assert "then Opus" not in orchestrator_text
|
|
579
|
+
assert "then Opus xhigh" not in refresh_text
|
|
580
|
+
|
|
581
|
+
|
|
477
582
|
def test_team_advisor_path_preserves_sol_routing_fields() -> None:
|
|
478
583
|
team_advisor_path = SCRIPTS_ROOT.parents[2] / ".agents" / "skills" / "team-advisor" / "SKILL.md"
|
|
479
584
|
sol_rung_path = SCRIPTS_ROOT.parent / "reference" / "sol-rung.md"
|