claude-dev-env 1.95.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_shared/advisor/CLAUDE.md +2 -2
- package/_shared/advisor/advisor-protocol.md +20 -20
- package/_shared/advisor/scripts/config/advisor_scripts_constants/model_tier_run_validator_constants.py +15 -12
- package/_shared/advisor/scripts/model_tier_run_validator.py +11 -10
- package/_shared/advisor/scripts/tests/test_model_tier_run_validator.py +25 -19
- package/_shared/advisor/scripts/tests/test_tier_model_ids.py +17 -17
- package/_shared/advisor/scripts/tier_model_ids.py +18 -18
- package/_shared/pr-loop/CLAUDE.md +1 -0
- package/_shared/pr-loop/scripts/CLAUDE.md +2 -1
- package/_shared/pr-loop/scripts/README.md +1 -0
- package/_shared/pr-loop/scripts/code_rules_gate.py +253 -1980
- package/_shared/pr-loop/scripts/code_rules_gate_parts/CLAUDE.md +32 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/__init__.py +7 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/added_line_maps.py +268 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/enforcer_loading.py +172 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/gate_arguments.py +70 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/gate_running.py +326 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/git_blob_readers.py +85 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/git_file_sets.py +331 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/staged_test_running.py +369 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/conftest.py +14 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_added_line_maps.py +118 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_enforcer_loading.py +17 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_gate_arguments.py +29 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_gate_running.py +99 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_git_blob_readers.py +69 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_git_file_sets.py +137 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_staged_test_running.py +116 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_violation_scoping.py +75 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_wrapper_plumb_check.py +49 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/violation_scoping.py +328 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/wrapper_plumb_check.py +206 -0
- package/_shared/pr-loop/scripts/pr_loop_shared_constants/code_rules_gate_constants.py +24 -17
- package/_shared/pr-loop/scripts/pr_loop_shared_constants/reviews_disabled_constants.py +1 -0
- package/_shared/pr-loop/scripts/reviews_disabled.py +19 -2
- package/_shared/pr-loop/scripts/test_code_rules_gate.py +278 -0
- package/_shared/pr-loop/scripts/tests/test_code_rules_gate_constants.py +6 -39
- package/_shared/pr-loop/scripts/tests/test_reviews_disabled.py +43 -0
- package/_shared/pr-loop/worker-spawn.md +186 -0
- package/agents/code-verifier.md +1 -1
- package/bin/ever-shipped-skills.mjs +3 -0
- package/bin/expand_home_directory_tokens.mjs +1 -1
- package/bin/install.mjs +5 -2
- package/hooks/advisory/refactor_guard.py +3 -4
- package/hooks/blocking/CLAUDE.md +7 -1
- package/hooks/blocking/block_main_commit.py +2 -2
- package/hooks/blocking/claude_md_orphan_file_blocker.py +75 -699
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/CLAUDE.md +28 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/__init__.py +1 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/config/__init__.py +1 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/config/orphan_blocker_constants.py +18 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/decision.py +81 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/references.py +307 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/scan_plan.py +124 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/subtree_scan.py +179 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/conftest.py +10 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_decision.py +34 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_references.py +42 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_scan_plan.py +27 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_subtree_scan.py +30 -0
- package/hooks/blocking/code_rules_boolean_mustcheck.py +1 -1
- package/hooks/blocking/code_rules_mock_completeness.py +1 -1
- package/hooks/blocking/code_rules_optional_params.py +2 -2
- package/hooks/blocking/code_rules_shared.py +1 -1
- package/hooks/blocking/code_rules_test_assertions.py +1 -1
- package/hooks/blocking/code_rules_typeddict_stub.py +1 -1
- package/hooks/blocking/gh_pr_author_enforcer.py +1 -1
- package/hooks/blocking/inventory_intent_records/CLAUDE.md +26 -0
- package/hooks/blocking/inventory_intent_records/__init__.py +1 -0
- package/hooks/blocking/inventory_intent_records/config/__init__.py +1 -0
- package/hooks/blocking/inventory_intent_records/config/intent_records_constants.py +20 -0
- package/hooks/blocking/inventory_intent_records/records.py +271 -0
- package/hooks/blocking/inventory_intent_records/tests/conftest.py +10 -0
- package/hooks/blocking/inventory_intent_records/tests/test_records.py +80 -0
- package/hooks/blocking/package_inventory_stale_blocker.py +54 -384
- package/hooks/blocking/package_inventory_stale_blocker_parts/CLAUDE.md +26 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/__init__.py +1 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/config/__init__.py +1 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/config/inventory_blocker_constants.py +16 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/decision.py +84 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/inventory_detection.py +307 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/tests/conftest.py +10 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/tests/test_decision.py +38 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/tests/test_inventory_detection.py +61 -0
- package/hooks/blocking/pii_payload_scan.py +138 -42
- package/hooks/blocking/pii_prevention_blocker.py +185 -291
- package/hooks/blocking/pii_prevention_blocker_parts/CLAUDE.md +24 -0
- package/hooks/blocking/pii_prevention_blocker_parts/__init__.py +1 -0
- package/hooks/blocking/pii_prevention_blocker_parts/config/__init__.py +1 -0
- package/hooks/blocking/pii_prevention_blocker_parts/config/repository_resolution_constants.py +28 -0
- package/hooks/blocking/pii_prevention_blocker_parts/repository_exemption.py +214 -0
- package/hooks/blocking/pii_prevention_blocker_parts/repository_resolution.py +208 -0
- package/hooks/blocking/pr_description_command_parser.py +8 -4
- package/hooks/blocking/precommit_code_rules_gate.py +3 -3
- package/hooks/blocking/tdd_enforcer.py +97 -608
- package/hooks/blocking/tdd_enforcer_parts/CLAUDE.md +30 -0
- package/hooks/blocking/tdd_enforcer_parts/__init__.py +1 -0
- package/hooks/blocking/tdd_enforcer_parts/candidate_paths.py +142 -0
- package/hooks/blocking/tdd_enforcer_parts/config/__init__.py +1 -0
- package/hooks/blocking/tdd_enforcer_parts/config/tdd_enforcer_constants.py +32 -0
- package/hooks/blocking/tdd_enforcer_parts/content_analysis.py +268 -0
- package/hooks/blocking/tdd_enforcer_parts/decisions.py +92 -0
- package/hooks/blocking/tdd_enforcer_parts/freshness.py +80 -0
- package/hooks/blocking/tdd_enforcer_parts/git_tracking.py +63 -0
- package/hooks/blocking/tdd_enforcer_parts/path_classification.py +119 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/conftest.py +10 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_candidate_paths.py +31 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_content_analysis.py +30 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_decisions.py +34 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_freshness.py +28 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_git_tracking.py +48 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_path_classification.py +36 -0
- package/hooks/blocking/test_inventory_deadlock_resolution.py +154 -0
- package/hooks/blocking/test_pii_payload_scan.py +168 -0
- package/hooks/blocking/test_tdd_enforcer_restore.py +108 -0
- package/hooks/blocking/test_verifier_verdict_minter.py +55 -158
- package/hooks/blocking/tests/conftest.py +10 -0
- package/hooks/blocking/tests/test_pii_prevention_blocker.py +260 -0
- package/hooks/blocking/tests/test_repository_exemption.py +105 -0
- package/hooks/blocking/tests/test_repository_resolution.py +108 -0
- package/hooks/diagnostic/hook_log_extractor.py +12 -10
- package/hooks/git-hooks/post_commit.py +3 -4
- package/hooks/hooks_constants/CLAUDE.md +2 -2
- package/hooks/hooks_constants/banned_identifiers_constants.py +0 -1
- package/hooks/hooks_constants/code_rules_path_utils_constants.py +1 -1
- package/hooks/hooks_constants/local_identity.py +59 -8
- package/hooks/hooks_constants/pii_prevention_constants.py +0 -6
- package/hooks/hooks_constants/test_local_identity.py +105 -3
- package/hooks/pyproject.toml +13 -36
- package/hooks/session/plugin_data_dir_cleanup.py +0 -1
- package/hooks/validation/mypy_validator.py +2 -2
- package/hooks/validators/health_check.py +1 -0
- package/hooks/validators/mypy_integration.py +2 -0
- package/hooks/validators/ruff_integration.py +3 -0
- package/hooks/workflow/auto_formatter.py +5 -4
- package/package.json +1 -1
- package/scripts/CLAUDE.md +4 -0
- package/scripts/dev_env_scripts_constants/CLAUDE.md +6 -4
- package/scripts/dev_env_scripts_constants/code_review_constants.py +71 -0
- package/scripts/dev_env_scripts_constants/grok_worker_constants.py +435 -0
- package/scripts/dev_env_scripts_constants/timing.py +7 -1
- package/scripts/grok_headless_runner.py +294 -0
- package/scripts/grok_worker_preflight.py +410 -0
- package/scripts/invoke_code_review.py +463 -0
- package/scripts/resolve_worker_spawn.py +619 -0
- package/scripts/spawn_grok_batch.py +672 -0
- package/scripts/test_grok_headless_runner.py +626 -0
- package/scripts/test_grok_worker_preflight.py +1054 -0
- package/scripts/test_invoke_code_review.py +672 -0
- package/scripts/test_resolve_worker_spawn.py +1014 -0
- package/scripts/test_spawn_grok_batch.py +1017 -0
- package/skills/CLAUDE.md +5 -3
- package/skills/_shared/pr-loop/scripts/build_audit_prompt.py +72 -13
- package/skills/_shared/pr-loop/scripts/build_fix_prompt.py +121 -14
- package/skills/_shared/pr-loop/scripts/skills_pr_loop_constants/path_resolver_constants.py +78 -0
- package/skills/_shared/pr-loop/scripts/test_build_audit_prompt.py +121 -0
- package/skills/_shared/pr-loop/scripts/test_build_fix_prompt.py +196 -6
- package/skills/autoconverge/CLAUDE.md +3 -3
- package/skills/autoconverge/SKILL.md +9 -3
- package/skills/autoconverge/reference/CLAUDE.md +2 -2
- package/skills/autoconverge/reference/convergence.md +33 -11
- package/skills/autoconverge/reference/stop-conditions.md +16 -5
- package/skills/autoconverge/workflow/CLAUDE.md +2 -1
- package/skills/autoconverge/workflow/converge.clean-audit.test.mjs +7 -2
- package/skills/autoconverge/workflow/converge.codex-gate.test.mjs +300 -0
- package/skills/autoconverge/workflow/converge.contract.test.mjs +5 -5
- package/skills/autoconverge/workflow/converge.copilot-gate.test.mjs +29 -29
- package/skills/autoconverge/workflow/converge.fix-progress.test.mjs +1 -1
- package/skills/autoconverge/workflow/converge.mjs +200 -16
- package/skills/bugteam/CLAUDE.md +2 -2
- package/skills/bugteam/CONSTRAINTS.md +3 -2
- package/skills/bugteam/PROMPTS.md +7 -6
- package/skills/bugteam/SKILL.md +18 -13
- package/skills/bugteam/reference/audit-and-teammates.md +215 -35
- package/skills/bugteam/reference/design-rationale.md +1 -1
- package/skills/bugteam/reference/obstacles/CLAUDE.md +1 -1
- package/skills/bugteam/reference/team-setup.md +8 -2
- package/skills/codex-review/CLAUDE.md +46 -0
- package/skills/codex-review/SKILL.md +181 -0
- package/skills/codex-review/reference/CLAUDE.md +15 -0
- package/skills/codex-review/reference/cli-contract.md +253 -0
- package/skills/codex-review/reference/loop-integration.md +118 -0
- package/skills/codex-review/scripts/codex_down_classifier.py +98 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/CLAUDE.md +18 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/__init__.py +1 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/classifier_constants.py +35 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/codex_usage_probe_constants.py +86 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/findings_constants.py +18 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/run_constants.py +45 -0
- package/skills/codex-review/scripts/codex_usage_probe.py +573 -0
- package/skills/codex-review/scripts/fixtures/auth_failure_synthetic.txt +1 -0
- package/skills/codex-review/scripts/fixtures/config_load_failure_v0.125.0.txt +1 -0
- package/skills/codex-review/scripts/fixtures/freeform_findings_v0.144.3.txt +6 -0
- package/skills/codex-review/scripts/fixtures/model_rejection_v0.125.0.jsonl +5 -0
- package/skills/codex-review/scripts/fixtures/structured_findings.txt +13 -0
- package/skills/codex-review/scripts/fixtures/success_stream_v0.144.3.jsonl +6 -0
- package/skills/codex-review/scripts/fixtures/unknown_failure_synthetic.txt +1 -0
- package/skills/codex-review/scripts/fixtures/usage_limit_synthetic.txt +1 -0
- package/skills/codex-review/scripts/parse_codex_findings.py +207 -0
- package/skills/codex-review/scripts/run_codex_review.py +415 -0
- package/skills/codex-review/scripts/test_codex_down_classifier.py +143 -0
- package/skills/codex-review/scripts/test_codex_usage_probe.py +678 -0
- package/skills/codex-review/scripts/test_parse_codex_findings.py +130 -0
- package/skills/codex-review/scripts/test_run_codex_review.py +812 -0
- package/skills/codex-review/test_skill_scaffold.py +192 -0
- package/skills/grok-spawn/CLAUDE.md +28 -0
- package/skills/grok-spawn/SKILL.md +226 -0
- package/skills/grok-spawn/reference/flag-profiles.md +132 -0
- package/skills/grok-spawn/reference/worker-briefs.md +152 -0
- package/skills/grokify/SKILL.md +9 -1
- package/skills/grokify/capability-claims.test.mjs +28 -0
- package/skills/grokify/evals/README.md +72 -0
- package/skills/grokify/evals/parse-payload.test.mjs +171 -0
- package/skills/grokify/evals/run-capability-evals.mjs +545 -0
- package/skills/orchestrator/SKILL.md +42 -29
- package/skills/orchestrator-refresh/SKILL.md +17 -9
- package/skills/pr-converge/SKILL.md +34 -13
- package/skills/pr-converge/reference/convergence-gates.md +42 -15
- package/skills/pr-converge/reference/fix-protocol.md +1 -1
- package/skills/pr-converge/reference/ground-rules.md +1 -1
- package/skills/pr-converge/reference/per-tick.md +130 -42
- package/skills/pr-converge/reference/state-schema.md +10 -0
- package/skills/pr-converge/scripts/CLAUDE.md +2 -0
- package/skills/pr-converge/scripts/_pr_converge_path_setup.py +5 -1
- package/skills/pr-converge/scripts/check_convergence.py +605 -29
- package/skills/pr-converge/scripts/check_convergence_availability.py +232 -0
- package/skills/pr-converge/scripts/check_convergence_gates.py +279 -235
- package/skills/pr-converge/scripts/check_convergence_thread_gates.py +1 -1
- package/skills/pr-converge/scripts/pr_converge_scripts_constants/convergence_gate_constants.py +36 -2
- package/skills/pr-converge/scripts/test__pr_converge_path_setup.py +4 -0
- package/skills/pr-converge/scripts/test_check_convergence.py +71 -3
- package/skills/pr-converge/scripts/test_check_convergence_availability.py +326 -0
- package/skills/pr-converge/scripts/test_check_convergence_codex.py +507 -0
- package/skills/pr-converge/scripts/test_check_convergence_contract.py +89 -17
- package/skills/pr-converge/scripts/test_check_convergence_fixture.py +179 -0
- package/skills/pr-converge/scripts/test_check_convergence_gates.py +84 -68
- package/skills/pr-converge/scripts/test_check_convergence_thread_gates.py +24 -0
- package/skills/pr-converge/test_step5_host_branch.py +106 -0
- package/skills/pr-loop-cloud-transport/SKILL.md +2 -0
- package/skills/reviewer-gates/SKILL.md +7 -5
- package/skills/team-advisor/SKILL.md +7 -7
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Opt-in live Grok capability evals (E1–E5).
|
|
4
|
+
*
|
|
5
|
+
* Never wired into package.json `test`. Requires GROK_CAPABILITY_EVALS=1 or --run.
|
|
6
|
+
*/
|
|
7
|
+
import { spawnSync } from 'node:child_process';
|
|
8
|
+
import {
|
|
9
|
+
existsSync,
|
|
10
|
+
mkdtempSync,
|
|
11
|
+
readFileSync,
|
|
12
|
+
rmSync,
|
|
13
|
+
writeFileSync,
|
|
14
|
+
} from 'node:fs';
|
|
15
|
+
import { tmpdir } from 'node:os';
|
|
16
|
+
import { join, resolve } from 'node:path';
|
|
17
|
+
import { fileURLToPath } from 'node:url';
|
|
18
|
+
|
|
19
|
+
const OPT_IN_ENV = 'GROK_CAPABILITY_EVALS';
|
|
20
|
+
const OPT_IN_VALUE = '1';
|
|
21
|
+
const RUN_FLAG = '--run';
|
|
22
|
+
const GROK_BINARY = process.env.GROK_BIN || 'grok';
|
|
23
|
+
const DEFAULT_MAX_TURNS = 12;
|
|
24
|
+
const SPAWN_MAX_TURNS = 20;
|
|
25
|
+
const SKILL_MAX_TURNS = 16;
|
|
26
|
+
const WRITE_MAX_TURNS = 12;
|
|
27
|
+
const WORKFLOW_MAX_TURNS = 8;
|
|
28
|
+
const PROBE_FILE_NAME = 'capability-probe-write.txt';
|
|
29
|
+
const PROBE_FILE_CONTENTS = 'capability-probe-ok';
|
|
30
|
+
const HOOKS_LOG_NAME = 'hooks.log';
|
|
31
|
+
const HOOKS_LOG_GLOBAL_SETTINGS_MARKER = 'global/settings';
|
|
32
|
+
const HOOKS_LOG_PRE_TOOL_USE_MARKER = 'pre_tool_use';
|
|
33
|
+
const SPAWN_MARKER = 'SPAWN_OK';
|
|
34
|
+
const SPAWN_SUBAGENT_TOOL = 'spawn_subagent';
|
|
35
|
+
const GROK_SPAWN_TIMEOUT_MS = 600000;
|
|
36
|
+
const SPAWN_MAX_BUFFER_BYTES = 20 * 1024 * 1024;
|
|
37
|
+
const WORKFLOW_RESULT_NO_TOOL = 'no_tool';
|
|
38
|
+
const SPAWN_TIMEOUT_ERROR_CODE = 'ETIMEDOUT';
|
|
39
|
+
const ENVELOPE_TEXT_FIELDS_BY_PRIORITY = ['result', 'text', 'message'];
|
|
40
|
+
const STREAM_EVENT_TEXT_FIELDS_BY_PRIORITY = ['result', 'text', 'content'];
|
|
41
|
+
|
|
42
|
+
const E1_PROMPT = `You are running a capability inventory. Do not edit files.
|
|
43
|
+
|
|
44
|
+
1. Inspect your available tools.
|
|
45
|
+
2. Check whether agents and skills directories exist under the user's Claude config (typical locations: ~/.claude/agents, ~/.claude/skills).
|
|
46
|
+
|
|
47
|
+
Reply with ONLY one JSON object and nothing else:
|
|
48
|
+
{
|
|
49
|
+
"can_spawn_subagent_tool": <true if a spawn_subagent tool is available>,
|
|
50
|
+
"tool_names": [<string tool names>],
|
|
51
|
+
"agents_dir_exists": <bool>,
|
|
52
|
+
"skills_dir_exists": <bool>
|
|
53
|
+
}`;
|
|
54
|
+
|
|
55
|
+
const E2_PROMPT = `You are measuring spawn_subagent.
|
|
56
|
+
|
|
57
|
+
1. Use spawn_subagent (or the equivalent agent-spawn tool) once.
|
|
58
|
+
2. Instruct the child to print exactly ${SPAWN_MARKER} on its first line, then list basenames from the agents directory under the user's Claude config (for example ~/.claude/agents).
|
|
59
|
+
3. Wait for the child to finish.
|
|
60
|
+
|
|
61
|
+
Reply with ONLY one JSON object and nothing else:
|
|
62
|
+
{
|
|
63
|
+
"spawn_succeeded": <true if the child ran and returned ${SPAWN_MARKER}>,
|
|
64
|
+
"child_excerpt": "<excerpt of child output that includes its first line>"
|
|
65
|
+
}`;
|
|
66
|
+
|
|
67
|
+
const E3_PROMPT = `You are measuring skill readability under an agent definition.
|
|
68
|
+
|
|
69
|
+
1. Read any one skill entrypoint under the user's Claude skills path (for example ~/.claude/skills/*/SKILL.md). Prefer a short file if several exist.
|
|
70
|
+
2. Confirm you loaded an agent definition for this run (this process was started with --agent).
|
|
71
|
+
|
|
72
|
+
Reply with ONLY one JSON object and nothing else:
|
|
73
|
+
{
|
|
74
|
+
"skill_read_ok": <true if you read a skill file's contents>,
|
|
75
|
+
"skill_path": "<path you read or empty string>",
|
|
76
|
+
"agent_definition_loaded": <true if an agent definition is active>
|
|
77
|
+
}`;
|
|
78
|
+
|
|
79
|
+
const E4_PROMPT = `You are measuring write capability in the current working directory.
|
|
80
|
+
|
|
81
|
+
1. Write a file named ${PROBE_FILE_NAME} in the current working directory with exact contents: ${PROBE_FILE_CONTENTS}
|
|
82
|
+
2. Do not write anything else.
|
|
83
|
+
|
|
84
|
+
Reply with ONLY one JSON object and nothing else:
|
|
85
|
+
{
|
|
86
|
+
"write_succeeded": <true if the write completed>,
|
|
87
|
+
"written_path": "<absolute or relative path written>"
|
|
88
|
+
}`;
|
|
89
|
+
|
|
90
|
+
const E5_PROMPT = `You are measuring whether a Claude/GSD Workflow tool is available.
|
|
91
|
+
|
|
92
|
+
Inspect your tool list. Do not invent a Workflow tool.
|
|
93
|
+
|
|
94
|
+
Reply with ONLY one JSON object and nothing else:
|
|
95
|
+
{
|
|
96
|
+
"has_workflow_tool": <true only if a Workflow tool is present>,
|
|
97
|
+
"result": <"has_tool" or "no_tool">
|
|
98
|
+
}`;
|
|
99
|
+
|
|
100
|
+
function isOptedIn(allArguments) {
|
|
101
|
+
if (allArguments.includes(RUN_FLAG)) {
|
|
102
|
+
return true;
|
|
103
|
+
}
|
|
104
|
+
return process.env[OPT_IN_ENV] === OPT_IN_VALUE;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function printOptInHelp() {
|
|
108
|
+
console.log(
|
|
109
|
+
[
|
|
110
|
+
'Grok capability evals are opt-in only (not part of npm test / CI).',
|
|
111
|
+
'',
|
|
112
|
+
'Run with either:',
|
|
113
|
+
` ${OPT_IN_ENV}=${OPT_IN_VALUE} node skills/grokify/evals/run-capability-evals.mjs`,
|
|
114
|
+
' node skills/grokify/evals/run-capability-evals.mjs --run',
|
|
115
|
+
'',
|
|
116
|
+
'See skills/grokify/evals/README.md.',
|
|
117
|
+
].join('\n'),
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function mintLeaderSocketPath(runDirectory, evalName) {
|
|
122
|
+
return join(runDirectory, `leader-${evalName}.sock`);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function writePromptFile(runDirectory, evalName, promptText) {
|
|
126
|
+
const promptPath = join(runDirectory, `${evalName}-prompt.md`);
|
|
127
|
+
writeFileSync(promptPath, promptText, 'utf8');
|
|
128
|
+
return promptPath;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function buildGrokArguments({
|
|
132
|
+
promptPath,
|
|
133
|
+
workingDirectory,
|
|
134
|
+
leaderSocketPath,
|
|
135
|
+
maxTurns,
|
|
136
|
+
agentName,
|
|
137
|
+
}) {
|
|
138
|
+
const allArguments = [
|
|
139
|
+
'--prompt-file',
|
|
140
|
+
promptPath,
|
|
141
|
+
'--cwd',
|
|
142
|
+
workingDirectory,
|
|
143
|
+
'--output-format',
|
|
144
|
+
'json',
|
|
145
|
+
'--always-approve',
|
|
146
|
+
'--max-turns',
|
|
147
|
+
String(maxTurns),
|
|
148
|
+
'--permission-mode',
|
|
149
|
+
'bypassPermissions',
|
|
150
|
+
'--leader-socket',
|
|
151
|
+
leaderSocketPath,
|
|
152
|
+
];
|
|
153
|
+
if (agentName) {
|
|
154
|
+
allArguments.push('--agent', agentName);
|
|
155
|
+
}
|
|
156
|
+
return allArguments;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function runGrok({
|
|
160
|
+
promptPath,
|
|
161
|
+
workingDirectory,
|
|
162
|
+
leaderSocketPath,
|
|
163
|
+
maxTurns,
|
|
164
|
+
agentName,
|
|
165
|
+
}) {
|
|
166
|
+
const allArguments = buildGrokArguments({
|
|
167
|
+
promptPath,
|
|
168
|
+
workingDirectory,
|
|
169
|
+
leaderSocketPath,
|
|
170
|
+
maxTurns,
|
|
171
|
+
agentName,
|
|
172
|
+
});
|
|
173
|
+
const processResult = spawnSync(GROK_BINARY, allArguments, {
|
|
174
|
+
encoding: 'utf8',
|
|
175
|
+
cwd: workingDirectory,
|
|
176
|
+
maxBuffer: SPAWN_MAX_BUFFER_BYTES,
|
|
177
|
+
timeout: GROK_SPAWN_TIMEOUT_MS,
|
|
178
|
+
windowsHide: true,
|
|
179
|
+
});
|
|
180
|
+
return {
|
|
181
|
+
exitCode: processResult.status,
|
|
182
|
+
stdout: processResult.stdout || '',
|
|
183
|
+
stderr: processResult.stderr || '',
|
|
184
|
+
error: processResult.error || null,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function isPlainObject(value) {
|
|
189
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function extractLastBalancedObject(text) {
|
|
193
|
+
const doubleQuote = '"';
|
|
194
|
+
const escapeCharacter = '\\';
|
|
195
|
+
let isInsideString = false;
|
|
196
|
+
let isNextCharacterEscaped = false;
|
|
197
|
+
let openBraceDepth = 0;
|
|
198
|
+
let objectStartIndex = -1;
|
|
199
|
+
let lastParsedObject = null;
|
|
200
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
201
|
+
const character = text[index];
|
|
202
|
+
if (isInsideString) {
|
|
203
|
+
if (isNextCharacterEscaped) {
|
|
204
|
+
isNextCharacterEscaped = false;
|
|
205
|
+
} else if (character === escapeCharacter) {
|
|
206
|
+
isNextCharacterEscaped = true;
|
|
207
|
+
} else if (character === doubleQuote) {
|
|
208
|
+
isInsideString = false;
|
|
209
|
+
}
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
if (character === doubleQuote) {
|
|
213
|
+
isInsideString = true;
|
|
214
|
+
} else if (character === '{') {
|
|
215
|
+
if (openBraceDepth === 0) {
|
|
216
|
+
objectStartIndex = index;
|
|
217
|
+
}
|
|
218
|
+
openBraceDepth += 1;
|
|
219
|
+
} else if (character === '}' && openBraceDepth > 0) {
|
|
220
|
+
openBraceDepth -= 1;
|
|
221
|
+
if (openBraceDepth === 0) {
|
|
222
|
+
try {
|
|
223
|
+
const parsed = JSON.parse(text.slice(objectStartIndex, index + 1));
|
|
224
|
+
if (isPlainObject(parsed)) {
|
|
225
|
+
lastParsedObject = parsed;
|
|
226
|
+
}
|
|
227
|
+
} catch {
|
|
228
|
+
// not a JSON object; keep scanning for a later balanced candidate
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return lastParsedObject;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function tryParseJsonObject(text) {
|
|
237
|
+
const trimmed = text.trim();
|
|
238
|
+
if (!trimmed) {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
try {
|
|
242
|
+
const parsed = JSON.parse(trimmed);
|
|
243
|
+
if (isPlainObject(parsed)) {
|
|
244
|
+
return parsed;
|
|
245
|
+
}
|
|
246
|
+
} catch {
|
|
247
|
+
// fall through to balanced-object extraction
|
|
248
|
+
}
|
|
249
|
+
return extractLastBalancedObject(trimmed);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function extractStringField(parsedEnvelope, fieldName) {
|
|
253
|
+
if (!parsedEnvelope || typeof parsedEnvelope !== 'object') {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
const fieldValue = parsedEnvelope[fieldName];
|
|
257
|
+
if (typeof fieldValue === 'string') {
|
|
258
|
+
return fieldValue;
|
|
259
|
+
}
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function pickStringFieldByPriority(candidate, fieldNames) {
|
|
264
|
+
for (const fieldName of fieldNames) {
|
|
265
|
+
const fieldValue = extractStringField(candidate, fieldName);
|
|
266
|
+
if (fieldValue !== null) {
|
|
267
|
+
return fieldValue;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export function extractResultText(stdout) {
|
|
274
|
+
const trimmed = stdout.trim();
|
|
275
|
+
if (!trimmed) {
|
|
276
|
+
return '';
|
|
277
|
+
}
|
|
278
|
+
try {
|
|
279
|
+
const parsed = JSON.parse(trimmed);
|
|
280
|
+
if (Array.isArray(parsed)) {
|
|
281
|
+
const resultEvent = parsed.findLast(
|
|
282
|
+
(eachEvent) =>
|
|
283
|
+
eachEvent &&
|
|
284
|
+
typeof eachEvent === 'object' &&
|
|
285
|
+
eachEvent.type === 'result' &&
|
|
286
|
+
typeof eachEvent.result === 'string',
|
|
287
|
+
);
|
|
288
|
+
if (resultEvent) {
|
|
289
|
+
return resultEvent.result;
|
|
290
|
+
}
|
|
291
|
+
const textChunks = [];
|
|
292
|
+
for (const eachEvent of parsed) {
|
|
293
|
+
if (!eachEvent || typeof eachEvent !== 'object') {
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
const eventText = pickStringFieldByPriority(
|
|
297
|
+
eachEvent,
|
|
298
|
+
STREAM_EVENT_TEXT_FIELDS_BY_PRIORITY,
|
|
299
|
+
);
|
|
300
|
+
if (eventText !== null) {
|
|
301
|
+
textChunks.push(eventText);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
if (textChunks.length > 0) {
|
|
305
|
+
return textChunks.join('\n');
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (parsed && typeof parsed === 'object') {
|
|
309
|
+
const envelopeText = pickStringFieldByPriority(
|
|
310
|
+
parsed,
|
|
311
|
+
ENVELOPE_TEXT_FIELDS_BY_PRIORITY,
|
|
312
|
+
);
|
|
313
|
+
if (envelopeText !== null) {
|
|
314
|
+
return envelopeText;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
} catch {
|
|
318
|
+
// stdout is plain text
|
|
319
|
+
}
|
|
320
|
+
return trimmed;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
export function parsePayload(stdout) {
|
|
324
|
+
const resultText = extractResultText(stdout);
|
|
325
|
+
return tryParseJsonObject(resultText) || tryParseJsonObject(stdout);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function assertCondition(condition, message) {
|
|
329
|
+
if (!condition) {
|
|
330
|
+
throw new Error(message);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function resolveDefaultAgentName() {
|
|
335
|
+
return process.env.GROK_CAPABILITY_EVAL_AGENT || 'Explore';
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function describeLaunchFailure(label, processResult) {
|
|
339
|
+
const launchError = processResult.error;
|
|
340
|
+
if (
|
|
341
|
+
launchError &&
|
|
342
|
+
typeof launchError === 'object' &&
|
|
343
|
+
launchError.code === SPAWN_TIMEOUT_ERROR_CODE
|
|
344
|
+
) {
|
|
345
|
+
return `${label}: grok timed out after ${GROK_SPAWN_TIMEOUT_MS}ms`;
|
|
346
|
+
}
|
|
347
|
+
if (launchError !== null) {
|
|
348
|
+
return `${label}: failed to launch grok: ${launchError}`;
|
|
349
|
+
}
|
|
350
|
+
return `${label}: grok exit ${processResult.exitCode}\n${processResult.stderr}\n${processResult.stdout}`;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function launchEval(runDirectory, { evalName, promptText, maxTurns, agentName }) {
|
|
354
|
+
const label = evalName.toUpperCase();
|
|
355
|
+
const promptPath = writePromptFile(runDirectory, evalName, promptText);
|
|
356
|
+
const processResult = runGrok({
|
|
357
|
+
promptPath,
|
|
358
|
+
workingDirectory: runDirectory,
|
|
359
|
+
leaderSocketPath: mintLeaderSocketPath(runDirectory, evalName),
|
|
360
|
+
maxTurns,
|
|
361
|
+
agentName,
|
|
362
|
+
});
|
|
363
|
+
if (processResult.error !== null || processResult.exitCode !== 0) {
|
|
364
|
+
throw new Error(describeLaunchFailure(label, processResult));
|
|
365
|
+
}
|
|
366
|
+
const payload = parsePayload(processResult.stdout);
|
|
367
|
+
assertCondition(
|
|
368
|
+
payload !== null,
|
|
369
|
+
`${label}: could not parse JSON payload from grok output`,
|
|
370
|
+
);
|
|
371
|
+
return payload;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function runEvalOne(runDirectory) {
|
|
375
|
+
const payload = launchEval(runDirectory, {
|
|
376
|
+
evalName: 'e1',
|
|
377
|
+
promptText: E1_PROMPT,
|
|
378
|
+
maxTurns: DEFAULT_MAX_TURNS,
|
|
379
|
+
});
|
|
380
|
+
assertCondition(
|
|
381
|
+
payload.can_spawn_subagent_tool === true,
|
|
382
|
+
`E1: expected can_spawn_subagent_tool === true, got ${JSON.stringify(payload.can_spawn_subagent_tool)}`,
|
|
383
|
+
);
|
|
384
|
+
const allToolNames = Array.isArray(payload.tool_names)
|
|
385
|
+
? payload.tool_names.map(String)
|
|
386
|
+
: [];
|
|
387
|
+
assertCondition(
|
|
388
|
+
allToolNames.includes(SPAWN_SUBAGENT_TOOL),
|
|
389
|
+
`E1: expected tool_names to include ${SPAWN_SUBAGENT_TOOL}, got ${JSON.stringify(allToolNames)}`,
|
|
390
|
+
);
|
|
391
|
+
return payload;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function runEvalTwo(runDirectory) {
|
|
395
|
+
const payload = launchEval(runDirectory, {
|
|
396
|
+
evalName: 'e2',
|
|
397
|
+
promptText: E2_PROMPT,
|
|
398
|
+
maxTurns: SPAWN_MAX_TURNS,
|
|
399
|
+
});
|
|
400
|
+
assertCondition(
|
|
401
|
+
payload.spawn_succeeded === true,
|
|
402
|
+
`E2: expected spawn_succeeded === true, got ${JSON.stringify(payload.spawn_succeeded)}`,
|
|
403
|
+
);
|
|
404
|
+
const childExcerpt =
|
|
405
|
+
typeof payload.child_excerpt === 'string' ? payload.child_excerpt : '';
|
|
406
|
+
assertCondition(
|
|
407
|
+
childExcerpt.includes(SPAWN_MARKER),
|
|
408
|
+
`E2: expected child_excerpt to include ${SPAWN_MARKER}, got ${JSON.stringify(payload.child_excerpt)}`,
|
|
409
|
+
);
|
|
410
|
+
return payload;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function runEvalThree(runDirectory) {
|
|
414
|
+
const payload = launchEval(runDirectory, {
|
|
415
|
+
evalName: 'e3',
|
|
416
|
+
promptText: E3_PROMPT,
|
|
417
|
+
maxTurns: SKILL_MAX_TURNS,
|
|
418
|
+
agentName: resolveDefaultAgentName(),
|
|
419
|
+
});
|
|
420
|
+
assertCondition(
|
|
421
|
+
payload.skill_read_ok === true,
|
|
422
|
+
`E3: expected skill_read_ok === true, got ${JSON.stringify(payload.skill_read_ok)}`,
|
|
423
|
+
);
|
|
424
|
+
assertCondition(
|
|
425
|
+
payload.agent_definition_loaded === true,
|
|
426
|
+
`E3: expected agent_definition_loaded === true, got ${JSON.stringify(payload.agent_definition_loaded)}`,
|
|
427
|
+
);
|
|
428
|
+
return payload;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function runEvalFour(runDirectory) {
|
|
432
|
+
const payload = launchEval(runDirectory, {
|
|
433
|
+
evalName: 'e4',
|
|
434
|
+
promptText: E4_PROMPT,
|
|
435
|
+
maxTurns: WRITE_MAX_TURNS,
|
|
436
|
+
});
|
|
437
|
+
const probePath = join(runDirectory, PROBE_FILE_NAME);
|
|
438
|
+
assertCondition(
|
|
439
|
+
existsSync(probePath),
|
|
440
|
+
`E4: probe file missing under the eval cwd (write_succeeded claim: ${JSON.stringify(payload.write_succeeded)})`,
|
|
441
|
+
);
|
|
442
|
+
const probeContents = readFileSync(probePath, 'utf8');
|
|
443
|
+
assertCondition(
|
|
444
|
+
probeContents.includes(PROBE_FILE_CONTENTS),
|
|
445
|
+
`E4: probe file contents mismatch: ${JSON.stringify(probeContents)}`,
|
|
446
|
+
);
|
|
447
|
+
const hooksLogPath = join(runDirectory, HOOKS_LOG_NAME);
|
|
448
|
+
let hooksNote = 'hooks.log not present (soft check skipped)';
|
|
449
|
+
if (existsSync(hooksLogPath)) {
|
|
450
|
+
const hooksLogText = readFileSync(hooksLogPath, 'utf8').toLowerCase();
|
|
451
|
+
const hasGlobalSettings = hooksLogText.includes(
|
|
452
|
+
HOOKS_LOG_GLOBAL_SETTINGS_MARKER,
|
|
453
|
+
);
|
|
454
|
+
const hasPreToolUse = hooksLogText.includes(HOOKS_LOG_PRE_TOOL_USE_MARKER);
|
|
455
|
+
hooksNote =
|
|
456
|
+
hasGlobalSettings && hasPreToolUse
|
|
457
|
+
? 'hooks.log contains global/settings and pre_tool_use'
|
|
458
|
+
: 'hooks.log present but missing expected markers (soft)';
|
|
459
|
+
}
|
|
460
|
+
return { payload, hooksNote };
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
export function isWorkflowToolAbsent(payload) {
|
|
464
|
+
if (!payload || typeof payload !== 'object') {
|
|
465
|
+
return false;
|
|
466
|
+
}
|
|
467
|
+
return (
|
|
468
|
+
payload.has_workflow_tool === false &&
|
|
469
|
+
payload.result === WORKFLOW_RESULT_NO_TOOL
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
function runEvalFive(runDirectory) {
|
|
474
|
+
const payload = launchEval(runDirectory, {
|
|
475
|
+
evalName: 'e5',
|
|
476
|
+
promptText: E5_PROMPT,
|
|
477
|
+
maxTurns: WORKFLOW_MAX_TURNS,
|
|
478
|
+
});
|
|
479
|
+
assertCondition(
|
|
480
|
+
isWorkflowToolAbsent(payload),
|
|
481
|
+
`E5: expected has_workflow_tool === false and result === "no_tool", got ${JSON.stringify(payload)}`,
|
|
482
|
+
);
|
|
483
|
+
return payload;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
function main() {
|
|
487
|
+
if (!isOptedIn(process.argv.slice(2))) {
|
|
488
|
+
printOptInHelp();
|
|
489
|
+
process.exit(0);
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
const runDirectory = mkdtempSync(join(tmpdir(), 'grok-capability-evals-'));
|
|
493
|
+
console.log(`run directory: ${runDirectory}`);
|
|
494
|
+
|
|
495
|
+
try {
|
|
496
|
+
console.log('E1 tool inventory...');
|
|
497
|
+
const e1Payload = runEvalOne(runDirectory);
|
|
498
|
+
console.log('E1 ok', JSON.stringify(e1Payload));
|
|
499
|
+
|
|
500
|
+
console.log('E2 spawn_subagent...');
|
|
501
|
+
const e2Payload = runEvalTwo(runDirectory);
|
|
502
|
+
console.log('E2 ok', JSON.stringify(e2Payload));
|
|
503
|
+
|
|
504
|
+
console.log('E3 --agent + skill read...');
|
|
505
|
+
const e3Payload = runEvalThree(runDirectory);
|
|
506
|
+
console.log('E3 ok', JSON.stringify(e3Payload));
|
|
507
|
+
|
|
508
|
+
console.log('E4 probe write...');
|
|
509
|
+
const e4Outcome = runEvalFour(runDirectory);
|
|
510
|
+
console.log(
|
|
511
|
+
'E4 ok',
|
|
512
|
+
JSON.stringify({
|
|
513
|
+
payload: e4Outcome.payload,
|
|
514
|
+
hooksNote: e4Outcome.hooksNote,
|
|
515
|
+
}),
|
|
516
|
+
);
|
|
517
|
+
|
|
518
|
+
console.log('E5 workflow tool absence...');
|
|
519
|
+
const e5Payload = runEvalFive(runDirectory);
|
|
520
|
+
console.log('E5 ok', JSON.stringify(e5Payload));
|
|
521
|
+
|
|
522
|
+
console.log('ALL CAPABILITY EVALS PASSED');
|
|
523
|
+
} catch (failure) {
|
|
524
|
+
console.error(String(failure && failure.stack ? failure.stack : failure));
|
|
525
|
+
process.exitCode = 1;
|
|
526
|
+
} finally {
|
|
527
|
+
try {
|
|
528
|
+
rmSync(runDirectory, { recursive: true, force: true });
|
|
529
|
+
} catch {
|
|
530
|
+
// best-effort cleanup
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function isDirectExecution() {
|
|
536
|
+
const entryArgument = process.argv[1];
|
|
537
|
+
if (!entryArgument) {
|
|
538
|
+
return false;
|
|
539
|
+
}
|
|
540
|
+
return resolve(fileURLToPath(import.meta.url)) === resolve(entryArgument);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
if (isDirectExecution()) {
|
|
544
|
+
main();
|
|
545
|
+
}
|
|
@@ -21,16 +21,12 @@ delegating to Sonnet-5 workers came out cheaper and faster
|
|
|
21
21
|
than a solo frontier agent held to the same verification rigor, with
|
|
22
22
|
84-98% of the team's input tokens billed at the worker rate.
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
shape with the tools Claude Code already has. Under this skill the session
|
|
27
|
-
is the orchestrator. In Claude Code the user always talks to the session and never to a
|
|
28
|
-
subagent, so the session is the user's sole interface: all user-facing
|
|
29
|
-
communication flows through it. It spawns and resumes executor subagents
|
|
24
|
+
Under this skill the session
|
|
25
|
+
is the orchestrator. It spawns and resumes executor subagents
|
|
30
26
|
— `clean-coder` and the like — and those executors do every bit of the
|
|
31
27
|
execution: the code edits, the build runs, the test runs. The orchestrating
|
|
32
28
|
session drives the plan and routes hard decisions to the shared advisor
|
|
33
|
-
(Claude: warm `session-advisor`;
|
|
29
|
+
(Claude: warm `session-advisor`; a third-party host: Claude CLI advisor via the chain runner).
|
|
34
30
|
|
|
35
31
|
## Gotchas
|
|
36
32
|
|
|
@@ -43,7 +39,7 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
43
39
|
wasted. Hand every code edit and every build or test run to an executor; keep
|
|
44
40
|
the orchestrating session's own tool use to orchestration and light
|
|
45
41
|
verification reads. Hard decisions go to the shared advisor (Claude:
|
|
46
|
-
`session-advisor` via SendMessage;
|
|
42
|
+
`session-advisor` via SendMessage; a third-party host: Claude CLI advisor via the chain).
|
|
47
43
|
- **Flat ad hoc spawns bypass routing.** Every execution task goes through a
|
|
48
44
|
workflow-backed spawn or workflow resume so the required agent type, model,
|
|
49
45
|
prompt packet, and sidecar metadata stay attached to the work.
|
|
@@ -55,25 +51,20 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
55
51
|
reach that agent later. A named agent is reachable by name.
|
|
56
52
|
- **Only the orchestrating session owns the shared advisor's lifecycle.** An
|
|
57
53
|
executor that finds the advisor unreachable (Claude warm agent, or Claude
|
|
58
|
-
CLI bind on
|
|
54
|
+
CLI bind on a third-party host) reports that upward; it never spawns a replacement itself.
|
|
59
55
|
|
|
60
56
|
## Process
|
|
61
57
|
|
|
62
58
|
1. **Check whether the refresh loop is already running this
|
|
63
59
|
session.** If it is, do not schedule a second loop. Reuse any live
|
|
64
|
-
shared advisor bind (Claude warm `session-advisor`, or
|
|
65
|
-
bind — run step 3 only when none exists yet), then skip straight to step 4.
|
|
66
|
-
|
|
67
|
-
2. **Register the discipline reminder.**
|
|
68
|
-
`ScheduleWakeup` at `delaySeconds:
|
|
69
|
-
where each refresh re-schedules the next one
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
`ScheduleWakeup` is not available. Either way the reminder is the
|
|
73
|
-
enforcement surface: each firing re-asserts the discipline while the run is
|
|
74
|
-
in flight.
|
|
75
|
-
|
|
76
|
-
3. **Bind the shared advisor before any executor.** Detect the host profile
|
|
60
|
+
shared advisor bind (Claude warm `session-advisor`, or a third-party host's
|
|
61
|
+
Claude CLI bind — run step 3 only when none exists yet), then skip straight to step 4.
|
|
62
|
+
|
|
63
|
+
2. **Register the discipline reminder.** schedule it with
|
|
64
|
+
`ScheduleWakeup` at `delaySeconds: 1800`, prompt `/orchestrator-refresh`,
|
|
65
|
+
where each refresh re-schedules the next one.
|
|
66
|
+
|
|
67
|
+
3. **Bind a shared advisor before any executor.** Detect the host profile
|
|
77
68
|
first (see Host profiles in
|
|
78
69
|
[`_shared/advisor/advisor-protocol.md`](../../_shared/advisor/advisor-protocol.md)).
|
|
79
70
|
On a **Claude host**, follow the Warm-up procedure in that doc (Agent spawn
|
|
@@ -85,18 +76,18 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
85
76
|
[`_shared/advisor/advisor-protocol.md`](../../_shared/advisor/advisor-protocol.md);
|
|
86
77
|
it is not a "spawn only at the floor" rule. Paste the **Claude host**
|
|
87
78
|
Advisor block from that doc, with the resolved agent name filled in, into
|
|
88
|
-
every executor's spawn prompt. On a **
|
|
79
|
+
every executor's spawn prompt. On a **third-party host**, skip the Agent spawn —
|
|
89
80
|
bind a max-tier Claude advisor through the protocol's CLI Claude-chain path
|
|
90
81
|
(Fable high, then Opus max; `claude_chain_runner.py` for account usage
|
|
91
|
-
failover). Paste the **
|
|
82
|
+
failover). Paste the **Third-party host** Advisor block from that doc into every
|
|
92
83
|
executor's spawn prompt — never the Claude SendMessage block. Every row in
|
|
93
84
|
the routing table is a consumer of the shared advisor, not just this
|
|
94
85
|
session. The orchestrating session owns the shared advisor's lifecycle end
|
|
95
86
|
to end (Agent spawn or CLI bind, drift handling per the shared doc,
|
|
96
87
|
shutdown at task end); executors only ever SendMessage the warm agent
|
|
97
88
|
(Claude) or report to this session so it can consult the CLI advisor
|
|
98
|
-
(
|
|
99
|
-
— do not answer the four signals as this
|
|
89
|
+
(a third-party host). When the CLI chain cannot bind, fail closed and report
|
|
90
|
+
to the user — do not answer the four signals as this third-party session.
|
|
100
91
|
|
|
101
92
|
4. **Orchestrate the task.** Hold the plan and the user conversation. Execute
|
|
102
93
|
workflow-backed spawns or resumes using the routing table below, and keep
|
|
@@ -114,7 +105,7 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
114
105
|
- The chosen approach is being reconsidered.
|
|
115
106
|
|
|
116
107
|
**Claude host:** consult the shared `session-advisor` via `SendMessage`.
|
|
117
|
-
**
|
|
108
|
+
**Third-party host:** executors report to this orchestrating session; this session
|
|
118
109
|
consults the bound Claude CLI advisor (`claude_chain_runner.py` +
|
|
119
110
|
`--resume`) and relays the four-signal reply (no Agent-tool
|
|
120
111
|
`session-advisor`, no SendMessage to one). Every consult gets back one of
|
|
@@ -153,10 +144,14 @@ Routing rules:
|
|
|
153
144
|
it.
|
|
154
145
|
- Exploration workflows return file paths, line numbers, and direct evidence;
|
|
155
146
|
they do not write code or mutate repo state.
|
|
147
|
+
- Fan-out worker fleets use the **grok-spawn** skill when that skill is
|
|
148
|
+
installed and grok is usable (`grok_worker_preflight.py` soft gate). The
|
|
149
|
+
Claude Code Agent tool remains the Claude-host alternative for in-process
|
|
150
|
+
workers.
|
|
156
151
|
|
|
157
152
|
## Agent reuse (non-negotiable)
|
|
158
153
|
|
|
159
|
-
- **Resume before you spawn, always.** A warm agent carries its context and
|
|
154
|
+
- **Resume before you spawn, always.** A warm agent (active within the past 59 minutes) carries its context and
|
|
160
155
|
cached tokens; a fresh spawn starts cold and pays to rebuild both.
|
|
161
156
|
Resume the existing workflow agent by name or `agentId` when it holds
|
|
162
157
|
relevant context. Prefer that path every time an existing workflow agent
|
|
@@ -166,6 +161,24 @@ Routing rules:
|
|
|
166
161
|
- **Name the agent to resume.** When a PLAN from the shared advisor fits a warm
|
|
167
162
|
agent, name which agent to resume and where.
|
|
168
163
|
|
|
164
|
+
## Task ledger discipline
|
|
165
|
+
|
|
166
|
+
The task list is the run's ledger, and it must be reconcilable against the live
|
|
167
|
+
agents at any moment. Four invariants hold at all times:
|
|
168
|
+
|
|
169
|
+
1. **No untracked work.** Every unit of delegated work has a task BEFORE its
|
|
170
|
+
executor spawns — TaskCreate first, then Agent.
|
|
171
|
+
2. **Ownership is live.** At spawn, set the task `in_progress` with `owner` =
|
|
172
|
+
the executor's agent name. One task, one owner.
|
|
173
|
+
3. **Completion follows evidence.** A task turns `completed` only when the
|
|
174
|
+
executor's result is back AND merged into run state — never on dispatch,
|
|
175
|
+
never on a self-report alone (see `workers-done-before-complete`).
|
|
176
|
+
4. **Dependencies mirror the plan.** Phase order is encoded as `blockedBy`
|
|
177
|
+
links, updated the moment the plan changes.
|
|
178
|
+
|
|
179
|
+
Reconcile on every state change (spawn, completion notification, plan change)
|
|
180
|
+
and on every `/orchestrator-refresh` firing.
|
|
181
|
+
|
|
169
182
|
## Constraints
|
|
170
183
|
|
|
171
184
|
- One `/orchestrator` per session; the invocation guard blocks a second
|
|
@@ -179,7 +192,7 @@ Routing rules:
|
|
|
179
192
|
session (see
|
|
180
193
|
[`_shared/advisor/advisor-protocol.md`](../../_shared/advisor/advisor-protocol.md))
|
|
181
194
|
— on Claude, executors consult a warm `session-advisor` via SendMessage; on
|
|
182
|
-
|
|
195
|
+
a third-party host, executors report here and this session relays the Claude CLI advisor.
|
|
183
196
|
Executors never spawn or respawn the advisor.
|
|
184
197
|
- Reuse a warm agent over a cold spawn whenever one holds relevant context.
|
|
185
198
|
|