claude-dev-env 1.94.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_shared/advisor/CLAUDE.md +2 -2
- package/_shared/advisor/advisor-protocol.md +41 -33
- package/_shared/advisor/scripts/config/advisor_scripts_constants/model_tier_run_validator_constants.py +17 -13
- package/_shared/advisor/scripts/model_tier_run_validator.py +27 -18
- package/_shared/advisor/scripts/tests/test_model_tier_run_validator.py +95 -25
- package/_shared/advisor/scripts/tests/test_tier_model_ids.py +17 -17
- package/_shared/advisor/scripts/tier_model_ids.py +18 -18
- package/_shared/pr-loop/CLAUDE.md +1 -0
- package/_shared/pr-loop/scripts/CLAUDE.md +2 -1
- package/_shared/pr-loop/scripts/README.md +1 -0
- package/_shared/pr-loop/scripts/code_rules_gate.py +253 -1980
- package/_shared/pr-loop/scripts/code_rules_gate_parts/CLAUDE.md +32 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/__init__.py +7 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/added_line_maps.py +268 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/enforcer_loading.py +172 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/gate_arguments.py +70 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/gate_running.py +326 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/git_blob_readers.py +85 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/git_file_sets.py +331 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/staged_test_running.py +369 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/conftest.py +14 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_added_line_maps.py +118 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_enforcer_loading.py +17 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_gate_arguments.py +29 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_gate_running.py +99 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_git_blob_readers.py +69 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_git_file_sets.py +137 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_staged_test_running.py +116 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_violation_scoping.py +75 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/tests/test_wrapper_plumb_check.py +49 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/violation_scoping.py +328 -0
- package/_shared/pr-loop/scripts/code_rules_gate_parts/wrapper_plumb_check.py +206 -0
- package/_shared/pr-loop/scripts/pr_loop_shared_constants/code_rules_gate_constants.py +24 -17
- package/_shared/pr-loop/scripts/pr_loop_shared_constants/reviews_disabled_constants.py +1 -0
- package/_shared/pr-loop/scripts/reviews_disabled.py +19 -2
- package/_shared/pr-loop/scripts/test_code_rules_gate.py +278 -0
- package/_shared/pr-loop/scripts/tests/test_code_rules_gate_constants.py +6 -39
- package/_shared/pr-loop/scripts/tests/test_reviews_disabled.py +43 -0
- package/_shared/pr-loop/worker-spawn.md +186 -0
- package/agents/code-verifier.md +1 -1
- package/bin/CLAUDE.md +10 -1
- package/bin/ever-shipped-skills.mjs +73 -0
- package/bin/expand_home_directory_tokens.mjs +1 -1
- package/bin/install.mjs +140 -7
- package/bin/install.prune.test.mjs +457 -0
- package/docs/CODE_RULES.md +1 -1
- package/hooks/advisory/refactor_guard.py +3 -4
- package/hooks/blocking/CLAUDE.md +7 -1
- package/hooks/blocking/block_main_commit.py +2 -2
- package/hooks/blocking/claude_md_orphan_file_blocker.py +75 -699
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/CLAUDE.md +28 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/__init__.py +1 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/config/__init__.py +1 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/config/orphan_blocker_constants.py +18 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/decision.py +81 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/references.py +307 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/scan_plan.py +124 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/subtree_scan.py +179 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/conftest.py +10 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_decision.py +34 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_references.py +42 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_scan_plan.py +27 -0
- package/hooks/blocking/claude_md_orphan_file_blocker_parts/tests/test_subtree_scan.py +30 -0
- package/hooks/blocking/code_rules_boolean_mustcheck.py +1 -1
- package/hooks/blocking/code_rules_enforcer.py +4 -0
- package/hooks/blocking/code_rules_mock_completeness.py +1 -1
- package/hooks/blocking/code_rules_optional_params.py +2 -2
- package/hooks/blocking/code_rules_shared.py +83 -1
- package/hooks/blocking/code_rules_test_assertions.py +1 -1
- package/hooks/blocking/code_rules_test_layout.py +9 -3
- package/hooks/blocking/code_rules_typeddict_stub.py +1 -1
- package/hooks/blocking/gh_pr_author_enforcer.py +1 -1
- package/hooks/blocking/inventory_intent_records/CLAUDE.md +26 -0
- package/hooks/blocking/inventory_intent_records/__init__.py +1 -0
- package/hooks/blocking/inventory_intent_records/config/__init__.py +1 -0
- package/hooks/blocking/inventory_intent_records/config/intent_records_constants.py +20 -0
- package/hooks/blocking/inventory_intent_records/records.py +271 -0
- package/hooks/blocking/inventory_intent_records/tests/conftest.py +10 -0
- package/hooks/blocking/inventory_intent_records/tests/test_records.py +80 -0
- package/hooks/blocking/package_inventory_stale_blocker.py +54 -384
- package/hooks/blocking/package_inventory_stale_blocker_parts/CLAUDE.md +26 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/__init__.py +1 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/config/__init__.py +1 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/config/inventory_blocker_constants.py +16 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/decision.py +84 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/inventory_detection.py +307 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/tests/conftest.py +10 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/tests/test_decision.py +38 -0
- package/hooks/blocking/package_inventory_stale_blocker_parts/tests/test_inventory_detection.py +61 -0
- package/hooks/blocking/pii_payload_scan.py +138 -42
- package/hooks/blocking/pii_prevention_blocker.py +185 -291
- package/hooks/blocking/pii_prevention_blocker_parts/CLAUDE.md +24 -0
- package/hooks/blocking/pii_prevention_blocker_parts/__init__.py +1 -0
- package/hooks/blocking/pii_prevention_blocker_parts/config/__init__.py +1 -0
- package/hooks/blocking/pii_prevention_blocker_parts/config/repository_resolution_constants.py +28 -0
- package/hooks/blocking/pii_prevention_blocker_parts/repository_exemption.py +214 -0
- package/hooks/blocking/pii_prevention_blocker_parts/repository_resolution.py +208 -0
- package/hooks/blocking/plain_language_blocker.py +138 -4
- package/hooks/blocking/pr_description_command_parser.py +8 -4
- package/hooks/blocking/precommit_code_rules_gate.py +3 -3
- package/hooks/blocking/sensitive_file_protector.py +114 -48
- package/hooks/blocking/tdd_enforcer.py +97 -601
- package/hooks/blocking/tdd_enforcer_parts/CLAUDE.md +30 -0
- package/hooks/blocking/tdd_enforcer_parts/__init__.py +1 -0
- package/hooks/blocking/tdd_enforcer_parts/candidate_paths.py +142 -0
- package/hooks/blocking/tdd_enforcer_parts/config/__init__.py +1 -0
- package/hooks/blocking/tdd_enforcer_parts/config/tdd_enforcer_constants.py +32 -0
- package/hooks/blocking/tdd_enforcer_parts/content_analysis.py +268 -0
- package/hooks/blocking/tdd_enforcer_parts/decisions.py +92 -0
- package/hooks/blocking/tdd_enforcer_parts/freshness.py +80 -0
- package/hooks/blocking/tdd_enforcer_parts/git_tracking.py +63 -0
- package/hooks/blocking/tdd_enforcer_parts/path_classification.py +119 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/conftest.py +10 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_candidate_paths.py +31 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_content_analysis.py +30 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_decisions.py +34 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_freshness.py +28 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_git_tracking.py +48 -0
- package/hooks/blocking/tdd_enforcer_parts/tests/test_path_classification.py +36 -0
- package/hooks/blocking/test_code_rules_enforcer_scratchpad.py +105 -0
- package/hooks/blocking/test_code_rules_shared.py +181 -0
- package/hooks/blocking/test_inventory_deadlock_resolution.py +154 -0
- package/hooks/blocking/test_pii_payload_scan.py +168 -0
- package/hooks/blocking/test_plain_language_blocker_allowlist.py +184 -0
- package/hooks/blocking/test_sensitive_file_protector.py +185 -0
- package/hooks/blocking/test_tdd_enforcer_restore.py +108 -0
- package/hooks/blocking/test_tdd_enforcer_scratchpad.py +105 -0
- package/hooks/blocking/tests/conftest.py +10 -0
- package/hooks/blocking/tests/test_pii_prevention_blocker.py +260 -0
- package/hooks/blocking/tests/test_repository_exemption.py +105 -0
- package/hooks/blocking/tests/test_repository_resolution.py +108 -0
- package/hooks/diagnostic/hook_log_extractor.py +12 -10
- package/hooks/git-hooks/post_commit.py +3 -4
- package/hooks/hooks_constants/CLAUDE.md +4 -2
- package/hooks/hooks_constants/banned_identifiers_constants.py +0 -1
- package/hooks/hooks_constants/code_rules_path_utils_constants.py +1 -1
- package/hooks/hooks_constants/harness_scratchpad_constants.py +17 -0
- package/hooks/hooks_constants/local_identity.py +59 -8
- package/hooks/hooks_constants/pii_prevention_constants.py +0 -6
- package/hooks/hooks_constants/plain_language_blocker_constants.py +5 -0
- package/hooks/hooks_constants/sensitive_file_protector_constants.py +42 -0
- package/hooks/hooks_constants/test_local_identity.py +105 -3
- package/hooks/pyproject.toml +52 -4
- package/hooks/session/plugin_data_dir_cleanup.py +0 -1
- package/hooks/validation/mypy_validator.py +2 -2
- package/hooks/validators/CLAUDE.md +1 -1
- package/hooks/validators/README.md +2 -0
- package/hooks/validators/health_check.py +1 -0
- package/hooks/validators/mypy_integration.py +2 -0
- package/hooks/validators/python_style_checks.py +114 -136
- package/hooks/validators/python_style_helpers.py +95 -0
- package/hooks/validators/ruff_integration.py +3 -0
- package/hooks/validators/test_python_style_checks.py +0 -164
- package/hooks/validators/test_python_style_checks_decorator_gap.py +119 -0
- package/hooks/validators/test_python_style_fixes.py +251 -0
- package/hooks/validators/test_python_style_helpers.py +125 -0
- package/hooks/workflow/auto_formatter.py +5 -4
- package/package.json +1 -1
- package/rules/CLAUDE.md +1 -0
- package/rules/anti-corollary-tests.md +69 -0
- package/rules/bdd.md +1 -3
- package/rules/code-reviews.md +1 -1
- package/rules/gh-paginate.md +1 -1
- package/rules/plain-language.md +2 -0
- package/scripts/CLAUDE.md +4 -0
- package/scripts/dev_env_scripts_constants/CLAUDE.md +6 -4
- package/scripts/dev_env_scripts_constants/code_review_constants.py +71 -0
- package/scripts/dev_env_scripts_constants/grok_worker_constants.py +435 -0
- package/scripts/dev_env_scripts_constants/timing.py +7 -1
- package/scripts/grok_headless_runner.py +294 -0
- package/scripts/grok_worker_preflight.py +410 -0
- package/scripts/invoke_code_review.py +463 -0
- package/scripts/resolve_worker_spawn.py +619 -0
- package/scripts/spawn_grok_batch.py +672 -0
- package/scripts/test_grok_headless_runner.py +626 -0
- package/scripts/test_grok_worker_preflight.py +1054 -0
- package/scripts/test_invoke_code_review.py +672 -0
- package/scripts/test_resolve_worker_spawn.py +1014 -0
- package/scripts/test_spawn_grok_batch.py +1017 -0
- package/skills/CLAUDE.md +6 -3
- package/skills/_shared/pr-loop/scripts/build_audit_prompt.py +72 -13
- package/skills/_shared/pr-loop/scripts/build_fix_prompt.py +121 -14
- package/skills/_shared/pr-loop/scripts/skills_pr_loop_constants/path_resolver_constants.py +78 -0
- package/skills/_shared/pr-loop/scripts/test_build_audit_prompt.py +121 -0
- package/skills/_shared/pr-loop/scripts/test_build_fix_prompt.py +196 -6
- package/skills/autoconverge/CLAUDE.md +3 -3
- package/skills/autoconverge/SKILL.md +9 -3
- package/skills/autoconverge/reference/CLAUDE.md +2 -2
- package/skills/autoconverge/reference/convergence.md +33 -11
- package/skills/autoconverge/reference/stop-conditions.md +16 -5
- package/skills/autoconverge/workflow/CLAUDE.md +2 -1
- package/skills/autoconverge/workflow/converge.clean-audit.test.mjs +7 -2
- package/skills/autoconverge/workflow/converge.codex-gate.test.mjs +300 -0
- package/skills/autoconverge/workflow/converge.contract.test.mjs +5 -5
- package/skills/autoconverge/workflow/converge.copilot-gate.test.mjs +29 -29
- package/skills/autoconverge/workflow/converge.fix-progress.test.mjs +1 -1
- package/skills/autoconverge/workflow/converge.mjs +202 -18
- package/skills/bugteam/CLAUDE.md +2 -2
- package/skills/bugteam/CONSTRAINTS.md +3 -2
- package/skills/bugteam/PROMPTS.md +7 -6
- package/skills/bugteam/SKILL.md +18 -13
- package/skills/bugteam/reference/README.md +2 -3
- package/skills/bugteam/reference/audit-and-teammates.md +215 -35
- package/skills/bugteam/reference/design-rationale.md +1 -1
- package/skills/bugteam/reference/obstacles/CLAUDE.md +1 -1
- package/skills/bugteam/reference/team-setup.md +8 -2
- package/skills/closeout/SKILL.md +153 -0
- package/skills/closeout/reference/handoff-prompt-template.md +72 -0
- package/skills/closeout/reference/issue-body-templates.md +108 -0
- package/skills/closeout/reference/pii-redaction-checklist.md +36 -0
- package/skills/codex-review/CLAUDE.md +46 -0
- package/skills/codex-review/SKILL.md +181 -0
- package/skills/codex-review/reference/CLAUDE.md +15 -0
- package/skills/codex-review/reference/cli-contract.md +253 -0
- package/skills/codex-review/reference/loop-integration.md +118 -0
- package/skills/codex-review/scripts/codex_down_classifier.py +98 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/CLAUDE.md +18 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/__init__.py +1 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/classifier_constants.py +35 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/codex_usage_probe_constants.py +86 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/findings_constants.py +18 -0
- package/skills/codex-review/scripts/codex_review_scripts_constants/run_constants.py +45 -0
- package/skills/codex-review/scripts/codex_usage_probe.py +573 -0
- package/skills/codex-review/scripts/fixtures/auth_failure_synthetic.txt +1 -0
- package/skills/codex-review/scripts/fixtures/config_load_failure_v0.125.0.txt +1 -0
- package/skills/codex-review/scripts/fixtures/freeform_findings_v0.144.3.txt +6 -0
- package/skills/codex-review/scripts/fixtures/model_rejection_v0.125.0.jsonl +5 -0
- package/skills/codex-review/scripts/fixtures/structured_findings.txt +13 -0
- package/skills/codex-review/scripts/fixtures/success_stream_v0.144.3.jsonl +6 -0
- package/skills/codex-review/scripts/fixtures/unknown_failure_synthetic.txt +1 -0
- package/skills/codex-review/scripts/fixtures/usage_limit_synthetic.txt +1 -0
- package/skills/codex-review/scripts/parse_codex_findings.py +207 -0
- package/skills/codex-review/scripts/run_codex_review.py +415 -0
- package/skills/codex-review/scripts/test_codex_down_classifier.py +143 -0
- package/skills/codex-review/scripts/test_codex_usage_probe.py +678 -0
- package/skills/codex-review/scripts/test_parse_codex_findings.py +130 -0
- package/skills/codex-review/scripts/test_run_codex_review.py +812 -0
- package/skills/codex-review/test_skill_scaffold.py +192 -0
- package/skills/grok-spawn/CLAUDE.md +28 -0
- package/skills/grok-spawn/SKILL.md +226 -0
- package/skills/grok-spawn/reference/flag-profiles.md +132 -0
- package/skills/grok-spawn/reference/worker-briefs.md +152 -0
- package/skills/grokify/SKILL.md +9 -1
- package/skills/grokify/capability-claims.test.mjs +28 -0
- package/skills/grokify/evals/README.md +72 -0
- package/skills/grokify/evals/parse-payload.test.mjs +171 -0
- package/skills/grokify/evals/run-capability-evals.mjs +545 -0
- package/skills/orchestrator/SKILL.md +32 -22
- package/skills/orchestrator-refresh/SKILL.md +12 -8
- package/skills/pr-converge/CLAUDE.md +1 -1
- package/skills/pr-converge/SKILL.md +34 -13
- package/skills/pr-converge/reference/convergence-gates.md +42 -15
- package/skills/pr-converge/reference/fix-protocol.md +1 -1
- package/skills/pr-converge/reference/ground-rules.md +1 -1
- package/skills/pr-converge/reference/per-tick.md +130 -42
- package/skills/pr-converge/reference/state-schema.md +10 -0
- package/skills/pr-converge/scripts/CLAUDE.md +2 -0
- package/skills/pr-converge/scripts/_pr_converge_path_setup.py +5 -1
- package/skills/pr-converge/scripts/check_convergence.py +605 -29
- package/skills/pr-converge/scripts/check_convergence_availability.py +232 -0
- package/skills/pr-converge/scripts/check_convergence_gates.py +279 -235
- package/skills/pr-converge/scripts/check_convergence_thread_gates.py +1 -1
- package/skills/pr-converge/scripts/pr_converge_scripts_constants/convergence_gate_constants.py +36 -2
- package/skills/pr-converge/scripts/test__pr_converge_path_setup.py +4 -0
- package/skills/pr-converge/scripts/test_check_convergence.py +71 -3
- package/skills/pr-converge/scripts/test_check_convergence_availability.py +326 -0
- package/skills/pr-converge/scripts/test_check_convergence_codex.py +507 -0
- package/skills/pr-converge/scripts/test_check_convergence_contract.py +89 -17
- package/skills/pr-converge/scripts/test_check_convergence_fixture.py +179 -0
- package/skills/pr-converge/scripts/test_check_convergence_gates.py +84 -68
- package/skills/pr-converge/scripts/test_check_convergence_thread_gates.py +24 -0
- package/skills/pr-converge/test_step5_host_branch.py +106 -0
- package/skills/pr-fix-protocol/SKILL.md +65 -0
- package/skills/pr-loop-cloud-transport/SKILL.md +2 -0
- package/skills/reviewer-gates/SKILL.md +7 -5
- package/skills/skill-builder/references/skill-modularity.md +1 -1
- package/skills/team-advisor/SKILL.md +16 -12
- package/system-prompts/software-engineer.xml +7 -6
- package/hooks/validators/test_verify_paths.py +0 -32
- package/hooks/validators/verify_paths.py +0 -57
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Opt-in live Grok capability evals (E1–E5).
|
|
4
|
+
*
|
|
5
|
+
* Never wired into package.json `test`. Requires GROK_CAPABILITY_EVALS=1 or --run.
|
|
6
|
+
*/
|
|
7
|
+
import { spawnSync } from 'node:child_process';
|
|
8
|
+
import {
|
|
9
|
+
existsSync,
|
|
10
|
+
mkdtempSync,
|
|
11
|
+
readFileSync,
|
|
12
|
+
rmSync,
|
|
13
|
+
writeFileSync,
|
|
14
|
+
} from 'node:fs';
|
|
15
|
+
import { tmpdir } from 'node:os';
|
|
16
|
+
import { join, resolve } from 'node:path';
|
|
17
|
+
import { fileURLToPath } from 'node:url';
|
|
18
|
+
|
|
19
|
+
const OPT_IN_ENV = 'GROK_CAPABILITY_EVALS';
|
|
20
|
+
const OPT_IN_VALUE = '1';
|
|
21
|
+
const RUN_FLAG = '--run';
|
|
22
|
+
const GROK_BINARY = process.env.GROK_BIN || 'grok';
|
|
23
|
+
const DEFAULT_MAX_TURNS = 12;
|
|
24
|
+
const SPAWN_MAX_TURNS = 20;
|
|
25
|
+
const SKILL_MAX_TURNS = 16;
|
|
26
|
+
const WRITE_MAX_TURNS = 12;
|
|
27
|
+
const WORKFLOW_MAX_TURNS = 8;
|
|
28
|
+
const PROBE_FILE_NAME = 'capability-probe-write.txt';
|
|
29
|
+
const PROBE_FILE_CONTENTS = 'capability-probe-ok';
|
|
30
|
+
const HOOKS_LOG_NAME = 'hooks.log';
|
|
31
|
+
const HOOKS_LOG_GLOBAL_SETTINGS_MARKER = 'global/settings';
|
|
32
|
+
const HOOKS_LOG_PRE_TOOL_USE_MARKER = 'pre_tool_use';
|
|
33
|
+
const SPAWN_MARKER = 'SPAWN_OK';
|
|
34
|
+
const SPAWN_SUBAGENT_TOOL = 'spawn_subagent';
|
|
35
|
+
const GROK_SPAWN_TIMEOUT_MS = 600000;
|
|
36
|
+
const SPAWN_MAX_BUFFER_BYTES = 20 * 1024 * 1024;
|
|
37
|
+
const WORKFLOW_RESULT_NO_TOOL = 'no_tool';
|
|
38
|
+
const SPAWN_TIMEOUT_ERROR_CODE = 'ETIMEDOUT';
|
|
39
|
+
const ENVELOPE_TEXT_FIELDS_BY_PRIORITY = ['result', 'text', 'message'];
|
|
40
|
+
const STREAM_EVENT_TEXT_FIELDS_BY_PRIORITY = ['result', 'text', 'content'];
|
|
41
|
+
|
|
42
|
+
const E1_PROMPT = `You are running a capability inventory. Do not edit files.
|
|
43
|
+
|
|
44
|
+
1. Inspect your available tools.
|
|
45
|
+
2. Check whether agents and skills directories exist under the user's Claude config (typical locations: ~/.claude/agents, ~/.claude/skills).
|
|
46
|
+
|
|
47
|
+
Reply with ONLY one JSON object and nothing else:
|
|
48
|
+
{
|
|
49
|
+
"can_spawn_subagent_tool": <true if a spawn_subagent tool is available>,
|
|
50
|
+
"tool_names": [<string tool names>],
|
|
51
|
+
"agents_dir_exists": <bool>,
|
|
52
|
+
"skills_dir_exists": <bool>
|
|
53
|
+
}`;
|
|
54
|
+
|
|
55
|
+
const E2_PROMPT = `You are measuring spawn_subagent.
|
|
56
|
+
|
|
57
|
+
1. Use spawn_subagent (or the equivalent agent-spawn tool) once.
|
|
58
|
+
2. Instruct the child to print exactly ${SPAWN_MARKER} on its first line, then list basenames from the agents directory under the user's Claude config (for example ~/.claude/agents).
|
|
59
|
+
3. Wait for the child to finish.
|
|
60
|
+
|
|
61
|
+
Reply with ONLY one JSON object and nothing else:
|
|
62
|
+
{
|
|
63
|
+
"spawn_succeeded": <true if the child ran and returned ${SPAWN_MARKER}>,
|
|
64
|
+
"child_excerpt": "<excerpt of child output that includes its first line>"
|
|
65
|
+
}`;
|
|
66
|
+
|
|
67
|
+
const E3_PROMPT = `You are measuring skill readability under an agent definition.
|
|
68
|
+
|
|
69
|
+
1. Read any one skill entrypoint under the user's Claude skills path (for example ~/.claude/skills/*/SKILL.md). Prefer a short file if several exist.
|
|
70
|
+
2. Confirm you loaded an agent definition for this run (this process was started with --agent).
|
|
71
|
+
|
|
72
|
+
Reply with ONLY one JSON object and nothing else:
|
|
73
|
+
{
|
|
74
|
+
"skill_read_ok": <true if you read a skill file's contents>,
|
|
75
|
+
"skill_path": "<path you read or empty string>",
|
|
76
|
+
"agent_definition_loaded": <true if an agent definition is active>
|
|
77
|
+
}`;
|
|
78
|
+
|
|
79
|
+
const E4_PROMPT = `You are measuring write capability in the current working directory.
|
|
80
|
+
|
|
81
|
+
1. Write a file named ${PROBE_FILE_NAME} in the current working directory with exact contents: ${PROBE_FILE_CONTENTS}
|
|
82
|
+
2. Do not write anything else.
|
|
83
|
+
|
|
84
|
+
Reply with ONLY one JSON object and nothing else:
|
|
85
|
+
{
|
|
86
|
+
"write_succeeded": <true if the write completed>,
|
|
87
|
+
"written_path": "<absolute or relative path written>"
|
|
88
|
+
}`;
|
|
89
|
+
|
|
90
|
+
const E5_PROMPT = `You are measuring whether a Claude/GSD Workflow tool is available.
|
|
91
|
+
|
|
92
|
+
Inspect your tool list. Do not invent a Workflow tool.
|
|
93
|
+
|
|
94
|
+
Reply with ONLY one JSON object and nothing else:
|
|
95
|
+
{
|
|
96
|
+
"has_workflow_tool": <true only if a Workflow tool is present>,
|
|
97
|
+
"result": <"has_tool" or "no_tool">
|
|
98
|
+
}`;
|
|
99
|
+
|
|
100
|
+
function isOptedIn(allArguments) {
|
|
101
|
+
if (allArguments.includes(RUN_FLAG)) {
|
|
102
|
+
return true;
|
|
103
|
+
}
|
|
104
|
+
return process.env[OPT_IN_ENV] === OPT_IN_VALUE;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function printOptInHelp() {
|
|
108
|
+
console.log(
|
|
109
|
+
[
|
|
110
|
+
'Grok capability evals are opt-in only (not part of npm test / CI).',
|
|
111
|
+
'',
|
|
112
|
+
'Run with either:',
|
|
113
|
+
` ${OPT_IN_ENV}=${OPT_IN_VALUE} node skills/grokify/evals/run-capability-evals.mjs`,
|
|
114
|
+
' node skills/grokify/evals/run-capability-evals.mjs --run',
|
|
115
|
+
'',
|
|
116
|
+
'See skills/grokify/evals/README.md.',
|
|
117
|
+
].join('\n'),
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function mintLeaderSocketPath(runDirectory, evalName) {
|
|
122
|
+
return join(runDirectory, `leader-${evalName}.sock`);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function writePromptFile(runDirectory, evalName, promptText) {
|
|
126
|
+
const promptPath = join(runDirectory, `${evalName}-prompt.md`);
|
|
127
|
+
writeFileSync(promptPath, promptText, 'utf8');
|
|
128
|
+
return promptPath;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function buildGrokArguments({
|
|
132
|
+
promptPath,
|
|
133
|
+
workingDirectory,
|
|
134
|
+
leaderSocketPath,
|
|
135
|
+
maxTurns,
|
|
136
|
+
agentName,
|
|
137
|
+
}) {
|
|
138
|
+
const allArguments = [
|
|
139
|
+
'--prompt-file',
|
|
140
|
+
promptPath,
|
|
141
|
+
'--cwd',
|
|
142
|
+
workingDirectory,
|
|
143
|
+
'--output-format',
|
|
144
|
+
'json',
|
|
145
|
+
'--always-approve',
|
|
146
|
+
'--max-turns',
|
|
147
|
+
String(maxTurns),
|
|
148
|
+
'--permission-mode',
|
|
149
|
+
'bypassPermissions',
|
|
150
|
+
'--leader-socket',
|
|
151
|
+
leaderSocketPath,
|
|
152
|
+
];
|
|
153
|
+
if (agentName) {
|
|
154
|
+
allArguments.push('--agent', agentName);
|
|
155
|
+
}
|
|
156
|
+
return allArguments;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function runGrok({
|
|
160
|
+
promptPath,
|
|
161
|
+
workingDirectory,
|
|
162
|
+
leaderSocketPath,
|
|
163
|
+
maxTurns,
|
|
164
|
+
agentName,
|
|
165
|
+
}) {
|
|
166
|
+
const allArguments = buildGrokArguments({
|
|
167
|
+
promptPath,
|
|
168
|
+
workingDirectory,
|
|
169
|
+
leaderSocketPath,
|
|
170
|
+
maxTurns,
|
|
171
|
+
agentName,
|
|
172
|
+
});
|
|
173
|
+
const processResult = spawnSync(GROK_BINARY, allArguments, {
|
|
174
|
+
encoding: 'utf8',
|
|
175
|
+
cwd: workingDirectory,
|
|
176
|
+
maxBuffer: SPAWN_MAX_BUFFER_BYTES,
|
|
177
|
+
timeout: GROK_SPAWN_TIMEOUT_MS,
|
|
178
|
+
windowsHide: true,
|
|
179
|
+
});
|
|
180
|
+
return {
|
|
181
|
+
exitCode: processResult.status,
|
|
182
|
+
stdout: processResult.stdout || '',
|
|
183
|
+
stderr: processResult.stderr || '',
|
|
184
|
+
error: processResult.error || null,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function isPlainObject(value) {
|
|
189
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function extractLastBalancedObject(text) {
|
|
193
|
+
const doubleQuote = '"';
|
|
194
|
+
const escapeCharacter = '\\';
|
|
195
|
+
let isInsideString = false;
|
|
196
|
+
let isNextCharacterEscaped = false;
|
|
197
|
+
let openBraceDepth = 0;
|
|
198
|
+
let objectStartIndex = -1;
|
|
199
|
+
let lastParsedObject = null;
|
|
200
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
201
|
+
const character = text[index];
|
|
202
|
+
if (isInsideString) {
|
|
203
|
+
if (isNextCharacterEscaped) {
|
|
204
|
+
isNextCharacterEscaped = false;
|
|
205
|
+
} else if (character === escapeCharacter) {
|
|
206
|
+
isNextCharacterEscaped = true;
|
|
207
|
+
} else if (character === doubleQuote) {
|
|
208
|
+
isInsideString = false;
|
|
209
|
+
}
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
if (character === doubleQuote) {
|
|
213
|
+
isInsideString = true;
|
|
214
|
+
} else if (character === '{') {
|
|
215
|
+
if (openBraceDepth === 0) {
|
|
216
|
+
objectStartIndex = index;
|
|
217
|
+
}
|
|
218
|
+
openBraceDepth += 1;
|
|
219
|
+
} else if (character === '}' && openBraceDepth > 0) {
|
|
220
|
+
openBraceDepth -= 1;
|
|
221
|
+
if (openBraceDepth === 0) {
|
|
222
|
+
try {
|
|
223
|
+
const parsed = JSON.parse(text.slice(objectStartIndex, index + 1));
|
|
224
|
+
if (isPlainObject(parsed)) {
|
|
225
|
+
lastParsedObject = parsed;
|
|
226
|
+
}
|
|
227
|
+
} catch {
|
|
228
|
+
// not a JSON object; keep scanning for a later balanced candidate
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return lastParsedObject;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function tryParseJsonObject(text) {
|
|
237
|
+
const trimmed = text.trim();
|
|
238
|
+
if (!trimmed) {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
try {
|
|
242
|
+
const parsed = JSON.parse(trimmed);
|
|
243
|
+
if (isPlainObject(parsed)) {
|
|
244
|
+
return parsed;
|
|
245
|
+
}
|
|
246
|
+
} catch {
|
|
247
|
+
// fall through to balanced-object extraction
|
|
248
|
+
}
|
|
249
|
+
return extractLastBalancedObject(trimmed);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function extractStringField(parsedEnvelope, fieldName) {
|
|
253
|
+
if (!parsedEnvelope || typeof parsedEnvelope !== 'object') {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
const fieldValue = parsedEnvelope[fieldName];
|
|
257
|
+
if (typeof fieldValue === 'string') {
|
|
258
|
+
return fieldValue;
|
|
259
|
+
}
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function pickStringFieldByPriority(candidate, fieldNames) {
|
|
264
|
+
for (const fieldName of fieldNames) {
|
|
265
|
+
const fieldValue = extractStringField(candidate, fieldName);
|
|
266
|
+
if (fieldValue !== null) {
|
|
267
|
+
return fieldValue;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export function extractResultText(stdout) {
|
|
274
|
+
const trimmed = stdout.trim();
|
|
275
|
+
if (!trimmed) {
|
|
276
|
+
return '';
|
|
277
|
+
}
|
|
278
|
+
try {
|
|
279
|
+
const parsed = JSON.parse(trimmed);
|
|
280
|
+
if (Array.isArray(parsed)) {
|
|
281
|
+
const resultEvent = parsed.findLast(
|
|
282
|
+
(eachEvent) =>
|
|
283
|
+
eachEvent &&
|
|
284
|
+
typeof eachEvent === 'object' &&
|
|
285
|
+
eachEvent.type === 'result' &&
|
|
286
|
+
typeof eachEvent.result === 'string',
|
|
287
|
+
);
|
|
288
|
+
if (resultEvent) {
|
|
289
|
+
return resultEvent.result;
|
|
290
|
+
}
|
|
291
|
+
const textChunks = [];
|
|
292
|
+
for (const eachEvent of parsed) {
|
|
293
|
+
if (!eachEvent || typeof eachEvent !== 'object') {
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
const eventText = pickStringFieldByPriority(
|
|
297
|
+
eachEvent,
|
|
298
|
+
STREAM_EVENT_TEXT_FIELDS_BY_PRIORITY,
|
|
299
|
+
);
|
|
300
|
+
if (eventText !== null) {
|
|
301
|
+
textChunks.push(eventText);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
if (textChunks.length > 0) {
|
|
305
|
+
return textChunks.join('\n');
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (parsed && typeof parsed === 'object') {
|
|
309
|
+
const envelopeText = pickStringFieldByPriority(
|
|
310
|
+
parsed,
|
|
311
|
+
ENVELOPE_TEXT_FIELDS_BY_PRIORITY,
|
|
312
|
+
);
|
|
313
|
+
if (envelopeText !== null) {
|
|
314
|
+
return envelopeText;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
} catch {
|
|
318
|
+
// stdout is plain text
|
|
319
|
+
}
|
|
320
|
+
return trimmed;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
export function parsePayload(stdout) {
|
|
324
|
+
const resultText = extractResultText(stdout);
|
|
325
|
+
return tryParseJsonObject(resultText) || tryParseJsonObject(stdout);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function assertCondition(condition, message) {
|
|
329
|
+
if (!condition) {
|
|
330
|
+
throw new Error(message);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function resolveDefaultAgentName() {
|
|
335
|
+
return process.env.GROK_CAPABILITY_EVAL_AGENT || 'Explore';
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function describeLaunchFailure(label, processResult) {
|
|
339
|
+
const launchError = processResult.error;
|
|
340
|
+
if (
|
|
341
|
+
launchError &&
|
|
342
|
+
typeof launchError === 'object' &&
|
|
343
|
+
launchError.code === SPAWN_TIMEOUT_ERROR_CODE
|
|
344
|
+
) {
|
|
345
|
+
return `${label}: grok timed out after ${GROK_SPAWN_TIMEOUT_MS}ms`;
|
|
346
|
+
}
|
|
347
|
+
if (launchError !== null) {
|
|
348
|
+
return `${label}: failed to launch grok: ${launchError}`;
|
|
349
|
+
}
|
|
350
|
+
return `${label}: grok exit ${processResult.exitCode}\n${processResult.stderr}\n${processResult.stdout}`;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function launchEval(runDirectory, { evalName, promptText, maxTurns, agentName }) {
|
|
354
|
+
const label = evalName.toUpperCase();
|
|
355
|
+
const promptPath = writePromptFile(runDirectory, evalName, promptText);
|
|
356
|
+
const processResult = runGrok({
|
|
357
|
+
promptPath,
|
|
358
|
+
workingDirectory: runDirectory,
|
|
359
|
+
leaderSocketPath: mintLeaderSocketPath(runDirectory, evalName),
|
|
360
|
+
maxTurns,
|
|
361
|
+
agentName,
|
|
362
|
+
});
|
|
363
|
+
if (processResult.error !== null || processResult.exitCode !== 0) {
|
|
364
|
+
throw new Error(describeLaunchFailure(label, processResult));
|
|
365
|
+
}
|
|
366
|
+
const payload = parsePayload(processResult.stdout);
|
|
367
|
+
assertCondition(
|
|
368
|
+
payload !== null,
|
|
369
|
+
`${label}: could not parse JSON payload from grok output`,
|
|
370
|
+
);
|
|
371
|
+
return payload;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function runEvalOne(runDirectory) {
|
|
375
|
+
const payload = launchEval(runDirectory, {
|
|
376
|
+
evalName: 'e1',
|
|
377
|
+
promptText: E1_PROMPT,
|
|
378
|
+
maxTurns: DEFAULT_MAX_TURNS,
|
|
379
|
+
});
|
|
380
|
+
assertCondition(
|
|
381
|
+
payload.can_spawn_subagent_tool === true,
|
|
382
|
+
`E1: expected can_spawn_subagent_tool === true, got ${JSON.stringify(payload.can_spawn_subagent_tool)}`,
|
|
383
|
+
);
|
|
384
|
+
const allToolNames = Array.isArray(payload.tool_names)
|
|
385
|
+
? payload.tool_names.map(String)
|
|
386
|
+
: [];
|
|
387
|
+
assertCondition(
|
|
388
|
+
allToolNames.includes(SPAWN_SUBAGENT_TOOL),
|
|
389
|
+
`E1: expected tool_names to include ${SPAWN_SUBAGENT_TOOL}, got ${JSON.stringify(allToolNames)}`,
|
|
390
|
+
);
|
|
391
|
+
return payload;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function runEvalTwo(runDirectory) {
|
|
395
|
+
const payload = launchEval(runDirectory, {
|
|
396
|
+
evalName: 'e2',
|
|
397
|
+
promptText: E2_PROMPT,
|
|
398
|
+
maxTurns: SPAWN_MAX_TURNS,
|
|
399
|
+
});
|
|
400
|
+
assertCondition(
|
|
401
|
+
payload.spawn_succeeded === true,
|
|
402
|
+
`E2: expected spawn_succeeded === true, got ${JSON.stringify(payload.spawn_succeeded)}`,
|
|
403
|
+
);
|
|
404
|
+
const childExcerpt =
|
|
405
|
+
typeof payload.child_excerpt === 'string' ? payload.child_excerpt : '';
|
|
406
|
+
assertCondition(
|
|
407
|
+
childExcerpt.includes(SPAWN_MARKER),
|
|
408
|
+
`E2: expected child_excerpt to include ${SPAWN_MARKER}, got ${JSON.stringify(payload.child_excerpt)}`,
|
|
409
|
+
);
|
|
410
|
+
return payload;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function runEvalThree(runDirectory) {
|
|
414
|
+
const payload = launchEval(runDirectory, {
|
|
415
|
+
evalName: 'e3',
|
|
416
|
+
promptText: E3_PROMPT,
|
|
417
|
+
maxTurns: SKILL_MAX_TURNS,
|
|
418
|
+
agentName: resolveDefaultAgentName(),
|
|
419
|
+
});
|
|
420
|
+
assertCondition(
|
|
421
|
+
payload.skill_read_ok === true,
|
|
422
|
+
`E3: expected skill_read_ok === true, got ${JSON.stringify(payload.skill_read_ok)}`,
|
|
423
|
+
);
|
|
424
|
+
assertCondition(
|
|
425
|
+
payload.agent_definition_loaded === true,
|
|
426
|
+
`E3: expected agent_definition_loaded === true, got ${JSON.stringify(payload.agent_definition_loaded)}`,
|
|
427
|
+
);
|
|
428
|
+
return payload;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function runEvalFour(runDirectory) {
|
|
432
|
+
const payload = launchEval(runDirectory, {
|
|
433
|
+
evalName: 'e4',
|
|
434
|
+
promptText: E4_PROMPT,
|
|
435
|
+
maxTurns: WRITE_MAX_TURNS,
|
|
436
|
+
});
|
|
437
|
+
const probePath = join(runDirectory, PROBE_FILE_NAME);
|
|
438
|
+
assertCondition(
|
|
439
|
+
existsSync(probePath),
|
|
440
|
+
`E4: probe file missing under the eval cwd (write_succeeded claim: ${JSON.stringify(payload.write_succeeded)})`,
|
|
441
|
+
);
|
|
442
|
+
const probeContents = readFileSync(probePath, 'utf8');
|
|
443
|
+
assertCondition(
|
|
444
|
+
probeContents.includes(PROBE_FILE_CONTENTS),
|
|
445
|
+
`E4: probe file contents mismatch: ${JSON.stringify(probeContents)}`,
|
|
446
|
+
);
|
|
447
|
+
const hooksLogPath = join(runDirectory, HOOKS_LOG_NAME);
|
|
448
|
+
let hooksNote = 'hooks.log not present (soft check skipped)';
|
|
449
|
+
if (existsSync(hooksLogPath)) {
|
|
450
|
+
const hooksLogText = readFileSync(hooksLogPath, 'utf8').toLowerCase();
|
|
451
|
+
const hasGlobalSettings = hooksLogText.includes(
|
|
452
|
+
HOOKS_LOG_GLOBAL_SETTINGS_MARKER,
|
|
453
|
+
);
|
|
454
|
+
const hasPreToolUse = hooksLogText.includes(HOOKS_LOG_PRE_TOOL_USE_MARKER);
|
|
455
|
+
hooksNote =
|
|
456
|
+
hasGlobalSettings && hasPreToolUse
|
|
457
|
+
? 'hooks.log contains global/settings and pre_tool_use'
|
|
458
|
+
: 'hooks.log present but missing expected markers (soft)';
|
|
459
|
+
}
|
|
460
|
+
return { payload, hooksNote };
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
export function isWorkflowToolAbsent(payload) {
|
|
464
|
+
if (!payload || typeof payload !== 'object') {
|
|
465
|
+
return false;
|
|
466
|
+
}
|
|
467
|
+
return (
|
|
468
|
+
payload.has_workflow_tool === false &&
|
|
469
|
+
payload.result === WORKFLOW_RESULT_NO_TOOL
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
function runEvalFive(runDirectory) {
|
|
474
|
+
const payload = launchEval(runDirectory, {
|
|
475
|
+
evalName: 'e5',
|
|
476
|
+
promptText: E5_PROMPT,
|
|
477
|
+
maxTurns: WORKFLOW_MAX_TURNS,
|
|
478
|
+
});
|
|
479
|
+
assertCondition(
|
|
480
|
+
isWorkflowToolAbsent(payload),
|
|
481
|
+
`E5: expected has_workflow_tool === false and result === "no_tool", got ${JSON.stringify(payload)}`,
|
|
482
|
+
);
|
|
483
|
+
return payload;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
function main() {
|
|
487
|
+
if (!isOptedIn(process.argv.slice(2))) {
|
|
488
|
+
printOptInHelp();
|
|
489
|
+
process.exit(0);
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
const runDirectory = mkdtempSync(join(tmpdir(), 'grok-capability-evals-'));
|
|
493
|
+
console.log(`run directory: ${runDirectory}`);
|
|
494
|
+
|
|
495
|
+
try {
|
|
496
|
+
console.log('E1 tool inventory...');
|
|
497
|
+
const e1Payload = runEvalOne(runDirectory);
|
|
498
|
+
console.log('E1 ok', JSON.stringify(e1Payload));
|
|
499
|
+
|
|
500
|
+
console.log('E2 spawn_subagent...');
|
|
501
|
+
const e2Payload = runEvalTwo(runDirectory);
|
|
502
|
+
console.log('E2 ok', JSON.stringify(e2Payload));
|
|
503
|
+
|
|
504
|
+
console.log('E3 --agent + skill read...');
|
|
505
|
+
const e3Payload = runEvalThree(runDirectory);
|
|
506
|
+
console.log('E3 ok', JSON.stringify(e3Payload));
|
|
507
|
+
|
|
508
|
+
console.log('E4 probe write...');
|
|
509
|
+
const e4Outcome = runEvalFour(runDirectory);
|
|
510
|
+
console.log(
|
|
511
|
+
'E4 ok',
|
|
512
|
+
JSON.stringify({
|
|
513
|
+
payload: e4Outcome.payload,
|
|
514
|
+
hooksNote: e4Outcome.hooksNote,
|
|
515
|
+
}),
|
|
516
|
+
);
|
|
517
|
+
|
|
518
|
+
console.log('E5 workflow tool absence...');
|
|
519
|
+
const e5Payload = runEvalFive(runDirectory);
|
|
520
|
+
console.log('E5 ok', JSON.stringify(e5Payload));
|
|
521
|
+
|
|
522
|
+
console.log('ALL CAPABILITY EVALS PASSED');
|
|
523
|
+
} catch (failure) {
|
|
524
|
+
console.error(String(failure && failure.stack ? failure.stack : failure));
|
|
525
|
+
process.exitCode = 1;
|
|
526
|
+
} finally {
|
|
527
|
+
try {
|
|
528
|
+
rmSync(runDirectory, { recursive: true, force: true });
|
|
529
|
+
} catch {
|
|
530
|
+
// best-effort cleanup
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function isDirectExecution() {
|
|
536
|
+
const entryArgument = process.argv[1];
|
|
537
|
+
if (!entryArgument) {
|
|
538
|
+
return false;
|
|
539
|
+
}
|
|
540
|
+
return resolve(fileURLToPath(import.meta.url)) === resolve(entryArgument);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
if (isDirectExecution()) {
|
|
544
|
+
main();
|
|
545
|
+
}
|
|
@@ -30,7 +30,7 @@ communication flows through it. It spawns and resumes executor subagents
|
|
|
30
30
|
— `clean-coder` and the like — and those executors do every bit of the
|
|
31
31
|
execution: the code edits, the build runs, the test runs. The orchestrating
|
|
32
32
|
session drives the plan and routes hard decisions to the shared advisor
|
|
33
|
-
(Claude: warm `session-advisor`;
|
|
33
|
+
(Claude: warm `session-advisor`; a third-party host: Claude CLI advisor via the chain runner).
|
|
34
34
|
|
|
35
35
|
## Gotchas
|
|
36
36
|
|
|
@@ -43,7 +43,7 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
43
43
|
wasted. Hand every code edit and every build or test run to an executor; keep
|
|
44
44
|
the orchestrating session's own tool use to orchestration and light
|
|
45
45
|
verification reads. Hard decisions go to the shared advisor (Claude:
|
|
46
|
-
`session-advisor` via SendMessage;
|
|
46
|
+
`session-advisor` via SendMessage; a third-party host: Claude CLI advisor via the chain).
|
|
47
47
|
- **Flat ad hoc spawns bypass routing.** Every execution task goes through a
|
|
48
48
|
workflow-backed spawn or workflow resume so the required agent type, model,
|
|
49
49
|
prompt packet, and sidecar metadata stay attached to the work.
|
|
@@ -54,15 +54,15 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
54
54
|
spawn returns an `agentId` (format `a...-...`); keep it so `SendMessage` can
|
|
55
55
|
reach that agent later. A named agent is reachable by name.
|
|
56
56
|
- **Only the orchestrating session owns the shared advisor's lifecycle.** An
|
|
57
|
-
executor that finds the advisor unreachable (Claude warm agent, or
|
|
58
|
-
|
|
57
|
+
executor that finds the advisor unreachable (Claude warm agent, or Claude
|
|
58
|
+
CLI bind on a third-party host) reports that upward; it never spawns a replacement itself.
|
|
59
59
|
|
|
60
60
|
## Process
|
|
61
61
|
|
|
62
62
|
1. **Check whether the refresh loop is already running this
|
|
63
63
|
session.** If it is, do not schedule a second loop. Reuse any live
|
|
64
|
-
shared advisor bind (Claude warm `session-advisor`, or
|
|
65
|
-
run step 3 only when none exists yet), then skip straight to step 4.
|
|
64
|
+
shared advisor bind (Claude warm `session-advisor`, or a third-party host's
|
|
65
|
+
Claude CLI bind — run step 3 only when none exists yet), then skip straight to step 4.
|
|
66
66
|
|
|
67
67
|
2. **Register the discipline reminder.** By default, schedule it with
|
|
68
68
|
`ScheduleWakeup` at `delaySeconds: 1200`, prompt `/orchestrator-refresh`,
|
|
@@ -85,15 +85,18 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
85
85
|
[`_shared/advisor/advisor-protocol.md`](../../_shared/advisor/advisor-protocol.md);
|
|
86
86
|
it is not a "spawn only at the floor" rule. Paste the **Claude host**
|
|
87
87
|
Advisor block from that doc, with the resolved agent name filled in, into
|
|
88
|
-
every executor's spawn prompt. On a **
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
shared advisor, not just this
|
|
94
|
-
shared advisor's lifecycle end
|
|
95
|
-
|
|
96
|
-
|
|
88
|
+
every executor's spawn prompt. On a **third-party host**, skip the Agent spawn —
|
|
89
|
+
bind a max-tier Claude advisor through the protocol's CLI Claude-chain path
|
|
90
|
+
(Fable high, then Opus max; `claude_chain_runner.py` for account usage
|
|
91
|
+
failover). Paste the **Third-party host** Advisor block from that doc into every
|
|
92
|
+
executor's spawn prompt — never the Claude SendMessage block. Every row in
|
|
93
|
+
the routing table is a consumer of the shared advisor, not just this
|
|
94
|
+
session. The orchestrating session owns the shared advisor's lifecycle end
|
|
95
|
+
to end (Agent spawn or CLI bind, drift handling per the shared doc,
|
|
96
|
+
shutdown at task end); executors only ever SendMessage the warm agent
|
|
97
|
+
(Claude) or report to this session so it can consult the CLI advisor
|
|
98
|
+
(a third-party host). When the CLI chain cannot bind, fail closed and report
|
|
99
|
+
to the user — do not answer the four signals as this third-party session.
|
|
97
100
|
|
|
98
101
|
4. **Orchestrate the task.** Hold the plan and the user conversation. Execute
|
|
99
102
|
workflow-backed spawns or resumes using the routing table below, and keep
|
|
@@ -111,11 +114,14 @@ session drives the plan and routes hard decisions to the shared advisor
|
|
|
111
114
|
- The chosen approach is being reconsidered.
|
|
112
115
|
|
|
113
116
|
**Claude host:** consult the shared `session-advisor` via `SendMessage`.
|
|
114
|
-
**
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
117
|
+
**Third-party host:** executors report to this orchestrating session; this session
|
|
118
|
+
consults the bound Claude CLI advisor (`claude_chain_runner.py` +
|
|
119
|
+
`--resume`) and relays the four-signal reply (no Agent-tool
|
|
120
|
+
`session-advisor`, no SendMessage to one). Every consult gets back one of
|
|
121
|
+
four signals (ENDORSE, CORRECTION, PLAN, or STOP). When the CLI advisor is
|
|
122
|
+
unreachable, fail closed and surface that to the user — do not invent a
|
|
123
|
+
signal as this session. See `agents/session-advisor.md` for what each
|
|
124
|
+
signal means and
|
|
119
125
|
[`_shared/advisor/advisor-protocol.md`](../../_shared/advisor/advisor-protocol.md)
|
|
120
126
|
for the consult format.
|
|
121
127
|
|
|
@@ -147,6 +153,10 @@ Routing rules:
|
|
|
147
153
|
it.
|
|
148
154
|
- Exploration workflows return file paths, line numbers, and direct evidence;
|
|
149
155
|
they do not write code or mutate repo state.
|
|
156
|
+
- Fan-out worker fleets use the **grok-spawn** skill when that skill is
|
|
157
|
+
installed and grok is usable (`grok_worker_preflight.py` soft gate). The
|
|
158
|
+
Claude Code Agent tool remains the Claude-host alternative for in-process
|
|
159
|
+
workers.
|
|
150
160
|
|
|
151
161
|
## Agent reuse (non-negotiable)
|
|
152
162
|
|
|
@@ -173,8 +183,8 @@ Routing rules:
|
|
|
173
183
|
session (see
|
|
174
184
|
[`_shared/advisor/advisor-protocol.md`](../../_shared/advisor/advisor-protocol.md))
|
|
175
185
|
— on Claude, executors consult a warm `session-advisor` via SendMessage; on
|
|
176
|
-
|
|
177
|
-
spawn or respawn the advisor.
|
|
186
|
+
a third-party host, executors report here and this session relays the Claude CLI advisor.
|
|
187
|
+
Executors never spawn or respawn the advisor.
|
|
178
188
|
- Reuse a warm agent over a cold spawn whenever one holds relevant context.
|
|
179
189
|
|
|
180
190
|
## File Index
|
|
@@ -4,7 +4,7 @@ description: >-
|
|
|
4
4
|
Fired by the /orchestrator loop reminder about every 20 minutes to
|
|
5
5
|
re-assert the advisor discipline mid-run: orchestrate, route hard decisions
|
|
6
6
|
to the shared advisor (ENDORSE / CORRECTION / PLAN / STOP — SendMessage on
|
|
7
|
-
Claude,
|
|
7
|
+
Claude, Claude CLI chain on a third-party host), reuse warm agents. Triggers:
|
|
8
8
|
'/orchestrator-refresh'.
|
|
9
9
|
---
|
|
10
10
|
|
|
@@ -12,8 +12,8 @@ description: >-
|
|
|
12
12
|
|
|
13
13
|
Detect the host profile first (see Host profiles in
|
|
14
14
|
[`_shared/advisor/advisor-protocol.md`](../../_shared/advisor/advisor-protocol.md)).
|
|
15
|
-
Re-assert the discipline for that host only — do not invent
|
|
16
|
-
`session-advisor` spawn on a
|
|
15
|
+
Re-assert the discipline for that host only — do not invent an Agent-tool
|
|
16
|
+
Claude `session-advisor` spawn on a third-party host.
|
|
17
17
|
|
|
18
18
|
1. **You are the orchestrator.** Orchestrate and hold the user conversation;
|
|
19
19
|
spawn executor subagents to do all the work — every code edit and build or
|
|
@@ -23,13 +23,17 @@ Re-assert the discipline for that host only — do not invent a Claude
|
|
|
23
23
|
`SendMessage` and receive one of four signals — ENDORSE, CORRECTION, PLAN,
|
|
24
24
|
or STOP. The orchestrating session routes its own hard decisions the same
|
|
25
25
|
way and keeps its tool use to orchestration and light verification reads.
|
|
26
|
-
- **
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
26
|
+
- **Third-party host:** the advisor is a max-tier Claude CLI bind owned by this
|
|
27
|
+
session (`claude_chain_runner.py`, Fable high then Opus max). Do **not**
|
|
28
|
+
spawn `session-advisor` via Agent and do **not** tell executors to
|
|
29
|
+
SendMessage a separate advisor agent. Executors report blockers to this
|
|
30
|
+
session; consult the Claude CLI advisor and relay ENDORSE / CORRECTION /
|
|
31
|
+
PLAN / STOP. When the CLI bind is unreachable, fail closed and report to
|
|
32
|
+
the user — do not answer the four signals as this third-party session.
|
|
30
33
|
3. **Resume before you spawn.** `SendMessage` an existing *executor* agent by
|
|
31
34
|
name or `agentId` to reuse its warm context; prefer that over a cold spawn.
|
|
32
|
-
(On
|
|
35
|
+
(On a third-party host this is executor reuse only — advisor re-bind stays on the CLI
|
|
36
|
+
chain path in the shared protocol.)
|
|
33
37
|
4. **Fresh spawn only for a genuine task switch.** No tool compacts or clears a
|
|
34
38
|
subagent's context, so a clean context comes from a fresh spawn — never tell
|
|
35
39
|
an agent to compact.
|