claude-dev-env 8.119.2 → 8.121.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/bin/ever-registered-hooks.mjs +1 -0
  2. package/hooks/advisory/auto_mode_denial_quick_fix.py +275 -27
  3. package/hooks/advisory/auto_mode_rule_flow_eval/auto_mode_rule_flow_cases.json +3475 -0
  4. package/hooks/advisory/auto_mode_rule_flow_eval/auto_mode_rule_flow_runner.py +849 -0
  5. package/hooks/advisory/auto_mode_rule_flow_eval/stub_allow_hook.py +20 -0
  6. package/hooks/advisory/auto_mode_rule_flow_eval/stub_hold_hook.py +54 -0
  7. package/hooks/advisory/auto_mode_rule_flow_eval/test_auto_mode_rule_flow_runner.py +143 -0
  8. package/hooks/advisory/test_auto_mode_denial_quick_fix.py +533 -9
  9. package/hooks/auto_mode_question_transcript.py +215 -0
  10. package/hooks/auto_mode_refusal_ledger.py +481 -0
  11. package/hooks/auto_mode_refusal_ledger_files.py +336 -0
  12. package/hooks/auto_mode_rule_steps.py +177 -0
  13. package/hooks/blocking/auto_mode_refusal_stop_gate.py +384 -0
  14. package/hooks/blocking/gh_api_budget_gate.py +286 -0
  15. package/hooks/blocking/test_auto_mode_refusal_stop_gate.py +484 -0
  16. package/hooks/blocking/test_bash_dispatcher_interpreter_starts.py +8 -3
  17. package/hooks/blocking/test_bash_pre_tool_use_dispatcher.py +1 -0
  18. package/hooks/blocking/test_gh_api_budget_gate.py +602 -0
  19. package/hooks/features/README.md +1 -1
  20. package/hooks/features/bash-dispatchers.md +3 -0
  21. package/hooks/features/session-context.md +5 -2
  22. package/hooks/hooks.json +5 -0
  23. package/hooks/hooks_constants/auto_mode_denial_quick_fix_constants.py +51 -3
  24. package/hooks/hooks_constants/auto_mode_refusal_ledger_constants.py +85 -0
  25. package/hooks/hooks_constants/auto_mode_refusal_stop_gate_constants.py +80 -0
  26. package/hooks/hooks_constants/auto_mode_rule_flow_runner_constants.py +225 -0
  27. package/hooks/hooks_constants/auto_mode_rule_steps_constants.py +82 -0
  28. package/hooks/hooks_constants/bash_pre_tool_use_dispatcher_constants.py +4 -0
  29. package/hooks/hooks_constants/gh_api_budget.py +186 -0
  30. package/hooks/hooks_constants/gh_api_budget_gate_constants.py +215 -0
  31. package/hooks/hooks_constants/gh_argument_lists.py +169 -0
  32. package/hooks/hooks_constants/gh_shell_loop_scan.py +386 -0
  33. package/hooks/hooks_constants/test_bash_pre_tool_use_dispatcher_constants.py +5 -1
  34. package/hooks/hooks_constants/test_gh_api_budget.py +125 -0
  35. package/hooks/hooks_constants/test_gh_argument_lists.py +49 -0
  36. package/hooks/hooks_constants/test_gh_shell_loop_scan.py +50 -0
  37. package/hooks/test_auto_mode_question_transcript.py +113 -0
  38. package/hooks/test_auto_mode_refusal_ledger.py +398 -0
  39. package/hooks/test_auto_mode_refusal_ledger_files.py +155 -0
  40. package/hooks/test_auto_mode_rule_steps.py +210 -0
  41. package/package.json +1 -1
  42. package/scripts/policy_lint/adapter_configuration.py +5 -0
  43. package/scripts/policy_lint/config/constants.py +1 -0
  44. package/scripts/tests/test_adapter_configuration.py +8 -0
@@ -0,0 +1,849 @@
1
+ #!/usr/bin/env python3
2
+ """Run the auto mode rule flow cases against a refusal hook and a Stop gate.
3
+
4
+ Each case is a list of events played in order in a fresh scratch home::
5
+
6
+ {"refusal": {...}, "name": "a"} -> PermissionDenied payload to the refusal hook
7
+ {"transcript": "ask", ...} -> answered AskUserQuestion records appended
8
+ {"stop": {...}} -> Stop payload to the Stop gate
9
+
10
+ A refusal event that adds one ledger file binds that file to its ``name``.
11
+ Expectations read only hook output (stdout JSON and exit code) and the
12
+ ``answer`` field of the ledger files, so a case never depends on the ledger's
13
+ id rule. A ledger file is any JSON object with an ``answer`` key under
14
+ ``<config>/state/auto-mode-refusals/<session_id>/``.
15
+
16
+ A case passes when every expectation holds, fails when one does not, and is an
17
+ error when a hook is missing, times out, exits with an unexpected code, or
18
+ prints output that is not a JSON object. ``HOME`` and ``CLAUDE_CONFIG_DIR``
19
+ always point into the scratch folder. Of the calling shell's variables, only
20
+ ``PATH``, ``LANG``, ``SYSTEMROOT``, ``TEMP`` and ``TMP`` reach a hook.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import argparse
26
+ import json
27
+ import os
28
+ import re
29
+ import subprocess
30
+ import sys
31
+ import tempfile
32
+ from dataclasses import dataclass, field
33
+ from datetime import datetime, timezone
34
+ from pathlib import Path
35
+
36
+ _hooks_root_directory = str(Path(__file__).resolve().parent.parent.parent)
37
+ if _hooks_root_directory not in sys.path:
38
+ sys.path.insert(0, _hooks_root_directory)
39
+
40
+ from hooks_constants.auto_mode_rule_flow_runner_constants import (
41
+ ALL_BASE_ENVIRONMENT_VARIABLE_NAMES,
42
+ ALLOW_STUB_FILE_NAME,
43
+ ALL_CASE_KINDS,
44
+ ALL_LEDGER_RELATIVE_PARTS,
45
+ ALL_QUESTION_OPTIONS,
46
+ ALL_REFUSAL_EXPECT_KEYS,
47
+ ALL_STOP_EXIT_CODES,
48
+ ALL_STOP_EXPECT_KEYS,
49
+ ANSWER_FIELD,
50
+ ANSWER_PAIR_SEPARATOR,
51
+ ANSWERED_RESULT_TEMPLATE,
52
+ ASK_TOOL_NAME,
53
+ ASK_TRANSCRIPT_KIND,
54
+ BASH_RESULT_TEXT,
55
+ BASH_TOOL_NAME,
56
+ BASH_TRANSCRIPT_KIND,
57
+ BLOCK_DECISION,
58
+ CALIBRATED_LINE,
59
+ CALIBRATION_PROBLEM_TEMPLATE,
60
+ CARD_RESULT_TEXT,
61
+ CARD_TRANSCRIPT_KIND,
62
+ CASE_FILE_NAME,
63
+ CONFIG_DIRECTORY_NAME,
64
+ CONFIG_DIRECTORY_VARIABLE,
65
+ DEFAULT_QUESTION_HEADER,
66
+ DEFAULT_REFUSAL_HOOK_RELATIVE_PATH,
67
+ DEFAULT_STOP_HOOK_RELATIVE_PATH,
68
+ ERROR_RULE_LINE,
69
+ ERROR_VERDICT,
70
+ ERRORS_KEY,
71
+ ERRORED_RESULT_CONTENT,
72
+ ERRORED_TOOL_USE_RESULT,
73
+ FAIL_VERDICT,
74
+ FLOW_MISSING_FILE,
75
+ FLOW_OFF,
76
+ FLOW_ON,
77
+ HOLD_EXIT_CODE,
78
+ HOLD_STUB_FILE_NAME,
79
+ HOME_VARIABLE,
80
+ HOOK_TIMEOUT_SECONDS,
81
+ LEDGER_PLACEHOLDER_PATTERN,
82
+ MAIN_SESSION_ID,
83
+ MISSING_PLAYBOOK_RELATIVE_PATH,
84
+ NO_BYTECODE_VALUE,
85
+ NO_BYTECODE_VARIABLE,
86
+ PASS_VERDICT,
87
+ PERMISSION_DENIED_EVENT_NAME,
88
+ PERMISSION_MODE,
89
+ PLAYBOOK_PLACEHOLDER,
90
+ PLAYBOOK_RELATIVE_PATH,
91
+ PLAYBOOK_TEXT,
92
+ PLAYBOOK_VARIABLE,
93
+ PROBLEM_LINE_INDENT,
94
+ REFUSAL_EVENT_KEY,
95
+ GIT_FILE_TEMPLATE,
96
+ RULE_REPOSITORY_NAME,
97
+ RULE_TOOL_RELATIVE_PATH,
98
+ RULE_TOOL_TEXT,
99
+ RULE_WORKTREE_NAME,
100
+ SCRATCH_GIT_NAME,
101
+ SCRATCH_HEAD_TEXT,
102
+ SCRATCH_PLACEHOLDER,
103
+ WORKTREE_COMMON_DIRECTORY_TEXT,
104
+ WORKTREE_BACK_POINTER_NAME,
105
+ WORKTREE_GIT_RELATIVE_PATH,
106
+ SHOULD_ACT_PASSED_KEY,
107
+ SHOULD_ACT_KIND,
108
+ SHOULD_STAY_QUIET_KIND,
109
+ SHOULD_STAY_QUIET_PASSED_KEY,
110
+ STOP_EVENT_KEY,
111
+ STOP_EVENT_NAME,
112
+ SUMMARY_LINE_TEMPLATE,
113
+ TIMED_OUT_EXIT_CODE,
114
+ TIMED_OUT_STDOUT,
115
+ TIMESTAMP_FORMAT,
116
+ TRANSCRIPT_EVENT_KEY,
117
+ TRANSCRIPT_FILE_NAME,
118
+ UNBOUND_LEDGER_TEMPLATE,
119
+ )
120
+ from hooks_constants.windows_filesystem import force_rmtree
121
+
122
+ JsonObject = dict[str, object]
123
+
124
+
125
+ class CaseFileInvalid(ValueError):
126
+ """The case file holds a case the runner cannot read."""
127
+
128
+
129
+ @dataclass(frozen=True)
130
+ class HookRun:
131
+ """One hook invocation.
132
+
133
+ Attributes:
134
+ exit_code: The process exit code, or None when the hook file is missing.
135
+ stdout: The text the hook printed on stdout.
136
+ """
137
+
138
+ exit_code: int | None
139
+ stdout: str
140
+
141
+
142
+ @dataclass(frozen=True)
143
+ class CaseOutcome:
144
+ """The graded result of one case.
145
+
146
+ Attributes:
147
+ case_id: The case id, such as ``R1``.
148
+ kind: ``should act`` or ``should stay quiet``.
149
+ verdict: ``pass``, ``fail`` or ``error``.
150
+ all_problems: One line per expectation that did not hold or hook that broke.
151
+ """
152
+
153
+ case_id: str
154
+ kind: str
155
+ verdict: str
156
+ all_problems: tuple[str, ...]
157
+
158
+
159
+ @dataclass
160
+ class _Scratch:
161
+ root: Path
162
+ refusal_hook: Path
163
+ stop_hook: Path
164
+ all_today_outputs: dict[str, str]
165
+ ledger_path_by_name: dict[str, Path] = field(default_factory=dict)
166
+ all_errors: list[str] = field(default_factory=list)
167
+ all_failures: list[str] = field(default_factory=list)
168
+ tool_use_counter: int = 0
169
+
170
+ @property
171
+ def config_directory(self) -> Path:
172
+ return self.root / CONFIG_DIRECTORY_NAME
173
+
174
+ @property
175
+ def transcript_path(self) -> Path:
176
+ return self.root / TRANSCRIPT_FILE_NAME
177
+
178
+ @property
179
+ def playbook_path(self) -> Path:
180
+ return self.root / PLAYBOOK_RELATIVE_PATH
181
+
182
+
183
+ def ledger_directory_for(config_directory: Path, session_id: str) -> Path:
184
+ """Return the folder that holds one session's ledger files.
185
+
186
+ Args:
187
+ config_directory: The scratch ``CLAUDE_CONFIG_DIR``.
188
+ session_id: The session the refusals belong to.
189
+
190
+ Returns:
191
+ ``<config>/state/auto-mode-refusals/<session_id>``.
192
+ """
193
+ return config_directory.joinpath(*ALL_LEDGER_RELATIVE_PARTS, session_id)
194
+
195
+
196
+ def read_ledger_answers(config_directory: Path, session_id: str) -> dict[Path, object]:
197
+ """Return each ledger file of a session mapped to its ``answer`` field.
198
+
199
+ ::
200
+
201
+ a.json {"answer": null, ...} -> {a.json: None}
202
+ q.json {"question": "...", ...} -> skipped, it has no answer key
203
+
204
+ Args:
205
+ config_directory: The scratch ``CLAUDE_CONFIG_DIR``.
206
+ session_id: The session whose folder is read.
207
+
208
+ Returns:
209
+ The answer of every JSON object in the folder that has an ``answer`` key.
210
+ """
211
+ ledger_directory = ledger_directory_for(config_directory, session_id)
212
+ answer_by_path: dict[Path, object] = {}
213
+ if not ledger_directory.is_dir():
214
+ return answer_by_path
215
+ for each_path in sorted(ledger_directory.glob("*.json")):
216
+ try:
217
+ parsed_file = json.loads(each_path.read_text(encoding="utf-8"))
218
+ except (OSError, ValueError):
219
+ continue
220
+ if isinstance(parsed_file, dict) and ANSWER_FIELD in parsed_file:
221
+ answer_by_path[each_path] = parsed_file[ANSWER_FIELD]
222
+ return answer_by_path
223
+
224
+
225
+ def _substituted(template: object, scratch: _Scratch) -> object:
226
+ if isinstance(template, str):
227
+ replaced_text = template.replace(PLAYBOOK_PLACEHOLDER, str(scratch.playbook_path))
228
+ replaced_text = replaced_text.replace(SCRATCH_PLACEHOLDER, str(scratch.root))
229
+ return re.sub(
230
+ LEDGER_PLACEHOLDER_PATTERN,
231
+ lambda each_match: _ledger_text_for(each_match.group("name"), scratch),
232
+ replaced_text,
233
+ )
234
+ if isinstance(template, list):
235
+ return [_substituted(each_entry, scratch) for each_entry in template]
236
+ if isinstance(template, dict):
237
+ return {
238
+ each_key: _substituted(each_entry, scratch) for each_key, each_entry in template.items()
239
+ }
240
+ return template
241
+
242
+
243
+ def _ledger_text_for(name: str, scratch: _Scratch) -> str:
244
+ bound_path = scratch.ledger_path_by_name.get(name)
245
+ return str(bound_path) if bound_path else UNBOUND_LEDGER_TEMPLATE.format(name=name)
246
+
247
+
248
+ def _hook_environment(event: JsonObject, scratch: _Scratch) -> dict[str, str]:
249
+ all_variables = {
250
+ each_name: os.environ[each_name]
251
+ for each_name in ALL_BASE_ENVIRONMENT_VARIABLE_NAMES
252
+ if each_name in os.environ
253
+ }
254
+ all_variables[HOME_VARIABLE] = str(scratch.root)
255
+ all_variables[CONFIG_DIRECTORY_VARIABLE] = str(scratch.config_directory)
256
+ all_variables[NO_BYTECODE_VARIABLE] = NO_BYTECODE_VALUE
257
+ flow_setting = event.get("flow", FLOW_ON)
258
+ playbook_path_by_flow = {
259
+ FLOW_ON: scratch.playbook_path,
260
+ FLOW_MISSING_FILE: scratch.root / MISSING_PLAYBOOK_RELATIVE_PATH,
261
+ }
262
+ if flow_setting in playbook_path_by_flow:
263
+ all_variables[PLAYBOOK_VARIABLE] = str(playbook_path_by_flow[flow_setting])
264
+ elif flow_setting != FLOW_OFF:
265
+ raise CaseFileInvalid(f"unknown flow setting {flow_setting!r}")
266
+ all_extra_variables = _substituted(event.get("env", {}), scratch)
267
+ if isinstance(all_extra_variables, dict):
268
+ all_variables.update(
269
+ {str(each_name): str(each_text) for each_name, each_text in all_extra_variables.items()}
270
+ )
271
+ return all_variables
272
+
273
+
274
+ def _run_hook(
275
+ hook_path: Path, payload: JsonObject, all_variables: dict[str, str], scratch: _Scratch
276
+ ) -> HookRun:
277
+ if not hook_path.is_file():
278
+ return HookRun(exit_code=None, stdout="")
279
+ try:
280
+ completed = subprocess.run(
281
+ [sys.executable, str(hook_path)],
282
+ input=json.dumps(payload),
283
+ capture_output=True,
284
+ text=True,
285
+ env=all_variables,
286
+ cwd=str(scratch.root),
287
+ timeout=HOOK_TIMEOUT_SECONDS,
288
+ check=False,
289
+ )
290
+ except subprocess.TimeoutExpired:
291
+ return HookRun(exit_code=TIMED_OUT_EXIT_CODE, stdout=TIMED_OUT_STDOUT)
292
+ return HookRun(exit_code=completed.returncode, stdout=completed.stdout)
293
+
294
+
295
+ def _parsed_stdout(hook_run: HookRun) -> JsonObject | None:
296
+ if not hook_run.stdout.strip():
297
+ return {}
298
+ try:
299
+ parsed_stdout = json.loads(hook_run.stdout)
300
+ except ValueError:
301
+ return None
302
+ return parsed_stdout if isinstance(parsed_stdout, dict) else None
303
+
304
+
305
+ def _check_ledger(expect: JsonObject, session_id: str, label: str, scratch: _Scratch) -> None:
306
+ answer_by_path = read_ledger_answers(scratch.config_directory, session_id)
307
+ if "entry_count" in expect and len(answer_by_path) != expect["entry_count"]:
308
+ scratch.all_failures.append(
309
+ f"{label}: expected {expect['entry_count']} ledger entries, found {len(answer_by_path)}"
310
+ )
311
+ all_expected_answers = expect.get("answers", {})
312
+ if not isinstance(all_expected_answers, dict):
313
+ raise CaseFileInvalid(f"{label}: answers must be an object")
314
+ for each_name, each_expected_answer in all_expected_answers.items():
315
+ bound_path = scratch.ledger_path_by_name.get(each_name)
316
+ if bound_path is None or bound_path not in answer_by_path:
317
+ scratch.all_failures.append(f"{label}: no ledger entry for {each_name}")
318
+ continue
319
+ if answer_by_path[bound_path] != each_expected_answer:
320
+ scratch.all_failures.append(
321
+ f"{label}: entry {each_name} answer is {answer_by_path[bound_path]!r}, "
322
+ f"expected {each_expected_answer!r}"
323
+ )
324
+
325
+
326
+ def _check_text(
327
+ text: str,
328
+ expect: JsonObject,
329
+ contains_key: str,
330
+ excludes_key: str,
331
+ label: str,
332
+ scratch: _Scratch,
333
+ ) -> None:
334
+ for each_needle in expect.get(contains_key, []):
335
+ if each_needle not in text:
336
+ scratch.all_failures.append(f"{label}: text lacks {each_needle!r}")
337
+ for each_needle in expect.get(excludes_key, []):
338
+ if each_needle in text:
339
+ scratch.all_failures.append(f"{label}: text holds {each_needle!r}")
340
+
341
+
342
+ def _play_refusal(event: JsonObject, label: str, scratch: _Scratch) -> None:
343
+ refusal_fields = _substituted(event[REFUSAL_EVENT_KEY], scratch)
344
+ if not isinstance(refusal_fields, dict):
345
+ raise CaseFileInvalid(f"{label}: refusal must be an object")
346
+ session_id = str(refusal_fields.get("session_id", MAIN_SESSION_ID))
347
+ scratch.tool_use_counter += 1
348
+ payload: JsonObject = {
349
+ "session_id": session_id,
350
+ "transcript_path": str(scratch.transcript_path),
351
+ "cwd": str(scratch.root),
352
+ "permission_mode": PERMISSION_MODE,
353
+ "hook_event_name": PERMISSION_DENIED_EVENT_NAME,
354
+ "tool_use_id": f"toolu_refused_{scratch.tool_use_counter}",
355
+ **refusal_fields,
356
+ }
357
+ before_paths = set(read_ledger_answers(scratch.config_directory, session_id))
358
+ hook_run = _run_hook(scratch.refusal_hook, payload, _hook_environment(event, scratch), scratch)
359
+ if hook_run.exit_code is None:
360
+ scratch.all_errors.append(f"{label}: refusal hook missing at {scratch.refusal_hook}")
361
+ return
362
+ parsed_stdout = _parsed_stdout(hook_run)
363
+ if hook_run.exit_code != 0 or parsed_stdout is None:
364
+ scratch.all_errors.append(
365
+ f"{label}: refusal hook exit {hook_run.exit_code}, stdout {hook_run.stdout[:200]!r}"
366
+ )
367
+ return
368
+ all_new_paths = sorted(
369
+ set(read_ledger_answers(scratch.config_directory, session_id)) - before_paths
370
+ )
371
+ if "name" in event and len(all_new_paths) == 1:
372
+ scratch.ledger_path_by_name[str(event["name"])] = all_new_paths[0]
373
+ expect = _substituted(event.get("expect", {}), scratch)
374
+ if not isinstance(expect, dict):
375
+ raise CaseFileInvalid(f"{label}: expect must be an object")
376
+ if "creates_entry" in expect and bool(all_new_paths) != expect["creates_entry"]:
377
+ scratch.all_failures.append(f"{label}: created {len(all_new_paths)} ledger entries")
378
+ if "name" in event and expect.get("creates_entry", True) and len(all_new_paths) != 1:
379
+ scratch.all_failures.append(
380
+ f"{label}: expected one new ledger entry named {event['name']}, found {len(all_new_paths)}"
381
+ )
382
+ hook_specific_output = parsed_stdout.get("hookSpecificOutput")
383
+ context_text = ""
384
+ if isinstance(hook_specific_output, dict):
385
+ context_text = str(hook_specific_output.get("additionalContext", ""))
386
+ _check_text(context_text, expect, "context_contains", "context_excludes", label, scratch)
387
+ if "stdout_equals_today" in expect:
388
+ today_stdout = scratch.all_today_outputs[str(expect["stdout_equals_today"])]
389
+ if hook_run.stdout != today_stdout:
390
+ scratch.all_failures.append(
391
+ f"{label}: stdout differs from today's output {expect['stdout_equals_today']}"
392
+ )
393
+ _check_ledger(expect, session_id, label, scratch)
394
+
395
+
396
+ def _play_stop(event: JsonObject, label: str, scratch: _Scratch) -> None:
397
+ stop_fields = _substituted(event[STOP_EVENT_KEY], scratch)
398
+ if not isinstance(stop_fields, dict):
399
+ raise CaseFileInvalid(f"{label}: stop must be an object")
400
+ session_id = str(stop_fields.get("session_id", MAIN_SESSION_ID))
401
+ payload: JsonObject = {
402
+ "session_id": session_id,
403
+ "transcript_path": str(scratch.transcript_path),
404
+ "cwd": str(scratch.root),
405
+ "permission_mode": PERMISSION_MODE,
406
+ "hook_event_name": STOP_EVENT_NAME,
407
+ "stop_hook_active": False,
408
+ **stop_fields,
409
+ }
410
+ hook_run = _run_hook(scratch.stop_hook, payload, _hook_environment(event, scratch), scratch)
411
+ if hook_run.exit_code is None:
412
+ scratch.all_errors.append(f"{label}: Stop gate missing at {scratch.stop_hook}")
413
+ return
414
+ parsed_stdout = _parsed_stdout(hook_run)
415
+ is_held = parsed_stdout is not None and parsed_stdout.get("decision") == BLOCK_DECISION
416
+ hold_reason = parsed_stdout.get("reason", "") if parsed_stdout else ""
417
+ is_broken = (
418
+ parsed_stdout is None
419
+ or hook_run.exit_code not in ALL_STOP_EXIT_CODES
420
+ or (hook_run.exit_code == HOLD_EXIT_CODE and not is_held)
421
+ or not isinstance(hold_reason, str)
422
+ )
423
+ if is_broken:
424
+ scratch.all_errors.append(
425
+ f"{label}: Stop gate exit {hook_run.exit_code}, stdout {hook_run.stdout[:200]!r}"
426
+ )
427
+ return
428
+ expect = _substituted(event.get("expect", {}), scratch)
429
+ if not isinstance(expect, dict):
430
+ raise CaseFileInvalid(f"{label}: expect must be an object")
431
+ if "held" in expect and is_held != expect["held"]:
432
+ scratch.all_failures.append(f"{label}: turn {'held' if is_held else 'ended'}")
433
+ if expect.get("stdout_empty") and hook_run.stdout.strip():
434
+ scratch.all_failures.append(f"{label}: Stop gate printed {hook_run.stdout[:120]!r}")
435
+ _check_text(str(hold_reason), expect, "reason_contains", "reason_excludes", label, scratch)
436
+ count_rule = expect.get("reason_holds_count_of")
437
+ if isinstance(count_rule, dict):
438
+ all_candidate_texts = count_rule.get("texts", [])
439
+ found_count = sum(1 for each_text in all_candidate_texts if each_text in str(hold_reason))
440
+ if found_count != count_rule.get("count"):
441
+ scratch.all_failures.append(
442
+ f"{label}: hold holds {found_count} of the listed texts, expected {count_rule.get('count')}"
443
+ )
444
+ _check_ledger(expect, session_id, label, scratch)
445
+
446
+
447
+ def _timestamp_fields(event: JsonObject) -> JsonObject:
448
+ if "timestamp" not in event:
449
+ return {"timestamp": datetime.now(timezone.utc).strftime(TIMESTAMP_FORMAT)}
450
+ if event["timestamp"] is None:
451
+ return {}
452
+ return {"timestamp": event["timestamp"]}
453
+
454
+
455
+ def transcript_records_for(
456
+ event: JsonObject, tool_use_id: str, session_id: str
457
+ ) -> list[JsonObject]:
458
+ """Return the two transcript records one transcript event appends.
459
+
460
+ ::
461
+
462
+ {"transcript": "ask", "questions": [{"question": "Q", "answer": "Allow it"}]}
463
+ -> assistant AskUserQuestion call, user result with answers {"Q": "Allow it"}
464
+ {"transcript": "bash", "command": "ls"} -> assistant Bash call, user result "ok"
465
+ {"transcript": "card", "tool": "mcp__x", "question": "Q"} -> card call, user result
466
+
467
+ The record shape follows an answered AskUserQuestion read from a session
468
+ transcript: ``toolUseResult`` holds ``questions`` and ``answers``.
469
+
470
+ Args:
471
+ event: The transcript event from the case file.
472
+ tool_use_id: The id that ties the call to its result.
473
+ session_id: The session the records belong to.
474
+
475
+ Returns:
476
+ The assistant call record and the user result record.
477
+ """
478
+ transcript_kind = event[TRANSCRIPT_EVENT_KEY]
479
+ timestamp_fields = _timestamp_fields(event)
480
+ tool_result_block: JsonObject = {"type": "tool_result", "tool_use_id": tool_use_id}
481
+ result_record: JsonObject = {"type": "user", "sessionId": session_id, **timestamp_fields}
482
+ if transcript_kind == ASK_TRANSCRIPT_KIND:
483
+ all_question_entries = event["questions"]
484
+ if not isinstance(all_question_entries, list):
485
+ raise CaseFileInvalid("ask questions must be a list")
486
+ all_questions = [
487
+ {
488
+ "question": each_entry["question"],
489
+ "header": each_entry.get("header", DEFAULT_QUESTION_HEADER),
490
+ "multiSelect": False,
491
+ "options": [dict(each_option) for each_option in ALL_QUESTION_OPTIONS],
492
+ }
493
+ for each_entry in all_question_entries
494
+ ]
495
+ tool_name, tool_input = ASK_TOOL_NAME, {"questions": all_questions}
496
+ if event.get("error"):
497
+ tool_result_block.update({"content": ERRORED_RESULT_CONTENT, "is_error": True})
498
+ result_record["toolUseResult"] = ERRORED_TOOL_USE_RESULT
499
+ else:
500
+ answer_by_question = {
501
+ each_entry["question"]: each_entry["answer"] for each_entry in all_question_entries
502
+ }
503
+ pairs_text = ANSWER_PAIR_SEPARATOR.join(
504
+ f'"{each_question}"="{each_answer}"'
505
+ for each_question, each_answer in answer_by_question.items()
506
+ )
507
+ tool_result_block["content"] = ANSWERED_RESULT_TEMPLATE.format(pairs=pairs_text)
508
+ result_record["toolUseResult"] = {
509
+ "questions": all_questions,
510
+ "answers": answer_by_question,
511
+ }
512
+ elif transcript_kind == BASH_TRANSCRIPT_KIND:
513
+ tool_name, tool_input = (
514
+ BASH_TOOL_NAME,
515
+ {"command": event["command"], "description": "Run a step"},
516
+ )
517
+ tool_result_block["content"] = BASH_RESULT_TEXT
518
+ result_record["toolUseResult"] = {
519
+ "stdout": BASH_RESULT_TEXT,
520
+ "stderr": "",
521
+ "interrupted": False,
522
+ "isImage": False,
523
+ }
524
+ elif transcript_kind == CARD_TRANSCRIPT_KIND:
525
+ tool_name, tool_input = (
526
+ str(event["tool"]),
527
+ {
528
+ "question": event["question"],
529
+ "options": [each_option["label"] for each_option in ALL_QUESTION_OPTIONS],
530
+ },
531
+ )
532
+ tool_result_block["content"] = CARD_RESULT_TEXT
533
+ result_record["toolUseResult"] = CARD_RESULT_TEXT
534
+ else:
535
+ raise CaseFileInvalid(f"unknown transcript kind {transcript_kind!r}")
536
+ call_record: JsonObject = {
537
+ "type": "assistant",
538
+ "sessionId": session_id,
539
+ **timestamp_fields,
540
+ "message": {
541
+ "role": "assistant",
542
+ "content": [
543
+ {"type": "tool_use", "id": tool_use_id, "name": tool_name, "input": tool_input}
544
+ ],
545
+ },
546
+ }
547
+ result_record["message"] = {"role": "user", "content": [tool_result_block]}
548
+ return [call_record, result_record]
549
+
550
+
551
+ def _play_transcript(event: JsonObject, scratch: _Scratch) -> None:
552
+ scratch.tool_use_counter += 1
553
+ resolved_event = _substituted(event, scratch)
554
+ if not isinstance(resolved_event, dict):
555
+ raise CaseFileInvalid("transcript event must be an object")
556
+ all_records = transcript_records_for(
557
+ resolved_event, f"toolu_case_{scratch.tool_use_counter}", MAIN_SESSION_ID
558
+ )
559
+ with scratch.transcript_path.open("a", encoding="utf-8") as transcript_file:
560
+ for each_record in all_records:
561
+ transcript_file.write(json.dumps(each_record) + "\n")
562
+
563
+
564
+ def _validate_case(case: JsonObject) -> None:
565
+ case_id = case.get("id")
566
+ if case.get("kind") not in ALL_CASE_KINDS:
567
+ raise CaseFileInvalid(f"{case_id}: kind must be one of {ALL_CASE_KINDS}")
568
+ all_events = case.get("events")
569
+ if not isinstance(all_events, list) or not all_events:
570
+ raise CaseFileInvalid(f"{case_id}: events must be a non-empty list")
571
+ for each_event in all_events:
572
+ expect_keys = set(each_event.get("expect", {}))
573
+ if REFUSAL_EVENT_KEY in each_event:
574
+ unknown_keys = expect_keys - ALL_REFUSAL_EXPECT_KEYS
575
+ elif STOP_EVENT_KEY in each_event:
576
+ unknown_keys = expect_keys - ALL_STOP_EXPECT_KEYS
577
+ elif TRANSCRIPT_EVENT_KEY in each_event:
578
+ unknown_keys = expect_keys
579
+ else:
580
+ raise CaseFileInvalid(f"{case_id}: event has no refusal, stop or transcript key")
581
+ if unknown_keys:
582
+ raise CaseFileInvalid(f"{case_id}: unknown expect keys {sorted(unknown_keys)}")
583
+
584
+
585
+ def _write_rule_repository(scratch_root: Path) -> None:
586
+ """Write the scratch agent-channels checkout and its rule worktree.
587
+
588
+ ::
589
+
590
+ agent-channels/.git/HEAD on claude/auto-mode-rule-own-branch-force-push
591
+ agent-channels/setup/add_auto_mode_rule.py
592
+ agent-channels/.git/worktrees/agent-channels-rule/{HEAD,commondir,gitdir}
593
+ agent-channels-rule/.git "gitdir: <the folder above>"
594
+
595
+ Args:
596
+ scratch_root: The case's scratch home.
597
+ """
598
+ repository_root = scratch_root / RULE_REPOSITORY_NAME
599
+ worktree_git_directory = repository_root / WORKTREE_GIT_RELATIVE_PATH
600
+ worktree_git_directory.mkdir(parents=True)
601
+ (repository_root / SCRATCH_GIT_NAME / "HEAD").write_text(SCRATCH_HEAD_TEXT, encoding="utf-8")
602
+ (worktree_git_directory / "HEAD").write_text(SCRATCH_HEAD_TEXT, encoding="utf-8")
603
+ (worktree_git_directory / "commondir").write_text(
604
+ WORKTREE_COMMON_DIRECTORY_TEXT, encoding="utf-8"
605
+ )
606
+ rule_tool_path = repository_root / RULE_TOOL_RELATIVE_PATH
607
+ rule_tool_path.parent.mkdir(parents=True)
608
+ rule_tool_path.write_text(RULE_TOOL_TEXT, encoding="utf-8")
609
+ worktree_directory = scratch_root / RULE_WORKTREE_NAME
610
+ worktree_directory.mkdir()
611
+ (worktree_directory / SCRATCH_GIT_NAME).write_text(
612
+ GIT_FILE_TEMPLATE.format(git_directory=worktree_git_directory), encoding="utf-8"
613
+ )
614
+ (worktree_git_directory / WORKTREE_BACK_POINTER_NAME).write_text(
615
+ f"{worktree_directory / SCRATCH_GIT_NAME}\n", encoding="utf-8"
616
+ )
617
+
618
+
619
+ def run_case(
620
+ case: JsonObject, refusal_hook: Path, stop_hook: Path, all_today_outputs: dict[str, str]
621
+ ) -> CaseOutcome:
622
+ """Play one case in a fresh scratch home and grade it.
623
+
624
+ Args:
625
+ case: One entry of the case file's ``cases`` list.
626
+ refusal_hook: The PermissionDenied hook script.
627
+ stop_hook: The Stop gate script.
628
+ all_today_outputs: The stored stdout of today's refusal hook, by name.
629
+
630
+ Returns:
631
+ The verdict and the problems found.
632
+ """
633
+ _validate_case(case)
634
+ scratch_root = Path(tempfile.mkdtemp(prefix="auto-mode-flow-case-"))
635
+ try:
636
+ scratch = _Scratch(scratch_root, refusal_hook, stop_hook, all_today_outputs)
637
+ scratch.config_directory.mkdir()
638
+ scratch.playbook_path.parent.mkdir(parents=True)
639
+ scratch.playbook_path.write_text(PLAYBOOK_TEXT, encoding="utf-8")
640
+ _write_rule_repository(scratch_root)
641
+ scratch.transcript_path.write_text("", encoding="utf-8")
642
+ for each_index, each_event in enumerate(case["events"], start=1):
643
+ label = f"event {each_index}"
644
+ if REFUSAL_EVENT_KEY in each_event:
645
+ _play_refusal(each_event, label, scratch)
646
+ elif STOP_EVENT_KEY in each_event:
647
+ _play_stop(each_event, label, scratch)
648
+ else:
649
+ _play_transcript(each_event, scratch)
650
+ finally:
651
+ force_rmtree(str(scratch_root))
652
+ if scratch.all_errors:
653
+ verdict = ERROR_VERDICT
654
+ elif scratch.all_failures:
655
+ verdict = FAIL_VERDICT
656
+ else:
657
+ verdict = PASS_VERDICT
658
+ return CaseOutcome(
659
+ case_id=str(case["id"]),
660
+ kind=str(case["kind"]),
661
+ verdict=verdict,
662
+ all_problems=tuple(scratch.all_errors + scratch.all_failures),
663
+ )
664
+
665
+
666
+ def load_case_file(case_file_path: Path) -> JsonObject:
667
+ """Read the case file.
668
+
669
+ Args:
670
+ case_file_path: Path to ``auto_mode_rule_flow_cases.json``.
671
+
672
+ Returns:
673
+ The parsed case file, with ``cases`` and ``today_outputs``.
674
+ """
675
+ parsed_file = json.loads(case_file_path.read_text(encoding="utf-8"))
676
+ if not isinstance(parsed_file, dict) or not isinstance(parsed_file.get("cases"), list):
677
+ raise CaseFileInvalid(f"{case_file_path} has no cases list")
678
+ return parsed_file
679
+
680
+
681
+ def run_all_cases(
682
+ case_file_path: Path,
683
+ refusal_hook: Path,
684
+ stop_hook: Path,
685
+ all_case_prefixes: tuple[str, ...],
686
+ ) -> list[CaseOutcome]:
687
+ """Run every case, or the cases whose id starts with one of the prefixes.
688
+
689
+ Args:
690
+ case_file_path: Path to the case file.
691
+ refusal_hook: The PermissionDenied hook script.
692
+ stop_hook: The Stop gate script.
693
+ all_case_prefixes: Id prefixes such as ``("R", "N3")``; empty runs every case.
694
+
695
+ Returns:
696
+ One outcome per case, in case file order.
697
+ """
698
+ case_file = load_case_file(case_file_path)
699
+ all_today_outputs = case_file.get("today_outputs", {})
700
+ if not isinstance(all_today_outputs, dict):
701
+ raise CaseFileInvalid("today_outputs must be an object")
702
+ return [
703
+ run_case(each_case, refusal_hook, stop_hook, all_today_outputs)
704
+ for each_case in case_file["cases"]
705
+ if not all_case_prefixes or str(each_case["id"]).startswith(all_case_prefixes)
706
+ ]
707
+
708
+
709
+ def score_summary(all_outcomes: list[CaseOutcome]) -> dict[str, int]:
710
+ """Count passes per kind and errors.
711
+
712
+ ::
713
+
714
+ R1 should act pass, R8 should stay quiet fail, Q1 should act error
715
+ -> should_act_passed 1 of 2, should_stay_quiet_passed 0 of 1, errors 1
716
+
717
+ Args:
718
+ all_outcomes: The outcomes from ``run_all_cases``.
719
+
720
+ Returns:
721
+ ``should_act_passed``, ``should_act_total``, ``should_stay_quiet_passed``,
722
+ ``should_stay_quiet_total`` and ``errors``.
723
+ """
724
+ return {
725
+ "should_act_passed": sum(
726
+ 1
727
+ for each in all_outcomes
728
+ if each.kind == SHOULD_ACT_KIND and each.verdict == PASS_VERDICT
729
+ ),
730
+ "should_act_total": sum(1 for each in all_outcomes if each.kind == SHOULD_ACT_KIND),
731
+ "should_stay_quiet_passed": sum(
732
+ 1
733
+ for each in all_outcomes
734
+ if each.kind == SHOULD_STAY_QUIET_KIND and each.verdict == PASS_VERDICT
735
+ ),
736
+ "should_stay_quiet_total": sum(
737
+ 1 for each in all_outcomes if each.kind == SHOULD_STAY_QUIET_KIND
738
+ ),
739
+ "errors": sum(1 for each in all_outcomes if each.verdict == ERROR_VERDICT),
740
+ }
741
+
742
+
743
+ def calibration_problems(case_file_path: Path) -> list[str]:
744
+ """Score the cases against the two stub hooks and name each miscalibration.
745
+
746
+ ::
747
+
748
+ stub that never acts -> must pass no ``should act`` case
749
+ stub that records and holds -> must pass no ``should stay quiet`` case
750
+ either stub -> must cause no error
751
+
752
+ Args:
753
+ case_file_path: Path to the case file.
754
+
755
+ Returns:
756
+ One line per calibration that does not hold; empty when both hold.
757
+ """
758
+ eval_directory = Path(__file__).resolve().parent
759
+ allow_stub = eval_directory / ALLOW_STUB_FILE_NAME
760
+ hold_stub = eval_directory / HOLD_STUB_FILE_NAME
761
+ allow_summary = score_summary(run_all_cases(case_file_path, allow_stub, allow_stub, ()))
762
+ hold_summary = score_summary(run_all_cases(case_file_path, hold_stub, hold_stub, ()))
763
+ all_checks = (
764
+ (ALLOW_STUB_FILE_NAME, SHOULD_ACT_PASSED_KEY, allow_summary),
765
+ (HOLD_STUB_FILE_NAME, SHOULD_STAY_QUIET_PASSED_KEY, hold_summary),
766
+ )
767
+ all_problems: list[str] = []
768
+ for each_stub_name, each_passed_key, each_summary in all_checks:
769
+ if each_summary[each_passed_key] or each_summary[ERRORS_KEY]:
770
+ all_problems.append(
771
+ CALIBRATION_PROBLEM_TEMPLATE.format(stub_name=each_stub_name, **each_summary)
772
+ )
773
+ return all_problems
774
+
775
+
776
+ def report_text_for(
777
+ all_outcomes: list[CaseOutcome], all_counts: dict[str, int], is_verbose: bool
778
+ ) -> str:
779
+ """Return the report the runner prints: one line per case, then the score.
780
+
781
+ ::
782
+
783
+ R1 should act pass
784
+ event 1: created 0 ledger entries (only when verbose)
785
+ should act passed 1 of 1; should stay quiet passed 0 of 0; errors 0
786
+
787
+ Args:
788
+ all_outcomes: The graded cases, in run order.
789
+ all_counts: The counts from ``score_summary``.
790
+ is_verbose: Whether to add each problem line under its case.
791
+
792
+ Returns:
793
+ The report text, each line ending in a newline.
794
+ """
795
+ all_lines: list[str] = []
796
+ for each_outcome in all_outcomes:
797
+ all_lines.append(
798
+ f"{each_outcome.case_id:5} {each_outcome.kind:17} {each_outcome.verdict}"
799
+ )
800
+ if is_verbose:
801
+ all_lines.extend(
802
+ PROBLEM_LINE_INDENT + each_problem for each_problem in each_outcome.all_problems
803
+ )
804
+ all_lines.append(SUMMARY_LINE_TEMPLATE.format(**all_counts))
805
+ all_lines.append(ERROR_RULE_LINE)
806
+ return "".join(each_line + "\n" for each_line in all_lines)
807
+
808
+
809
+ def main() -> int:
810
+ hooks_root = Path(_hooks_root_directory)
811
+ parser = argparse.ArgumentParser(description="Run the auto mode rule flow cases.")
812
+ parser.add_argument(
813
+ "--refusal-hook", type=Path, default=hooks_root / DEFAULT_REFUSAL_HOOK_RELATIVE_PATH
814
+ )
815
+ parser.add_argument(
816
+ "--stop-hook", type=Path, default=hooks_root / DEFAULT_STOP_HOOK_RELATIVE_PATH
817
+ )
818
+ parser.add_argument(
819
+ "--cases", type=Path, default=Path(__file__).resolve().parent / CASE_FILE_NAME
820
+ )
821
+ parser.add_argument("--only", default="", help="comma-separated case id prefixes, such as R,N3")
822
+ parser.add_argument("--verbose", action="store_true", help="print each problem line")
823
+ parser.add_argument(
824
+ "--calibrate", action="store_true", help="score the cases against both stub hooks"
825
+ )
826
+ arguments = parser.parse_args()
827
+ if arguments.calibrate:
828
+ all_problems = calibration_problems(arguments.cases)
829
+ sys.stdout.write("".join(each_line + "\n" for each_line in all_problems or [CALIBRATED_LINE]))
830
+ sys.stdout.flush()
831
+ return 1 if all_problems else 0
832
+ all_case_prefixes = tuple(each for each in arguments.only.split(",") if each)
833
+ all_outcomes = run_all_cases(
834
+ arguments.cases,
835
+ arguments.refusal_hook.resolve(),
836
+ arguments.stop_hook.resolve(),
837
+ all_case_prefixes,
838
+ )
839
+ summary = score_summary(all_outcomes)
840
+ sys.stdout.write(report_text_for(all_outcomes, summary, arguments.verbose))
841
+ sys.stdout.flush()
842
+ is_all_passed = summary["should_act_passed"] + summary["should_stay_quiet_passed"] == len(
843
+ all_outcomes
844
+ )
845
+ return 0 if is_all_passed else 1
846
+
847
+
848
+ if __name__ == "__main__":
849
+ raise SystemExit(main())