claude-dev-env 8.124.0 → 8.125.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/profile-stop-gate.mjs +5 -7
- package/bin/profile-stop-gate.test.mjs +29 -1
- package/docs/agents-guidance/asks.md +1 -1
- package/docs/rule-guides/question-presentation.md +1 -1
- package/hooks/blocking/closing_reply_judge.py +332 -0
- package/hooks/blocking/closing_reply_shapes.py +365 -0
- package/hooks/blocking/conftest.py +84 -0
- package/hooks/blocking/open_tasks_stop_gate.py +247 -10
- package/hooks/blocking/open_tasks_transcript.py +25 -2
- package/hooks/blocking/open_tasks_transcript_records.py +10 -0
- package/hooks/blocking/question_brief_gate.py +6 -4
- package/hooks/blocking/test_closing_reply_judge.py +246 -0
- package/hooks/blocking/test_closing_reply_shapes.py +247 -0
- package/hooks/blocking/test_open_tasks_stop_gate.py +562 -0
- package/hooks/blocking/test_open_tasks_transcript.py +23 -0
- package/hooks/blocking/test_open_tasks_transcript_records.py +14 -0
- package/hooks/blocking/test_question_brief_gate.py +22 -0
- package/hooks/features/conduct-gates.md +4 -3
- package/hooks/hooks.json +1 -1
- package/hooks/hooks_constants/closing_reply_judge_constants.py +143 -0
- package/hooks/hooks_constants/open_tasks_stop_gate_constants.py +93 -0
- package/hooks/hooks_constants/test_hook_time_budget.py +7 -0
- package/hooks/once_per_session.py +65 -1
- package/hooks/test_once_per_session.py +35 -0
- package/package.json +1 -1
|
@@ -141,10 +141,8 @@ function stopHooks(settingsPath) {
|
|
|
141
141
|
return stopGroups.flatMap((eachGroup) => (Array.isArray(eachGroup?.hooks) ? eachGroup.hooks : []));
|
|
142
142
|
}
|
|
143
143
|
|
|
144
|
-
function
|
|
145
|
-
return stopHooks(settingsPath)
|
|
146
|
-
.filter((eachHook) => isGateHook(eachHook, claudeHome))
|
|
147
|
-
.map((eachHook) => eachHook.command);
|
|
144
|
+
function profileGateHooks(settingsPath, claudeHome) {
|
|
145
|
+
return stopHooks(settingsPath).filter((eachHook) => isGateHook(eachHook, claudeHome));
|
|
148
146
|
}
|
|
149
147
|
|
|
150
148
|
/**
|
|
@@ -170,7 +168,7 @@ export function profileRunsItsOwnGate(settingsPath, claudeHome) {
|
|
|
170
168
|
*
|
|
171
169
|
* A profile keeps its own settings and hooks. The merge adds the one gate
|
|
172
170
|
* entry, in the spelling the main settings use, only where it is missing;
|
|
173
|
-
* an entry left with an older interpreter spelling is replaced. The file as
|
|
171
|
+
* an entry left with an older interpreter spelling or timeout is replaced. The file as
|
|
174
172
|
* it was before either change is copied to a timestamped `.bak`. A profile
|
|
175
173
|
* that already runs its own copy of the gate script is left as it is.
|
|
176
174
|
*
|
|
@@ -187,8 +185,8 @@ export function addOpenTasksGateToProfiles(options) {
|
|
|
187
185
|
(eachPath) => !profileRunsItsOwnGate(eachPath, options.claudeHome),
|
|
188
186
|
);
|
|
189
187
|
const allStalePaths = allTargetPaths.filter(
|
|
190
|
-
(eachPath) =>
|
|
191
|
-
.some((
|
|
188
|
+
(eachPath) => profileGateHooks(eachPath, options.claudeHome)
|
|
189
|
+
.some((eachHook) => eachHook.command !== gateHook.command || eachHook.timeout !== gateHook.timeout),
|
|
192
190
|
);
|
|
193
191
|
const originalTextByStalePath = new Map(allStalePaths.map((eachPath) => [eachPath, readFileSync(eachPath, 'utf8')]));
|
|
194
192
|
removeOpenTasksGateFromProfiles({ claudeHome: options.claudeHome, allProfileSettingsPaths: allStalePaths });
|
|
@@ -2,7 +2,10 @@ import { test } from 'node:test';
|
|
|
2
2
|
import { strict as assert } from 'node:assert';
|
|
3
3
|
import { lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
|
-
import { join } from 'node:path';
|
|
5
|
+
import { dirname, join } from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
|
|
8
|
+
import { mergeHooksIntoSettings } from './install.mjs';
|
|
6
9
|
|
|
7
10
|
import {
|
|
8
11
|
addOpenTasksGateToProfiles,
|
|
@@ -25,6 +28,13 @@ function makeHome(context) {
|
|
|
25
28
|
return { homeDirectory, claudeHome, mainSettingsPath, gateCommand };
|
|
26
29
|
}
|
|
27
30
|
|
|
31
|
+
function gateTimeouts(settingsPath) {
|
|
32
|
+
const settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
|
|
33
|
+
return (settings.hooks?.Stop ?? []).flatMap((eachGroup) => eachGroup.hooks
|
|
34
|
+
.filter((eachHook) => eachHook.command.includes('open_tasks_stop_gate.py'))
|
|
35
|
+
.map((eachHook) => eachHook.timeout));
|
|
36
|
+
}
|
|
37
|
+
|
|
28
38
|
function writeProfile(homeDirectory, name, settings) {
|
|
29
39
|
const profileDirectory = join(homeDirectory, '.claude-profiles', name);
|
|
30
40
|
mkdirSync(profileDirectory, { recursive: true });
|
|
@@ -183,3 +193,21 @@ test('a settings.json that does not parse may hold the gate when its text names
|
|
|
183
193
|
assert.equal(unparsedSettingsMayHoldGate(noGatePath, claudeHome), false);
|
|
184
194
|
assert.equal(unparsedSettingsMayHoldGate(join(homeDirectory, 'missing', 'settings.json'), claudeHome), true);
|
|
185
195
|
});
|
|
196
|
+
|
|
197
|
+
test('should raise a profile gate timeout of 10 to 45 on install', (context) => {
|
|
198
|
+
const { homeDirectory, claudeHome, mainSettingsPath, gateCommand } = makeHome(context);
|
|
199
|
+
const profilePath = writeProfile(homeDirectory, 'older-install', {
|
|
200
|
+
hooks: { Stop: [{ hooks: [{ type: 'command', command: gateCommand, timeout: 10 }] }] },
|
|
201
|
+
});
|
|
202
|
+
const hooksConfigPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'hooks', 'hooks.json');
|
|
203
|
+
const hooksConfig = JSON.parse(readFileSync(hooksConfigPath, 'utf8'));
|
|
204
|
+
const mainSettings = JSON.parse(readFileSync(mainSettingsPath, 'utf8'));
|
|
205
|
+
|
|
206
|
+
mergeHooksIntoSettings(mainSettings, hooksConfig, claudeHome, 'python3', null, homeDirectory);
|
|
207
|
+
writeFileSync(mainSettingsPath, JSON.stringify(mainSettings, null, 4));
|
|
208
|
+
addOpenTasksGateToProfiles({ mainSettingsPath, claudeHome, allProfileSettingsPaths: [profilePath], homeDirectory });
|
|
209
|
+
|
|
210
|
+
assert.deepEqual(gateTimeouts(mainSettingsPath), [45]);
|
|
211
|
+
assert.deepEqual(gateTimeouts(profilePath), [45]);
|
|
212
|
+
assert.deepEqual(stopCommands(profilePath), [gateCommand]);
|
|
213
|
+
});
|
|
@@ -45,7 +45,7 @@ The gate holds a turn once when all three hold:
|
|
|
45
45
|
- The turn ran a work tool: any tool but a tracker call, the session title tool, `AskUserQuestion`, a hearthbot decision card, Codex `request_user_input`, `ToolSearch`, `TaskList`, or `TaskGet`. A turn that only asks the user a question passes.
|
|
46
46
|
- The turn made no tracker call: TaskCreate, TaskUpdate, TodoWrite, the Codex `update_plan`, or the Claude Code desktop `mcp__ccd_turn__update_plan`.
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
It also holds a turn once when the closing reply promises a later report the prompt did not ask for, or lists open decisions in chat when the turn sent no question tool call. In Claude Code one small model call judges the reply, and Codex and Cursor use phrase checks. [Conduct gates](../../hooks/features/conduct-gates.md) describes both. A held turn passes, so the gate adds at most one step to a turn.
|
|
49
49
|
|
|
50
50
|
In Claude Code, `task_subject_outcome_gate.py` denies a task list item named as a procedure step. [Conduct gates](../../hooks/features/conduct-gates.md) lists what it matches.
|
|
51
51
|
|
|
@@ -16,7 +16,7 @@ Remove completed items from the task list before the turn ends, so the list show
|
|
|
16
16
|
|
|
17
17
|
A file or page the user reads can carry the context for a question. The question itself goes through the question tool, where the user answers it. On Claude, the `sent_file_question_gate.py` PreToolUse hook refuses a `SendUserFile` call or an `Artifact` page publish when a Markdown, text, or HTML file it sends asks the user something: a "Questions for you" or "Decisions needed" section, or a sentence that ends with a question mark and names the reader, asks for a go-ahead, or offers a choice. The refusal quotes the question. Ask it through the question tool, remove it from the file, and send the file again. A page that collects the answer itself, such as option cards, declares the `db` or `artifact` capability that saves the answer, and the hook lets it through.
|
|
18
18
|
|
|
19
|
-
The `open_tasks_stop_gate.py` Stop hook holds the turn once when completed items sit beside open ones on the session's own task list. It also holds the turn once when the closing reply, or a Markdown, text, or HTML file the turn sent, says something waits on the user and the turn ran no read call or sent no question tool call.
|
|
19
|
+
The `open_tasks_stop_gate.py` Stop hook holds the turn once when completed items sit beside open ones on the session's own task list. It also holds the turn once when the closing reply, or a Markdown, text, or HTML file the turn sent, says something waits on the user and the turn ran no read call or sent no question tool call. It also holds the turn once when the closing reply lists 3 or more open items, such as a deferred list or next steps, under a lead that names open work, and the turn sent no question tool call. The hold tells the agent to act on each item it can settle and to send each item that needs the user's judgment through the question tool. It holds the turn once, too, when the closing reply promises the user a later report that no typed prompt asked for. A held reply's text counts toward the limits below, so the first question call after such a hold can be refused once; the same call sent again passes.
|
|
20
20
|
|
|
21
21
|
## Several items, one question at a time
|
|
22
22
|
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""Ask one small model call whether a closing reply promises a later report or lists open decisions.
|
|
2
|
+
|
|
3
|
+
The open tasks Stop gate sends the turn's prompt and its closing reply to
|
|
4
|
+
``claude -p --model haiku --effort medium`` with the prompt on stdin, and reads
|
|
5
|
+
back one JSON verdict::
|
|
6
|
+
|
|
7
|
+
in: {"prompt_kind": "typed", "user_prompt": "...", "reply": "...", "question_tool_used": false}
|
|
8
|
+
out: {"promised_report": true, "promise_span": "...", "decisions_as_list": false, "list_span": ""}
|
|
9
|
+
|
|
10
|
+
The prompt kind tells the model where the prompt came from::
|
|
11
|
+
|
|
12
|
+
"<task-notification>...<summary>Lint ended</summary>..." -> task_notification, "Lint ended"
|
|
13
|
+
"<command-name>/review</command-name><command-args>12</command-args>" -> slash_command, "/review 12"
|
|
14
|
+
any other prompt -> typed, its first 4,000 characters
|
|
15
|
+
|
|
16
|
+
A verdict counts only when it is the four-key object, bare or in one ``json``
|
|
17
|
+
code fence, and each raised flag names a nonempty span found in the reply.
|
|
18
|
+
Two code gates then drop the list flag: the turn used the question tool, or
|
|
19
|
+
the prompt asked for a list.
|
|
20
|
+
|
|
21
|
+
The child runs in a new empty folder with no tools, no hooks, no MCP servers,
|
|
22
|
+
no CLAUDE.md files, no auto memory, ``CLAUDE_TURN_REVIEW_CHILD=1``, and none of
|
|
23
|
+
the parent session's id, messaging, or ingress variables. Every failure gives
|
|
24
|
+
no verdict and one line in ``~/.claude/logs/closing-reply-judge.log``::
|
|
25
|
+
|
|
26
|
+
the child times out, cannot start, or exits with another code than 0
|
|
27
|
+
stdout is not one JSON object with a string result, or is_error is set
|
|
28
|
+
the result is not the verdict shape, or a span is not in the reply
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import datetime
|
|
34
|
+
import json
|
|
35
|
+
import os
|
|
36
|
+
import shutil
|
|
37
|
+
import subprocess
|
|
38
|
+
import tempfile
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import NamedTuple
|
|
41
|
+
|
|
42
|
+
from closing_reply_shapes import prompt_asks_for_list
|
|
43
|
+
from hooks_constants.closing_reply_judge_constants import (
|
|
44
|
+
ALL_JUDGE_COMMAND_FLAGS,
|
|
45
|
+
ALL_JUDGE_ENVIRONMENT_SETTINGS,
|
|
46
|
+
ALL_LOG_PATH_PARTS,
|
|
47
|
+
ALL_VERDICT_KEYS,
|
|
48
|
+
ANSWER_FENCE_PATTERN,
|
|
49
|
+
ANSWER_IS_ERROR_KEY,
|
|
50
|
+
ANSWER_RESULT_KEY,
|
|
51
|
+
CHILD_DECODE_ERRORS,
|
|
52
|
+
CHILD_TEXT_ENCODING,
|
|
53
|
+
CLAUDE_EXECUTABLE_NAME,
|
|
54
|
+
COMMAND_ARGS_PATTERN,
|
|
55
|
+
COMMAND_NAME_PATTERN,
|
|
56
|
+
COMMAND_PART_SEPARATOR,
|
|
57
|
+
COMMAND_START_PATTERN,
|
|
58
|
+
DECISIONS_AS_LIST_KEY,
|
|
59
|
+
ERROR_ANSWER_REASON,
|
|
60
|
+
EXIT_CODE_REASON_TEMPLATE,
|
|
61
|
+
JUDGE_DATA_INDENT,
|
|
62
|
+
JUDGE_DATA_PLACEHOLDER,
|
|
63
|
+
JUDGE_PROMPT_TEMPLATE,
|
|
64
|
+
JUDGE_TIMEOUT_SECONDS,
|
|
65
|
+
LIST_SPAN_KEY,
|
|
66
|
+
LOG_LINE_TEMPLATE,
|
|
67
|
+
NO_VERDICT_LOG_TEMPLATE,
|
|
68
|
+
NOT_JSON_REASON,
|
|
69
|
+
NOTIFICATION_SUMMARY_PATTERN,
|
|
70
|
+
PROMISE_SPAN_KEY,
|
|
71
|
+
PROMISED_REPORT_KEY,
|
|
72
|
+
PROMPT_KIND_KEY,
|
|
73
|
+
QUESTION_TOOL_USED_KEY,
|
|
74
|
+
REPLY_KEY,
|
|
75
|
+
SLASH_COMMAND_PROMPT_KIND,
|
|
76
|
+
START_FAILURE_REASON,
|
|
77
|
+
TASK_NOTIFICATION_PREFIX,
|
|
78
|
+
TASK_NOTIFICATION_PROMPT_KIND,
|
|
79
|
+
TIMEOUT_REASON,
|
|
80
|
+
TYPED_PROMPT_CHARACTER_LIMIT,
|
|
81
|
+
TYPED_PROMPT_KIND,
|
|
82
|
+
USER_PROMPT_KEY,
|
|
83
|
+
VERDICT_SHAPE_REASON,
|
|
84
|
+
)
|
|
85
|
+
from hooks_constants.subprocess_window import hidden_window_creation_flags
|
|
86
|
+
from hooks_constants.turn_review_gate_constants import (
|
|
87
|
+
ALL_PARENT_SESSION_VARIABLES,
|
|
88
|
+
REVIEW_CHILD_ENVIRONMENT_VARIABLE,
|
|
89
|
+
REVIEW_CHILD_MARKER,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class JudgeVerdict(NamedTuple):
|
|
94
|
+
"""The spans the judge raised: the promised report and the open-decision list, each empty when not raised."""
|
|
95
|
+
|
|
96
|
+
promise_span: str
|
|
97
|
+
list_span: str
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _log(diagnostic_text: str) -> None:
|
|
101
|
+
timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
102
|
+
try:
|
|
103
|
+
log_path = Path.home().joinpath(*ALL_LOG_PATH_PARTS)
|
|
104
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
with log_path.open(
|
|
106
|
+
"a", encoding=CHILD_TEXT_ENCODING, errors=CHILD_DECODE_ERRORS
|
|
107
|
+
) as log_file:
|
|
108
|
+
log_file.write(LOG_LINE_TEMPLATE.format(timestamp=timestamp, text=diagnostic_text))
|
|
109
|
+
except (OSError, RuntimeError):
|
|
110
|
+
return
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def log_no_verdict(reason: str) -> None:
|
|
114
|
+
"""Write one line to the judge log that names why the judge gave no verdict."""
|
|
115
|
+
_log(NO_VERDICT_LOG_TEMPLATE.format(reason=reason))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _prompt_kind_and_text(prompt_text: str) -> tuple[str, str]:
|
|
119
|
+
stripped_prompt = prompt_text.lstrip()
|
|
120
|
+
if stripped_prompt.startswith(TASK_NOTIFICATION_PREFIX):
|
|
121
|
+
summary_match = NOTIFICATION_SUMMARY_PATTERN.search(stripped_prompt)
|
|
122
|
+
return TASK_NOTIFICATION_PROMPT_KIND, summary_match.group(
|
|
123
|
+
1
|
|
124
|
+
).strip() if summary_match else ""
|
|
125
|
+
if COMMAND_START_PATTERN.match(stripped_prompt):
|
|
126
|
+
all_command_parts = (
|
|
127
|
+
each_match.group(1).strip()
|
|
128
|
+
for each_match in (
|
|
129
|
+
COMMAND_NAME_PATTERN.search(stripped_prompt),
|
|
130
|
+
COMMAND_ARGS_PATTERN.search(stripped_prompt),
|
|
131
|
+
)
|
|
132
|
+
if each_match is not None
|
|
133
|
+
)
|
|
134
|
+
return SLASH_COMMAND_PROMPT_KIND, COMMAND_PART_SEPARATOR.join(
|
|
135
|
+
each_part for each_part in all_command_parts if each_part
|
|
136
|
+
)
|
|
137
|
+
return TYPED_PROMPT_KIND, prompt_text[:TYPED_PROMPT_CHARACTER_LIMIT]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def judge_input(prompt_text: str, reply_text: str, has_question_tool_call: bool) -> dict[str, object]:
|
|
141
|
+
"""Return the data the judge reads: the prompt kind, the prompt text, the reply, and the question-tool flag.
|
|
142
|
+
|
|
143
|
+
::
|
|
144
|
+
|
|
145
|
+
"<task-notification>...<summary>Lint ended</summary>" -> task_notification, "Lint ended"
|
|
146
|
+
"<command-name>/review</command-name>" -> slash_command, "/review"
|
|
147
|
+
"Fix the build" -> typed, "Fix the build"
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
prompt_text: The prompt that started the turn.
|
|
151
|
+
reply_text: The closing reply the turn ends with.
|
|
152
|
+
has_question_tool_call: Whether the turn called the question tool.
|
|
153
|
+
"""
|
|
154
|
+
prompt_kind, user_prompt = _prompt_kind_and_text(prompt_text)
|
|
155
|
+
return {
|
|
156
|
+
PROMPT_KIND_KEY: prompt_kind,
|
|
157
|
+
USER_PROMPT_KEY: user_prompt,
|
|
158
|
+
REPLY_KEY: reply_text,
|
|
159
|
+
QUESTION_TOOL_USED_KEY: bool(has_question_tool_call),
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def judge_prompt(prompt_text: str, reply_text: str, has_question_tool_call: bool) -> str:
|
|
164
|
+
"""Return the full judge prompt, with the ``judge_input`` data as indented JSON in place of ``$DATA``."""
|
|
165
|
+
judge_data = json.dumps(
|
|
166
|
+
judge_input(prompt_text, reply_text, has_question_tool_call),
|
|
167
|
+
ensure_ascii=False,
|
|
168
|
+
indent=JUDGE_DATA_INDENT,
|
|
169
|
+
)
|
|
170
|
+
return JUDGE_PROMPT_TEMPLATE.replace(JUDGE_DATA_PLACEHOLDER, judge_data)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def judge_command(claude_path: str) -> list[str]:
|
|
174
|
+
"""Return the ``claude -p`` command for the judge: Haiku at medium effort, JSON out, no tools, hooks, or MCP."""
|
|
175
|
+
return [claude_path, *ALL_JUDGE_COMMAND_FLAGS]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def child_environment() -> dict[str, str]:
|
|
179
|
+
"""Return this process's environment without the parent session's variables, marked as a review child."""
|
|
180
|
+
all_child_variables = {
|
|
181
|
+
each_name: each_value
|
|
182
|
+
for each_name, each_value in os.environ.items()
|
|
183
|
+
if each_name not in ALL_PARENT_SESSION_VARIABLES
|
|
184
|
+
}
|
|
185
|
+
all_child_variables[REVIEW_CHILD_ENVIRONMENT_VARIABLE] = REVIEW_CHILD_MARKER
|
|
186
|
+
all_child_variables.update(ALL_JUDGE_ENVIRONMENT_SETTINGS)
|
|
187
|
+
return all_child_variables
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def is_judge_available() -> bool:
|
|
191
|
+
"""Return True when a ``claude`` executable is on PATH."""
|
|
192
|
+
return shutil.which(CLAUDE_EXECUTABLE_NAME) is not None
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _answer_result(stdout_text: str) -> str | None:
|
|
196
|
+
try:
|
|
197
|
+
answer = json.loads(stdout_text)
|
|
198
|
+
except (ValueError, RecursionError):
|
|
199
|
+
log_no_verdict(NOT_JSON_REASON)
|
|
200
|
+
return None
|
|
201
|
+
if not isinstance(answer, dict) or not isinstance(answer.get(ANSWER_RESULT_KEY), str):
|
|
202
|
+
log_no_verdict(NOT_JSON_REASON)
|
|
203
|
+
return None
|
|
204
|
+
if answer.get(ANSWER_IS_ERROR_KEY) is True:
|
|
205
|
+
log_no_verdict(ERROR_ANSWER_REASON)
|
|
206
|
+
return None
|
|
207
|
+
return answer[ANSWER_RESULT_KEY]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _claude_result(claude_path: str, prompt: str, timeout_seconds: float) -> str | None:
|
|
211
|
+
try:
|
|
212
|
+
with tempfile.TemporaryDirectory() as empty_folder:
|
|
213
|
+
completed_process = subprocess.run(
|
|
214
|
+
judge_command(claude_path),
|
|
215
|
+
check=False,
|
|
216
|
+
capture_output=True,
|
|
217
|
+
text=True,
|
|
218
|
+
encoding=CHILD_TEXT_ENCODING,
|
|
219
|
+
errors=CHILD_DECODE_ERRORS,
|
|
220
|
+
timeout=timeout_seconds,
|
|
221
|
+
cwd=empty_folder,
|
|
222
|
+
env=child_environment(),
|
|
223
|
+
creationflags=hidden_window_creation_flags(),
|
|
224
|
+
input=prompt,
|
|
225
|
+
)
|
|
226
|
+
except subprocess.TimeoutExpired:
|
|
227
|
+
log_no_verdict(TIMEOUT_REASON)
|
|
228
|
+
return None
|
|
229
|
+
except OSError:
|
|
230
|
+
log_no_verdict(START_FAILURE_REASON)
|
|
231
|
+
return None
|
|
232
|
+
if completed_process.returncode != 0:
|
|
233
|
+
log_no_verdict(EXIT_CODE_REASON_TEMPLATE.format(exit_code=completed_process.returncode))
|
|
234
|
+
return None
|
|
235
|
+
return _answer_result(completed_process.stdout)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _raised_span(
|
|
239
|
+
all_verdict_fields: dict[str, object], flag_key: str, span_key: str, reply_text: str
|
|
240
|
+
) -> str | None:
|
|
241
|
+
is_raised = all_verdict_fields[flag_key]
|
|
242
|
+
span = all_verdict_fields[span_key]
|
|
243
|
+
if not isinstance(is_raised, bool) or not isinstance(span, str):
|
|
244
|
+
return None
|
|
245
|
+
if not is_raised:
|
|
246
|
+
return ""
|
|
247
|
+
return span if span and span in reply_text else None
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def parsed_verdict(result_text: str, reply_text: str) -> JudgeVerdict | None:
|
|
251
|
+
"""Return the verdict in the judge's result text, or None when it is not the verdict shape.
|
|
252
|
+
|
|
253
|
+
::
|
|
254
|
+
|
|
255
|
+
'{"promised_report": true, "promise_span": "I will report back", ...}' -> JudgeVerdict("I will report back", "")
|
|
256
|
+
the same object inside one json code fence -> the same verdict
|
|
257
|
+
a raised flag whose span is empty or not in the reply -> None
|
|
258
|
+
prose, an extra key, or a string in place of a bool -> None
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
result_text: The ``result`` string of the ``claude -p`` JSON answer.
|
|
262
|
+
reply_text: The closing reply the judge read.
|
|
263
|
+
"""
|
|
264
|
+
fence_match = ANSWER_FENCE_PATTERN.match(result_text)
|
|
265
|
+
verdict_text = fence_match.group(1) if fence_match else result_text.strip()
|
|
266
|
+
try:
|
|
267
|
+
all_verdict_fields = json.loads(verdict_text)
|
|
268
|
+
except (ValueError, RecursionError):
|
|
269
|
+
return None
|
|
270
|
+
if not isinstance(all_verdict_fields, dict) or set(all_verdict_fields) != ALL_VERDICT_KEYS:
|
|
271
|
+
return None
|
|
272
|
+
promise_span = _raised_span(
|
|
273
|
+
all_verdict_fields, PROMISED_REPORT_KEY, PROMISE_SPAN_KEY, reply_text
|
|
274
|
+
)
|
|
275
|
+
list_span = _raised_span(all_verdict_fields, DECISIONS_AS_LIST_KEY, LIST_SPAN_KEY, reply_text)
|
|
276
|
+
if promise_span is None or list_span is None:
|
|
277
|
+
return None
|
|
278
|
+
return JudgeVerdict(promise_span, list_span)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def gated_verdict(
|
|
282
|
+
verdict: JudgeVerdict, prompt_text: str, has_question_tool_call: bool
|
|
283
|
+
) -> JudgeVerdict:
|
|
284
|
+
"""Return the verdict with its list span dropped when the turn used the question tool or the prompt asked for a list.
|
|
285
|
+
|
|
286
|
+
::
|
|
287
|
+
|
|
288
|
+
list span "Open calls:", no question tool, prompt "fix the build" -> the list span stays
|
|
289
|
+
the same verdict after an AskUserQuestion call -> list span ""
|
|
290
|
+
the same verdict after the prompt "list the open calls as bullets" -> list span ""
|
|
291
|
+
"""
|
|
292
|
+
if has_question_tool_call or prompt_asks_for_list(prompt_text):
|
|
293
|
+
return verdict._replace(list_span="")
|
|
294
|
+
return verdict
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def judged_verdict(
|
|
298
|
+
prompt_text: str,
|
|
299
|
+
reply_text: str,
|
|
300
|
+
has_question_tool_call: bool,
|
|
301
|
+
timeout_seconds: float | None = None,
|
|
302
|
+
) -> JudgeVerdict | None:
|
|
303
|
+
"""Run the judge once on a closing reply and return its gated verdict, or None when it gave none.
|
|
304
|
+
|
|
305
|
+
::
|
|
306
|
+
|
|
307
|
+
a reply that promises a report later -> JudgeVerdict("<the promise>", "")
|
|
308
|
+
a reply with no fault -> JudgeVerdict("", "")
|
|
309
|
+
no claude on PATH, a timeout, a bad answer, or a span not in the reply -> None
|
|
310
|
+
|
|
311
|
+
Args:
|
|
312
|
+
prompt_text: The prompt that started the turn.
|
|
313
|
+
reply_text: The closing reply the turn ends with.
|
|
314
|
+
has_question_tool_call: Whether the turn called the question tool.
|
|
315
|
+
timeout_seconds: How long the child may run; ``JUDGE_TIMEOUT_SECONDS`` when None.
|
|
316
|
+
"""
|
|
317
|
+
claude_path = shutil.which(CLAUDE_EXECUTABLE_NAME)
|
|
318
|
+
if claude_path is None:
|
|
319
|
+
log_no_verdict(START_FAILURE_REASON)
|
|
320
|
+
return None
|
|
321
|
+
result_text = _claude_result(
|
|
322
|
+
claude_path,
|
|
323
|
+
judge_prompt(prompt_text, reply_text, has_question_tool_call),
|
|
324
|
+
JUDGE_TIMEOUT_SECONDS if timeout_seconds is None else timeout_seconds,
|
|
325
|
+
)
|
|
326
|
+
if result_text is None:
|
|
327
|
+
return None
|
|
328
|
+
verdict = parsed_verdict(result_text, reply_text)
|
|
329
|
+
if verdict is None:
|
|
330
|
+
log_no_verdict(VERDICT_SHAPE_REASON)
|
|
331
|
+
return None
|
|
332
|
+
return gated_verdict(verdict, prompt_text, has_question_tool_call)
|