claude-dev-env 8.89.0 → 8.90.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/pr-visual/README.md +29 -0
- package/.agents/skills/pr-visual/SKILL.md +62 -0
- package/.agents/skills/pr-visual/assets/icons.symbols +27 -0
- package/.agents/skills/pr-visual/assets/viewer.html +147 -0
- package/.agents/skills/pr-visual/fixtures/bar.scene.json +193 -0
- package/.agents/skills/pr-visual/fixtures/flow.scene.json +201 -0
- package/.agents/skills/pr-visual/fixtures/head/metrics.json +16 -0
- package/.agents/skills/pr-visual/fixtures/integrated.scene.json +239 -0
- package/.agents/skills/pr-visual/fixtures/line.scene.json +193 -0
- package/.agents/skills/pr-visual/fixtures/scatter.scene.json +193 -0
- package/.agents/skills/pr-visual/fixtures/snapshot.json +21 -0
- package/.agents/skills/pr-visual/references/evidence.md +9 -0
- package/.agents/skills/pr-visual/references/formats.md +30 -0
- package/.agents/skills/pr-visual/references/scene-contract.md +48 -0
- package/.agents/skills/pr-visual/references/visual-contract.md +23 -0
- package/.agents/skills/pr-visual/scripts/collect.py +111 -0
- package/.agents/skills/pr-visual/scripts/metrics.py +99 -0
- package/.agents/skills/pr-visual/scripts/render.py +133 -0
- package/.agents/skills/pr-visual/scripts/test_collect.py +58 -0
- package/.agents/skills/pr-visual/scripts/test_metrics.py +47 -0
- package/.agents/skills/pr-visual/scripts/test_render.py +77 -0
- package/.agents/skills/pr-visual/scripts/verify.py +218 -0
- package/.agents/skills/pr-visual/scripts/visual_config/config/constants.py +104 -0
- package/bin/codex-skill-load-block.mjs +0 -25
- package/bin/codex-skill-load-block.test.mjs +5 -17
- package/bin/codex-trimmed-sol.test.mjs +0 -1
- package/bin/cursor-native-hooks.test.mjs +3 -3
- package/bin/ever-shipped-skills.mjs +1 -0
- package/bin/install-deepstack-plugin.mjs +42 -25
- package/bin/install-deepstack-plugin.test.mjs +37 -11
- package/bin/install-pstack-plugin.mjs +58 -11
- package/bin/install-pstack-plugin.test.mjs +54 -21
- package/bin/install.agents-home.test.mjs +0 -1
- package/bin/install.codex-rules.test.mjs +0 -2
- package/bin/install.cursor-rules.test.mjs +90 -26
- package/bin/install.mjs +30 -36
- package/bin/install.pr-visual.test.mjs +40 -0
- package/bin/install.prompt-context.test.mjs +3 -4
- package/bin/install.prune.test.mjs +0 -1
- package/bin/install.question-presentation.test.mjs +0 -4
- package/bin/install.settings-defaults.test.mjs +0 -1
- package/bin/install.shared-settings.test.mjs +0 -1
- package/bin/install.test.mjs +14 -44
- package/bin/install.transaction.test.mjs +0 -8
- package/bin/install.uninstall-transaction.test.mjs +0 -2
- package/bin/install.update-keeps-hooks.test.mjs +0 -1
- package/docs/BDD_SCENARIO_QUALITY.md +1 -1
- package/docs/BDD_TEST_LAYOUT.md +1 -1
- package/docs/CODE_RULES.md +1 -1
- package/docs/rule-guides/deepstack-wording.md +3 -5
- package/docs/rule-guides/session-title.md +5 -4
- package/hooks/blocking/session_title_turn_gate.py +287 -0
- package/hooks/blocking/test_session_title_format_gate.py +5 -0
- package/hooks/blocking/test_session_title_turn_gate.py +264 -0
- package/hooks/features/conduct-gates.md +3 -0
- package/hooks/features/session-context.md +0 -4
- package/hooks/hooks.json +5 -5
- package/hooks/hooks_constants/mod_handoff_constants.py +0 -2
- package/hooks/hooks_constants/session_title_constants.py +41 -3
- package/hooks/lifecycle/nested_project_hooks.py +2 -2
- package/hooks/lifecycle/test_nested_project_hooks.py +2 -2
- package/hooks/session/compaction_user_messages.py +2 -2
- package/package.json +2 -2
- package/rules/deepstack-wording.md +3 -3
- package/scripts/ci/install-playtest.mjs +58 -314
- package/scripts/ci/install-playtest.test.mjs +24 -6
- package/scripts/policy_lint/adapter_configuration.py +1 -0
- package/scripts/policy_lint/config/constants.py +1 -0
- package/scripts/test_deepstack_review_criterion.py +2 -3
- package/scripts/test_deepstack_wording_rule.py +6 -4
- package/scripts/tests/test_adapter_configuration.py +8 -0
- package/.agents/skills/deepen/SKILL.md +0 -26
- package/commands/deepen.md +0 -14
- package/docs/decisions/deepstack-gate.md +0 -32
- package/hooks/hooks_constants/deepen_first_prompt_constants.py +0 -187
- package/hooks/session/deepen_default_rewrite.py +0 -128
- package/hooks/session/deepen_first_prompt.py +0 -383
- package/hooks/session/deepen_role_gate.py +0 -171
- package/hooks/session/deepen_turn_marker.py +0 -75
- package/hooks/session/fixtures/deepstack_replay.json +0 -15
- package/hooks/session/grade_deepstack_replay.py +0 -78
- package/hooks/session/test_deepen_default_rewrite.py +0 -90
- package/hooks/session/test_deepen_first_prompt.py +0 -452
- package/hooks/session/test_deepen_role_gate.py +0 -83
- package/hooks/session/test_deepen_turn_marker.py +0 -59
- package/hooks/session/test_grade_deepstack_replay.py +0 -47
|
@@ -1,452 +0,0 @@
|
|
|
1
|
-
"""Behavior tests for first-task deepening."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import json
|
|
6
|
-
import os
|
|
7
|
-
import sys
|
|
8
|
-
import tempfile
|
|
9
|
-
from io import StringIO
|
|
10
|
-
from pathlib import Path
|
|
11
|
-
from unittest.mock import patch
|
|
12
|
-
|
|
13
|
-
import pytest
|
|
14
|
-
|
|
15
|
-
_SESSION_DIR = Path(__file__).resolve().parent
|
|
16
|
-
_HOOKS_ROOT = _SESSION_DIR.parent
|
|
17
|
-
for each_sys_path_entry in (str(_SESSION_DIR), str(_HOOKS_ROOT)):
|
|
18
|
-
if each_sys_path_entry not in sys.path:
|
|
19
|
-
sys.path.insert(0, each_sys_path_entry)
|
|
20
|
-
|
|
21
|
-
import deepen_first_prompt as deepen
|
|
22
|
-
|
|
23
|
-
from hooks_constants.deepen_first_prompt_constants import (
|
|
24
|
-
DEEPEN_AGENT_MARKER,
|
|
25
|
-
DEEPEN_MARKER,
|
|
26
|
-
DEEPEN_OUTPUT_JSON,
|
|
27
|
-
DEEPEN_OUTPUT_REWRITE,
|
|
28
|
-
DEEPEN_REWRITE_ENV_VAR,
|
|
29
|
-
DEEPEN_TURN_MARKER_SUFFIX,
|
|
30
|
-
REFUSAL_DROPPED_HARD_CONSTRAINT,
|
|
31
|
-
REFUSAL_EXAMPLE_NOUN_IN_SCOPE,
|
|
32
|
-
REFUSAL_MISSING_ROLE_LINES,
|
|
33
|
-
REFUSAL_RESTATES_FOUR_QUESTIONS,
|
|
34
|
-
ALL_FOUR_QUESTION_STEMS,
|
|
35
|
-
SKIP_DEEPEN_ENV_VAR,
|
|
36
|
-
USER_PROMPT_SUBMIT_EVENT_NAME,
|
|
37
|
-
CURSOR_BEFORE_SUBMIT_PROMPT_EVENT_NAME,
|
|
38
|
-
)
|
|
39
|
-
from hooks_constants.skill_loaded_reminder_constants import CLAUDE_SUBAGENT_PROMPT_PREFIX
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def _payload(prompt: str, transcript_path: str | None = None) -> dict[str, object]:
|
|
43
|
-
fields: dict[str, object] = {
|
|
44
|
-
"hook_event_name": USER_PROMPT_SUBMIT_EVENT_NAME,
|
|
45
|
-
"prompt": prompt,
|
|
46
|
-
"session_id": "session-one",
|
|
47
|
-
}
|
|
48
|
-
if transcript_path is not None:
|
|
49
|
-
fields["transcript_path"] = transcript_path
|
|
50
|
-
return fields
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def _run_main(payload: dict[str, object]) -> str:
|
|
54
|
-
captured_stdout = StringIO()
|
|
55
|
-
with patch("sys.stdin", StringIO(json.dumps(payload))), patch("sys.stdout", captured_stdout):
|
|
56
|
-
deepen.main()
|
|
57
|
-
return captured_stdout.getvalue()
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
def _user_transcript_line() -> str:
|
|
61
|
-
return json.dumps({"type": "user", "message": {"role": "user", "content": "earlier task"}})
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def _cursor_transcript_line(role: str, text: str) -> str:
|
|
65
|
-
return json.dumps({"role": role, "message": {"content": [{"type": "text", "text": text}]}})
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def _role_rewrite(body: str) -> str:
|
|
69
|
-
return (
|
|
70
|
-
f"{DEEPEN_MARKER}\n"
|
|
71
|
-
"SCOPE (role): Ship a contrast change the screen can show\n"
|
|
72
|
-
"EXAMPLES (illustration only): none\n"
|
|
73
|
-
"ZOOM-IN BLOCKER: none\n"
|
|
74
|
-
f"{body}\n"
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
_FILLED_FOUR_ANSWERS = (
|
|
79
|
-
"Omitted: ship contrast tokens with the theme.\n"
|
|
80
|
-
"Source: src/settings/theme.ts\n"
|
|
81
|
-
"Methods: existing token file and a visual check.\n"
|
|
82
|
-
"Proof: screenshot of the settings page in dark mode."
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
class TestSkipReasonFor:
|
|
87
|
-
def test_should_skip_a_thanks_prompt(self) -> None:
|
|
88
|
-
assert deepen.skip_reason_for(_payload("thanks")) is deepen.SkipReason.TRIVIAL
|
|
89
|
-
|
|
90
|
-
def test_should_skip_a_yes_prompt(self) -> None:
|
|
91
|
-
assert deepen.skip_reason_for(_payload("Yes")) is deepen.SkipReason.TRIVIAL
|
|
92
|
-
|
|
93
|
-
def test_should_skip_status(self) -> None:
|
|
94
|
-
assert deepen.skip_reason_for(_payload("status")) is deepen.SkipReason.TRIVIAL
|
|
95
|
-
|
|
96
|
-
def test_should_skip_a_slash_command_other_than_deepen(self) -> None:
|
|
97
|
-
assert deepen.skip_reason_for(_payload("/sr-loop")) is deepen.SkipReason.SLASH_COMMAND
|
|
98
|
-
|
|
99
|
-
def test_should_skip_bare_deepen_with_no_task(self) -> None:
|
|
100
|
-
assert deepen.skip_reason_for(_payload("/deepen")) is deepen.SkipReason.SLASH_COMMAND
|
|
101
|
-
|
|
102
|
-
def test_should_allow_deepen_with_a_task(self) -> None:
|
|
103
|
-
assert deepen.skip_reason_for(_payload("/deepen fix the login page")) is None
|
|
104
|
-
|
|
105
|
-
def test_should_skip_a_loop_iteration(self) -> None:
|
|
106
|
-
assert (
|
|
107
|
-
deepen.skip_reason_for(_payload("/loop until the tests pass"))
|
|
108
|
-
is deepen.SkipReason.LOOP_OR_HELPER
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
def test_should_skip_a_helper_report(self) -> None:
|
|
112
|
-
assert (
|
|
113
|
-
deepen.skip_reason_for(_payload("<task-notification> helper finished"))
|
|
114
|
-
is deepen.SkipReason.LOOP_OR_HELPER
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
def test_should_skip_a_spawn_rewrite_prefix(self) -> None:
|
|
118
|
-
prompt = CLAUDE_SUBAGENT_PROMPT_PREFIX + "\n\nReply leaf."
|
|
119
|
-
assert deepen.skip_reason_for(_payload(prompt)) is deepen.SkipReason.SPAWN_REWRITE
|
|
120
|
-
|
|
121
|
-
def test_should_skip_when_the_prompt_already_carries_the_marker(self) -> None:
|
|
122
|
-
assert (
|
|
123
|
-
deepen.skip_reason_for(_payload(f"{DEEPEN_MARKER}\n\nDo the job."))
|
|
124
|
-
is deepen.SkipReason.MARKER_PRESENT
|
|
125
|
-
)
|
|
126
|
-
|
|
127
|
-
def test_should_skip_the_deepen_agent_prompt(self) -> None:
|
|
128
|
-
prompt = f"{DEEPEN_AGENT_MARKER}\nAnswer the four deepening questions."
|
|
129
|
-
assert deepen.skip_reason_for(_payload(prompt)) is deepen.SkipReason.DEEPEN_AGENT
|
|
130
|
-
|
|
131
|
-
def test_should_skip_when_the_skip_flag_is_set(
|
|
132
|
-
self, monkeypatch: pytest.MonkeyPatch
|
|
133
|
-
) -> None:
|
|
134
|
-
monkeypatch.setenv(SKIP_DEEPEN_ENV_VAR, "1")
|
|
135
|
-
assert (
|
|
136
|
-
deepen.skip_reason_for(_payload("Add dark mode to settings"))
|
|
137
|
-
is deepen.SkipReason.SKIP_FLAG
|
|
138
|
-
)
|
|
139
|
-
|
|
140
|
-
def test_should_skip_a_print_mode_session(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
141
|
-
monkeypatch.setenv("CLAUDE_CODE_NONINTERACTIVE", "true")
|
|
142
|
-
assert (
|
|
143
|
-
deepen.skip_reason_for(_payload("Add dark mode to settings"))
|
|
144
|
-
is deepen.SkipReason.NONINTERACTIVE
|
|
145
|
-
)
|
|
146
|
-
|
|
147
|
-
def test_should_skip_codex_exec(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
148
|
-
monkeypatch.setenv("CODEX_EXEC", "1")
|
|
149
|
-
assert (
|
|
150
|
-
deepen.skip_reason_for(_payload("Add dark mode to settings"))
|
|
151
|
-
is deepen.SkipReason.NONINTERACTIVE
|
|
152
|
-
)
|
|
153
|
-
|
|
154
|
-
def test_should_skip_turn_two_when_the_transcript_has_a_user_line(
|
|
155
|
-
self, tmp_path: Path
|
|
156
|
-
) -> None:
|
|
157
|
-
transcript_path = tmp_path / "session.jsonl"
|
|
158
|
-
transcript_path.write_text(_user_transcript_line() + "\n", encoding="utf-8")
|
|
159
|
-
assert (
|
|
160
|
-
deepen.skip_reason_for(_payload("Add dark mode", str(transcript_path)))
|
|
161
|
-
is deepen.SkipReason.LATER_TURN
|
|
162
|
-
)
|
|
163
|
-
|
|
164
|
-
def test_should_deepen_a_first_task(self) -> None:
|
|
165
|
-
assert deepen.skip_reason_for(_payload("Add dark mode to the settings page")) is None
|
|
166
|
-
|
|
167
|
-
def test_should_deepen_a_first_task_from_the_cursor_prompt_event(self) -> None:
|
|
168
|
-
fields = _payload("Add dark mode to the settings page")
|
|
169
|
-
fields["hook_event_name"] = CURSOR_BEFORE_SUBMIT_PROMPT_EVENT_NAME
|
|
170
|
-
assert deepen.skip_reason_for(fields) is None
|
|
171
|
-
|
|
172
|
-
def test_should_skip_when_a_cursor_transcript_already_has_a_user_turn(self, tmp_path: Path) -> None:
|
|
173
|
-
cursor_transcript_path = tmp_path / "cursor.jsonl"
|
|
174
|
-
cursor_transcript_path.write_text(_cursor_transcript_line("user", "earlier task"), encoding="utf-8")
|
|
175
|
-
fields = _payload("Add dark mode to the settings page", str(cursor_transcript_path))
|
|
176
|
-
fields["hook_event_name"] = CURSOR_BEFORE_SUBMIT_PROMPT_EVENT_NAME
|
|
177
|
-
fields.pop("session_id")
|
|
178
|
-
assert deepen.skip_reason_for(fields) is deepen.SkipReason.LATER_TURN
|
|
179
|
-
|
|
180
|
-
def test_should_skip_any_other_event(self) -> None:
|
|
181
|
-
fields = _payload("Add dark mode to the settings page")
|
|
182
|
-
fields["hook_event_name"] = "stop"
|
|
183
|
-
assert deepen.skip_reason_for(fields) is deepen.SkipReason.LATER_TURN
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
class TestDecideFirstPrompt:
|
|
187
|
-
def test_should_emit_additional_context_with_the_four_answers_for_a_first_task(
|
|
188
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
189
|
-
) -> None:
|
|
190
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite(_FILLED_FOUR_ANSWERS))
|
|
191
|
-
transcript_path = tmp_path / "fresh.jsonl"
|
|
192
|
-
decision = deepen.decide_first_prompt(
|
|
193
|
-
_payload("Add dark mode to the settings page", str(transcript_path))
|
|
194
|
-
)
|
|
195
|
-
assert isinstance(decision, deepen.Deepen)
|
|
196
|
-
assert DEEPEN_MARKER in decision.rewritten_prompt
|
|
197
|
-
assert "Omitted: ship contrast tokens" in decision.rewritten_prompt
|
|
198
|
-
assert "src/settings/theme.ts" in decision.rewritten_prompt
|
|
199
|
-
assert "visual check" in decision.rewritten_prompt
|
|
200
|
-
assert "screenshot of the settings page" in decision.additional_context
|
|
201
|
-
assert decision.additional_context.startswith(DEEPEN_MARKER)
|
|
202
|
-
|
|
203
|
-
def test_should_keep_a_skip_silent_on_the_command_hook(
|
|
204
|
-
self, monkeypatch: pytest.MonkeyPatch
|
|
205
|
-
) -> None:
|
|
206
|
-
monkeypatch.delenv("CDE_DEEPEN_OUTPUT", raising=False)
|
|
207
|
-
assert _run_main(_payload("thanks")) == ""
|
|
208
|
-
|
|
209
|
-
def test_should_write_a_turn_marker_so_turn_two_stays_silent(
|
|
210
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
211
|
-
) -> None:
|
|
212
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
213
|
-
transcript_path = tmp_path / "fresh.jsonl"
|
|
214
|
-
first_stdout = _run_main(_payload("Add dark mode to the settings page", str(transcript_path)))
|
|
215
|
-
assert DEEPEN_MARKER in first_stdout
|
|
216
|
-
marker_path = Path(str(transcript_path) + DEEPEN_TURN_MARKER_SUFFIX)
|
|
217
|
-
assert marker_path.is_file()
|
|
218
|
-
second_stdout = _run_main(_payload("and also the billing page", str(transcript_path)))
|
|
219
|
-
assert second_stdout == ""
|
|
220
|
-
|
|
221
|
-
def test_should_inject_once_when_two_copies_see_the_same_cursor_prompt(
|
|
222
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
223
|
-
) -> None:
|
|
224
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
225
|
-
monkeypatch.delenv("CDE_DEEPEN_OUTPUT", raising=False)
|
|
226
|
-
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
|
227
|
-
cursor_fields = _payload("Add dark mode to the settings page")
|
|
228
|
-
cursor_fields["hook_event_name"] = CURSOR_BEFORE_SUBMIT_PROMPT_EVENT_NAME
|
|
229
|
-
cursor_fields["transcript_path"] = None
|
|
230
|
-
cursor_fields["session_id"] = "cursor-conversation"
|
|
231
|
-
all_outputs = [_run_main(cursor_fields), _run_main(cursor_fields)]
|
|
232
|
-
assert sum(DEEPEN_MARKER in each_output for each_output in all_outputs) == 1
|
|
233
|
-
|
|
234
|
-
def test_should_keep_cursor_turn_two_silent_once_the_transcript_appears(
|
|
235
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
236
|
-
) -> None:
|
|
237
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
238
|
-
monkeypatch.delenv("CDE_DEEPEN_OUTPUT", raising=False)
|
|
239
|
-
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
|
240
|
-
first_fields = _payload("Add dark mode to the settings page")
|
|
241
|
-
first_fields["hook_event_name"] = CURSOR_BEFORE_SUBMIT_PROMPT_EVENT_NAME
|
|
242
|
-
first_fields["transcript_path"] = None
|
|
243
|
-
first_fields["session_id"] = "cursor-conversation"
|
|
244
|
-
assert DEEPEN_MARKER in _run_main(first_fields)
|
|
245
|
-
cursor_transcript_path = tmp_path / "cursor-conversation.jsonl"
|
|
246
|
-
cursor_transcript_path.write_text(
|
|
247
|
-
"\n".join(
|
|
248
|
-
[
|
|
249
|
-
_cursor_transcript_line("user", "Add dark mode to the settings page"),
|
|
250
|
-
_cursor_transcript_line("assistant", "Done."),
|
|
251
|
-
]
|
|
252
|
-
),
|
|
253
|
-
encoding="utf-8",
|
|
254
|
-
)
|
|
255
|
-
second_fields = dict(first_fields)
|
|
256
|
-
second_fields["prompt"] = "and also the billing page"
|
|
257
|
-
second_fields["transcript_path"] = str(cursor_transcript_path)
|
|
258
|
-
assert _run_main(second_fields) == ""
|
|
259
|
-
|
|
260
|
-
def test_should_print_rewrite_text_for_the_claude_mod(
|
|
261
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
262
|
-
) -> None:
|
|
263
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
264
|
-
monkeypatch.setenv("CDE_DEEPEN_OUTPUT", DEEPEN_OUTPUT_REWRITE)
|
|
265
|
-
transcript_path = tmp_path / "fresh.jsonl"
|
|
266
|
-
printed = _run_main(_payload("Add dark mode to the settings page", str(transcript_path)))
|
|
267
|
-
assert printed.startswith(DEEPEN_MARKER)
|
|
268
|
-
assert "Deeper task body" in printed
|
|
269
|
-
assert "hookSpecificOutput" not in printed
|
|
270
|
-
|
|
271
|
-
def test_should_print_json_decisions_for_the_playtest(
|
|
272
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
273
|
-
) -> None:
|
|
274
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
275
|
-
monkeypatch.setenv("CDE_DEEPEN_OUTPUT", DEEPEN_OUTPUT_JSON)
|
|
276
|
-
transcript_path = tmp_path / "fresh.jsonl"
|
|
277
|
-
deepen_payload = json.loads(
|
|
278
|
-
_run_main(_payload("Add dark mode to the settings page", str(transcript_path)))
|
|
279
|
-
)
|
|
280
|
-
assert deepen_payload["decision"] == "deepen"
|
|
281
|
-
skip_payload = json.loads(_run_main(_payload("thanks")))
|
|
282
|
-
assert skip_payload == {"decision": "skip", "reason": "trivial"}
|
|
283
|
-
|
|
284
|
-
def test_should_stay_silent_when_the_claude_mod_is_on(
|
|
285
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
286
|
-
) -> None:
|
|
287
|
-
config_directory = tmp_path / "claude-config"
|
|
288
|
-
config_directory.mkdir()
|
|
289
|
-
(config_directory / "settings.json").write_text(
|
|
290
|
-
json.dumps({"enabledPlugins": {"deepen-first-prompt@claude-dev-env": True}}),
|
|
291
|
-
encoding="utf-8",
|
|
292
|
-
)
|
|
293
|
-
monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(config_directory))
|
|
294
|
-
monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path))
|
|
295
|
-
monkeypatch.setenv("CLAUDECODE", "1")
|
|
296
|
-
monkeypatch.delenv("CLAUDE_CODE_REMOTE", raising=False)
|
|
297
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
298
|
-
transcript_path = tmp_path / "fresh.jsonl"
|
|
299
|
-
assert _run_main(_payload("Add dark mode to the settings page", str(transcript_path))) == ""
|
|
300
|
-
|
|
301
|
-
def test_should_emit_context_for_codex_when_the_claude_mod_is_on(
|
|
302
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
303
|
-
) -> None:
|
|
304
|
-
config_directory = tmp_path / "claude-config"
|
|
305
|
-
config_directory.mkdir()
|
|
306
|
-
(config_directory / "settings.json").write_text(
|
|
307
|
-
json.dumps({"enabledPlugins": {"deepen-first-prompt@claude-dev-env": True}}),
|
|
308
|
-
encoding="utf-8",
|
|
309
|
-
)
|
|
310
|
-
monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(config_directory))
|
|
311
|
-
monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path))
|
|
312
|
-
monkeypatch.delenv("CLAUDECODE", raising=False)
|
|
313
|
-
monkeypatch.delenv("CLAUDE_CODE_REMOTE", raising=False)
|
|
314
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
315
|
-
transcript_path = tmp_path / "codex-fresh.jsonl"
|
|
316
|
-
printed = _run_main(_payload("Add dark mode to the settings page", str(transcript_path)))
|
|
317
|
-
envelope = json.loads(printed)
|
|
318
|
-
assert envelope["hookSpecificOutput"]["hookEventName"] == USER_PROMPT_SUBMIT_EVENT_NAME
|
|
319
|
-
assert "Deeper task body" in envelope["hookSpecificOutput"]["additionalContext"]
|
|
320
|
-
|
|
321
|
-
def test_should_still_emit_context_in_a_cloud_session_when_the_mod_is_on(
|
|
322
|
-
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
323
|
-
) -> None:
|
|
324
|
-
config_directory = tmp_path / "claude-config"
|
|
325
|
-
config_directory.mkdir()
|
|
326
|
-
(config_directory / "settings.json").write_text(
|
|
327
|
-
json.dumps({"enabledPlugins": {"deepen-first-prompt@claude-dev-env": True}}),
|
|
328
|
-
encoding="utf-8",
|
|
329
|
-
)
|
|
330
|
-
monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(config_directory))
|
|
331
|
-
monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path))
|
|
332
|
-
monkeypatch.setenv("CLAUDE_CODE_REMOTE", "true")
|
|
333
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("Deeper task body"))
|
|
334
|
-
transcript_path = tmp_path / "fresh.jsonl"
|
|
335
|
-
printed = _run_main(_payload("Add dark mode to the settings page", str(transcript_path)))
|
|
336
|
-
envelope = json.loads(printed)
|
|
337
|
-
assert envelope["hookSpecificOutput"]["hookEventName"] == USER_PROMPT_SUBMIT_EVENT_NAME
|
|
338
|
-
assert "Deeper task body" in envelope["hookSpecificOutput"]["additionalContext"]
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
def test_should_strip_the_deepen_slash_before_rewriting() -> None:
|
|
342
|
-
assert deepen.task_text_for_deepen("/deepen fix the login page") == "fix the login page"
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
def test_should_refuse_the_paste_path_as_the_known_bad(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
346
|
-
pasted_questions = "\n".join(ALL_FOUR_QUESTION_STEMS)
|
|
347
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite(pasted_questions))
|
|
348
|
-
decision = deepen.decide_first_prompt(_payload("Add dark mode to the settings page"))
|
|
349
|
-
assert isinstance(decision, deepen.Refuse)
|
|
350
|
-
assert decision.reason == REFUSAL_RESTATES_FOUR_QUESTIONS
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
def test_should_deepen_with_the_default_rewrite_when_nothing_is_configured() -> None:
|
|
354
|
-
decision = deepen.decide_first_prompt(_payload("Add dark mode to the settings page"))
|
|
355
|
-
assert isinstance(decision, deepen.Deepen)
|
|
356
|
-
assert "SCOPE (role): Add dark mode to the settings page" in decision.rewritten_prompt
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
def test_should_refuse_the_filled_four_answer_stub_without_role_lines(
|
|
360
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
361
|
-
) -> None:
|
|
362
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _FILLED_FOUR_ANSWERS)
|
|
363
|
-
decision = deepen.decide_first_prompt(_payload("Add dark mode to the settings page"))
|
|
364
|
-
assert isinstance(decision, deepen.Refuse)
|
|
365
|
-
assert decision.reason == REFUSAL_MISSING_ROLE_LINES
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
def test_should_reject_the_wrong_twin_that_names_the_example_in_scope(
|
|
369
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
370
|
-
) -> None:
|
|
371
|
-
task = "Keep the build green. For example, the login failure."
|
|
372
|
-
wrong_twin = (
|
|
373
|
-
f"{DEEPEN_MARKER}\n"
|
|
374
|
-
"SCOPE (role): Fix the login failure\n"
|
|
375
|
-
"EXAMPLES (illustration only): login failure\n"
|
|
376
|
-
"ZOOM-IN BLOCKER: none\n"
|
|
377
|
-
"Read the workflow log and repair the shared gate.\n"
|
|
378
|
-
)
|
|
379
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, wrong_twin)
|
|
380
|
-
decision = deepen.decide_first_prompt(_payload(task))
|
|
381
|
-
assert isinstance(decision, deepen.Refuse)
|
|
382
|
-
assert decision.reason == REFUSAL_EXAMPLE_NOUN_IN_SCOPE
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
def test_should_accept_a_role_scope_that_leaves_the_example_noun_out(
|
|
386
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
387
|
-
) -> None:
|
|
388
|
-
task = "Keep the build green. For example, the login failure."
|
|
389
|
-
role_rewrite = (
|
|
390
|
-
f"{DEEPEN_MARKER}\n"
|
|
391
|
-
"SCOPE (role): Keep every build green across the repository\n"
|
|
392
|
-
"EXAMPLES (illustration only): login failure\n"
|
|
393
|
-
"ZOOM-IN BLOCKER: none\n"
|
|
394
|
-
"Read the workflow log and repair the shared gate.\n"
|
|
395
|
-
)
|
|
396
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, role_rewrite)
|
|
397
|
-
decision = deepen.decide_first_prompt(_payload(task))
|
|
398
|
-
assert isinstance(decision, deepen.Deepen)
|
|
399
|
-
assert "SCOPE (role): Keep every build green across the repository" in decision.rewritten_prompt
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
def test_should_reject_a_rewrite_that_drops_a_hard_constraint(
|
|
403
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
404
|
-
) -> None:
|
|
405
|
-
task = "Ship the page.\n\nHARD CONSTRAINTS\n- Never open a pull request.\n"
|
|
406
|
-
loosened = (
|
|
407
|
-
f"{DEEPEN_MARKER}\n"
|
|
408
|
-
"SCOPE (role): Ship the page the caller asked for\n"
|
|
409
|
-
"EXAMPLES (illustration only): none\n"
|
|
410
|
-
"ZOOM-IN BLOCKER: none\n"
|
|
411
|
-
"Open a pull request when the branch is pushed.\n"
|
|
412
|
-
)
|
|
413
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, loosened)
|
|
414
|
-
decision = deepen.decide_first_prompt(_payload(task))
|
|
415
|
-
assert isinstance(decision, deepen.Refuse)
|
|
416
|
-
assert decision.reason == REFUSAL_DROPPED_HARD_CONSTRAINT
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
def test_should_keep_a_hard_constraint_line_in_an_accepted_rewrite(
|
|
420
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
421
|
-
) -> None:
|
|
422
|
-
task = "Ship the page.\n\nHARD CONSTRAINTS\n- Never open a pull request.\n"
|
|
423
|
-
kept = (
|
|
424
|
-
f"{DEEPEN_MARKER}\n"
|
|
425
|
-
"SCOPE (role): Ship the page the caller asked for\n"
|
|
426
|
-
"EXAMPLES (illustration only): none\n"
|
|
427
|
-
"ZOOM-IN BLOCKER: none\n"
|
|
428
|
-
"- Never open a pull request.\n"
|
|
429
|
-
)
|
|
430
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, kept)
|
|
431
|
-
decision = deepen.decide_first_prompt(_payload(task))
|
|
432
|
-
assert isinstance(decision, deepen.Deepen)
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
def test_should_print_a_json_refusal_for_the_paste_path(
|
|
436
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
437
|
-
) -> None:
|
|
438
|
-
monkeypatch.setenv("CDE_DEEPEN_OUTPUT", DEEPEN_OUTPUT_JSON)
|
|
439
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _role_rewrite("\n".join(ALL_FOUR_QUESTION_STEMS)))
|
|
440
|
-
printed = _run_main(_payload("Add dark mode to the settings page"))
|
|
441
|
-
assert json.loads(printed) == {
|
|
442
|
-
"decision": "refuse",
|
|
443
|
-
"reason": REFUSAL_RESTATES_FOUR_QUESTIONS,
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
def test_should_stay_silent_in_rewrite_mode_when_the_gate_refuses(
|
|
448
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
449
|
-
) -> None:
|
|
450
|
-
monkeypatch.setenv("CDE_DEEPEN_OUTPUT", DEEPEN_OUTPUT_REWRITE)
|
|
451
|
-
monkeypatch.setenv(DEEPEN_REWRITE_ENV_VAR, _FILLED_FOUR_ANSWERS)
|
|
452
|
-
assert _run_main(_payload("Add dark mode to the settings page")) == ""
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
"""Behavior tests for the first-task rewrite refusal gate."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import sys
|
|
6
|
-
from pathlib import Path
|
|
7
|
-
|
|
8
|
-
_SESSION_DIR = Path(__file__).resolve().parent
|
|
9
|
-
_HOOKS_ROOT = _SESSION_DIR.parent
|
|
10
|
-
for each_sys_path_entry in (str(_SESSION_DIR), str(_HOOKS_ROOT)):
|
|
11
|
-
if each_sys_path_entry not in sys.path:
|
|
12
|
-
sys.path.insert(0, each_sys_path_entry)
|
|
13
|
-
|
|
14
|
-
from deepen_role_gate import refusal_reason
|
|
15
|
-
from hooks_constants.deepen_first_prompt_constants import (
|
|
16
|
-
ALL_FOUR_QUESTION_STEMS,
|
|
17
|
-
REFUSAL_DROPPED_HARD_CONSTRAINT,
|
|
18
|
-
REFUSAL_EXAMPLE_NOUN_IN_SCOPE,
|
|
19
|
-
REFUSAL_MISSING_ROLE_LINES,
|
|
20
|
-
REFUSAL_RESTATES_FOUR_QUESTIONS,
|
|
21
|
-
)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def test_accepts_a_rewrite_with_role_lines_and_preserved_constraint() -> None:
|
|
25
|
-
original_task = "Fix the settings page.\nHARD CONSTRAINTS\n- Keep the save button."
|
|
26
|
-
rewritten_prompt = (
|
|
27
|
-
"SCOPE (role): Fix the settings page.\n"
|
|
28
|
-
"EXAMPLES (illustration only): none\n"
|
|
29
|
-
"ZOOM-IN BLOCKER: none\n"
|
|
30
|
-
"Keep the save button."
|
|
31
|
-
)
|
|
32
|
-
|
|
33
|
-
assert refusal_reason(original_task, rewritten_prompt) is None
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def test_refuses_a_rewrite_that_repeats_all_four_questions() -> None:
|
|
37
|
-
rewritten_prompt = (
|
|
38
|
-
"SCOPE (role): Fix the settings page.\n"
|
|
39
|
-
"EXAMPLES (illustration only): none\n"
|
|
40
|
-
"ZOOM-IN BLOCKER: none\n"
|
|
41
|
-
+ "\n".join(ALL_FOUR_QUESTION_STEMS)
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
assert refusal_reason("Fix the settings page.", rewritten_prompt) == (
|
|
45
|
-
REFUSAL_RESTATES_FOUR_QUESTIONS
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def test_refuses_a_rewrite_with_a_blank_role_line() -> None:
|
|
50
|
-
rewritten_prompt = (
|
|
51
|
-
"SCOPE (role): Fix the settings page.\n"
|
|
52
|
-
"EXAMPLES (illustration only): none\n"
|
|
53
|
-
"ZOOM-IN BLOCKER: "
|
|
54
|
-
)
|
|
55
|
-
|
|
56
|
-
assert refusal_reason("Fix the settings page.", rewritten_prompt) == (
|
|
57
|
-
REFUSAL_MISSING_ROLE_LINES
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def test_refuses_an_example_noun_in_scope_without_a_blocker() -> None:
|
|
62
|
-
rewritten_prompt = (
|
|
63
|
-
"SCOPE (role): Add contrast to the settings page.\n"
|
|
64
|
-
"EXAMPLES (illustration only): contrast tokens\n"
|
|
65
|
-
"ZOOM-IN BLOCKER: none"
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
assert refusal_reason("Fix the settings page.", rewritten_prompt) == (
|
|
69
|
-
REFUSAL_EXAMPLE_NOUN_IN_SCOPE
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def test_refuses_a_rewrite_that_drops_a_hard_constraint() -> None:
|
|
74
|
-
original_task = "Fix the settings page.\nHARD CONSTRAINTS\n- Keep the save button."
|
|
75
|
-
rewritten_prompt = (
|
|
76
|
-
"SCOPE (role): Fix the settings page.\n"
|
|
77
|
-
"EXAMPLES (illustration only): none\n"
|
|
78
|
-
"ZOOM-IN BLOCKER: none"
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
assert refusal_reason(original_task, rewritten_prompt) == (
|
|
82
|
-
REFUSAL_DROPPED_HARD_CONSTRAINT
|
|
83
|
-
)
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
"""Tests for the deepen turn-one marker."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import sys
|
|
6
|
-
import tempfile
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
|
-
import pytest
|
|
10
|
-
|
|
11
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
12
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
13
|
-
|
|
14
|
-
import deepen_turn_marker as marker
|
|
15
|
-
from hooks_constants.deepen_first_prompt_constants import (
|
|
16
|
-
DEEPEN_SESSION_MARKER_PREFIX,
|
|
17
|
-
DEEPEN_TURN_MARKER_SUFFIX,
|
|
18
|
-
)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def test_should_name_the_marker_after_the_transcript_when_there_is_one(tmp_path: Path) -> None:
|
|
22
|
-
transcript_path = str(tmp_path / "session.jsonl")
|
|
23
|
-
assert marker.turn_marker_path({"transcript_path": transcript_path, "session_id": "s"}) == Path(
|
|
24
|
-
transcript_path + DEEPEN_TURN_MARKER_SUFFIX
|
|
25
|
-
)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def test_should_name_the_marker_after_the_session_in_the_temp_folder(
|
|
29
|
-
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
30
|
-
) -> None:
|
|
31
|
-
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
|
32
|
-
assert marker.turn_marker_path({"transcript_path": None, "session_id": "abc"}) == (
|
|
33
|
-
tmp_path / f"{DEEPEN_SESSION_MARKER_PREFIX}abc"
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def test_should_let_only_the_first_claim_win(tmp_path: Path) -> None:
|
|
38
|
-
fields = {"transcript_path": str(tmp_path / "session.jsonl")}
|
|
39
|
-
assert marker.claim_turn_marker(fields) is True
|
|
40
|
-
assert marker.claim_turn_marker(fields) is False
|
|
41
|
-
assert marker.turn_marker_exists(fields)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def test_should_allow_a_claim_when_the_payload_names_no_session() -> None:
|
|
45
|
-
assert marker.claim_turn_marker({}) is True
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def test_should_allow_a_claim_when_the_marker_cannot_be_written(tmp_path: Path) -> None:
|
|
49
|
-
fields = {"transcript_path": str(tmp_path / "missing-folder" / "session.jsonl")}
|
|
50
|
-
assert marker.claim_turn_marker(fields) is True
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def test_should_see_the_session_marker_once_a_transcript_path_appears(
|
|
54
|
-
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
55
|
-
) -> None:
|
|
56
|
-
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
|
57
|
-
assert marker.claim_turn_marker({"transcript_path": None, "session_id": "abc"}) is True
|
|
58
|
-
later_fields = {"transcript_path": str(tmp_path / "cursor.jsonl"), "session_id": "abc"}
|
|
59
|
-
assert marker.turn_marker_exists(later_fields)
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
"""The grader scores a replay. It does not rewrite the task."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import json
|
|
6
|
-
import subprocess
|
|
7
|
-
import sys
|
|
8
|
-
from pathlib import Path
|
|
9
|
-
|
|
10
|
-
_SESSION_DIR = Path(__file__).resolve().parent
|
|
11
|
-
_HOOKS_ROOT = _SESSION_DIR.parent
|
|
12
|
-
for each_sys_path_entry in (str(_SESSION_DIR), str(_HOOKS_ROOT)):
|
|
13
|
-
if each_sys_path_entry not in sys.path:
|
|
14
|
-
sys.path.insert(0, each_sys_path_entry)
|
|
15
|
-
|
|
16
|
-
import grade_deepstack_replay as grader
|
|
17
|
-
|
|
18
|
-
_FIXTURE_PATH = _SESSION_DIR / "fixtures" / "deepstack_replay.json"
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def test_should_score_the_login_failure_example_without_rewriting() -> None:
|
|
22
|
-
fixture = json.loads(_FIXTURE_PATH.read_text(encoding="utf-8"))
|
|
23
|
-
all_scores = grader.score_fixture(fixture)
|
|
24
|
-
assert [each_score["name"] for each_score in all_scores] == ["known_bad", "known_good"]
|
|
25
|
-
assert all_scores[0]["score"] == "refuse"
|
|
26
|
-
assert all_scores[0]["reason"] == "example-noun-in-scope"
|
|
27
|
-
assert all_scores[1]["score"] == "accept"
|
|
28
|
-
assert all(each_score["matched"] == "yes" for each_score in all_scores)
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def test_should_leave_the_task_text_unchanged_in_the_fixture() -> None:
|
|
32
|
-
fixture_text = _FIXTURE_PATH.read_text(encoding="utf-8")
|
|
33
|
-
grader.score_fixture(json.loads(fixture_text))
|
|
34
|
-
assert _FIXTURE_PATH.read_text(encoding="utf-8") == fixture_text
|
|
35
|
-
assert "For example, the login failure." in fixture_text
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def test_should_exit_zero_when_the_fixture_matches() -> None:
|
|
39
|
-
completed = subprocess.run(
|
|
40
|
-
[sys.executable, str(_SESSION_DIR / "grade_deepstack_replay.py"), str(_FIXTURE_PATH)],
|
|
41
|
-
capture_output=True,
|
|
42
|
-
text=True,
|
|
43
|
-
check=False,
|
|
44
|
-
)
|
|
45
|
-
assert completed.returncode == 0
|
|
46
|
-
assert "example-noun-in-scope" in completed.stdout
|
|
47
|
-
assert '"score": "accept"' in completed.stdout
|