claude-dev-env 8.89.0 → 8.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/pr-visual/README.md +29 -0
- package/.agents/skills/pr-visual/SKILL.md +62 -0
- package/.agents/skills/pr-visual/assets/icons.symbols +27 -0
- package/.agents/skills/pr-visual/assets/viewer.html +147 -0
- package/.agents/skills/pr-visual/fixtures/bar.scene.json +193 -0
- package/.agents/skills/pr-visual/fixtures/flow.scene.json +201 -0
- package/.agents/skills/pr-visual/fixtures/head/metrics.json +16 -0
- package/.agents/skills/pr-visual/fixtures/integrated.scene.json +239 -0
- package/.agents/skills/pr-visual/fixtures/line.scene.json +193 -0
- package/.agents/skills/pr-visual/fixtures/scatter.scene.json +193 -0
- package/.agents/skills/pr-visual/fixtures/snapshot.json +21 -0
- package/.agents/skills/pr-visual/references/evidence.md +9 -0
- package/.agents/skills/pr-visual/references/formats.md +30 -0
- package/.agents/skills/pr-visual/references/scene-contract.md +48 -0
- package/.agents/skills/pr-visual/references/visual-contract.md +23 -0
- package/.agents/skills/pr-visual/scripts/collect.py +111 -0
- package/.agents/skills/pr-visual/scripts/metrics.py +99 -0
- package/.agents/skills/pr-visual/scripts/render.py +133 -0
- package/.agents/skills/pr-visual/scripts/test_collect.py +58 -0
- package/.agents/skills/pr-visual/scripts/test_metrics.py +47 -0
- package/.agents/skills/pr-visual/scripts/test_render.py +77 -0
- package/.agents/skills/pr-visual/scripts/verify.py +218 -0
- package/.agents/skills/pr-visual/scripts/visual_config/config/constants.py +104 -0
- package/bin/codex-skill-load-block.mjs +0 -25
- package/bin/codex-skill-load-block.test.mjs +5 -17
- package/bin/codex-trimmed-sol.test.mjs +0 -1
- package/bin/cursor-native-hooks.test.mjs +3 -3
- package/bin/ever-shipped-skills.mjs +1 -0
- package/bin/install-deepstack-plugin.mjs +42 -25
- package/bin/install-deepstack-plugin.test.mjs +37 -11
- package/bin/install-pstack-plugin.mjs +58 -11
- package/bin/install-pstack-plugin.test.mjs +54 -21
- package/bin/install.agents-home.test.mjs +0 -1
- package/bin/install.codex-rules.test.mjs +0 -2
- package/bin/install.cursor-rules.test.mjs +90 -26
- package/bin/install.mjs +30 -36
- package/bin/install.pr-visual.test.mjs +40 -0
- package/bin/install.prompt-context.test.mjs +3 -4
- package/bin/install.prune.test.mjs +0 -1
- package/bin/install.question-presentation.test.mjs +0 -4
- package/bin/install.settings-defaults.test.mjs +0 -1
- package/bin/install.shared-settings.test.mjs +0 -1
- package/bin/install.test.mjs +14 -44
- package/bin/install.transaction.test.mjs +0 -8
- package/bin/install.uninstall-transaction.test.mjs +0 -2
- package/bin/install.update-keeps-hooks.test.mjs +0 -1
- package/docs/BDD_SCENARIO_QUALITY.md +1 -1
- package/docs/BDD_TEST_LAYOUT.md +1 -1
- package/docs/CODE_RULES.md +1 -1
- package/docs/rule-guides/deepstack-wording.md +3 -5
- package/docs/rule-guides/session-title.md +5 -4
- package/hooks/blocking/session_title_turn_gate.py +287 -0
- package/hooks/blocking/test_session_title_format_gate.py +5 -0
- package/hooks/blocking/test_session_title_turn_gate.py +264 -0
- package/hooks/features/conduct-gates.md +3 -0
- package/hooks/features/session-context.md +0 -4
- package/hooks/hooks.json +5 -5
- package/hooks/hooks_constants/mod_handoff_constants.py +0 -2
- package/hooks/hooks_constants/session_title_constants.py +41 -3
- package/hooks/lifecycle/nested_project_hooks.py +2 -2
- package/hooks/lifecycle/test_nested_project_hooks.py +2 -2
- package/hooks/session/compaction_user_messages.py +2 -2
- package/package.json +2 -2
- package/rules/deepstack-wording.md +3 -3
- package/scripts/ci/install-playtest.mjs +58 -314
- package/scripts/ci/install-playtest.test.mjs +24 -6
- package/scripts/policy_lint/adapter_configuration.py +1 -0
- package/scripts/policy_lint/config/constants.py +1 -0
- package/scripts/test_deepstack_review_criterion.py +2 -3
- package/scripts/test_deepstack_wording_rule.py +6 -4
- package/scripts/tests/test_adapter_configuration.py +8 -0
- package/.agents/skills/deepen/SKILL.md +0 -26
- package/commands/deepen.md +0 -14
- package/docs/decisions/deepstack-gate.md +0 -32
- package/hooks/hooks_constants/deepen_first_prompt_constants.py +0 -187
- package/hooks/session/deepen_default_rewrite.py +0 -128
- package/hooks/session/deepen_first_prompt.py +0 -383
- package/hooks/session/deepen_role_gate.py +0 -171
- package/hooks/session/deepen_turn_marker.py +0 -75
- package/hooks/session/fixtures/deepstack_replay.json +0 -15
- package/hooks/session/grade_deepstack_replay.py +0 -78
- package/hooks/session/test_deepen_default_rewrite.py +0 -90
- package/hooks/session/test_deepen_first_prompt.py +0 -452
- package/hooks/session/test_deepen_role_gate.py +0 -83
- package/hooks/session/test_deepen_turn_marker.py +0 -59
- package/hooks/session/test_grade_deepstack_replay.py +0 -47
|
@@ -1,171 +0,0 @@
|
|
|
1
|
-
"""Refusal checks for a first-task rewrite.
|
|
2
|
-
|
|
3
|
-
The rewriter lives in ``deepen_first_prompt.py``. This module only scores a
|
|
4
|
-
candidate. A grader imports ``refusal_reason`` and does not rewrite.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import re
|
|
10
|
-
|
|
11
|
-
from hooks_constants.deepen_first_prompt_constants import (
|
|
12
|
-
ALL_EXAMPLE_NOUN_STOPWORDS,
|
|
13
|
-
ALL_FOUR_QUESTION_STEMS,
|
|
14
|
-
EXAMPLE_NOUN_PATTERN,
|
|
15
|
-
BULLET_PREFIX,
|
|
16
|
-
HARD_CONSTRAINTS_LABEL,
|
|
17
|
-
REFUSAL_DROPPED_HARD_CONSTRAINT,
|
|
18
|
-
REFUSAL_EXAMPLE_NOUN_IN_SCOPE,
|
|
19
|
-
REFUSAL_MISSING_ROLE_LINES,
|
|
20
|
-
REFUSAL_RESTATES_FOUR_QUESTIONS,
|
|
21
|
-
ROLE_BLOCKER_LABEL,
|
|
22
|
-
ROLE_BLOCKER_NONE,
|
|
23
|
-
ROLE_EXAMPLES_LABEL,
|
|
24
|
-
ROLE_SCOPE_LABEL,
|
|
25
|
-
)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def _labeled_text(rewritten_prompt: str, label: str) -> str | None:
|
|
29
|
-
"""Return the text after ``label`` on its line, or None when the line is absent."""
|
|
30
|
-
prefix = label
|
|
31
|
-
for each_line in rewritten_prompt.splitlines():
|
|
32
|
-
stripped_line = each_line.strip()
|
|
33
|
-
if stripped_line.startswith(prefix):
|
|
34
|
-
return stripped_line[len(prefix) :].strip()
|
|
35
|
-
return None
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def _restates_four_questions(rewritten_prompt: str) -> bool:
|
|
39
|
-
"""Return True when the rewrite pastes every canonical question stem."""
|
|
40
|
-
return all(each_stem in rewritten_prompt for each_stem in ALL_FOUR_QUESTION_STEMS)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def _role_lines_are_missing(rewritten_prompt: str) -> bool:
|
|
44
|
-
"""Return True when any of the three role lines is absent or blank."""
|
|
45
|
-
for each_label in (ROLE_SCOPE_LABEL, ROLE_EXAMPLES_LABEL, ROLE_BLOCKER_LABEL):
|
|
46
|
-
labeled_text = _labeled_text(rewritten_prompt, each_label)
|
|
47
|
-
if labeled_text is None or not labeled_text:
|
|
48
|
-
return True
|
|
49
|
-
return False
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def _example_nouns(examples_text: str) -> frozenset[str]:
|
|
53
|
-
"""Return content words from the examples line. ``none`` yields an empty set."""
|
|
54
|
-
if examples_text.strip().rstrip(".").lower() == ROLE_BLOCKER_NONE:
|
|
55
|
-
return frozenset()
|
|
56
|
-
noun_pattern = re.compile(EXAMPLE_NOUN_PATTERN)
|
|
57
|
-
return frozenset(
|
|
58
|
-
each_match.group(0).lower()
|
|
59
|
-
for each_match in noun_pattern.finditer(examples_text)
|
|
60
|
-
if each_match.group(0).lower() not in ALL_EXAMPLE_NOUN_STOPWORDS
|
|
61
|
-
)
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def _scope_names_example_noun(scope_text: str, examples_text: str, blocker_text: str) -> bool:
|
|
65
|
-
"""Return True when the scope line names an example noun and the blocker is none."""
|
|
66
|
-
if blocker_text.strip().rstrip(".").lower() != ROLE_BLOCKER_NONE:
|
|
67
|
-
return False
|
|
68
|
-
for each_noun in _example_nouns(examples_text):
|
|
69
|
-
if re.search(
|
|
70
|
-
rf"(?i)(?<![A-Za-z0-9]){re.escape(each_noun)}(?![A-Za-z0-9])",
|
|
71
|
-
scope_text,
|
|
72
|
-
):
|
|
73
|
-
return True
|
|
74
|
-
return False
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def _is_constraints_heading(stripped_line: str) -> bool:
|
|
78
|
-
"""Return whether the line opens the hard-constraints block."""
|
|
79
|
-
return stripped_line.upper().startswith(HARD_CONSTRAINTS_LABEL)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def _bullet_body(stripped_line: str) -> str | None:
|
|
83
|
-
"""Return the text after a bullet marker, or None when the line is not a bullet."""
|
|
84
|
-
if not stripped_line.startswith(BULLET_PREFIX):
|
|
85
|
-
return None
|
|
86
|
-
return stripped_line[len(BULLET_PREFIX) :].strip()
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _constraint_step(
|
|
90
|
-
stripped_line: str,
|
|
91
|
-
collecting: bool,
|
|
92
|
-
has_sentences: bool,
|
|
93
|
-
) -> tuple[str | None, bool, bool]:
|
|
94
|
-
"""Return the sentence to keep, whether collection stays on, and whether to stop."""
|
|
95
|
-
if not collecting:
|
|
96
|
-
return None, _is_constraints_heading(stripped_line), False
|
|
97
|
-
bullet_body = _bullet_body(stripped_line)
|
|
98
|
-
if bullet_body is not None:
|
|
99
|
-
return bullet_body, True, False
|
|
100
|
-
if stripped_line == "" and not has_sentences:
|
|
101
|
-
return None, True, False
|
|
102
|
-
return None, collecting, True
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def _constraint_sentences(original_task: str) -> tuple[str, ...]:
|
|
106
|
-
"""Return bullet sentences under the hard-constraints heading."""
|
|
107
|
-
collecting = False
|
|
108
|
-
all_sentences: list[str] = []
|
|
109
|
-
for each_line in original_task.splitlines():
|
|
110
|
-
sentence, collecting, finished = _constraint_step(
|
|
111
|
-
each_line.strip(),
|
|
112
|
-
collecting,
|
|
113
|
-
bool(all_sentences),
|
|
114
|
-
)
|
|
115
|
-
if finished:
|
|
116
|
-
break
|
|
117
|
-
if sentence is not None:
|
|
118
|
-
all_sentences.append(sentence)
|
|
119
|
-
return tuple(all_sentences)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def _line_body(stripped_line: str) -> str | None:
|
|
123
|
-
"""Return one rewrite line with a leading bullet marker removed."""
|
|
124
|
-
if not stripped_line:
|
|
125
|
-
return None
|
|
126
|
-
bullet_body = _bullet_body(stripped_line)
|
|
127
|
-
if bullet_body is not None:
|
|
128
|
-
return bullet_body
|
|
129
|
-
return stripped_line
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
def _rewrite_line_bodies(rewritten_prompt: str) -> frozenset[str]:
|
|
133
|
-
"""Return each rewrite line with a leading bullet marker removed."""
|
|
134
|
-
all_bodies: set[str] = set()
|
|
135
|
-
for each_line in rewritten_prompt.splitlines():
|
|
136
|
-
body = _line_body(each_line.strip())
|
|
137
|
-
if body is not None:
|
|
138
|
-
all_bodies.add(body)
|
|
139
|
-
return frozenset(all_bodies)
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
def _dropped_hard_constraint(original_task: str, rewritten_prompt: str) -> bool:
|
|
143
|
-
"""Return True when a hard-constraint sentence is absent as its own line."""
|
|
144
|
-
rewrite_bodies = _rewrite_line_bodies(rewritten_prompt)
|
|
145
|
-
return any(
|
|
146
|
-
each_sentence not in rewrite_bodies for each_sentence in _constraint_sentences(original_task)
|
|
147
|
-
)
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
def refusal_reason(original_task: str, rewritten_prompt: str) -> str | None:
|
|
151
|
-
"""Return the refusal reason for one candidate rewrite, or None when it can ship.
|
|
152
|
-
|
|
153
|
-
Args:
|
|
154
|
-
original_task: The task text the rewrite must keep constraints from.
|
|
155
|
-
rewritten_prompt: The candidate worker prompt.
|
|
156
|
-
|
|
157
|
-
Returns:
|
|
158
|
-
A refusal reason string, or None when the candidate passes.
|
|
159
|
-
"""
|
|
160
|
-
if _restates_four_questions(rewritten_prompt):
|
|
161
|
-
return REFUSAL_RESTATES_FOUR_QUESTIONS
|
|
162
|
-
if _role_lines_are_missing(rewritten_prompt):
|
|
163
|
-
return REFUSAL_MISSING_ROLE_LINES
|
|
164
|
-
scope_text = _labeled_text(rewritten_prompt, ROLE_SCOPE_LABEL) or ""
|
|
165
|
-
examples_text = _labeled_text(rewritten_prompt, ROLE_EXAMPLES_LABEL) or ""
|
|
166
|
-
blocker_text = _labeled_text(rewritten_prompt, ROLE_BLOCKER_LABEL) or ""
|
|
167
|
-
if _scope_names_example_noun(scope_text, examples_text, blocker_text):
|
|
168
|
-
return REFUSAL_EXAMPLE_NOUN_IN_SCOPE
|
|
169
|
-
if _dropped_hard_constraint(original_task, rewritten_prompt):
|
|
170
|
-
return REFUSAL_DROPPED_HARD_CONSTRAINT
|
|
171
|
-
return None
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
"""Turn-one marker for the first-task deepen hook.
|
|
2
|
-
|
|
3
|
-
The marker records that a session already deepened its first task, so later
|
|
4
|
-
prompts stay untouched. A host can run two copies of the hook for one prompt:
|
|
5
|
-
Cursor runs its own ``beforeSubmitPrompt`` entry and the ``UserPromptSubmit``
|
|
6
|
-
entry it imports from the Claude Code settings, side by side. Each copy claims
|
|
7
|
-
the marker by creating the file only when it does not exist yet, so exactly
|
|
8
|
-
one copy wins and injects.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
from __future__ import annotations
|
|
12
|
-
|
|
13
|
-
import os
|
|
14
|
-
import tempfile
|
|
15
|
-
from pathlib import Path
|
|
16
|
-
|
|
17
|
-
from hooks_constants.deepen_first_prompt_constants import (
|
|
18
|
-
DEEPEN_MARKER,
|
|
19
|
-
DEEPEN_SESSION_MARKER_PREFIX,
|
|
20
|
-
DEEPEN_TURN_MARKER_SUFFIX,
|
|
21
|
-
SESSION_ID_FIELD_NAME,
|
|
22
|
-
TRANSCRIPT_PATH_FIELD_NAME,
|
|
23
|
-
UTF8_ENCODING,
|
|
24
|
-
)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def _all_turn_marker_paths(all_hook_fields: dict[str, object]) -> list[Path]:
|
|
28
|
-
all_marker_paths: list[Path] = []
|
|
29
|
-
transcript_path = all_hook_fields.get(TRANSCRIPT_PATH_FIELD_NAME)
|
|
30
|
-
if isinstance(transcript_path, str) and transcript_path:
|
|
31
|
-
all_marker_paths.append(Path(transcript_path + DEEPEN_TURN_MARKER_SUFFIX))
|
|
32
|
-
session_id = all_hook_fields.get(SESSION_ID_FIELD_NAME)
|
|
33
|
-
if isinstance(session_id, str) and session_id:
|
|
34
|
-
all_marker_paths.append(Path(tempfile.gettempdir()) / f"{DEEPEN_SESSION_MARKER_PREFIX}{session_id}")
|
|
35
|
-
return all_marker_paths
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def turn_marker_path(all_hook_fields: dict[str, object]) -> Path | None:
|
|
39
|
-
"""Return the marker path for this session, or None when the payload names none.
|
|
40
|
-
|
|
41
|
-
The transcript path wins when the payload has one. Cursor sends no transcript
|
|
42
|
-
path before the first reply, so the session id names a file in the system
|
|
43
|
-
temp folder instead.
|
|
44
|
-
"""
|
|
45
|
-
all_marker_paths = _all_turn_marker_paths(all_hook_fields)
|
|
46
|
-
return all_marker_paths[0] if all_marker_paths else None
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def turn_marker_exists(all_hook_fields: dict[str, object]) -> bool:
|
|
50
|
-
"""Return True when this session already deepened a task.
|
|
51
|
-
|
|
52
|
-
Both marker locations count. Cursor's first prompt claims the session marker
|
|
53
|
-
because it has no transcript path yet, and a later prompt carries one.
|
|
54
|
-
"""
|
|
55
|
-
return any(each_marker_path.is_file() for each_marker_path in _all_turn_marker_paths(all_hook_fields))
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def claim_turn_marker(all_hook_fields: dict[str, object]) -> bool:
|
|
59
|
-
"""Create the marker, and return False when another copy created it first.
|
|
60
|
-
|
|
61
|
-
A payload with no transcript path or session id, or a marker that cannot be
|
|
62
|
-
written, returns True: the hook then deepens without a guard, as before.
|
|
63
|
-
"""
|
|
64
|
-
marker_path = turn_marker_path(all_hook_fields)
|
|
65
|
-
if marker_path is None:
|
|
66
|
-
return True
|
|
67
|
-
try:
|
|
68
|
-
marker_descriptor = os.open(marker_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
69
|
-
except FileExistsError:
|
|
70
|
-
return False
|
|
71
|
-
except OSError:
|
|
72
|
-
return True
|
|
73
|
-
with os.fdopen(marker_descriptor, "w", encoding=UTF8_ENCODING) as marker_file:
|
|
74
|
-
marker_file.write(DEEPEN_MARKER)
|
|
75
|
-
return True
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"task": "Keep the build green. For example, the login failure.\n\nHARD CONSTRAINTS\n- Never open a pull request.\n",
|
|
3
|
-
"cases": [
|
|
4
|
-
{
|
|
5
|
-
"name": "known_bad",
|
|
6
|
-
"expect": "refuse",
|
|
7
|
-
"rewrittenPrompt": "CDE_DEEPEN:v1\nSCOPE (role): Fix the login failure\nEXAMPLES (illustration only): login failure\nZOOM-IN BLOCKER: none\n- Never open a pull request.\nRead the workflow log and repair the shared gate.\n"
|
|
8
|
-
},
|
|
9
|
-
{
|
|
10
|
-
"name": "known_good",
|
|
11
|
-
"expect": "accept",
|
|
12
|
-
"rewrittenPrompt": "CDE_DEEPEN:v1\nSCOPE (role): Keep every build green across the repository\nEXAMPLES (illustration only): login failure\nZOOM-IN BLOCKER: none\n- Never open a pull request.\nRead the workflow log and repair the shared gate.\n"
|
|
13
|
-
}
|
|
14
|
-
]
|
|
15
|
-
}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Score a replay fixture. This script does not rewrite the task."""
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
import json
|
|
7
|
-
import sys
|
|
8
|
-
from pathlib import Path
|
|
9
|
-
|
|
10
|
-
_SESSION_DIR = Path(__file__).resolve().parent
|
|
11
|
-
_HOOKS_ROOT = _SESSION_DIR.parent
|
|
12
|
-
for each_sys_path_entry in (str(_SESSION_DIR), str(_HOOKS_ROOT)):
|
|
13
|
-
if each_sys_path_entry not in sys.path:
|
|
14
|
-
sys.path.insert(0, each_sys_path_entry)
|
|
15
|
-
|
|
16
|
-
from deepen_role_gate import refusal_reason
|
|
17
|
-
from hooks_constants.deepen_first_prompt_constants import (
|
|
18
|
-
SCORE_ACCEPT,
|
|
19
|
-
SCORE_REFUSE,
|
|
20
|
-
UTF8_ENCODING,
|
|
21
|
-
)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def score_replay(task_text: str, rewritten_prompt: str) -> dict[str, str]:
|
|
25
|
-
"""Return accept, or refuse plus the gate reason. Never rewrite the task."""
|
|
26
|
-
reason = refusal_reason(task_text, rewritten_prompt)
|
|
27
|
-
if reason is None:
|
|
28
|
-
return {"score": SCORE_ACCEPT}
|
|
29
|
-
return {"score": SCORE_REFUSE, "reason": reason}
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def score_fixture(all_replay_fields: dict[str, object]) -> list[dict[str, str]]:
|
|
33
|
-
"""Score each case in a replay fixture against the task it names."""
|
|
34
|
-
task_text = all_replay_fields["task"]
|
|
35
|
-
if not isinstance(task_text, str):
|
|
36
|
-
raise TypeError("fixture task must be a string")
|
|
37
|
-
cases = all_replay_fields["cases"]
|
|
38
|
-
if not isinstance(cases, list):
|
|
39
|
-
raise TypeError("fixture cases must be a list")
|
|
40
|
-
all_scores: list[dict[str, str]] = []
|
|
41
|
-
for each_case in cases:
|
|
42
|
-
if not isinstance(each_case, dict):
|
|
43
|
-
raise TypeError("each fixture case must be an object")
|
|
44
|
-
case_name = each_case["name"]
|
|
45
|
-
rewritten_prompt = each_case["rewrittenPrompt"]
|
|
46
|
-
expected_score = each_case["expect"]
|
|
47
|
-
if not isinstance(case_name, str) or not isinstance(rewritten_prompt, str):
|
|
48
|
-
raise TypeError("case name and rewrittenPrompt must be strings")
|
|
49
|
-
if not isinstance(expected_score, str):
|
|
50
|
-
raise TypeError("case expect must be a string")
|
|
51
|
-
scored = score_replay(task_text, rewritten_prompt)
|
|
52
|
-
scored["name"] = case_name
|
|
53
|
-
scored["expect"] = expected_score
|
|
54
|
-
scored["matched"] = "yes" if scored["score"] == expected_score else "no"
|
|
55
|
-
all_scores.append(scored)
|
|
56
|
-
return all_scores
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def main(all_arguments: list[str]) -> int:
|
|
60
|
-
"""Read one fixture path and print a score line per case."""
|
|
61
|
-
if len(all_arguments) != 1:
|
|
62
|
-
sys.stderr.write("usage: grade_deepstack_replay.py <fixture.json>\n")
|
|
63
|
-
return 2
|
|
64
|
-
fixture_path = Path(all_arguments[0])
|
|
65
|
-
fixture = json.loads(fixture_path.read_text(encoding=UTF8_ENCODING))
|
|
66
|
-
if not isinstance(fixture, dict):
|
|
67
|
-
sys.stderr.write("fixture root must be an object\n")
|
|
68
|
-
return 2
|
|
69
|
-
all_scores = score_fixture(fixture)
|
|
70
|
-
for each_score in all_scores:
|
|
71
|
-
sys.stdout.write(json.dumps(each_score) + "\n")
|
|
72
|
-
if any(each_score["matched"] != "yes" for each_score in all_scores):
|
|
73
|
-
return 1
|
|
74
|
-
return 0
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if __name__ == "__main__":
|
|
78
|
-
raise SystemExit(main(sys.argv[1:]))
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
"""Tests for the role-shaped default rewrite."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import sys
|
|
6
|
-
from pathlib import Path
|
|
7
|
-
|
|
8
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
9
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
10
|
-
|
|
11
|
-
from deepen_default_rewrite import role_default_rewrite
|
|
12
|
-
from deepen_role_gate import refusal_reason
|
|
13
|
-
from hooks_constants.deepen_first_prompt_constants import ALL_FOUR_QUESTION_STEMS, DEEPEN_MARKER
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def _role_line(rewrite: str, label: str) -> str:
|
|
17
|
-
return next(each for each in rewrite.splitlines() if each.startswith(label))[len(label) :].strip()
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def test_should_pass_the_gate_for_a_plain_task() -> None:
|
|
21
|
-
task = "Add dark mode to the settings page"
|
|
22
|
-
rewrite = role_default_rewrite(task)
|
|
23
|
-
assert rewrite.startswith(DEEPEN_MARKER)
|
|
24
|
-
assert refusal_reason(task, rewrite) is None
|
|
25
|
-
assert _role_line(rewrite, "SCOPE (role):") == task
|
|
26
|
-
assert _role_line(rewrite, "EXAMPLES (illustration only):") == "none"
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def test_should_not_paste_the_four_question_stems() -> None:
|
|
30
|
-
rewrite = role_default_rewrite("Add dark mode to the settings page")
|
|
31
|
-
assert not any(each_stem in rewrite for each_stem in ALL_FOUR_QUESTION_STEMS)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def test_should_move_a_marked_example_out_of_the_scope() -> None:
|
|
35
|
-
task = "fix the flaky tests, e.g. test_auth and test_cache, and keep CI fast"
|
|
36
|
-
rewrite = role_default_rewrite(task)
|
|
37
|
-
assert refusal_reason(task, rewrite) is None
|
|
38
|
-
assert _role_line(rewrite, "SCOPE (role):") == "fix the flaky tests and keep CI fast"
|
|
39
|
-
assert _role_line(rewrite, "EXAMPLES (illustration only):") == "test_auth and test_cache"
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def test_should_ask_before_narrowing_when_the_scope_names_the_example() -> None:
|
|
43
|
-
task = "Fix the login failure. For example, the login failure on Safari."
|
|
44
|
-
rewrite = role_default_rewrite(task)
|
|
45
|
-
assert refusal_reason(task, rewrite) is None
|
|
46
|
-
assert _role_line(rewrite, "ZOOM-IN BLOCKER:").startswith("the scope names something")
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def test_should_keep_every_hard_constraint_line() -> None:
|
|
50
|
-
task = "Keep the build green. For example, the login failure.\n\nHARD CONSTRAINTS\n- Never open a pull request.\n"
|
|
51
|
-
rewrite = role_default_rewrite(task)
|
|
52
|
-
assert refusal_reason(task, rewrite) is None
|
|
53
|
-
assert "- Never open a pull request." in rewrite.splitlines()
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
def test_should_keep_every_task_sentence_in_scope_when_the_task_opens_with_context() -> None:
|
|
57
|
-
task = "We have a Django app. Add CSV export to the reports page, e.g. the monthly sales report, and keep the existing URLs."
|
|
58
|
-
rewrite = role_default_rewrite(task)
|
|
59
|
-
assert refusal_reason(task, rewrite) is None
|
|
60
|
-
assert _role_line(rewrite, "SCOPE (role):") == (
|
|
61
|
-
"We have a Django app. Add CSV export to the reports page and keep the existing URLs."
|
|
62
|
-
)
|
|
63
|
-
assert _role_line(rewrite, "EXAMPLES (illustration only):") == "the monthly sales report"
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def test_should_clip_a_long_scope_and_point_at_the_full_task() -> None:
|
|
67
|
-
task = "Refactor the module. " * 40
|
|
68
|
-
rewrite = role_default_rewrite(task.strip())
|
|
69
|
-
scope = _role_line(rewrite, "SCOPE (role):")
|
|
70
|
-
assert scope.endswith("(the rest is in the task below)")
|
|
71
|
-
assert len(scope) < len(task)
|
|
72
|
-
assert refusal_reason(task.strip(), rewrite) is None
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def test_should_keep_a_marker_word_inside_a_longer_word_in_the_scope() -> None:
|
|
76
|
-
task = "Search the codebase for examples of deprecated API calls and replace each one with the new client."
|
|
77
|
-
rewrite = role_default_rewrite(task)
|
|
78
|
-
assert refusal_reason(task, rewrite) is None
|
|
79
|
-
assert _role_line(rewrite, "SCOPE (role):") == task
|
|
80
|
-
assert _role_line(rewrite, "EXAMPLES (illustration only):") == "none"
|
|
81
|
-
assumption_task = "Avoid such assumptions in the parser and add input validation."
|
|
82
|
-
assert _role_line(role_default_rewrite(assumption_task), "SCOPE (role):") == assumption_task
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
def test_should_move_every_marked_example_in_a_sentence_out_of_the_scope() -> None:
|
|
86
|
-
task = "Speed up the slow pages such as the dashboard, and cache reads, e.g. the user lookup, and keep the API stable."
|
|
87
|
-
rewrite = role_default_rewrite(task)
|
|
88
|
-
assert refusal_reason(task, rewrite) is None
|
|
89
|
-
assert _role_line(rewrite, "SCOPE (role):") == "Speed up the slow pages and cache reads and keep the API stable."
|
|
90
|
-
assert _role_line(rewrite, "EXAMPLES (illustration only):") == "the dashboard; the user lookup"
|