claude-dev-env 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/install.mjs +1 -0
- package/hooks/blocking/prompt-workflow-stop-guard.py +14 -0
- package/hooks/blocking/prompt_workflow_gate_config.py +117 -0
- package/hooks/blocking/prompt_workflow_gate_core.py +30 -107
- package/hooks/blocking/test_prompt_workflow_gate_core.py +51 -0
- package/hooks/blocking/test_prompt_workflow_stop_guard.py +38 -4
- package/package.json +1 -1
- package/skills/prompt-generator/REFINEMENT_PIPELINE_RUNBOOK.md +3 -1
- package/skills/prompt-generator/SKILL.md +5 -0
- package/skills/prompt-generator/TARGET_OUTPUT.md +7 -0
- package/skills/prompt-generator/evals/prompt-generator.json +37 -0
package/bin/install.mjs
CHANGED
|
@@ -29,6 +29,7 @@ const INSTALL_GROUPS = {
|
|
|
29
29
|
description: 'Prompt engineering tools',
|
|
30
30
|
skills: ['prompt-generator', 'agent-prompt'],
|
|
31
31
|
includeHookFiles: [
|
|
32
|
+
'blocking/prompt_workflow_gate_config.py',
|
|
32
33
|
'blocking/prompt_workflow_gate_core.py',
|
|
33
34
|
'blocking/prompt-workflow-stop-guard.py',
|
|
34
35
|
'HOOK_SPECS_PROMPT_WORKFLOW.md',
|
|
@@ -18,6 +18,7 @@ from prompt_workflow_gate_core import (
|
|
|
18
18
|
is_prompt_workflow_response,
|
|
19
19
|
missing_context_control_signals,
|
|
20
20
|
missing_checklist_rows,
|
|
21
|
+
missing_required_xml_sections,
|
|
21
22
|
missing_scope_anchors,
|
|
22
23
|
)
|
|
23
24
|
|
|
@@ -150,10 +151,23 @@ def _check_negative_keywords_in_artifact(assistant_message: str) -> dict | None:
|
|
|
150
151
|
),
|
|
151
152
|
)
|
|
152
153
|
|
|
154
|
+
def _check_required_xml_sections(assistant_message: str) -> dict | None:
|
|
155
|
+
missing_sections = missing_required_xml_sections(assistant_message)
|
|
156
|
+
if not missing_sections:
|
|
157
|
+
return None
|
|
158
|
+
return _build_block(
|
|
159
|
+
brief_label="retrying: include all required XML sections",
|
|
160
|
+
full_reason=(
|
|
161
|
+
"PROMPT-WORKFLOW GATE: Fenced XML artifact missing required sections: "
|
|
162
|
+
+ ", ".join(missing_sections)
|
|
163
|
+
),
|
|
164
|
+
)
|
|
165
|
+
|
|
153
166
|
def _evaluate_workflow_gates(assistant_message: str) -> dict | None:
|
|
154
167
|
if not is_prompt_workflow_response(assistant_message):
|
|
155
168
|
return None
|
|
156
169
|
workflow_gate_checks: tuple[Callable[[str], dict | None], ...] = (
|
|
170
|
+
_check_required_xml_sections,
|
|
157
171
|
_check_missing_checklist_rows,
|
|
158
172
|
_check_missing_scope_anchors,
|
|
159
173
|
_check_missing_context_signals,
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Static lists and compiled regexes for prompt-workflow gate checks.
|
|
2
|
+
|
|
3
|
+
Edit this file to change scope anchors, checklist rows, markers, or keyword lists
|
|
4
|
+
without touching gate logic in prompt_workflow_gate_core.py.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
REQUIRED_SCOPE_ANCHORS: tuple[str, ...] = (
|
|
12
|
+
"target_local_roots",
|
|
13
|
+
"target_canonical_roots",
|
|
14
|
+
"target_file_globs",
|
|
15
|
+
"comparison_basis",
|
|
16
|
+
"completion_boundary",
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
REQUIRED_CHECKLIST_ROWS: tuple[str, ...] = (
|
|
20
|
+
"structured_scoped_instructions",
|
|
21
|
+
"sequential_steps_present",
|
|
22
|
+
"positive_framing",
|
|
23
|
+
"acceptance_criteria_defined",
|
|
24
|
+
"safety_reversibility_language",
|
|
25
|
+
"reversible_action_and_safety_check_guidance",
|
|
26
|
+
"concrete_output_contract",
|
|
27
|
+
"scope_boundary_present",
|
|
28
|
+
"explicit_scope_anchors_present",
|
|
29
|
+
"all_instructions_artifact_bound",
|
|
30
|
+
"scope_terms_explicit_and_anchored",
|
|
31
|
+
"completion_boundary_measurable",
|
|
32
|
+
"citation_grounding_policy_present",
|
|
33
|
+
"source_priority_rules_present",
|
|
34
|
+
"artifact_language_confidence",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
AMBIGUOUS_SCOPE_TERMS: tuple[str, ...] = (
|
|
38
|
+
"this session",
|
|
39
|
+
"current files",
|
|
40
|
+
"here",
|
|
41
|
+
"above",
|
|
42
|
+
"as needed",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
INTERNAL_OBJECT_MARKERS: tuple[str, ...] = (
|
|
46
|
+
'"pipeline_mode": "internal_section_refinement_with_final_audit"',
|
|
47
|
+
'"scope_block": {',
|
|
48
|
+
'"required_sections": [',
|
|
49
|
+
'"section_output_contract": {',
|
|
50
|
+
'"merge_output_contract": {',
|
|
51
|
+
'"audit_output_contract": {',
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
PROMPT_WORKFLOW_RESPONSE_MARKERS: tuple[str, ...] = (
|
|
55
|
+
"checklist_results",
|
|
56
|
+
"overall_status",
|
|
57
|
+
"scope anchors",
|
|
58
|
+
"target_local_roots",
|
|
59
|
+
"target_canonical_roots",
|
|
60
|
+
"target_file_globs",
|
|
61
|
+
"comparison_basis",
|
|
62
|
+
"completion_boundary",
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
DEBUG_INTENT_MARKERS: tuple[str, ...] = (
|
|
66
|
+
"debug",
|
|
67
|
+
"show internal",
|
|
68
|
+
"raw internal object",
|
|
69
|
+
"pipeline object",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
NEGATIVE_KEYWORDS_IN_ARTIFACT: tuple[str, ...] = (
|
|
73
|
+
"no",
|
|
74
|
+
"not",
|
|
75
|
+
"don't",
|
|
76
|
+
"do not",
|
|
77
|
+
"never",
|
|
78
|
+
"avoid",
|
|
79
|
+
"without",
|
|
80
|
+
"refrain",
|
|
81
|
+
"stop",
|
|
82
|
+
"prevent",
|
|
83
|
+
"exclude",
|
|
84
|
+
"prohibit",
|
|
85
|
+
"forbid",
|
|
86
|
+
"reject",
|
|
87
|
+
"cannot",
|
|
88
|
+
"unless",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
NEGATIVE_INDIRECT_PATTERNS_IN_ARTIFACT: tuple[str, ...] = (
|
|
92
|
+
r"instead of\s+\w+",
|
|
93
|
+
r"rather than\s+\w+",
|
|
94
|
+
r"as opposed to\s+\w+",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
REQUIRED_XML_SECTIONS: tuple[str, ...] = (
|
|
98
|
+
"role",
|
|
99
|
+
"context",
|
|
100
|
+
"instructions",
|
|
101
|
+
"constraints",
|
|
102
|
+
"output_format",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
FENCED_XML_BLOCK_PATTERN: re.Pattern[str] = re.compile(
|
|
106
|
+
r"```xml\s*\n(.*?)```", re.DOTALL
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
COMPILED_NEGATIVE_KEYWORD_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
|
|
110
|
+
re.compile(rf"\b{re.escape(keyword)}\b", re.IGNORECASE)
|
|
111
|
+
for keyword in NEGATIVE_KEYWORDS_IN_ARTIFACT
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
COMPILED_NEGATIVE_INDIRECT_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
|
|
115
|
+
re.compile(pattern, re.IGNORECASE)
|
|
116
|
+
for pattern in NEGATIVE_INDIRECT_PATTERNS_IN_ARTIFACT
|
|
117
|
+
)
|
|
@@ -6,110 +6,17 @@ from __future__ import annotations
|
|
|
6
6
|
import re
|
|
7
7
|
from typing import Iterable
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
REQUIRED_CHECKLIST_ROWS
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
"positive_framing",
|
|
21
|
-
"acceptance_criteria_defined",
|
|
22
|
-
"safety_reversibility_language",
|
|
23
|
-
"reversible_action_and_safety_check_guidance",
|
|
24
|
-
"concrete_output_contract",
|
|
25
|
-
"scope_boundary_present",
|
|
26
|
-
"explicit_scope_anchors_present",
|
|
27
|
-
"all_instructions_artifact_bound",
|
|
28
|
-
"scope_terms_explicit_and_anchored",
|
|
29
|
-
"completion_boundary_measurable",
|
|
30
|
-
"citation_grounding_policy_present",
|
|
31
|
-
"source_priority_rules_present",
|
|
32
|
-
"artifact_language_confidence",
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
REQUIRED_CONTEXT_CONTROL_SIGNALS: tuple[str, ...] = (
|
|
36
|
-
"base_minimal_instruction_layer: true",
|
|
37
|
-
"on_demand_skill_loading: true",
|
|
38
|
-
)
|
|
39
|
-
|
|
40
|
-
AMBIGUOUS_SCOPE_TERMS: tuple[str, ...] = (
|
|
41
|
-
"this session",
|
|
42
|
-
"current files",
|
|
43
|
-
"here",
|
|
44
|
-
"above",
|
|
45
|
-
"as needed",
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
INTERNAL_OBJECT_MARKERS: tuple[str, ...] = (
|
|
49
|
-
'"pipeline_mode": "internal_section_refinement_with_final_audit"',
|
|
50
|
-
'"scope_block": {',
|
|
51
|
-
'"required_sections": [',
|
|
52
|
-
'"section_output_contract": {',
|
|
53
|
-
'"merge_output_contract": {',
|
|
54
|
-
'"audit_output_contract": {',
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
PROMPT_WORKFLOW_RESPONSE_MARKERS: tuple[str, ...] = (
|
|
58
|
-
"checklist_results",
|
|
59
|
-
"overall_status",
|
|
60
|
-
"scope anchors",
|
|
61
|
-
"target_local_roots",
|
|
62
|
-
"target_canonical_roots",
|
|
63
|
-
"target_file_globs",
|
|
64
|
-
"comparison_basis",
|
|
65
|
-
"completion_boundary",
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
DEBUG_INTENT_MARKERS: tuple[str, ...] = (
|
|
69
|
-
"debug",
|
|
70
|
-
"show internal",
|
|
71
|
-
"raw internal object",
|
|
72
|
-
"pipeline object",
|
|
73
|
-
)
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
NEGATIVE_KEYWORDS_IN_ARTIFACT: tuple[str, ...] = (
|
|
77
|
-
"no",
|
|
78
|
-
"not",
|
|
79
|
-
"don't",
|
|
80
|
-
"do not",
|
|
81
|
-
"never",
|
|
82
|
-
"avoid",
|
|
83
|
-
"without",
|
|
84
|
-
"refrain",
|
|
85
|
-
"stop",
|
|
86
|
-
"prevent",
|
|
87
|
-
"exclude",
|
|
88
|
-
"prohibit",
|
|
89
|
-
"forbid",
|
|
90
|
-
"reject",
|
|
91
|
-
"cannot",
|
|
92
|
-
"unless",
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
NEGATIVE_INDIRECT_PATTERNS_IN_ARTIFACT: tuple[str, ...] = (
|
|
96
|
-
r"instead of\s+\w+",
|
|
97
|
-
r"rather than\s+\w+",
|
|
98
|
-
r"as opposed to\s+\w+",
|
|
99
|
-
)
|
|
100
|
-
|
|
101
|
-
COMPILED_NEGATIVE_KEYWORD_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
|
|
102
|
-
re.compile(rf"\b{re.escape(keyword)}\b", re.IGNORECASE)
|
|
103
|
-
for keyword in NEGATIVE_KEYWORDS_IN_ARTIFACT
|
|
104
|
-
)
|
|
105
|
-
|
|
106
|
-
COMPILED_NEGATIVE_INDIRECT_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
|
|
107
|
-
re.compile(pattern, re.IGNORECASE)
|
|
108
|
-
for pattern in NEGATIVE_INDIRECT_PATTERNS_IN_ARTIFACT
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
FENCED_XML_BLOCK_PATTERN: re.Pattern[str] = re.compile(
|
|
112
|
-
r"```xml\s*\n(.*?)```", re.DOTALL
|
|
9
|
+
from prompt_workflow_gate_config import (
|
|
10
|
+
AMBIGUOUS_SCOPE_TERMS,
|
|
11
|
+
COMPILED_NEGATIVE_INDIRECT_PATTERNS,
|
|
12
|
+
COMPILED_NEGATIVE_KEYWORD_PATTERNS,
|
|
13
|
+
DEBUG_INTENT_MARKERS,
|
|
14
|
+
FENCED_XML_BLOCK_PATTERN,
|
|
15
|
+
INTERNAL_OBJECT_MARKERS,
|
|
16
|
+
PROMPT_WORKFLOW_RESPONSE_MARKERS,
|
|
17
|
+
REQUIRED_CHECKLIST_ROWS,
|
|
18
|
+
REQUIRED_SCOPE_ANCHORS,
|
|
19
|
+
REQUIRED_XML_SECTIONS,
|
|
113
20
|
)
|
|
114
21
|
|
|
115
22
|
|
|
@@ -118,6 +25,19 @@ def extract_fenced_xml_content(text: str) -> str:
|
|
|
118
25
|
return "\n".join(all_matches)
|
|
119
26
|
|
|
120
27
|
|
|
28
|
+
def missing_required_xml_sections(text: str) -> list[str]:
|
|
29
|
+
fenced_body = extract_fenced_xml_content(text)
|
|
30
|
+
if not fenced_body.strip():
|
|
31
|
+
return []
|
|
32
|
+
missing_sections: list[str] = []
|
|
33
|
+
for section_name in REQUIRED_XML_SECTIONS:
|
|
34
|
+
open_tag = re.compile(rf"<{re.escape(section_name)}(\s[^>]*)?>")
|
|
35
|
+
close_tag = re.compile(rf"</{re.escape(section_name)}>")
|
|
36
|
+
if not open_tag.search(fenced_body) or not close_tag.search(fenced_body):
|
|
37
|
+
missing_sections.append(section_name)
|
|
38
|
+
return missing_sections
|
|
39
|
+
|
|
40
|
+
|
|
121
41
|
def find_negative_keywords_in_fenced_xml(
|
|
122
42
|
text: str,
|
|
123
43
|
) -> list[dict[str, str | int]]:
|
|
@@ -192,6 +112,9 @@ def is_prompt_workflow_response(text: str) -> bool:
|
|
|
192
112
|
|
|
193
113
|
|
|
194
114
|
def missing_context_control_signals(text: str) -> list[str]:
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
115
|
+
required_signals: tuple[str, ...] = (
|
|
116
|
+
"base_minimal_instruction_layer: true",
|
|
117
|
+
"on_demand_skill_loading: true",
|
|
118
|
+
)
|
|
119
|
+
lowered = text.lower()
|
|
120
|
+
return [signal for signal in required_signals if signal not in lowered]
|
|
@@ -7,6 +7,7 @@ from prompt_workflow_gate_core import (
|
|
|
7
7
|
is_prompt_workflow_response,
|
|
8
8
|
missing_context_control_signals,
|
|
9
9
|
missing_checklist_rows,
|
|
10
|
+
missing_required_xml_sections,
|
|
10
11
|
missing_scope_anchors,
|
|
11
12
|
)
|
|
12
13
|
|
|
@@ -52,3 +53,53 @@ def test_ambiguous_scope_terms_detected() -> None:
|
|
|
52
53
|
terms = find_ambiguous_scope_terms(text)
|
|
53
54
|
assert "this session" in terms
|
|
54
55
|
assert "current files" in terms
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _fenced_xml(body: str) -> str:
|
|
59
|
+
return f"```xml\n{body}\n```"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_missing_required_xml_sections_all_present_returns_empty() -> None:
|
|
63
|
+
body = (
|
|
64
|
+
"<role>R.</role>\n"
|
|
65
|
+
"<context>C.</context>\n"
|
|
66
|
+
"<instructions>I.</instructions>\n"
|
|
67
|
+
"<constraints>Co.</constraints>\n"
|
|
68
|
+
"<output_format>O.</output_format>\n"
|
|
69
|
+
)
|
|
70
|
+
assert missing_required_xml_sections(_fenced_xml(body)) == []
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_missing_required_xml_sections_missing_context() -> None:
|
|
74
|
+
body = (
|
|
75
|
+
"<role>R.</role>\n"
|
|
76
|
+
"<instructions>I.</instructions>\n"
|
|
77
|
+
"<constraints>Co.</constraints>\n"
|
|
78
|
+
"<output_format>O.</output_format>\n"
|
|
79
|
+
)
|
|
80
|
+
assert missing_required_xml_sections(_fenced_xml(body)) == ["context"]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_missing_required_xml_sections_missing_role_and_output_format() -> None:
|
|
84
|
+
body = (
|
|
85
|
+
"<context>C.</context>\n"
|
|
86
|
+
"<instructions>I.</instructions>\n"
|
|
87
|
+
"<constraints>Co.</constraints>\n"
|
|
88
|
+
)
|
|
89
|
+
missing = missing_required_xml_sections(_fenced_xml(body))
|
|
90
|
+
assert missing == ["role", "output_format"]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_missing_required_xml_sections_no_fence_returns_empty() -> None:
|
|
94
|
+
assert missing_required_xml_sections("no fenced xml here") == []
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_missing_required_xml_sections_prose_without_tags_counts_as_missing() -> None:
|
|
98
|
+
body = (
|
|
99
|
+
"<role>R.</role>\n"
|
|
100
|
+
"context appears in prose but has no tags.\n"
|
|
101
|
+
"<instructions>I.</instructions>\n"
|
|
102
|
+
"<constraints>Co.</constraints>\n"
|
|
103
|
+
"<output_format>O.</output_format>\n"
|
|
104
|
+
)
|
|
105
|
+
assert missing_required_xml_sections(_fenced_xml(body)) == ["context"]
|
|
@@ -118,6 +118,16 @@ def test_blocks_ambiguous_scope_phrasing() -> None:
|
|
|
118
118
|
assert response["decision"] == "block"
|
|
119
119
|
assert "Ambiguous scope phrasing detected" in response["reason"]
|
|
120
120
|
|
|
121
|
+
def _wrap_five_section_scaffold(inner_body: str) -> str:
|
|
122
|
+
return (
|
|
123
|
+
"<role>Test role sentence one.</role>\n"
|
|
124
|
+
"<context>Test context sentence one.</context>\n"
|
|
125
|
+
f"{inner_body}\n"
|
|
126
|
+
"<constraints>Test constraints sentence one.</constraints>\n"
|
|
127
|
+
"<output_format>Test output format sentence one.</output_format>\n"
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
121
131
|
def _build_prompt_workflow_message_with_fenced_xml(fenced_xml_body: str) -> str:
|
|
122
132
|
return (
|
|
123
133
|
"Audit: pass 15/15\n"
|
|
@@ -137,7 +147,9 @@ def _build_prompt_workflow_message_with_fenced_xml(fenced_xml_body: str) -> str:
|
|
|
137
147
|
|
|
138
148
|
|
|
139
149
|
def test_allows_positive_phrasing_inside_fenced_xml() -> None:
|
|
140
|
-
fenced_content =
|
|
150
|
+
fenced_content = _wrap_five_section_scaffold(
|
|
151
|
+
"<instructions>Ensure all functions have explicit return types.</instructions>"
|
|
152
|
+
)
|
|
141
153
|
payload = {
|
|
142
154
|
"last_assistant_message": _build_prompt_workflow_message_with_fenced_xml(fenced_content),
|
|
143
155
|
}
|
|
@@ -172,7 +184,9 @@ def test_blocks_banned_pattern_inside_fenced_xml(
|
|
|
172
184
|
fenced_xml_content: str,
|
|
173
185
|
) -> None:
|
|
174
186
|
payload = {
|
|
175
|
-
"last_assistant_message": _build_prompt_workflow_message_with_fenced_xml(
|
|
187
|
+
"last_assistant_message": _build_prompt_workflow_message_with_fenced_xml(
|
|
188
|
+
_wrap_five_section_scaffold(fenced_xml_content)
|
|
189
|
+
),
|
|
176
190
|
}
|
|
177
191
|
result = _run_hook(payload)
|
|
178
192
|
response = json.loads(result.stdout)
|
|
@@ -180,12 +194,15 @@ def test_blocks_banned_pattern_inside_fenced_xml(
|
|
|
180
194
|
|
|
181
195
|
|
|
182
196
|
def test_permits_negative_keywords_outside_fenced_xml() -> None:
|
|
197
|
+
fenced_inner = _wrap_five_section_scaffold(
|
|
198
|
+
"<instructions>Ensure all functions have explicit return types.</instructions>"
|
|
199
|
+
)
|
|
183
200
|
message = (
|
|
184
201
|
"Audit: pass 15/15\n"
|
|
185
202
|
"Do not skip the audit line.\n"
|
|
186
203
|
"```xml\n"
|
|
187
|
-
|
|
188
|
-
"```\n"
|
|
204
|
+
+ fenced_inner
|
|
205
|
+
+ "\n```\n"
|
|
189
206
|
"overall_status: pass\n"
|
|
190
207
|
+ _full_checklist_rows()
|
|
191
208
|
+ "target_local_roots\n"
|
|
@@ -201,6 +218,23 @@ def test_permits_negative_keywords_outside_fenced_xml() -> None:
|
|
|
201
218
|
assert result.stdout.strip() == ""
|
|
202
219
|
|
|
203
220
|
|
|
221
|
+
def test_blocks_when_fenced_xml_missing_context_section() -> None:
|
|
222
|
+
fenced_body = (
|
|
223
|
+
"<role>Test role sentence one.</role>\n"
|
|
224
|
+
"<instructions>Test instructions sentence one.</instructions>\n"
|
|
225
|
+
"<constraints>Test constraints sentence one.</constraints>\n"
|
|
226
|
+
"<output_format>Test output format sentence one.</output_format>\n"
|
|
227
|
+
)
|
|
228
|
+
payload = {
|
|
229
|
+
"last_assistant_message": _build_prompt_workflow_message_with_fenced_xml(fenced_body),
|
|
230
|
+
}
|
|
231
|
+
result = _run_hook(payload)
|
|
232
|
+
response = json.loads(result.stdout)
|
|
233
|
+
assert response["decision"] == "block"
|
|
234
|
+
assert "context" in response["reason"]
|
|
235
|
+
assert "include all required XML sections" in response["systemMessage"]
|
|
236
|
+
|
|
237
|
+
|
|
204
238
|
def test_allows_fully_structured_prompt_workflow_output() -> None:
|
|
205
239
|
payload = {
|
|
206
240
|
"last_assistant_message": (
|
package/package.json
CHANGED
|
@@ -26,7 +26,7 @@ Use this command:
|
|
|
26
26
|
- `target_file_globs`
|
|
27
27
|
- `comparison_basis`
|
|
28
28
|
- `completion_boundary`
|
|
29
|
-
- XML scaffold includes all sections:
|
|
29
|
+
- XML scaffold includes all sections — verified by the Stop hook at runtime; each required section tag must have both an opening and a closing tag:
|
|
30
30
|
- `<role>`
|
|
31
31
|
- `<context>`
|
|
32
32
|
- `<instructions>`
|
|
@@ -129,6 +129,7 @@ If `overall_status` is `fail`:
|
|
|
129
129
|
Validate fail-closed runtime gates:
|
|
130
130
|
|
|
131
131
|
1. **Stop leakage/scope/checklist gate**
|
|
132
|
+
- **Section-presence gate (Stop)** — Block responses where the fenced XML artifact is missing any of the five required section tag pairs: `role`, `context`, `instructions`, `constraints`, `output_format`.
|
|
132
133
|
- Block responses that leak raw internal refinement object fields unless debug intent is explicit.
|
|
133
134
|
- Block responses missing deterministic checklist rows when audit output is present.
|
|
134
135
|
- Block responses using ambiguous scope phrasing in scope-bound sections.
|
|
@@ -148,6 +149,7 @@ Validate fail-closed runtime gates:
|
|
|
148
149
|
- Missing required scope anchors (when Stop guard applies)
|
|
149
150
|
- Raw internal object leakage without debug intent
|
|
150
151
|
- Missing required checklist rows in audit output
|
|
152
|
+
- Missing required XML sections (`role`, `context`, `instructions`, `constraints`, `output_format`) in the fenced artifact (opening and closing tags)
|
|
151
153
|
- Ambiguous scope terms in scope-bound text
|
|
152
154
|
- Negative keywords inside fenced XML artifacts
|
|
153
155
|
- Hedging language inside fenced XML artifacts
|
|
@@ -185,6 +185,7 @@ Expand the light self-check with this internal checklist when useful:
|
|
|
185
185
|
- [ ] Emotion-informed framing is present: collaborative language, explicit success criteria, and explicit permission to express uncertainty ("say so if unsure")
|
|
186
186
|
- [ ] Constraints are surfaced upfront (proactive constraint awareness) so the model can incorporate them into its plan, and each non-obvious constraint carries its motivation
|
|
187
187
|
- [ ] Self-correction chaining is considered when the prompt must hold up over time (generate → review → refine)
|
|
188
|
+
- [ ] All five required XML sections (`<role>`, `<context>`, `<instructions>`, `<constraints>`, `<output_format>`) are present with both opening and closing tags in the fenced artifact
|
|
188
189
|
|
|
189
190
|
### 9. Deliver (orchestrator)
|
|
190
191
|
|
|
@@ -196,6 +197,8 @@ Audit: pass 15/15
|
|
|
196
197
|
|
|
197
198
|
(or `fail N/15 — …`), immediately followed by **one** fenced XML block; **send boundary** is immediately after the closing fence so the user receives a copy-ready pair (audit line + artifact) in one assistant message before the conversation continues.
|
|
198
199
|
|
|
200
|
+
**Render-survival:** When the fenced XML uses tag names that **collide with HTML5 elements** (`context`, `section`, `summary`, `details`, `header`, `footer`, `main`, `aside`, `article`, `nav`, `figure`), or when the artifact is **very large**, **write the artifact to a file** and give the user the path together with the usual one-line audit. Add a brief **section inventory** (confirming the five required sections) so the user can trust the file even if the inline fence would render poorly. Details: **TARGET_OUTPUT.md — Structural invariant E**.
|
|
201
|
+
|
|
199
202
|
### 10. Default refinement mode (subagent-internal)
|
|
200
203
|
|
|
201
204
|
For non-trivial requests, run inside the drafting subagent (use **draft-only** when the user explicitly asks for a quick draft / no refinement loop):
|
|
@@ -212,6 +215,8 @@ Required section list is immutable for this pipeline: `role`, `context`, `instru
|
|
|
212
215
|
|
|
213
216
|
**Two-tier validation — tier 2:** The `15` in `Audit: pass 15/15` counts these **compliance** rows (stable ids for hooks). Tier 1 is the **light self-check** in §8—keep the steps separate so models do not merge them.
|
|
214
217
|
|
|
218
|
+
**Runtime Stop hook:** In addition to the 15-row internal audit, the `prompt-workflow-stop-guard` Stop hook enforces **section presence** on prompt-workflow responses: any fenced Markdown XML block must include opening and closing tags for `role`, `context`, `instructions`, `constraints`, and `output_format`. Missing tags trigger a retry before the user sees a passing turn. Pair this with **Structural invariant E** in `TARGET_OUTPUT.md` so users still receive intact XML when chat renderers strip HTML-named tags.
|
|
219
|
+
|
|
215
220
|
| # | Row name |
|
|
216
221
|
|---|----------|
|
|
217
222
|
| 1 | structured_scoped_instructions |
|
|
@@ -87,6 +87,13 @@ This file is the **target output spec** for eval-driven iteration of the `prompt
|
|
|
87
87
|
- Place residual uncertainty only in `<open_question>` elements (one topic per tag) with a clear decision you need from the executor or user.
|
|
88
88
|
- Use definitive phrasing inside instructions (e.g. “Run tests in `packages/foo` with `pytest tests/`”) so each step reads like an executable checklist.
|
|
89
89
|
|
|
90
|
+
## Structural invariant E — Render-survival for XML sections
|
|
91
|
+
|
|
92
|
+
- **Problem:** Tag names used for prompt XML sections can overlap **HTML5 element names**. Chat renderers may treat those tokens as HTML and hide or alter the content between tags. High-risk examples include: `context`, `section`, `summary`, `details`, `header`, `footer`, `main`, `aside`, `article`, `nav`, `figure`. The raw assistant text may be complete while the **rendered** message looks like sections are missing (notably `<context>`).
|
|
93
|
+
- **Primary mitigation:** When the fenced XML artifact **contains any tag whose local name is on that HTML-collision list**, or when the artifact is **large enough that render truncation is likely**, the orchestrator **must write the full artifact to a file** (default: under `data/prompts/` or a path the user supplied earlier) and **paste the absolute file path** in the chat message. Pair the path with a **short section inventory** confirming all five required sections (`role`, `context`, `instructions`, `constraints`, `output_format`) are present in the file.
|
|
94
|
+
- **Fallback when file write is unavailable:** Escape the **opening angle bracket** of colliding tags (for example `<context>` — user restores `<` when pasting) or use another distinctive wrapper **documented in the same message**, so the user can recover literal XML. State explicitly that the user should restore brackets when copying into another system.
|
|
95
|
+
- **Structural safety net:** Regardless of renderer behavior, the **Stop hook section-presence gate** blocks any prompt-workflow response whose fenced XML is missing any required opening/closing section tag pair. Methodology: [Anthropic — Agent Skills: evaluation and iteration](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices#evaluation-and-iteration).
|
|
96
|
+
|
|
90
97
|
## XML artifact (minimum sections)
|
|
91
98
|
|
|
92
99
|
Include at least:
|
|
@@ -133,6 +133,43 @@
|
|
|
133
133
|
"Example: 'Ensure all functions have explicit return types' passes; 'Do not leave return types implicit' fails; 'Avoid missing return types' fails",
|
|
134
134
|
"Applies to all sections inside the fenced block: <role>, <context>, <instructions>, <constraints>, <output_format>"
|
|
135
135
|
]
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
"id": 10,
|
|
139
|
+
"name": "required_sections_present_in_artifact",
|
|
140
|
+
"scenario": "Section completeness gate (render-survival)",
|
|
141
|
+
"prompt": "/prompt-generator Write a system prompt for a Python linting agent that auto-fixes code style issues in this repo",
|
|
142
|
+
"files": [],
|
|
143
|
+
"expected_behavior": [
|
|
144
|
+
"Fenced XML block contains opening and closing tags for all five required sections: role, context, instructions, constraints, output_format",
|
|
145
|
+
"Each required section contains substantive content (minimum one sentence each)",
|
|
146
|
+
"The Stop hook section-presence check passes for this output (no missing section tags)",
|
|
147
|
+
"Sections appear in order: role first, output_format last among the five required sections"
|
|
148
|
+
]
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
"id": 11,
|
|
152
|
+
"name": "section_missing_triggers_hook_block",
|
|
153
|
+
"scenario": "Section completeness gate — failure path",
|
|
154
|
+
"prompt": "Synthetic eval: assistant final message is prompt-workflow shaped (overall_status, checklist, scope anchors, runtime signals) with a fenced Markdown XML block whose body omits the entire context section (no context opening/closing tags); observer asserts Stop hook behavior and successful retry.",
|
|
155
|
+
"files": [],
|
|
156
|
+
"expected_behavior": [
|
|
157
|
+
"The Stop hook runs _check_required_xml_sections and returns a block decision naming context as a missing section",
|
|
158
|
+
"The model retry includes all five required sections with both opening and closing tags",
|
|
159
|
+
"The retry output passes the section-presence gate (empty missing list from missing_required_xml_sections)"
|
|
160
|
+
]
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
"id": 12,
|
|
164
|
+
"name": "render_survival_file_fallback",
|
|
165
|
+
"scenario": "Render-layer mitigation",
|
|
166
|
+
"prompt": "/prompt-generator Write a comprehensive agent prompt for migrating a large Prisma schema and all related API routes, with step-by-step rollout, rollback, and verification — artifact sized like the migration prompt that triggered chat render stripping.",
|
|
167
|
+
"files": [],
|
|
168
|
+
"expected_behavior": [
|
|
169
|
+
"When the artifact exceeds a size threshold or contains XML section tag names that collide with HTML5 elements (context, section, summary, details, header, footer, main, aside, article, nav, figure), the orchestrator writes the full artifact to a file under data/prompts/ or a user-specified path",
|
|
170
|
+
"The file contains the complete XML with all tags preserved as literal text",
|
|
171
|
+
"The user-facing message states the file path and briefly inventories which required sections the artifact contains"
|
|
172
|
+
]
|
|
136
173
|
}
|
|
137
174
|
]
|
|
138
175
|
}
|