ma-agents 3.13.2 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agents.js +7 -0
- package/lib/bmad-cache/bmb/.claude-plugin/marketplace.json +3 -3
- package/lib/bmad-cache/bmb/_git_preserved/index +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-b388526edf307cf56f8e65673466cff074105abf.idx +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/{pack-8f8b045fef5af6911495cf3b2a89f1ed75e120f7.pack → pack-b388526edf307cf56f8e65673466cff074105abf.pack} +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-b388526edf307cf56f8e65673466cff074105abf.rev +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/packed-refs +1 -1
- package/lib/bmad-cache/bmb/_git_preserved/refs/heads/main +1 -1
- package/lib/bmad-cache/bmb/_git_preserved/shallow +1 -1
- package/lib/bmad-cache/bmb/package-lock.json +2 -2
- package/lib/bmad-cache/bmb/package.json +1 -1
- package/lib/bmad-cache/bmb/samples/bmad-agent-code-coach/scripts/wake.py +78 -0
- package/lib/bmad-cache/bmb/samples/bmad-agent-creative-muse/scripts/wake.py +78 -0
- package/lib/bmad-cache/bmb/samples/bmad-agent-sentinel/scripts/wake.py +78 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/customize-template.toml +3 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/init-sanctum-template.py +11 -5
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/report-shell.html +1073 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/wake-template.py +78 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/customize.toml +48 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/count_tokens.py +78 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass.py +258 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/process-template.py +20 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/render_report.py +387 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/assets/adapter-claude-code.json +9 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/aggregate_benchmark.py +236 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/memlog.py +197 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/run_evals.py +513 -390
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/run_triggers.py +347 -251
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/tests/test_env_isolation.py +79 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/tests/test_trigger_detection.py +123 -0
- package/lib/bmad-cache/bmb/skills/bmad-module-builder/scripts/tests/test-validate-module.py +82 -4
- package/lib/bmad-cache/bmb/skills/bmad-module-builder/scripts/validate-module.py +55 -11
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/assets/customize-template.toml +3 -5
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/assets/report-shell.html +860 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/customize.toml +59 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/count_tokens.py +77 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/init_skill.py +132 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/prepass-prompt-metrics.py +186 -221
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/quick_validate.py +117 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/render_report.py +387 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/tests/test_canon_sync.py +46 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/tests/test_count_tokens.py +180 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/tests/test_render_report.py +171 -0
- package/lib/bmad-cache/cache-manifest.json +4 -4
- package/lib/bmad-extension/.claude-plugin/marketplace.json.template +1 -1
- package/lib/bmad-extension-plugin/.claude-plugin/marketplace.json +2 -2
- package/lib/bmad.js +6 -1
- package/lib/templates/instruction-block-git.template.md +25 -25
- package/lib/templates/instruction-block-onprem.template.md +86 -86
- package/lib/templates/instruction-block-universal.template.md +29 -29
- package/package.json +3 -3
- package/lib/.bmad-extension-plugin.build-22612-1782454014750/.claude-plugin/marketplace.json +0 -109
- package/lib/.bmad-extension-plugin.build-22612-1782454014750/skills/module-help.csv +0 -62
- package/lib/.bmad-extension-plugin.build-22612-1782454014750/skills/module.yaml +0 -20
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-8f8b045fef5af6911495cf3b2a89f1ed75e120f7.idx +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-8f8b045fef5af6911495cf3b2a89f1ed75e120f7.rev +0 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/references/sample-init-sanctum.py +0 -274
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/generate-html-report.py +0 -534
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-execution-deps.py +0 -337
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-prompt-metrics.py +0 -425
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-sanctum-architecture.py +0 -385
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-structure-capabilities.py +0 -482
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/assets/Dockerfile +0 -29
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/docker_setup.py +0 -115
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/generate_report.py +0 -184
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/pty_runner.py +0 -171
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/utils.py +0 -260
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/extract-report-json.py +0 -287
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/generate-html-report.py +0 -588
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/prepass-execution-deps.py +0 -288
package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-structure-capabilities.py
DELETED
|
@@ -1,482 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Deterministic pre-pass for agent structure and capabilities scanner.
|
|
3
|
-
|
|
4
|
-
Extracts structural metadata from a BMad agent skill that the LLM scanner
|
|
5
|
-
can use instead of reading all files itself. Covers:
|
|
6
|
-
- Frontmatter parsing and validation
|
|
7
|
-
- Section inventory (H2/H3 headers)
|
|
8
|
-
- Template artifact detection
|
|
9
|
-
- Agent name validation (kebab-case, must contain 'agent')
|
|
10
|
-
- Required agent sections (stateless vs memory agent bootloader detection)
|
|
11
|
-
- Memory path consistency checking
|
|
12
|
-
- Language/directness pattern grep
|
|
13
|
-
- On Exit / Exiting section detection (invalid)
|
|
14
|
-
- Capability file scanning in references/ directory
|
|
15
|
-
"""
|
|
16
|
-
|
|
17
|
-
# /// script
|
|
18
|
-
# requires-python = ">=3.9"
|
|
19
|
-
# dependencies = [
|
|
20
|
-
# "pyyaml>=6.0",
|
|
21
|
-
# ]
|
|
22
|
-
# ///
|
|
23
|
-
|
|
24
|
-
from __future__ import annotations
|
|
25
|
-
|
|
26
|
-
import argparse
|
|
27
|
-
import json
|
|
28
|
-
import re
|
|
29
|
-
import sys
|
|
30
|
-
from datetime import datetime, timezone
|
|
31
|
-
from pathlib import Path
|
|
32
|
-
|
|
33
|
-
try:
|
|
34
|
-
import yaml
|
|
35
|
-
except ImportError:
|
|
36
|
-
print("Error: pyyaml required. Run with: uv run prepass-structure-capabilities.py", file=sys.stderr)
|
|
37
|
-
sys.exit(2)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
# Template artifacts that should NOT appear in finalized skills
|
|
41
|
-
TEMPLATE_ARTIFACTS = [
|
|
42
|
-
r'\{if-complex-workflow\}', r'\{/if-complex-workflow\}',
|
|
43
|
-
r'\{if-simple-workflow\}', r'\{/if-simple-workflow\}',
|
|
44
|
-
r'\{if-simple-utility\}', r'\{/if-simple-utility\}',
|
|
45
|
-
r'\{if-module\}', r'\{/if-module\}',
|
|
46
|
-
r'\{if-headless\}', r'\{/if-headless\}',
|
|
47
|
-
r'\{if-autonomous\}', r'\{/if-autonomous\}',
|
|
48
|
-
r'\{if-memory\}', r'\{/if-memory\}',
|
|
49
|
-
r'\{if-memory-agent\}', r'\{/if-memory-agent\}',
|
|
50
|
-
r'\{if-stateless-agent\}', r'\{/if-stateless-agent\}',
|
|
51
|
-
r'\{if-evolvable\}', r'\{/if-evolvable\}',
|
|
52
|
-
r'\{if-pulse\}', r'\{/if-pulse\}',
|
|
53
|
-
r'\{displayName\}', r'\{skillName\}',
|
|
54
|
-
]
|
|
55
|
-
# Runtime variables that ARE expected (not artifacts)
|
|
56
|
-
RUNTIME_VARS = {
|
|
57
|
-
'{user_name}', '{communication_language}', '{document_output_language}',
|
|
58
|
-
'{project-root}', '{output_folder}', '{planning_artifacts}',
|
|
59
|
-
'{headless_mode}',
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
# Directness anti-patterns
|
|
63
|
-
DIRECTNESS_PATTERNS = [
|
|
64
|
-
(r'\byou should\b', 'Suggestive "you should" — use direct imperative'),
|
|
65
|
-
(r'\bplease\b(?! note)', 'Polite "please" — use direct imperative'),
|
|
66
|
-
(r'\bhandle appropriately\b', 'Ambiguous "handle appropriately" — specify how'),
|
|
67
|
-
(r'\bwhen ready\b', 'Vague "when ready" — specify testable condition'),
|
|
68
|
-
]
|
|
69
|
-
|
|
70
|
-
# Invalid sections
|
|
71
|
-
INVALID_SECTIONS = [
|
|
72
|
-
(r'^##\s+On\s+Exit\b', 'On Exit section found — no exit hooks exist in the system, this will never run'),
|
|
73
|
-
(r'^##\s+Exiting\b', 'Exiting section found — no exit hooks exist in the system, this will never run'),
|
|
74
|
-
]
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def parse_frontmatter(content: str) -> tuple[dict | None, list[dict]]:
|
|
78
|
-
"""Parse YAML frontmatter and validate."""
|
|
79
|
-
findings = []
|
|
80
|
-
fm_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
|
|
81
|
-
if not fm_match:
|
|
82
|
-
findings.append({
|
|
83
|
-
'file': 'SKILL.md', 'line': 1,
|
|
84
|
-
'severity': 'critical', 'category': 'frontmatter',
|
|
85
|
-
'issue': 'No YAML frontmatter found',
|
|
86
|
-
})
|
|
87
|
-
return None, findings
|
|
88
|
-
|
|
89
|
-
try:
|
|
90
|
-
fm = yaml.safe_load(fm_match.group(1))
|
|
91
|
-
except yaml.YAMLError as e:
|
|
92
|
-
findings.append({
|
|
93
|
-
'file': 'SKILL.md', 'line': 1,
|
|
94
|
-
'severity': 'critical', 'category': 'frontmatter',
|
|
95
|
-
'issue': f'Invalid YAML frontmatter: {e}',
|
|
96
|
-
})
|
|
97
|
-
return None, findings
|
|
98
|
-
|
|
99
|
-
if not isinstance(fm, dict):
|
|
100
|
-
findings.append({
|
|
101
|
-
'file': 'SKILL.md', 'line': 1,
|
|
102
|
-
'severity': 'critical', 'category': 'frontmatter',
|
|
103
|
-
'issue': 'Frontmatter is not a YAML mapping',
|
|
104
|
-
})
|
|
105
|
-
return None, findings
|
|
106
|
-
|
|
107
|
-
# name check
|
|
108
|
-
name = fm.get('name')
|
|
109
|
-
if not name:
|
|
110
|
-
findings.append({
|
|
111
|
-
'file': 'SKILL.md', 'line': 1,
|
|
112
|
-
'severity': 'critical', 'category': 'frontmatter',
|
|
113
|
-
'issue': 'Missing "name" field in frontmatter',
|
|
114
|
-
})
|
|
115
|
-
elif not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name):
|
|
116
|
-
findings.append({
|
|
117
|
-
'file': 'SKILL.md', 'line': 1,
|
|
118
|
-
'severity': 'high', 'category': 'frontmatter',
|
|
119
|
-
'issue': f'Name "{name}" is not kebab-case',
|
|
120
|
-
})
|
|
121
|
-
elif 'agent' not in name.split('-'):
|
|
122
|
-
findings.append({
|
|
123
|
-
'file': 'SKILL.md', 'line': 1,
|
|
124
|
-
'severity': 'medium', 'category': 'frontmatter',
|
|
125
|
-
'issue': f'Name "{name}" should contain "agent" (e.g., agent-{{name}} or {{code}}-agent-{{name}})',
|
|
126
|
-
})
|
|
127
|
-
|
|
128
|
-
# description check
|
|
129
|
-
desc = fm.get('description')
|
|
130
|
-
if not desc:
|
|
131
|
-
findings.append({
|
|
132
|
-
'file': 'SKILL.md', 'line': 1,
|
|
133
|
-
'severity': 'high', 'category': 'frontmatter',
|
|
134
|
-
'issue': 'Missing "description" field in frontmatter',
|
|
135
|
-
})
|
|
136
|
-
elif 'Use when' not in desc and 'use when' not in desc:
|
|
137
|
-
findings.append({
|
|
138
|
-
'file': 'SKILL.md', 'line': 1,
|
|
139
|
-
'severity': 'medium', 'category': 'frontmatter',
|
|
140
|
-
'issue': 'Description missing "Use when..." trigger phrase',
|
|
141
|
-
})
|
|
142
|
-
|
|
143
|
-
# Extra fields check — only name and description allowed for agents
|
|
144
|
-
allowed = {'name', 'description'}
|
|
145
|
-
extra = set(fm.keys()) - allowed
|
|
146
|
-
if extra:
|
|
147
|
-
findings.append({
|
|
148
|
-
'file': 'SKILL.md', 'line': 1,
|
|
149
|
-
'severity': 'low', 'category': 'frontmatter',
|
|
150
|
-
'issue': f'Extra frontmatter fields: {", ".join(sorted(extra))}',
|
|
151
|
-
})
|
|
152
|
-
|
|
153
|
-
return fm, findings
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
def extract_sections(content: str) -> list[dict]:
|
|
157
|
-
"""Extract all H2/H3 headers with line numbers."""
|
|
158
|
-
sections = []
|
|
159
|
-
for i, line in enumerate(content.split('\n'), 1):
|
|
160
|
-
m = re.match(r'^(#{2,3})\s+(.+)$', line)
|
|
161
|
-
if m:
|
|
162
|
-
sections.append({
|
|
163
|
-
'level': len(m.group(1)),
|
|
164
|
-
'title': m.group(2).strip(),
|
|
165
|
-
'line': i,
|
|
166
|
-
})
|
|
167
|
-
return sections
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
def detect_memory_agent(skill_path: Path, content: str) -> bool:
|
|
171
|
-
"""Detect if this is a memory agent bootloader (vs stateless agent).
|
|
172
|
-
|
|
173
|
-
Memory agents have assets/ with sanctum template files and contain
|
|
174
|
-
Three Laws / Sacred Truth in their SKILL.md.
|
|
175
|
-
"""
|
|
176
|
-
assets_dir = skill_path / 'assets'
|
|
177
|
-
has_templates = (
|
|
178
|
-
assets_dir.exists()
|
|
179
|
-
and any(f.name.endswith('-template.md') for f in assets_dir.iterdir() if f.is_file())
|
|
180
|
-
)
|
|
181
|
-
has_three_laws = 'First Law:' in content and 'Second Law:' in content
|
|
182
|
-
has_sacred_truth = 'Sacred Truth' in content
|
|
183
|
-
return has_templates or (has_three_laws and has_sacred_truth)
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
def check_required_sections(sections: list[dict], is_memory_agent: bool) -> list[dict]:
|
|
187
|
-
"""Check for required and invalid sections."""
|
|
188
|
-
findings = []
|
|
189
|
-
h2_titles = [s['title'] for s in sections if s['level'] == 2]
|
|
190
|
-
|
|
191
|
-
if is_memory_agent:
|
|
192
|
-
# Memory agent bootloaders have a different required structure
|
|
193
|
-
required = ['The Three Laws', 'The Sacred Truth', 'On Activation']
|
|
194
|
-
for req in required:
|
|
195
|
-
if req not in h2_titles:
|
|
196
|
-
findings.append({
|
|
197
|
-
'file': 'SKILL.md', 'line': 1,
|
|
198
|
-
'severity': 'high', 'category': 'sections',
|
|
199
|
-
'issue': f'Missing ## {req} section (required for memory agent bootloader)',
|
|
200
|
-
})
|
|
201
|
-
else:
|
|
202
|
-
# Stateless agents use the traditional full structure
|
|
203
|
-
required = ['Overview', 'Identity', 'Communication Style', 'Principles', 'On Activation']
|
|
204
|
-
for req in required:
|
|
205
|
-
if req not in h2_titles:
|
|
206
|
-
findings.append({
|
|
207
|
-
'file': 'SKILL.md', 'line': 1,
|
|
208
|
-
'severity': 'high', 'category': 'sections',
|
|
209
|
-
'issue': f'Missing ## {req} section',
|
|
210
|
-
})
|
|
211
|
-
|
|
212
|
-
# Invalid sections (both types)
|
|
213
|
-
for s in sections:
|
|
214
|
-
if s['level'] == 2:
|
|
215
|
-
for pattern, message in INVALID_SECTIONS:
|
|
216
|
-
if re.match(pattern, f"## {s['title']}"):
|
|
217
|
-
findings.append({
|
|
218
|
-
'file': 'SKILL.md', 'line': s['line'],
|
|
219
|
-
'severity': 'high', 'category': 'invalid-section',
|
|
220
|
-
'issue': message,
|
|
221
|
-
})
|
|
222
|
-
|
|
223
|
-
return findings
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
def find_template_artifacts(filepath: Path, rel_path: str) -> list[dict]:
|
|
227
|
-
"""Scan for orphaned template substitution artifacts."""
|
|
228
|
-
findings = []
|
|
229
|
-
content = filepath.read_text(encoding='utf-8')
|
|
230
|
-
|
|
231
|
-
for pattern in TEMPLATE_ARTIFACTS:
|
|
232
|
-
for m in re.finditer(pattern, content):
|
|
233
|
-
matched = m.group()
|
|
234
|
-
if matched in RUNTIME_VARS:
|
|
235
|
-
continue
|
|
236
|
-
line_num = content[:m.start()].count('\n') + 1
|
|
237
|
-
findings.append({
|
|
238
|
-
'file': rel_path, 'line': line_num,
|
|
239
|
-
'severity': 'high', 'category': 'artifacts',
|
|
240
|
-
'issue': f'Orphaned template artifact: {matched}',
|
|
241
|
-
'fix': 'Resolve or remove this template conditional/placeholder',
|
|
242
|
-
})
|
|
243
|
-
|
|
244
|
-
return findings
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
def extract_memory_paths(skill_path: Path) -> tuple[list[str], list[dict]]:
|
|
248
|
-
"""Extract all memory path references across files and check consistency."""
|
|
249
|
-
findings = []
|
|
250
|
-
memory_paths = set()
|
|
251
|
-
|
|
252
|
-
# Memory path patterns
|
|
253
|
-
mem_pattern = re.compile(r'memory/[\w\-/]+(?:\.\w+)?')
|
|
254
|
-
|
|
255
|
-
files_to_scan = []
|
|
256
|
-
|
|
257
|
-
skill_md = skill_path / 'SKILL.md'
|
|
258
|
-
if skill_md.exists():
|
|
259
|
-
files_to_scan.append(('SKILL.md', skill_md))
|
|
260
|
-
|
|
261
|
-
for subdir in ['prompts', 'resources', 'references']:
|
|
262
|
-
d = skill_path / subdir
|
|
263
|
-
if d.exists():
|
|
264
|
-
for f in sorted(d.iterdir()):
|
|
265
|
-
if f.is_file() and f.suffix in ('.md', '.json', '.yaml', '.yml'):
|
|
266
|
-
files_to_scan.append((f'{subdir}/{f.name}', f))
|
|
267
|
-
|
|
268
|
-
for rel_path, filepath in files_to_scan:
|
|
269
|
-
content = filepath.read_text(encoding='utf-8')
|
|
270
|
-
for m in mem_pattern.finditer(content):
|
|
271
|
-
memory_paths.add(m.group())
|
|
272
|
-
|
|
273
|
-
sorted_paths = sorted(memory_paths)
|
|
274
|
-
|
|
275
|
-
# Check for inconsistent formats
|
|
276
|
-
prefixes = set()
|
|
277
|
-
for p in sorted_paths:
|
|
278
|
-
prefix = p.split('/')[0]
|
|
279
|
-
prefixes.add(prefix)
|
|
280
|
-
|
|
281
|
-
memory_prefixes = {p for p in prefixes if 'memory' in p.lower()}
|
|
282
|
-
|
|
283
|
-
if len(memory_prefixes) > 1:
|
|
284
|
-
findings.append({
|
|
285
|
-
'file': 'multiple', 'line': 0,
|
|
286
|
-
'severity': 'medium', 'category': 'memory-paths',
|
|
287
|
-
'issue': f'Inconsistent memory path prefixes: {", ".join(sorted(memory_prefixes))}',
|
|
288
|
-
})
|
|
289
|
-
|
|
290
|
-
return sorted_paths, findings
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
def check_prompt_basics(skill_path: Path) -> tuple[list[dict], list[dict]]:
|
|
294
|
-
"""Check each prompt file for config header and progression conditions."""
|
|
295
|
-
findings = []
|
|
296
|
-
prompt_details = []
|
|
297
|
-
skip_files = {'SKILL.md'}
|
|
298
|
-
|
|
299
|
-
prompt_files = [f for f in sorted(skill_path.iterdir())
|
|
300
|
-
if f.is_file() and f.suffix == '.md' and f.name not in skip_files]
|
|
301
|
-
|
|
302
|
-
# Also scan references/ for capability prompts (memory agents keep prompts here)
|
|
303
|
-
refs_dir = skill_path / 'references'
|
|
304
|
-
if refs_dir.exists():
|
|
305
|
-
prompt_files.extend(
|
|
306
|
-
f for f in sorted(refs_dir.iterdir())
|
|
307
|
-
if f.is_file() and f.suffix == '.md'
|
|
308
|
-
)
|
|
309
|
-
|
|
310
|
-
if not prompt_files:
|
|
311
|
-
return prompt_details, findings
|
|
312
|
-
|
|
313
|
-
for f in prompt_files:
|
|
314
|
-
content = f.read_text(encoding='utf-8')
|
|
315
|
-
rel_path = f.name
|
|
316
|
-
detail = {'file': f.name, 'has_config_header': False, 'has_progression': False}
|
|
317
|
-
|
|
318
|
-
# Config header check
|
|
319
|
-
if '{communication_language}' in content or '{document_output_language}' in content:
|
|
320
|
-
detail['has_config_header'] = True
|
|
321
|
-
else:
|
|
322
|
-
findings.append({
|
|
323
|
-
'file': rel_path, 'line': 1,
|
|
324
|
-
'severity': 'medium', 'category': 'config-header',
|
|
325
|
-
'issue': 'No config header with language variables found',
|
|
326
|
-
})
|
|
327
|
-
|
|
328
|
-
# Progression condition check
|
|
329
|
-
lower = content.lower()
|
|
330
|
-
prog_keywords = ['progress', 'advance', 'move to', 'next stage', 'when complete',
|
|
331
|
-
'proceed to', 'transition', 'completion criteria']
|
|
332
|
-
if any(kw in lower for kw in prog_keywords):
|
|
333
|
-
detail['has_progression'] = True
|
|
334
|
-
else:
|
|
335
|
-
findings.append({
|
|
336
|
-
'file': rel_path, 'line': len(content.split('\n')),
|
|
337
|
-
'severity': 'high', 'category': 'progression',
|
|
338
|
-
'issue': 'No progression condition keywords found',
|
|
339
|
-
})
|
|
340
|
-
|
|
341
|
-
# Directness checks
|
|
342
|
-
for pattern, message in DIRECTNESS_PATTERNS:
|
|
343
|
-
for m in re.finditer(pattern, content, re.IGNORECASE):
|
|
344
|
-
line_num = content[:m.start()].count('\n') + 1
|
|
345
|
-
findings.append({
|
|
346
|
-
'file': rel_path, 'line': line_num,
|
|
347
|
-
'severity': 'low', 'category': 'language',
|
|
348
|
-
'issue': message,
|
|
349
|
-
})
|
|
350
|
-
|
|
351
|
-
# Template artifacts
|
|
352
|
-
findings.extend(find_template_artifacts(f, rel_path))
|
|
353
|
-
|
|
354
|
-
prompt_details.append(detail)
|
|
355
|
-
|
|
356
|
-
return prompt_details, findings
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
def scan_structure_capabilities(skill_path: Path) -> dict:
|
|
360
|
-
"""Run all deterministic agent structure and capability checks."""
|
|
361
|
-
all_findings = []
|
|
362
|
-
|
|
363
|
-
# Read SKILL.md
|
|
364
|
-
skill_md = skill_path / 'SKILL.md'
|
|
365
|
-
if not skill_md.exists():
|
|
366
|
-
return {
|
|
367
|
-
'scanner': 'structure-capabilities-prepass',
|
|
368
|
-
'script': 'prepass-structure-capabilities.py',
|
|
369
|
-
'version': '1.0.0',
|
|
370
|
-
'skill_path': str(skill_path),
|
|
371
|
-
'timestamp': datetime.now(timezone.utc).isoformat(),
|
|
372
|
-
'status': 'fail',
|
|
373
|
-
'issues': [{'file': 'SKILL.md', 'line': 1, 'severity': 'critical',
|
|
374
|
-
'category': 'missing-file', 'issue': 'SKILL.md does not exist'}],
|
|
375
|
-
'summary': {'total_issues': 1, 'by_severity': {'critical': 1, 'high': 0, 'medium': 0, 'low': 0}},
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
skill_content = skill_md.read_text(encoding='utf-8')
|
|
379
|
-
|
|
380
|
-
# Detect agent type
|
|
381
|
-
is_memory_agent = detect_memory_agent(skill_path, skill_content)
|
|
382
|
-
|
|
383
|
-
# Frontmatter
|
|
384
|
-
frontmatter, fm_findings = parse_frontmatter(skill_content)
|
|
385
|
-
all_findings.extend(fm_findings)
|
|
386
|
-
|
|
387
|
-
# Sections
|
|
388
|
-
sections = extract_sections(skill_content)
|
|
389
|
-
section_findings = check_required_sections(sections, is_memory_agent)
|
|
390
|
-
all_findings.extend(section_findings)
|
|
391
|
-
|
|
392
|
-
# Template artifacts in SKILL.md
|
|
393
|
-
all_findings.extend(find_template_artifacts(skill_md, 'SKILL.md'))
|
|
394
|
-
|
|
395
|
-
# Directness checks in SKILL.md
|
|
396
|
-
for pattern, message in DIRECTNESS_PATTERNS:
|
|
397
|
-
for m in re.finditer(pattern, skill_content, re.IGNORECASE):
|
|
398
|
-
line_num = skill_content[:m.start()].count('\n') + 1
|
|
399
|
-
all_findings.append({
|
|
400
|
-
'file': 'SKILL.md', 'line': line_num,
|
|
401
|
-
'severity': 'low', 'category': 'language',
|
|
402
|
-
'issue': message,
|
|
403
|
-
})
|
|
404
|
-
|
|
405
|
-
# Memory path consistency
|
|
406
|
-
memory_paths, memory_findings = extract_memory_paths(skill_path)
|
|
407
|
-
all_findings.extend(memory_findings)
|
|
408
|
-
|
|
409
|
-
# Prompt basics
|
|
410
|
-
prompt_details, prompt_findings = check_prompt_basics(skill_path)
|
|
411
|
-
all_findings.extend(prompt_findings)
|
|
412
|
-
|
|
413
|
-
# Build severity summary
|
|
414
|
-
by_severity = {'critical': 0, 'high': 0, 'medium': 0, 'low': 0}
|
|
415
|
-
for f in all_findings:
|
|
416
|
-
sev = f['severity']
|
|
417
|
-
if sev in by_severity:
|
|
418
|
-
by_severity[sev] += 1
|
|
419
|
-
|
|
420
|
-
status = 'pass'
|
|
421
|
-
if by_severity['critical'] > 0:
|
|
422
|
-
status = 'fail'
|
|
423
|
-
elif by_severity['high'] > 0:
|
|
424
|
-
status = 'warning'
|
|
425
|
-
|
|
426
|
-
return {
|
|
427
|
-
'scanner': 'structure-capabilities-prepass',
|
|
428
|
-
'script': 'prepass-structure-capabilities.py',
|
|
429
|
-
'version': '1.0.0',
|
|
430
|
-
'skill_path': str(skill_path),
|
|
431
|
-
'timestamp': datetime.now(timezone.utc).isoformat(),
|
|
432
|
-
'status': status,
|
|
433
|
-
'metadata': {
|
|
434
|
-
'frontmatter': frontmatter,
|
|
435
|
-
'sections': sections,
|
|
436
|
-
'is_memory_agent': is_memory_agent,
|
|
437
|
-
},
|
|
438
|
-
'prompt_details': prompt_details,
|
|
439
|
-
'memory_paths': memory_paths,
|
|
440
|
-
'issues': all_findings,
|
|
441
|
-
'summary': {
|
|
442
|
-
'total_issues': len(all_findings),
|
|
443
|
-
'by_severity': by_severity,
|
|
444
|
-
},
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
def main() -> int:
|
|
449
|
-
parser = argparse.ArgumentParser(
|
|
450
|
-
description='Deterministic pre-pass for agent structure and capabilities scanning',
|
|
451
|
-
)
|
|
452
|
-
parser.add_argument(
|
|
453
|
-
'skill_path',
|
|
454
|
-
type=Path,
|
|
455
|
-
help='Path to the skill directory to scan',
|
|
456
|
-
)
|
|
457
|
-
parser.add_argument(
|
|
458
|
-
'--output', '-o',
|
|
459
|
-
type=Path,
|
|
460
|
-
help='Write JSON output to file instead of stdout',
|
|
461
|
-
)
|
|
462
|
-
args = parser.parse_args()
|
|
463
|
-
|
|
464
|
-
if not args.skill_path.is_dir():
|
|
465
|
-
print(f"Error: {args.skill_path} is not a directory", file=sys.stderr)
|
|
466
|
-
return 2
|
|
467
|
-
|
|
468
|
-
result = scan_structure_capabilities(args.skill_path)
|
|
469
|
-
output = json.dumps(result, indent=2)
|
|
470
|
-
|
|
471
|
-
if args.output:
|
|
472
|
-
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
473
|
-
args.output.write_text(output)
|
|
474
|
-
print(f"Results written to {args.output}", file=sys.stderr)
|
|
475
|
-
else:
|
|
476
|
-
print(output)
|
|
477
|
-
|
|
478
|
-
return 0 if result['status'] == 'pass' else 1
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
if __name__ == '__main__':
|
|
482
|
-
sys.exit(main())
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
FROM node:20-bookworm-slim
|
|
2
|
-
|
|
3
|
-
ENV DEBIAN_FRONTEND=noninteractive
|
|
4
|
-
|
|
5
|
-
RUN apt-get update \
|
|
6
|
-
&& apt-get install -y --no-install-recommends \
|
|
7
|
-
git \
|
|
8
|
-
python3 \
|
|
9
|
-
python3-pip \
|
|
10
|
-
ca-certificates \
|
|
11
|
-
curl \
|
|
12
|
-
jq \
|
|
13
|
-
rsync \
|
|
14
|
-
&& rm -rf /var/lib/apt/lists/*
|
|
15
|
-
|
|
16
|
-
RUN npm install -g @anthropic-ai/claude-code
|
|
17
|
-
|
|
18
|
-
RUN useradd -ms /bin/bash evaluator \
|
|
19
|
-
&& mkdir -p /workspace /project /output /home/evaluator/.claude \
|
|
20
|
-
&& chown -R evaluator:evaluator /workspace /output /home/evaluator
|
|
21
|
-
|
|
22
|
-
USER evaluator
|
|
23
|
-
WORKDIR /workspace
|
|
24
|
-
|
|
25
|
-
ENV HOME=/home/evaluator
|
|
26
|
-
ENV CLAUDE_CONFIG_DIR=/home/evaluator/.claude
|
|
27
|
-
ENV PATH=/home/evaluator/.local/bin:$PATH
|
|
28
|
-
|
|
29
|
-
CMD ["bash"]
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# /// script
|
|
3
|
-
# requires-python = ">=3.9"
|
|
4
|
-
# ///
|
|
5
|
-
"""Detect Docker and build the bmad-eval-runner image when needed.
|
|
6
|
-
|
|
7
|
-
Usage:
|
|
8
|
-
python3 docker_setup.py --check # exit 0 if image is ready, 1 otherwise
|
|
9
|
-
python3 docker_setup.py --build # build the image (no-op if present)
|
|
10
|
-
python3 docker_setup.py --rebuild # force rebuild
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
from __future__ import annotations
|
|
14
|
-
|
|
15
|
-
import argparse
|
|
16
|
-
import json
|
|
17
|
-
import shutil
|
|
18
|
-
import subprocess
|
|
19
|
-
import sys
|
|
20
|
-
from pathlib import Path
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
IMAGE_TAG = "bmad-eval-runner:latest"
|
|
24
|
-
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
25
|
-
DOCKERFILE = SCRIPT_DIR.parent / "assets" / "Dockerfile"
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def docker_available() -> tuple[bool, str]:
|
|
29
|
-
if shutil.which("docker") is None:
|
|
30
|
-
return False, "docker CLI not found on PATH"
|
|
31
|
-
try:
|
|
32
|
-
result = subprocess.run(
|
|
33
|
-
["docker", "info"],
|
|
34
|
-
capture_output=True,
|
|
35
|
-
text=True,
|
|
36
|
-
timeout=5,
|
|
37
|
-
)
|
|
38
|
-
if result.returncode != 0:
|
|
39
|
-
return False, f"`docker info` failed: {result.stderr.strip().splitlines()[-1] if result.stderr.strip() else 'unknown'}"
|
|
40
|
-
return True, "ok"
|
|
41
|
-
except subprocess.TimeoutExpired:
|
|
42
|
-
return False, "`docker info` timed out"
|
|
43
|
-
except Exception as e:
|
|
44
|
-
return False, f"docker check error: {e}"
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def image_present(tag: str = IMAGE_TAG) -> bool:
|
|
48
|
-
try:
|
|
49
|
-
result = subprocess.run(
|
|
50
|
-
["docker", "image", "inspect", tag],
|
|
51
|
-
stdout=subprocess.DEVNULL,
|
|
52
|
-
stderr=subprocess.DEVNULL,
|
|
53
|
-
timeout=10,
|
|
54
|
-
)
|
|
55
|
-
return result.returncode == 0
|
|
56
|
-
except Exception:
|
|
57
|
-
return False
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
def build_image(tag: str = IMAGE_TAG, force: bool = False, verbose: bool = True) -> int:
|
|
61
|
-
if not DOCKERFILE.is_file():
|
|
62
|
-
print(f"Dockerfile missing at {DOCKERFILE}", file=sys.stderr)
|
|
63
|
-
return 2
|
|
64
|
-
|
|
65
|
-
cmd = ["docker", "build", "-t", tag, "-f", str(DOCKERFILE), str(DOCKERFILE.parent)]
|
|
66
|
-
if force:
|
|
67
|
-
cmd.insert(2, "--no-cache")
|
|
68
|
-
|
|
69
|
-
if verbose:
|
|
70
|
-
print(f"Building {tag} from {DOCKERFILE} ...", file=sys.stderr)
|
|
71
|
-
|
|
72
|
-
proc = subprocess.run(cmd, stdout=sys.stderr if verbose else subprocess.DEVNULL, stderr=sys.stderr)
|
|
73
|
-
return proc.returncode
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
def main() -> int:
|
|
77
|
-
parser = argparse.ArgumentParser(description="Manage the bmad-eval-runner Docker image")
|
|
78
|
-
group = parser.add_mutually_exclusive_group(required=True)
|
|
79
|
-
group.add_argument("--check", action="store_true", help="Report status as JSON; exit 0 if image is ready")
|
|
80
|
-
group.add_argument("--build", action="store_true", help="Build the image (no-op if already present)")
|
|
81
|
-
group.add_argument("--rebuild", action="store_true", help="Force rebuild")
|
|
82
|
-
parser.add_argument("--quiet", action="store_true")
|
|
83
|
-
args = parser.parse_args()
|
|
84
|
-
|
|
85
|
-
available, reason = docker_available()
|
|
86
|
-
present = image_present() if available else False
|
|
87
|
-
|
|
88
|
-
if args.check:
|
|
89
|
-
print(json.dumps({
|
|
90
|
-
"docker_available": available,
|
|
91
|
-
"docker_reason": reason,
|
|
92
|
-
"image_present": present,
|
|
93
|
-
"image_tag": IMAGE_TAG,
|
|
94
|
-
}, indent=2))
|
|
95
|
-
return 0 if (available and present) else 1
|
|
96
|
-
|
|
97
|
-
if not available:
|
|
98
|
-
print(f"Docker is not available: {reason}", file=sys.stderr)
|
|
99
|
-
return 3
|
|
100
|
-
|
|
101
|
-
if args.rebuild:
|
|
102
|
-
return build_image(force=True, verbose=not args.quiet)
|
|
103
|
-
|
|
104
|
-
if args.build:
|
|
105
|
-
if present:
|
|
106
|
-
if not args.quiet:
|
|
107
|
-
print(f"{IMAGE_TAG} already present; skipping build (use --rebuild to force).", file=sys.stderr)
|
|
108
|
-
return 0
|
|
109
|
-
return build_image(force=False, verbose=not args.quiet)
|
|
110
|
-
|
|
111
|
-
return 0
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
if __name__ == "__main__":
|
|
115
|
-
sys.exit(main())
|