ma-agents 3.13.2 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agents.js +7 -0
- package/lib/bmad-cache/bmb/.claude-plugin/marketplace.json +3 -3
- package/lib/bmad-cache/bmb/_git_preserved/index +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-b388526edf307cf56f8e65673466cff074105abf.idx +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/{pack-8f8b045fef5af6911495cf3b2a89f1ed75e120f7.pack → pack-b388526edf307cf56f8e65673466cff074105abf.pack} +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-b388526edf307cf56f8e65673466cff074105abf.rev +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/packed-refs +1 -1
- package/lib/bmad-cache/bmb/_git_preserved/refs/heads/main +1 -1
- package/lib/bmad-cache/bmb/_git_preserved/shallow +1 -1
- package/lib/bmad-cache/bmb/package-lock.json +2 -2
- package/lib/bmad-cache/bmb/package.json +1 -1
- package/lib/bmad-cache/bmb/samples/bmad-agent-code-coach/scripts/wake.py +78 -0
- package/lib/bmad-cache/bmb/samples/bmad-agent-creative-muse/scripts/wake.py +78 -0
- package/lib/bmad-cache/bmb/samples/bmad-agent-sentinel/scripts/wake.py +78 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/customize-template.toml +3 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/init-sanctum-template.py +11 -5
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/report-shell.html +1073 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/assets/wake-template.py +78 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/customize.toml +48 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/count_tokens.py +78 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass.py +258 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/process-template.py +20 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/render_report.py +387 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/assets/adapter-claude-code.json +9 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/aggregate_benchmark.py +236 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/memlog.py +197 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/run_evals.py +513 -390
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/run_triggers.py +347 -251
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/tests/test_env_isolation.py +79 -0
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/tests/test_trigger_detection.py +123 -0
- package/lib/bmad-cache/bmb/skills/bmad-module-builder/scripts/tests/test-validate-module.py +82 -4
- package/lib/bmad-cache/bmb/skills/bmad-module-builder/scripts/validate-module.py +55 -11
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/assets/customize-template.toml +3 -5
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/assets/report-shell.html +860 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/customize.toml +59 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/count_tokens.py +77 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/init_skill.py +132 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/prepass-prompt-metrics.py +186 -221
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/quick_validate.py +117 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/render_report.py +387 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/tests/test_canon_sync.py +46 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/tests/test_count_tokens.py +180 -0
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/tests/test_render_report.py +171 -0
- package/lib/bmad-cache/cache-manifest.json +4 -4
- package/lib/bmad-extension/.claude-plugin/marketplace.json.template +1 -1
- package/lib/bmad-extension-plugin/.claude-plugin/marketplace.json +2 -2
- package/lib/bmad.js +6 -1
- package/lib/templates/instruction-block-git.template.md +25 -25
- package/lib/templates/instruction-block-onprem.template.md +86 -86
- package/lib/templates/instruction-block-universal.template.md +29 -29
- package/package.json +3 -3
- package/lib/.bmad-extension-plugin.build-22612-1782454014750/.claude-plugin/marketplace.json +0 -109
- package/lib/.bmad-extension-plugin.build-22612-1782454014750/skills/module-help.csv +0 -62
- package/lib/.bmad-extension-plugin.build-22612-1782454014750/skills/module.yaml +0 -20
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-8f8b045fef5af6911495cf3b2a89f1ed75e120f7.idx +0 -0
- package/lib/bmad-cache/bmb/_git_preserved/objects/pack/pack-8f8b045fef5af6911495cf3b2a89f1ed75e120f7.rev +0 -0
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/references/sample-init-sanctum.py +0 -274
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/generate-html-report.py +0 -534
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-execution-deps.py +0 -337
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-prompt-metrics.py +0 -425
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-sanctum-architecture.py +0 -385
- package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/prepass-structure-capabilities.py +0 -482
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/assets/Dockerfile +0 -29
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/docker_setup.py +0 -115
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/generate_report.py +0 -184
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/pty_runner.py +0 -171
- package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/utils.py +0 -260
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/extract-report-json.py +0 -287
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/generate-html-report.py +0 -588
- package/lib/bmad-cache/bmb/skills/bmad-workflow-builder/scripts/prepass-execution-deps.py +0 -288
|
@@ -1,24 +1,28 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""Deterministic pre-pass for prompt craft scanner.
|
|
3
|
-
|
|
4
|
-
Extracts metrics and flagged patterns from SKILL.md and prompt files
|
|
5
|
-
so the LLM scanner can work from compact data instead of reading raw files.
|
|
6
|
-
|
|
7
|
-
Covers:
|
|
8
|
-
- SKILL.md line count and section inventory
|
|
9
|
-
- Overview section size
|
|
10
|
-
- Inline data detection (tables, fenced code blocks)
|
|
11
|
-
- Defensive padding pattern grep
|
|
12
|
-
- Meta-explanation pattern grep
|
|
13
|
-
- Back-reference detection ("as described above")
|
|
14
|
-
- Config header and progression condition presence per prompt
|
|
15
|
-
- File-level token estimates (chars / 4 rough approximation)
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
2
|
# /// script
|
|
19
3
|
# requires-python = ">=3.9"
|
|
4
|
+
# dependencies = ["tiktoken"]
|
|
20
5
|
# ///
|
|
6
|
+
"""Deterministic prompt-metrics pre-pass for the Analyze scanners.
|
|
7
|
+
|
|
8
|
+
Reads SKILL.md, root-level prompt files, and references, and emits one compact
|
|
9
|
+
JSON object the LLM scanners read instead of the raw files. Length is reported
|
|
10
|
+
as tiktoken token counts via count_tokens (cl100k_base, chars//4 fallback);
|
|
11
|
+
there is no line-count gate anywhere in this script.
|
|
12
|
+
|
|
13
|
+
What it surfaces per file:
|
|
14
|
+
- token count and the counting method (tiktoken or fallback)
|
|
15
|
+
- frontmatter facts (name, description, description length, angle-bracket flag)
|
|
16
|
+
- section inventory (heading level + title)
|
|
17
|
+
- structural signals scanners care about: tables, fenced blocks, defensive
|
|
18
|
+
padding, meta-explanation, back-references, config header, progression cues
|
|
21
19
|
|
|
20
|
+
Budgets the scanners compare against: SKILL.md ~1500-2500 tokens,
|
|
21
|
+
multi-branch reference ~4500, single-purpose reference ~9000.
|
|
22
|
+
|
|
23
|
+
Usage:
|
|
24
|
+
prepass-prompt-metrics.py <skill-dir> [--output FILE]
|
|
25
|
+
"""
|
|
22
26
|
from __future__ import annotations
|
|
23
27
|
|
|
24
28
|
import argparse
|
|
@@ -28,258 +32,219 @@ import sys
|
|
|
28
32
|
from datetime import datetime, timezone
|
|
29
33
|
from pathlib import Path
|
|
30
34
|
|
|
35
|
+
# Reuse the single length metric rather than reimplementing token counting.
|
|
36
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
37
|
+
try:
|
|
38
|
+
from count_tokens import count_tokens
|
|
39
|
+
except Exception: # pragma: no cover - count_tokens ships alongside this script
|
|
40
|
+
def count_tokens(text: str) -> tuple[int, str]:
|
|
41
|
+
return len(text) // 4, "fallback"
|
|
42
|
+
|
|
31
43
|
|
|
32
|
-
# Defensive padding / filler patterns
|
|
33
44
|
WASTE_PATTERNS = [
|
|
34
|
-
(r
|
|
35
|
-
(r"\b[Dd]on'?t forget (?:to|that)\b",
|
|
36
|
-
(r
|
|
37
|
-
(r
|
|
38
|
-
(r
|
|
39
|
-
(r
|
|
40
|
-
(r
|
|
41
|
-
(r
|
|
42
|
-
(r
|
|
43
|
-
(r
|
|
44
|
-
(r
|
|
45
|
-
(r"\b[Ll]et'?s (?:think about|begin|start)\b", 'filler', "Filler: \"let's think/begin\""),
|
|
46
|
-
(r'\b[Nn]ow we(?:\'ll| will)\b', 'filler', "Filler: \"now we'll\""),
|
|
45
|
+
(r"\b[Mm]ake sure (?:to|you)\b", "defensive-padding", 'Defensive: "make sure to/you"'),
|
|
46
|
+
(r"\b[Dd]on'?t forget (?:to|that)\b", "defensive-padding", 'Defensive: "don\'t forget"'),
|
|
47
|
+
(r"\b[Rr]emember (?:to|that)\b", "defensive-padding", 'Defensive: "remember to/that"'),
|
|
48
|
+
(r"\b[Bb]e sure to\b", "defensive-padding", 'Defensive: "be sure to"'),
|
|
49
|
+
(r"\b[Pp]lease ensure\b", "defensive-padding", 'Defensive: "please ensure"'),
|
|
50
|
+
(r"\b[Ii]t is important (?:to|that)\b", "defensive-padding", 'Defensive: "it is important"'),
|
|
51
|
+
(r"\b[Yy]ou are an AI\b", "meta-explanation", 'Meta: "you are an AI"'),
|
|
52
|
+
(r"\b[Aa]s a language model\b", "meta-explanation", 'Meta: "as a language model"'),
|
|
53
|
+
(r"\b[Aa]s an AI assistant\b", "meta-explanation", 'Meta: "as an AI assistant"'),
|
|
54
|
+
(r"\b[Tt]his (?:workflow|skill|process) is designed to\b", "meta-explanation", 'Meta: "this is designed to"'),
|
|
55
|
+
(r"\b[Tt]he purpose of this (?:section|step) is\b", "meta-explanation", 'Meta: "the purpose of this is"'),
|
|
47
56
|
]
|
|
48
57
|
|
|
49
|
-
# Back-reference patterns (self-containment risk)
|
|
50
58
|
BACKREF_PATTERNS = [
|
|
51
|
-
(r
|
|
52
|
-
(r
|
|
53
|
-
(r
|
|
54
|
-
(r
|
|
55
|
-
(r'\brefer to (?:the )?(?:above|overview|SKILL)\b', 'Back-reference: "refer to above/overview"'),
|
|
59
|
+
(r"\bas described above\b", 'Back-reference: "as described above"'),
|
|
60
|
+
(r"\bas mentioned (?:above|in|earlier)\b", 'Back-reference: "as mentioned above/earlier"'),
|
|
61
|
+
(r"\bsee (?:above|the overview)\b", 'Back-reference: "see above/the overview"'),
|
|
62
|
+
(r"\brefer to (?:the )?(?:above|overview|SKILL)\b", 'Back-reference: "refer to above/overview"'),
|
|
56
63
|
]
|
|
57
64
|
|
|
65
|
+
ALLCAPS_PATTERN = re.compile(r"\b(?:ALWAYS|NEVER|MUST|DO NOT|CRITICAL|REQUIRED)\b")
|
|
66
|
+
NUMBERED_PREFIX = re.compile(r"^\d{2}[-_]")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def split_frontmatter(content: str) -> tuple[dict, str]:
|
|
70
|
+
"""Return (frontmatter dict, body). Empty dict when there is no frontmatter."""
|
|
71
|
+
lines = content.splitlines()
|
|
72
|
+
if not lines or lines[0].strip() != "---":
|
|
73
|
+
return {}, content
|
|
74
|
+
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
|
75
|
+
if end is None:
|
|
76
|
+
return {}, content
|
|
77
|
+
meta: dict[str, str] = {}
|
|
78
|
+
for line in lines[1:end]:
|
|
79
|
+
if ":" in line:
|
|
80
|
+
k, v = line.split(":", 1)
|
|
81
|
+
meta[k.strip()] = v.strip()
|
|
82
|
+
return meta, "\n".join(lines[end + 1:])
|
|
83
|
+
|
|
58
84
|
|
|
59
85
|
def count_tables(content: str) -> tuple[int, int]:
|
|
60
|
-
|
|
61
|
-
table_count = 0
|
|
62
|
-
table_lines = 0
|
|
86
|
+
count = rows = 0
|
|
63
87
|
in_table = False
|
|
64
|
-
for line in content.split(
|
|
65
|
-
if
|
|
88
|
+
for line in content.split("\n"):
|
|
89
|
+
if re.match(r"^\s*\|", line):
|
|
66
90
|
if not in_table:
|
|
67
|
-
|
|
91
|
+
count += 1
|
|
68
92
|
in_table = True
|
|
69
|
-
|
|
93
|
+
rows += 1
|
|
70
94
|
else:
|
|
71
95
|
in_table = False
|
|
72
|
-
return
|
|
96
|
+
return count, rows
|
|
73
97
|
|
|
74
98
|
|
|
75
|
-
def
|
|
76
|
-
|
|
77
|
-
block_count = 0
|
|
78
|
-
block_lines = 0
|
|
99
|
+
def count_fenced(content: str) -> int:
|
|
100
|
+
blocks = 0
|
|
79
101
|
in_block = False
|
|
80
|
-
for line in content.split(
|
|
81
|
-
if line.strip().startswith(
|
|
102
|
+
for line in content.split("\n"):
|
|
103
|
+
if line.strip().startswith("```"):
|
|
104
|
+
in_block = not in_block
|
|
82
105
|
if in_block:
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
line_num = content[:m.start()].count('\n') + 1
|
|
133
|
-
waste_matches.append({
|
|
134
|
-
'line': line_num,
|
|
135
|
-
'category': category,
|
|
136
|
-
'pattern': label,
|
|
137
|
-
'context': lines[line_num - 1].strip()[:100],
|
|
138
|
-
})
|
|
139
|
-
|
|
140
|
-
backref_matches = []
|
|
141
|
-
for pattern, label in BACKREF_PATTERNS:
|
|
142
|
-
for m in re.finditer(pattern, content, re.IGNORECASE):
|
|
143
|
-
line_num = content[:m.start()].count('\n') + 1
|
|
144
|
-
backref_matches.append({
|
|
145
|
-
'line': line_num,
|
|
146
|
-
'pattern': label,
|
|
147
|
-
'context': lines[line_num - 1].strip()[:100],
|
|
148
|
-
})
|
|
149
|
-
|
|
150
|
-
# Config header
|
|
151
|
-
has_config_header = '{communication_language}' in content or '{document_output_language}' in content
|
|
152
|
-
|
|
153
|
-
# Progression condition
|
|
154
|
-
prog_keywords = ['progress', 'advance', 'move to', 'next stage',
|
|
155
|
-
'when complete', 'proceed to', 'transition', 'completion criteria']
|
|
156
|
-
has_progression = any(kw in content.lower() for kw in prog_keywords)
|
|
157
|
-
|
|
158
|
-
result = {
|
|
159
|
-
'file': rel_path,
|
|
160
|
-
'line_count': line_count,
|
|
161
|
-
'token_estimate': token_estimate,
|
|
162
|
-
'sections': sections,
|
|
163
|
-
'table_count': table_count,
|
|
164
|
-
'table_lines': table_lines,
|
|
165
|
-
'fenced_block_count': block_count,
|
|
166
|
-
'fenced_block_lines': block_lines,
|
|
167
|
-
'waste_patterns': waste_matches,
|
|
168
|
-
'back_references': backref_matches,
|
|
169
|
-
'has_config_header': has_config_header,
|
|
170
|
-
'has_progression': has_progression,
|
|
106
|
+
blocks += 1
|
|
107
|
+
return blocks
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def grep(content: str, lines: list[str], patterns, ignore_case: bool = False) -> list[dict]:
|
|
111
|
+
flags = re.IGNORECASE if ignore_case else 0
|
|
112
|
+
hits = []
|
|
113
|
+
for entry in patterns:
|
|
114
|
+
pattern, *rest = entry
|
|
115
|
+
if len(rest) == 2:
|
|
116
|
+
category, label = rest
|
|
117
|
+
else:
|
|
118
|
+
category, label = None, rest[0]
|
|
119
|
+
for m in re.finditer(pattern, content, flags):
|
|
120
|
+
ln = content[: m.start()].count("\n") + 1
|
|
121
|
+
hit = {"line": ln, "pattern": label, "context": lines[ln - 1].strip()[:100]}
|
|
122
|
+
if category:
|
|
123
|
+
hit["category"] = category
|
|
124
|
+
hits.append(hit)
|
|
125
|
+
return hits
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def scan_file(filepath: Path, rel_path: str) -> dict:
|
|
129
|
+
content = filepath.read_text(encoding="utf-8")
|
|
130
|
+
lines = content.split("\n")
|
|
131
|
+
meta, body = split_frontmatter(content)
|
|
132
|
+
tokens, method = count_tokens(content)
|
|
133
|
+
|
|
134
|
+
sections = [
|
|
135
|
+
{"level": len(m.group(1)), "title": m.group(2).strip()}
|
|
136
|
+
for m in (re.match(r"^(#{2,4})\s+(.+)$", ln) for ln in lines)
|
|
137
|
+
if m
|
|
138
|
+
]
|
|
139
|
+
|
|
140
|
+
table_count, table_rows = count_tables(content)
|
|
141
|
+
allcaps = len(ALLCAPS_PATTERN.findall(content))
|
|
142
|
+
|
|
143
|
+
data = {
|
|
144
|
+
"file": rel_path,
|
|
145
|
+
"tokens": tokens,
|
|
146
|
+
"token_method": method,
|
|
147
|
+
"sections": sections,
|
|
148
|
+
"table_count": table_count,
|
|
149
|
+
"table_rows": table_rows,
|
|
150
|
+
"fenced_block_count": count_fenced(content),
|
|
151
|
+
"allcaps_directive_count": allcaps,
|
|
152
|
+
"numbered_prefix_filename": bool(NUMBERED_PREFIX.match(filepath.name)),
|
|
153
|
+
"waste_patterns": grep(content, lines, WASTE_PATTERNS),
|
|
154
|
+
"back_references": grep(content, lines, BACKREF_PATTERNS, ignore_case=True),
|
|
171
155
|
}
|
|
172
156
|
|
|
173
|
-
|
|
157
|
+
if meta:
|
|
158
|
+
desc = meta.get("description", "")
|
|
159
|
+
data["frontmatter"] = {
|
|
160
|
+
"name": meta.get("name", ""),
|
|
161
|
+
"description": desc,
|
|
162
|
+
"description_chars": len(desc),
|
|
163
|
+
"description_has_angle_brackets": "<" in desc or ">" in desc,
|
|
164
|
+
"keys": sorted(meta.keys()),
|
|
165
|
+
}
|
|
166
|
+
return data
|
|
174
167
|
|
|
175
168
|
|
|
176
|
-
def
|
|
177
|
-
"""Extract metrics from all prompt-relevant files."""
|
|
169
|
+
def scan(skill_path: Path) -> dict:
|
|
178
170
|
files_data = []
|
|
179
171
|
|
|
180
|
-
|
|
181
|
-
skill_md = skill_path / 'SKILL.md'
|
|
172
|
+
skill_md = skill_path / "SKILL.md"
|
|
182
173
|
if skill_md.exists():
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
data['is_skill_md'] = True
|
|
187
|
-
files_data.append(data)
|
|
174
|
+
d = scan_file(skill_md, "SKILL.md")
|
|
175
|
+
d["is_skill_md"] = True
|
|
176
|
+
files_data.append(d)
|
|
188
177
|
|
|
189
|
-
# Prompt files at skill root (non-SKILL.md .md files)
|
|
190
178
|
for f in sorted(skill_path.iterdir()):
|
|
191
|
-
if f.is_file() and f.suffix ==
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
files_data.append(
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
179
|
+
if f.is_file() and f.suffix == ".md" and f.name != "SKILL.md":
|
|
180
|
+
d = scan_file(f, f.name)
|
|
181
|
+
d["is_skill_md"] = False
|
|
182
|
+
files_data.append(d)
|
|
183
|
+
|
|
184
|
+
references = {}
|
|
185
|
+
ref_dir = skill_path / "references"
|
|
186
|
+
if ref_dir.exists():
|
|
187
|
+
for f in sorted(ref_dir.iterdir()):
|
|
188
|
+
if f.is_file() and f.suffix in (".md", ".json", ".yaml", ".yml"):
|
|
189
|
+
tokens, method = count_tokens(f.read_text(encoding="utf-8"))
|
|
190
|
+
references[f.name] = {
|
|
191
|
+
"tokens": tokens,
|
|
192
|
+
"token_method": method,
|
|
193
|
+
"numbered_prefix_filename": bool(NUMBERED_PREFIX.match(f.name)),
|
|
206
194
|
}
|
|
207
195
|
|
|
208
|
-
|
|
209
|
-
total_waste = sum(len(f['waste_patterns']) for f in files_data)
|
|
210
|
-
total_backrefs = sum(len(f['back_references']) for f in files_data)
|
|
211
|
-
total_tokens = sum(f['token_estimate'] for f in files_data)
|
|
212
|
-
prompts_with_config = sum(1 for f in files_data if not f.get('is_skill_md') and f['has_config_header'])
|
|
213
|
-
prompts_with_progression = sum(1 for f in files_data if not f.get('is_skill_md') and f['has_progression'])
|
|
214
|
-
total_prompts = sum(1 for f in files_data if not f.get('is_skill_md'))
|
|
215
|
-
|
|
216
|
-
skill_md_data = next((f for f in files_data if f.get('is_skill_md')), None)
|
|
196
|
+
skill_md_data = next((f for f in files_data if f.get("is_skill_md")), None)
|
|
217
197
|
|
|
218
198
|
return {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
'overview_lines': skill_md_data.get('overview_lines', 0) if skill_md_data else 0,
|
|
229
|
-
'table_count': skill_md_data['table_count'] if skill_md_data else 0,
|
|
230
|
-
'table_lines': skill_md_data['table_lines'] if skill_md_data else 0,
|
|
231
|
-
'fenced_block_count': skill_md_data['fenced_block_count'] if skill_md_data else 0,
|
|
232
|
-
'fenced_block_lines': skill_md_data['fenced_block_lines'] if skill_md_data else 0,
|
|
233
|
-
'section_count': len(skill_md_data['sections']) if skill_md_data else 0,
|
|
199
|
+
"scanner": "prompt-metrics-prepass",
|
|
200
|
+
"script": "prepass-prompt-metrics.py",
|
|
201
|
+
"version": "2.0.0",
|
|
202
|
+
"skill_path": str(skill_path),
|
|
203
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
204
|
+
"budgets": {
|
|
205
|
+
"skill_md_tokens": [1500, 2500],
|
|
206
|
+
"multi_branch_reference_tokens": 4500,
|
|
207
|
+
"single_purpose_reference_tokens": 9000,
|
|
234
208
|
},
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
209
|
+
"skill_md": {
|
|
210
|
+
"tokens": skill_md_data["tokens"] if skill_md_data else 0,
|
|
211
|
+
"token_method": skill_md_data["token_method"] if skill_md_data else "fallback",
|
|
212
|
+
"section_count": len(skill_md_data["sections"]) if skill_md_data else 0,
|
|
213
|
+
"frontmatter": skill_md_data.get("frontmatter") if skill_md_data else None,
|
|
239
214
|
},
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
215
|
+
"aggregate": {
|
|
216
|
+
"total_files_scanned": len(files_data),
|
|
217
|
+
"total_tokens": sum(f["tokens"] for f in files_data),
|
|
218
|
+
"total_waste_patterns": sum(len(f["waste_patterns"]) for f in files_data),
|
|
219
|
+
"total_back_references": sum(len(f["back_references"]) for f in files_data),
|
|
220
|
+
"files_with_numbered_prefix": sum(
|
|
221
|
+
1 for f in files_data if f["numbered_prefix_filename"]
|
|
222
|
+
) + sum(1 for r in references.values() if r["numbered_prefix_filename"]),
|
|
245
223
|
},
|
|
246
|
-
|
|
247
|
-
|
|
224
|
+
"reference_sizes": references,
|
|
225
|
+
"files": files_data,
|
|
248
226
|
}
|
|
249
227
|
|
|
250
228
|
|
|
251
|
-
def main() -> int:
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
)
|
|
255
|
-
|
|
256
|
-
'skill_path',
|
|
257
|
-
type=Path,
|
|
258
|
-
help='Path to the skill directory to scan',
|
|
259
|
-
)
|
|
260
|
-
parser.add_argument(
|
|
261
|
-
'--output', '-o',
|
|
262
|
-
type=Path,
|
|
263
|
-
help='Write JSON output to file instead of stdout',
|
|
264
|
-
)
|
|
265
|
-
args = parser.parse_args()
|
|
229
|
+
def main(argv: list[str] | None = None) -> int:
|
|
230
|
+
p = argparse.ArgumentParser(description="Token-based prompt metrics for the Analyze scanners")
|
|
231
|
+
p.add_argument("skill_path", type=Path, help="path to the skill directory to scan")
|
|
232
|
+
p.add_argument("--output", "-o", type=Path, help="write JSON to a file instead of stdout")
|
|
233
|
+
args = p.parse_args(argv)
|
|
266
234
|
|
|
267
235
|
if not args.skill_path.is_dir():
|
|
268
|
-
print(f"
|
|
236
|
+
print(f"error: {args.skill_path} is not a directory", file=sys.stderr)
|
|
269
237
|
return 2
|
|
270
238
|
|
|
271
|
-
|
|
272
|
-
output = json.dumps(result, indent=2)
|
|
273
|
-
|
|
239
|
+
output = json.dumps(scan(args.skill_path), indent=2)
|
|
274
240
|
if args.output:
|
|
275
241
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
276
242
|
args.output.write_text(output)
|
|
277
|
-
print(f"
|
|
243
|
+
print(f"results written to {args.output}", file=sys.stderr)
|
|
278
244
|
else:
|
|
279
245
|
print(output)
|
|
280
|
-
|
|
281
246
|
return 0
|
|
282
247
|
|
|
283
248
|
|
|
284
|
-
if __name__ ==
|
|
249
|
+
if __name__ == "__main__":
|
|
285
250
|
sys.exit(main())
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# /// script
|
|
3
|
+
# requires-python = ">=3.9"
|
|
4
|
+
# ///
|
|
5
|
+
"""quick_validate — structural lint for a skill's SKILL.md frontmatter.
|
|
6
|
+
|
|
7
|
+
Checks the few things a structural error makes obvious: the frontmatter parses,
|
|
8
|
+
it carries only allowed keys, the name is hyphen-case and within length, and the
|
|
9
|
+
description is present, within bounds, and free of angle brackets (which break
|
|
10
|
+
the router). The allowed-key set is configurable, never baked to one provider:
|
|
11
|
+
pass --allow-key to extend it or --allow-keys to replace it.
|
|
12
|
+
|
|
13
|
+
Exit code is 0 when every check passes and 1 when any check fails, so a build or
|
|
14
|
+
CI step can gate on it. Findings print as one JSON object on stdout.
|
|
15
|
+
|
|
16
|
+
Usage:
|
|
17
|
+
quick_validate.py <skill-dir-or-SKILL.md>
|
|
18
|
+
quick_validate.py <path> --allow-key license --allow-key version
|
|
19
|
+
quick_validate.py <path> --allow-keys name,description
|
|
20
|
+
quick_validate.py <path> --max-name 64 --max-desc 1024
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import json
|
|
26
|
+
import re
|
|
27
|
+
import sys
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
DEFAULT_ALLOWED_KEYS = ["name", "description"]
|
|
31
|
+
HYPHEN_CASE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def split_frontmatter(content: str):
|
|
35
|
+
"""Return (ok, frontmatter dict, error). ok is False when there is no parseable block."""
|
|
36
|
+
lines = content.splitlines()
|
|
37
|
+
if not lines or lines[0].strip() != "---":
|
|
38
|
+
return False, {}, "no frontmatter block (file does not open with ---)"
|
|
39
|
+
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
|
40
|
+
if end is None:
|
|
41
|
+
return False, {}, "frontmatter block is not terminated with a closing ---"
|
|
42
|
+
meta: dict[str, str] = {}
|
|
43
|
+
for line in lines[1:end]:
|
|
44
|
+
if not line.strip():
|
|
45
|
+
continue
|
|
46
|
+
if ":" not in line:
|
|
47
|
+
return False, {}, f"frontmatter line is not key: value -> {line.strip()!r}"
|
|
48
|
+
k, v = line.split(":", 1)
|
|
49
|
+
meta[k.strip()] = v.strip()
|
|
50
|
+
return True, meta, ""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def validate(content: str, allowed_keys, max_name: int, max_desc: int) -> list[dict]:
|
|
54
|
+
errors: list[dict] = []
|
|
55
|
+
|
|
56
|
+
ok, meta, parse_error = split_frontmatter(content)
|
|
57
|
+
if not ok:
|
|
58
|
+
return [{"check": "frontmatter", "message": parse_error}]
|
|
59
|
+
|
|
60
|
+
extra = [k for k in meta if k not in allowed_keys]
|
|
61
|
+
if extra:
|
|
62
|
+
errors.append({
|
|
63
|
+
"check": "allowed-keys",
|
|
64
|
+
"message": f"unexpected frontmatter keys: {', '.join(sorted(extra))}; allowed: {', '.join(allowed_keys)}",
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
name = meta.get("name", "")
|
|
68
|
+
if not name:
|
|
69
|
+
errors.append({"check": "name", "message": "name is missing or empty"})
|
|
70
|
+
else:
|
|
71
|
+
if not HYPHEN_CASE.match(name):
|
|
72
|
+
errors.append({"check": "name", "message": f"name {name!r} is not hyphen-case (lowercase, digits, single hyphens)"})
|
|
73
|
+
if len(name) > max_name:
|
|
74
|
+
errors.append({"check": "name", "message": f"name is {len(name)} chars, over the {max_name} limit"})
|
|
75
|
+
|
|
76
|
+
desc = meta.get("description", "")
|
|
77
|
+
if not desc:
|
|
78
|
+
errors.append({"check": "description", "message": "description is missing or empty"})
|
|
79
|
+
else:
|
|
80
|
+
if len(desc) > max_desc:
|
|
81
|
+
errors.append({"check": "description", "message": f"description is {len(desc)} chars, over the {max_desc} limit"})
|
|
82
|
+
if "<" in desc or ">" in desc:
|
|
83
|
+
errors.append({"check": "description", "message": "description contains angle brackets, which break router matching"})
|
|
84
|
+
|
|
85
|
+
return errors
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def resolve_skill_md(path: Path) -> Path:
|
|
89
|
+
return path / "SKILL.md" if path.is_dir() else path
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def main(argv: list[str] | None = None) -> int:
|
|
93
|
+
p = argparse.ArgumentParser(description="Structural lint for a skill's SKILL.md frontmatter")
|
|
94
|
+
p.add_argument("path", type=Path, help="skill directory or a SKILL.md file")
|
|
95
|
+
p.add_argument("--allow-key", action="append", default=[], help="add one key to the allowed set (repeatable)")
|
|
96
|
+
p.add_argument("--allow-keys", help="comma-separated set that REPLACES the default allowed keys")
|
|
97
|
+
p.add_argument("--max-name", type=int, default=64, help="max name length (default 64)")
|
|
98
|
+
p.add_argument("--max-desc", type=int, default=1024, help="max description length (default 1024)")
|
|
99
|
+
args = p.parse_args(argv)
|
|
100
|
+
|
|
101
|
+
skill_md = resolve_skill_md(args.path)
|
|
102
|
+
if not skill_md.is_file():
|
|
103
|
+
print(json.dumps({"ok": False, "errors": [{"check": "path", "message": f"{skill_md} not found"}]}))
|
|
104
|
+
return 1
|
|
105
|
+
|
|
106
|
+
if args.allow_keys:
|
|
107
|
+
allowed = [k.strip() for k in args.allow_keys.split(",") if k.strip()]
|
|
108
|
+
else:
|
|
109
|
+
allowed = list(DEFAULT_ALLOWED_KEYS) + list(args.allow_key)
|
|
110
|
+
|
|
111
|
+
errors = validate(skill_md.read_text(encoding="utf-8"), allowed, args.max_name, args.max_desc)
|
|
112
|
+
print(json.dumps({"ok": not errors, "file": str(skill_md), "errors": errors}))
|
|
113
|
+
return 0 if not errors else 1
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__":
|
|
117
|
+
sys.exit(main())
|