@intentsolutionsio/skill-creator 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +17 -0
- package/README.md +55 -0
- package/package.json +38 -0
- package/scripts/validate-skill.py +1132 -0
- package/skills/agent-creator/SKILL.md +305 -0
- package/skills/agent-creator/references/anthropic-agent-spec.md +89 -0
- package/skills/skill-creator/SKILL.md +267 -0
- package/skills/skill-creator/agents/analyzer.md +279 -0
- package/skills/skill-creator/agents/comparator.md +207 -0
- package/skills/skill-creator/agents/grader.md +228 -0
- package/skills/skill-creator/assets/eval_review.html +146 -0
- package/skills/skill-creator/eval-viewer/generate_review.py +471 -0
- package/skills/skill-creator/eval-viewer/viewer.html +1325 -0
- package/skills/skill-creator/references/advanced-eval-workflow.md +320 -0
- package/skills/skill-creator/references/anthropic-comparison.md +93 -0
- package/skills/skill-creator/references/ard-template.md +47 -0
- package/skills/skill-creator/references/creation-guide.md +305 -0
- package/skills/skill-creator/references/errors-template.md +27 -0
- package/skills/skill-creator/references/examples-template.md +40 -0
- package/skills/skill-creator/references/frontmatter-spec.md +531 -0
- package/skills/skill-creator/references/implementation-template.md +42 -0
- package/skills/skill-creator/references/output-patterns.md +193 -0
- package/skills/skill-creator/references/prd-template.md +55 -0
- package/skills/skill-creator/references/schemas.md +430 -0
- package/skills/skill-creator/references/source-of-truth.md +658 -0
- package/skills/skill-creator/references/validation-rules.md +528 -0
- package/skills/skill-creator/references/workflows.md +233 -0
- package/skills/skill-creator/scripts/__init__.py +0 -0
- package/skills/skill-creator/scripts/aggregate_benchmark.py +401 -0
- package/skills/skill-creator/scripts/generate_report.py +326 -0
- package/skills/skill-creator/scripts/improve_description.py +247 -0
- package/skills/skill-creator/scripts/package_skill.py +136 -0
- package/skills/skill-creator/scripts/quick_validate.py +103 -0
- package/skills/skill-creator/scripts/run_eval.py +344 -0
- package/skills/skill-creator/scripts/run_loop.py +329 -0
- package/skills/skill-creator/scripts/utils.py +47 -0
- package/skills/skill-creator/scripts/validate-skill.py +87 -0
- package/skills/skill-creator/templates/agent-template.md +99 -0
- package/skills/skill-creator/templates/skill-template.md +122 -0
|
@@ -0,0 +1,1132 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Skill Validator v5.1 (Anthropic Best Practices 2026) - Two-Tier Validation + 100-Point Grading
|
|
4
|
+
|
|
5
|
+
Validates SKILL.md files against:
|
|
6
|
+
- Standard tier (DEFAULT): AgentSkills.io minimum (name defaults to dir name, description recommended)
|
|
7
|
+
- Enterprise tier: Standard + identity fields, scoped tools, sections, disclosure
|
|
8
|
+
- 100-Point grading: Intent Solutions marketplace rubric (--grade flag)
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
python validate-skill.py path/to/SKILL.md # Standard (default)
|
|
12
|
+
python validate-skill.py --enterprise path/to/SKILL.md # Enterprise tier
|
|
13
|
+
python validate-skill.py --grade path/to/SKILL.md # 100-point grading
|
|
14
|
+
python validate-skill.py --grade --json path/to/SKILL.md # JSON grade output
|
|
15
|
+
python validate-skill.py --json path/to/SKILL.md # JSON output
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import json
|
|
20
|
+
import re
|
|
21
|
+
import sys
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Dict, List, Tuple
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
import yaml
|
|
27
|
+
except ImportError:
|
|
28
|
+
print("ERROR: pyyaml required. Install: pip install pyyaml", file=sys.stderr)
|
|
29
|
+
sys.exit(1)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# === CONSTANTS ===
|
|
33
|
+
|
|
34
|
+
VALID_TOOLS = {
|
|
35
|
+
"Read", "Write", "Edit", "Bash", "Glob", "Grep",
|
|
36
|
+
"WebFetch", "WebSearch", "Task", "NotebookEdit",
|
|
37
|
+
"AskUserQuestion", "Skill",
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
KNOWN_FRONTMATTER_FIELDS = {
|
|
41
|
+
# AgentSkills.io spec
|
|
42
|
+
"name", "description", "license", "compatibility", "metadata", "allowed-tools",
|
|
43
|
+
# Top-level identity fields (marketplace standard)
|
|
44
|
+
"version", "author", "compatible-with", "tags",
|
|
45
|
+
# Claude Code extensions
|
|
46
|
+
"argument-hint", "disable-model-invocation", "user-invocable", "model",
|
|
47
|
+
"context", "agent", "hooks",
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
DEPRECATED_FIELDS = {
|
|
51
|
+
"when_to_use": "Move content to description",
|
|
52
|
+
"mode": "Use disable-model-invocation instead",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
VALID_PLATFORMS = {
|
|
56
|
+
"claude-code", "codex", "openclaw", "aider", "continue", "cursor", "windsurf",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
RE_FRONTMATTER = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)$", re.DOTALL)
|
|
60
|
+
RE_FIRST_PERSON = re.compile(r"\b(I can|I will|I'm going to|I help|I'm)\b", re.IGNORECASE)
|
|
61
|
+
RE_SECOND_PERSON = re.compile(r"\b(You can|You should|You will)\b", re.IGNORECASE)
|
|
62
|
+
RE_HARDCODED_MODEL = re.compile(r"claude-[\w]+-\d{8}")
|
|
63
|
+
RE_CONSECUTIVE_HYPHENS = re.compile(r"--")
|
|
64
|
+
RE_WINDOWS_PATH = re.compile(r"[A-Za-z]:\\\\", re.IGNORECASE)
|
|
65
|
+
RE_ARGUMENTS = re.compile(r"\$ARGUMENTS|\$\d+|\$ARGUMENTS\[\d+\]")
|
|
66
|
+
RE_DYNAMIC_CONTEXT = re.compile(r"!`[^`]+`")
|
|
67
|
+
|
|
68
|
+
RE_XML_TAG = re.compile(r"[<>]")
|
|
69
|
+
RE_TIME_SENSITIVE = [
|
|
70
|
+
re.compile(r"\b(20\d{2}[-/]\d{2}[-/]\d{2})\b"), # dates like 2025-01-01
|
|
71
|
+
re.compile(r"\b(v\d+\.\d+\.\d+)\b", re.IGNORECASE), # version numbers like v1.2.3
|
|
72
|
+
re.compile(r"\b(as of|since|after|before) (January|February|March|April|May|June|July|August|September|October|November|December)\b", re.IGNORECASE),
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
ABSOLUTE_PATH_PATTERNS = [
|
|
76
|
+
(re.compile(r"/home/\w+/"), "/home/..."),
|
|
77
|
+
(re.compile(r"/Users/\w+/"), "/Users/..."),
|
|
78
|
+
(re.compile(r"[A-Za-z]:\\\\Users\\\\", re.IGNORECASE), "C:\\Users\\..."),
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def parse_frontmatter(content: str) -> Tuple[dict, str]:
|
|
83
|
+
"""Parse YAML frontmatter from SKILL.md content."""
|
|
84
|
+
m = RE_FRONTMATTER.match(content)
|
|
85
|
+
if not m:
|
|
86
|
+
raise ValueError("Invalid or absent YAML frontmatter (missing --- delimiters)")
|
|
87
|
+
front_str, body = m.groups()
|
|
88
|
+
try:
|
|
89
|
+
data = yaml.safe_load(front_str) or {}
|
|
90
|
+
except yaml.YAMLError as e:
|
|
91
|
+
raise ValueError(f"YAML parse error: {e}")
|
|
92
|
+
return data, body
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def validate_name(fm: dict, path: Path, enterprise: bool = False) -> Tuple[List[str], List[str]]:
|
|
96
|
+
"""Validate the name field."""
|
|
97
|
+
errors, warnings = [], []
|
|
98
|
+
if "name" not in fm:
|
|
99
|
+
if enterprise:
|
|
100
|
+
errors.append("Missing required field: 'name'")
|
|
101
|
+
else:
|
|
102
|
+
warnings.append(f"INFO: 'name' not set - defaults to directory name '{path.parent.name}'")
|
|
103
|
+
return errors, warnings
|
|
104
|
+
|
|
105
|
+
name = str(fm["name"]).strip()
|
|
106
|
+
if not name:
|
|
107
|
+
errors.append("'name' must be non-empty")
|
|
108
|
+
return errors, warnings
|
|
109
|
+
|
|
110
|
+
if len(name) > 64:
|
|
111
|
+
errors.append(f"'name' exceeds 64 characters ({len(name)})")
|
|
112
|
+
|
|
113
|
+
if not re.match(r"^[a-z]([a-z0-9-]*[a-z0-9])?$", name):
|
|
114
|
+
errors.append(f"'name' must be kebab-case (lowercase, hyphens): '{name}'")
|
|
115
|
+
elif RE_CONSECUTIVE_HYPHENS.search(name):
|
|
116
|
+
errors.append(f"'name' has consecutive hyphens: '{name}'")
|
|
117
|
+
|
|
118
|
+
if name.startswith("-") or name.endswith("-"):
|
|
119
|
+
errors.append(f"'name' must not start or end with hyphen: '{name}'")
|
|
120
|
+
|
|
121
|
+
if name != path.parent.name:
|
|
122
|
+
warnings.append(f"'name' '{name}' differs from directory '{path.parent.name}'")
|
|
123
|
+
|
|
124
|
+
if RE_XML_TAG.search(name):
|
|
125
|
+
errors.append(f"'name' must not contain XML tags (< or >): '{name}'")
|
|
126
|
+
|
|
127
|
+
return errors, warnings
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def validate_description(fm: dict, enterprise: bool = False) -> Tuple[List[str], List[str]]:
|
|
131
|
+
"""Validate the description field."""
|
|
132
|
+
errors, warnings = [], []
|
|
133
|
+
if "description" not in fm:
|
|
134
|
+
if enterprise:
|
|
135
|
+
errors.append("Missing required field: 'description'")
|
|
136
|
+
else:
|
|
137
|
+
warnings.append("'description' is recommended for discovery - add trigger phrases and use-cases")
|
|
138
|
+
return errors, warnings
|
|
139
|
+
|
|
140
|
+
desc = str(fm["description"]).strip()
|
|
141
|
+
if not desc:
|
|
142
|
+
errors.append("'description' must be non-empty")
|
|
143
|
+
return errors, warnings
|
|
144
|
+
|
|
145
|
+
if len(desc) > 1024:
|
|
146
|
+
errors.append(f"'description' exceeds 1024 characters ({len(desc)})")
|
|
147
|
+
if len(desc) < 20:
|
|
148
|
+
warnings.append(f"'description' is very short ({len(desc)} chars) - add keywords for discovery")
|
|
149
|
+
if len(desc) > 500:
|
|
150
|
+
warnings.append(f"'description' is long ({len(desc)} chars) - impacts token budget")
|
|
151
|
+
|
|
152
|
+
if RE_FIRST_PERSON.search(desc):
|
|
153
|
+
if enterprise:
|
|
154
|
+
errors.append("'description' must not use first person (I can, I will, I'm, I help)")
|
|
155
|
+
else:
|
|
156
|
+
warnings.append("'description' should not use first person (I can, I will, I'm, I help)")
|
|
157
|
+
if RE_SECOND_PERSON.search(desc):
|
|
158
|
+
if enterprise:
|
|
159
|
+
errors.append("'description' must not use second person (You can, You should, You will)")
|
|
160
|
+
else:
|
|
161
|
+
warnings.append("'description' should not use second person (You can, You should, You will)")
|
|
162
|
+
|
|
163
|
+
if RE_XML_TAG.search(desc):
|
|
164
|
+
errors.append("'description' must not contain XML tags (< or >)")
|
|
165
|
+
|
|
166
|
+
return errors, warnings
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def validate_tools(fm: dict, enterprise: bool) -> Tuple[List[str], List[str]]:
|
|
170
|
+
"""Validate allowed-tools field."""
|
|
171
|
+
errors, warnings = [], []
|
|
172
|
+
if "allowed-tools" not in fm:
|
|
173
|
+
if enterprise:
|
|
174
|
+
warnings.append("'allowed-tools' not set (Enterprise recommends scoped tools)")
|
|
175
|
+
return errors, warnings
|
|
176
|
+
|
|
177
|
+
tools_val = fm["allowed-tools"]
|
|
178
|
+
if isinstance(tools_val, list):
|
|
179
|
+
tools = [str(t).strip() for t in tools_val]
|
|
180
|
+
else:
|
|
181
|
+
raw = str(tools_val)
|
|
182
|
+
tools = re.findall(r"[A-Za-z_][\w-]*(?:\([^)]*\))?", raw)
|
|
183
|
+
|
|
184
|
+
if not tools or tools == [""]:
|
|
185
|
+
errors.append("'allowed-tools' is empty")
|
|
186
|
+
return errors, warnings
|
|
187
|
+
|
|
188
|
+
for tool in tools:
|
|
189
|
+
if ":" in tool and "(" not in tool:
|
|
190
|
+
continue # MCP tool reference
|
|
191
|
+
base_tool = tool.split("(")[0].strip()
|
|
192
|
+
if base_tool not in VALID_TOOLS:
|
|
193
|
+
errors.append(f"Unknown tool: '{base_tool}'")
|
|
194
|
+
if "(" in tool and not tool.endswith(")"):
|
|
195
|
+
errors.append(f"Invalid tool scope syntax: '{tool}'")
|
|
196
|
+
|
|
197
|
+
if "Bash" in tools:
|
|
198
|
+
if enterprise:
|
|
199
|
+
errors.append("Unscoped 'Bash' forbidden in Enterprise tier - use Bash(git:*) etc.")
|
|
200
|
+
else:
|
|
201
|
+
warnings.append("Unscoped 'Bash' - consider scoping: Bash(git:*), Bash(npm:*)")
|
|
202
|
+
|
|
203
|
+
return errors, warnings
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def validate_optional_fields(fm: dict) -> Tuple[List[str], List[str]]:
|
|
207
|
+
"""Validate optional and extension fields."""
|
|
208
|
+
errors, warnings = [], []
|
|
209
|
+
|
|
210
|
+
# Model validation
|
|
211
|
+
if "model" in fm:
|
|
212
|
+
model = str(fm["model"]).strip()
|
|
213
|
+
valid_models = {"inherit", "sonnet", "haiku", "opus"}
|
|
214
|
+
if model not in valid_models and not model.startswith("claude-"):
|
|
215
|
+
warnings.append(f"'model' value '{model}' - expected: inherit, sonnet, haiku, opus, or claude-* ID")
|
|
216
|
+
if RE_HARDCODED_MODEL.match(model):
|
|
217
|
+
warnings.append(f"Hardcoded model ID '{model}' - prefer 'inherit' or short names (sonnet, opus)")
|
|
218
|
+
|
|
219
|
+
# Context + agent validation
|
|
220
|
+
if "context" in fm:
|
|
221
|
+
if str(fm["context"]) != "fork":
|
|
222
|
+
errors.append("'context' must be 'fork' if set")
|
|
223
|
+
if "agent" in fm and "context" not in fm:
|
|
224
|
+
warnings.append("'agent' set without 'context: fork' - agent requires fork context")
|
|
225
|
+
|
|
226
|
+
# Boolean fields
|
|
227
|
+
for field in ("disable-model-invocation", "user-invocable"):
|
|
228
|
+
if field in fm and not isinstance(fm[field], bool):
|
|
229
|
+
errors.append(f"'{field}' must be boolean, got: {type(fm[field]).__name__}")
|
|
230
|
+
|
|
231
|
+
# Conflicting invocation controls
|
|
232
|
+
if fm.get("disable-model-invocation") is True and fm.get("user-invocable") is False:
|
|
233
|
+
errors.append(
|
|
234
|
+
"Conflicting: 'disable-model-invocation: true' + 'user-invocable: false' "
|
|
235
|
+
"makes skill unreachable by both user and model"
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
# Compatibility length
|
|
239
|
+
if "compatibility" in fm:
|
|
240
|
+
compat = str(fm["compatibility"]).strip()
|
|
241
|
+
if len(compat) > 500:
|
|
242
|
+
errors.append(f"'compatibility' exceeds 500 characters ({len(compat)})")
|
|
243
|
+
|
|
244
|
+
# compatible-with validation
|
|
245
|
+
if "compatible-with" in fm:
|
|
246
|
+
compat_with = fm["compatible-with"]
|
|
247
|
+
if isinstance(compat_with, str):
|
|
248
|
+
platforms = [p.strip() for p in compat_with.split(",")]
|
|
249
|
+
elif isinstance(compat_with, list):
|
|
250
|
+
platforms = [str(p).strip() for p in compat_with]
|
|
251
|
+
else:
|
|
252
|
+
platforms = []
|
|
253
|
+
warnings.append("'compatible-with' should be a comma-separated string or list")
|
|
254
|
+
for p in platforms:
|
|
255
|
+
if p and p not in VALID_PLATFORMS:
|
|
256
|
+
warnings.append(f"Unknown platform in compatible-with: '{p}'")
|
|
257
|
+
|
|
258
|
+
# tags validation
|
|
259
|
+
if "tags" in fm:
|
|
260
|
+
tags = fm["tags"]
|
|
261
|
+
if not isinstance(tags, list):
|
|
262
|
+
warnings.append("'tags' should be an array/list of strings")
|
|
263
|
+
|
|
264
|
+
# Version validation (top-level)
|
|
265
|
+
if "version" in fm:
|
|
266
|
+
version = str(fm["version"]).strip()
|
|
267
|
+
if version and not re.match(r"^\d+\.\d+\.\d+", version):
|
|
268
|
+
warnings.append(f"Version should be semver (X.Y.Z): '{version}'")
|
|
269
|
+
|
|
270
|
+
# Author validation (top-level)
|
|
271
|
+
if "author" in fm:
|
|
272
|
+
author = str(fm["author"]).strip()
|
|
273
|
+
if not author:
|
|
274
|
+
warnings.append("'author' is empty")
|
|
275
|
+
elif "@" not in author:
|
|
276
|
+
warnings.append("'author' should include email (Name <email>)")
|
|
277
|
+
|
|
278
|
+
# Deprecated fields
|
|
279
|
+
for field, msg in DEPRECATED_FIELDS.items():
|
|
280
|
+
if field in fm:
|
|
281
|
+
warnings.append(f"Deprecated field '{field}': {msg}")
|
|
282
|
+
|
|
283
|
+
# Check if author/version are nested in metadata (warn to move top-level)
|
|
284
|
+
metadata = fm.get("metadata", {})
|
|
285
|
+
if isinstance(metadata, dict):
|
|
286
|
+
for field in ("author", "version", "license", "tags"):
|
|
287
|
+
if field in metadata and field not in fm:
|
|
288
|
+
warnings.append(
|
|
289
|
+
f"'{field}' found in metadata block - move to top-level for marketplace scoring"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
# Unknown fields
|
|
293
|
+
all_known = KNOWN_FRONTMATTER_FIELDS | DEPRECATED_FIELDS.keys()
|
|
294
|
+
for field in fm:
|
|
295
|
+
if field not in all_known:
|
|
296
|
+
warnings.append(f"Unknown frontmatter field: '{field}'")
|
|
297
|
+
|
|
298
|
+
return errors, warnings
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def validate_enterprise_metadata(fm: dict) -> Tuple[List[str], List[str]]:
|
|
302
|
+
"""Enterprise tier: check author and version at top-level (or metadata fallback)."""
|
|
303
|
+
errors, warnings = [], []
|
|
304
|
+
metadata = fm.get("metadata", {})
|
|
305
|
+
if not isinstance(metadata, dict):
|
|
306
|
+
metadata = {}
|
|
307
|
+
|
|
308
|
+
# Check author (top-level preferred, metadata fallback accepted with warning)
|
|
309
|
+
has_author = "author" in fm
|
|
310
|
+
has_author_meta = "author" in metadata
|
|
311
|
+
if not has_author and not has_author_meta:
|
|
312
|
+
warnings.append("Enterprise: 'author' recommended (top-level field)")
|
|
313
|
+
elif has_author_meta and not has_author:
|
|
314
|
+
warnings.append("'author' in metadata block - move to top-level for marketplace scoring")
|
|
315
|
+
|
|
316
|
+
# Check version (top-level preferred, metadata fallback accepted with warning)
|
|
317
|
+
has_version = "version" in fm
|
|
318
|
+
has_version_meta = "version" in metadata
|
|
319
|
+
if not has_version and not has_version_meta:
|
|
320
|
+
warnings.append("Enterprise: 'version' recommended (top-level field)")
|
|
321
|
+
elif has_version_meta and not has_version:
|
|
322
|
+
warnings.append("'version' in metadata block - move to top-level for marketplace scoring")
|
|
323
|
+
|
|
324
|
+
# Check license
|
|
325
|
+
if "license" not in fm:
|
|
326
|
+
warnings.append("Enterprise: 'license' recommended (top-level field)")
|
|
327
|
+
|
|
328
|
+
return errors, warnings
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def validate_body(body: str, path: Path, enterprise: bool) -> Tuple[List[str], List[str], List[str]]:
|
|
332
|
+
"""Validate SKILL.md body content."""
|
|
333
|
+
errors, warnings, info = [], [], []
|
|
334
|
+
lines = body.splitlines()
|
|
335
|
+
line_count = len(lines)
|
|
336
|
+
word_count = len(body.split())
|
|
337
|
+
|
|
338
|
+
if line_count > 500:
|
|
339
|
+
errors.append(f"Body has {line_count} lines (max 500)")
|
|
340
|
+
elif line_count > 400:
|
|
341
|
+
warnings.append(f"Body has {line_count} lines (approaching 500 limit)")
|
|
342
|
+
|
|
343
|
+
if word_count > 5000:
|
|
344
|
+
warnings.append(f"Body has {word_count} words - consider splitting to references/")
|
|
345
|
+
|
|
346
|
+
# Strip code blocks for path checks
|
|
347
|
+
body_no_code = re.sub(r"```.*?```", "", body, flags=re.DOTALL)
|
|
348
|
+
body_no_code = re.sub(r"`[^`]+`", "", body_no_code)
|
|
349
|
+
|
|
350
|
+
for pattern, desc in ABSOLUTE_PATH_PATTERNS:
|
|
351
|
+
if pattern.search(body_no_code):
|
|
352
|
+
errors.append(f"Contains absolute path ({desc}) - use ${{CLAUDE_SKILL_DIR}}")
|
|
353
|
+
|
|
354
|
+
if RE_WINDOWS_PATH.search(body_no_code):
|
|
355
|
+
errors.append("Contains Windows-style path - use Unix paths or ${CLAUDE_SKILL_DIR}")
|
|
356
|
+
|
|
357
|
+
if not re.search(r"^# \S", body, re.MULTILINE):
|
|
358
|
+
if enterprise:
|
|
359
|
+
warnings.append("Missing H1 title (# Title)")
|
|
360
|
+
|
|
361
|
+
if enterprise:
|
|
362
|
+
instruction_headings = ("## Instructions", "## Steps", "## Your task", "## Usage", "## Workflow")
|
|
363
|
+
has_instructions = any(h in body for h in instruction_headings)
|
|
364
|
+
if not has_instructions:
|
|
365
|
+
warnings.append("Missing '## Instructions' section (or ## Steps/Your task/Usage/Workflow)")
|
|
366
|
+
else:
|
|
367
|
+
instr_match = re.search(
|
|
368
|
+
r"## (?:Instructions|Steps|Your task|Usage|Workflow)(.*?)(?=\n## |\Z)", body, re.DOTALL
|
|
369
|
+
)
|
|
370
|
+
if instr_match:
|
|
371
|
+
instr = instr_match.group(1)
|
|
372
|
+
has_steps = (
|
|
373
|
+
re.search(r"(?m)^\s*\d+\.\s+", instr)
|
|
374
|
+
or re.search(r"(?mi)^\s*#{2,6}\s*step\s*\d+", instr)
|
|
375
|
+
)
|
|
376
|
+
if not has_steps:
|
|
377
|
+
warnings.append("Instructions should have numbered steps or ### Step N headings")
|
|
378
|
+
|
|
379
|
+
if "## Examples" not in body and "## Example" not in body:
|
|
380
|
+
warnings.append("Missing '## Examples' section")
|
|
381
|
+
|
|
382
|
+
if "## Error" not in body and "error" not in body.lower():
|
|
383
|
+
info.append("Consider adding error handling documentation")
|
|
384
|
+
|
|
385
|
+
skill_dir = path.parent.resolve()
|
|
386
|
+
refs_dir = skill_dir / "references"
|
|
387
|
+
if line_count > 300 and not refs_dir.exists():
|
|
388
|
+
warnings.append(
|
|
389
|
+
f"SKILL.md is {line_count} lines with no references/ directory - "
|
|
390
|
+
"consider splitting heavy content"
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
if "{baseDir}/../" in body:
|
|
394
|
+
errors.append("Path escape detected: {baseDir}/../ - references must stay within skill directory")
|
|
395
|
+
|
|
396
|
+
nested_refs = re.findall(r"\{baseDir\}/references/\S+/\S+/", body)
|
|
397
|
+
if nested_refs:
|
|
398
|
+
warnings.append(f"Deeply nested reference paths detected: {nested_refs[0]} - keep references one level deep")
|
|
399
|
+
|
|
400
|
+
dynamic_cmds = RE_DYNAMIC_CONTEXT.findall(body)
|
|
401
|
+
if dynamic_cmds:
|
|
402
|
+
info.append(f"Dynamic context injection detected ({len(dynamic_cmds)} command(s))")
|
|
403
|
+
|
|
404
|
+
if RE_ARGUMENTS.search(body):
|
|
405
|
+
try:
|
|
406
|
+
content = path.read_text(encoding="utf-8")
|
|
407
|
+
if "argument-hint" not in content.split("---")[1]:
|
|
408
|
+
info.append("Uses $ARGUMENTS but no 'argument-hint' in frontmatter")
|
|
409
|
+
except Exception:
|
|
410
|
+
pass
|
|
411
|
+
|
|
412
|
+
# Time-sensitive information detection (skip code blocks)
|
|
413
|
+
stripped_body = re.sub(r"```[\s\S]*?```", "", body)
|
|
414
|
+
stripped_body = re.sub(r"`[^`]+`", "", stripped_body)
|
|
415
|
+
for pattern in RE_TIME_SENSITIVE:
|
|
416
|
+
if pattern.search(stripped_body):
|
|
417
|
+
warnings.append("Body may contain time-sensitive information (dates, versions) that could go stale")
|
|
418
|
+
break
|
|
419
|
+
|
|
420
|
+
return errors, warnings, info
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def validate_resources(body: str, path: Path) -> List[str]:
|
|
424
|
+
"""Validate that referenced resources exist."""
|
|
425
|
+
errors = []
|
|
426
|
+
skill_dir = path.parent.resolve()
|
|
427
|
+
|
|
428
|
+
for subdir in ("scripts", "references", "templates", "assets"):
|
|
429
|
+
# Check both ${CLAUDE_SKILL_DIR} and {baseDir} references
|
|
430
|
+
for pattern in [
|
|
431
|
+
rf"\$\{{CLAUDE_SKILL_DIR\}}/{subdir}/([\w\-./]+)",
|
|
432
|
+
rf"\{{baseDir\}}/{subdir}/([\w\-./]+)",
|
|
433
|
+
]:
|
|
434
|
+
for match in re.finditer(pattern, body):
|
|
435
|
+
rel_path = match.group(1)
|
|
436
|
+
full_path = skill_dir / subdir / rel_path
|
|
437
|
+
if not full_path.exists():
|
|
438
|
+
errors.append(f"Resource not found: {subdir}/{rel_path}")
|
|
439
|
+
|
|
440
|
+
return errors
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def calculate_disclosure_score(fm: dict, body: str, path: Path) -> int:
|
|
444
|
+
"""Calculate progressive disclosure score (0-6)."""
|
|
445
|
+
score = 0
|
|
446
|
+
lines = len(body.splitlines())
|
|
447
|
+
skill_dir = path.parent.resolve()
|
|
448
|
+
|
|
449
|
+
if lines < 200:
|
|
450
|
+
score += 2
|
|
451
|
+
elif lines < 400:
|
|
452
|
+
score += 1
|
|
453
|
+
|
|
454
|
+
desc = str(fm.get("description", ""))
|
|
455
|
+
if len(desc) < 200:
|
|
456
|
+
score += 1
|
|
457
|
+
|
|
458
|
+
if (skill_dir / "references").exists():
|
|
459
|
+
score += 1
|
|
460
|
+
if (skill_dir / "scripts").exists():
|
|
461
|
+
score += 1
|
|
462
|
+
if lines <= 500:
|
|
463
|
+
score += 1
|
|
464
|
+
|
|
465
|
+
return min(score, 6)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
# === 100-POINT GRADING RUBRIC ===
|
|
469
|
+
#
|
|
470
|
+
# Ported from marketplace validator (scripts/validate-skills-schema.py)
|
|
471
|
+
# Grade Scale:
|
|
472
|
+
# A (90-100): Production-ready
|
|
473
|
+
# B (80-89): Good, minor improvements needed
|
|
474
|
+
# C (70-79): Adequate, has gaps
|
|
475
|
+
# D (60-69): Needs significant work
|
|
476
|
+
# F (<60): Major revision required
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def score_progressive_disclosure(path: Path, body: str, fm: dict) -> dict:
|
|
480
|
+
"""Progressive Disclosure Architecture (30 pts max)."""
|
|
481
|
+
breakdown = {}
|
|
482
|
+
lines = len(body.splitlines())
|
|
483
|
+
skill_dir = path.parent
|
|
484
|
+
|
|
485
|
+
# Token Economy (10 pts)
|
|
486
|
+
if lines <= 150:
|
|
487
|
+
breakdown["token_economy"] = (10, f"Excellent: {lines} lines")
|
|
488
|
+
elif lines <= 300:
|
|
489
|
+
breakdown["token_economy"] = (7, f"Good: {lines} lines (target <=150)")
|
|
490
|
+
elif lines <= 500:
|
|
491
|
+
breakdown["token_economy"] = (4, f"Acceptable: {lines} lines (target <=150)")
|
|
492
|
+
else:
|
|
493
|
+
breakdown["token_economy"] = (0, f"Too long: {lines} lines (target <=150)")
|
|
494
|
+
|
|
495
|
+
# Layered Structure (10 pts)
|
|
496
|
+
refs_dir = skill_dir / "references"
|
|
497
|
+
if refs_dir.exists():
|
|
498
|
+
ref_files = list(refs_dir.glob("*.md"))
|
|
499
|
+
if ref_files:
|
|
500
|
+
breakdown["layered_structure"] = (10, f"Has references/ with {len(ref_files)} files")
|
|
501
|
+
else:
|
|
502
|
+
breakdown["layered_structure"] = (3, "references/ exists but empty")
|
|
503
|
+
else:
|
|
504
|
+
if lines <= 100:
|
|
505
|
+
breakdown["layered_structure"] = (8, "No references/ (acceptable for short skill)")
|
|
506
|
+
elif lines <= 200:
|
|
507
|
+
breakdown["layered_structure"] = (4, "No references/ (should extract content)")
|
|
508
|
+
else:
|
|
509
|
+
breakdown["layered_structure"] = (0, "No references/ (long skill needs extraction)")
|
|
510
|
+
|
|
511
|
+
# Reference Depth (5 pts)
|
|
512
|
+
if refs_dir.exists():
|
|
513
|
+
nested_dirs = [d for d in refs_dir.iterdir() if d.is_dir()]
|
|
514
|
+
if not nested_dirs:
|
|
515
|
+
breakdown["reference_depth"] = (5, "References are flat (good)")
|
|
516
|
+
else:
|
|
517
|
+
breakdown["reference_depth"] = (2, f"Nested dirs in references/: {len(nested_dirs)}")
|
|
518
|
+
else:
|
|
519
|
+
breakdown["reference_depth"] = (5, "N/A - no references/")
|
|
520
|
+
|
|
521
|
+
# Navigation Signals (5 pts)
|
|
522
|
+
has_toc = bool(re.search(r"(?mi)^##?\s*(table of contents|contents|toc)\b", body))
|
|
523
|
+
has_nav_links = bool(re.search(r"\[.*?\]\(#.*?\)", body))
|
|
524
|
+
if lines <= 100:
|
|
525
|
+
breakdown["navigation_signals"] = (5, "Short file, TOC optional")
|
|
526
|
+
elif has_toc or has_nav_links:
|
|
527
|
+
breakdown["navigation_signals"] = (5, "Has navigation/TOC")
|
|
528
|
+
else:
|
|
529
|
+
breakdown["navigation_signals"] = (0, "Long file needs TOC/navigation")
|
|
530
|
+
|
|
531
|
+
total = sum(v[0] for v in breakdown.values())
|
|
532
|
+
return {"score": total, "max": 30, "breakdown": breakdown}
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def score_ease_of_use(path: Path, body: str, fm: dict) -> dict:
|
|
536
|
+
"""Ease of Use (25 pts max)."""
|
|
537
|
+
breakdown = {}
|
|
538
|
+
desc = str(fm.get("description", "")).lower()
|
|
539
|
+
|
|
540
|
+
# Metadata Quality (10 pts)
|
|
541
|
+
meta_score = 0
|
|
542
|
+
meta_notes = []
|
|
543
|
+
if fm.get("name"):
|
|
544
|
+
meta_score += 2
|
|
545
|
+
else:
|
|
546
|
+
meta_notes.append("missing name")
|
|
547
|
+
if fm.get("description") and len(str(fm.get("description", ""))) >= 50:
|
|
548
|
+
meta_score += 3
|
|
549
|
+
else:
|
|
550
|
+
meta_notes.append("description too short")
|
|
551
|
+
if fm.get("version"):
|
|
552
|
+
meta_score += 2
|
|
553
|
+
else:
|
|
554
|
+
meta_notes.append("missing version")
|
|
555
|
+
if fm.get("allowed-tools"):
|
|
556
|
+
meta_score += 2
|
|
557
|
+
else:
|
|
558
|
+
meta_notes.append("missing allowed-tools")
|
|
559
|
+
if fm.get("author") and "@" in str(fm.get("author", "")):
|
|
560
|
+
meta_score += 1
|
|
561
|
+
breakdown["metadata_quality"] = (meta_score, ", ".join(meta_notes) if meta_notes else "Complete metadata")
|
|
562
|
+
|
|
563
|
+
# Discoverability (6 pts)
|
|
564
|
+
disc_score = 0
|
|
565
|
+
disc_notes = []
|
|
566
|
+
if "use when" in desc:
|
|
567
|
+
disc_score += 3
|
|
568
|
+
disc_notes.append("has 'Use when'")
|
|
569
|
+
if "trigger with" in desc or "trigger phrase" in desc:
|
|
570
|
+
disc_score += 3
|
|
571
|
+
disc_notes.append("has trigger phrases")
|
|
572
|
+
if not disc_notes:
|
|
573
|
+
disc_notes.append("missing discovery cues")
|
|
574
|
+
breakdown["discoverability"] = (disc_score, ", ".join(disc_notes))
|
|
575
|
+
|
|
576
|
+
# Terminology Consistency (4 pts)
|
|
577
|
+
name = str(fm.get("name", ""))
|
|
578
|
+
folder = path.parent.name
|
|
579
|
+
term_score = 4
|
|
580
|
+
term_notes = []
|
|
581
|
+
if name and name != folder:
|
|
582
|
+
term_score -= 2
|
|
583
|
+
term_notes.append("name differs from folder")
|
|
584
|
+
if any(w.isupper() and len(w) > 3 for w in str(fm.get("description", "")).split()):
|
|
585
|
+
term_score -= 1
|
|
586
|
+
term_notes.append("inconsistent casing")
|
|
587
|
+
breakdown["terminology"] = (max(0, term_score), ", ".join(term_notes) if term_notes else "Consistent terminology")
|
|
588
|
+
|
|
589
|
+
# Workflow Clarity (5 pts)
|
|
590
|
+
workflow_score = 0
|
|
591
|
+
workflow_notes = []
|
|
592
|
+
if re.search(r"(?m)^\s*1\.\s+", body):
|
|
593
|
+
workflow_score += 3
|
|
594
|
+
workflow_notes.append("has numbered steps")
|
|
595
|
+
section_count = len(re.findall(r"(?m)^##\s+", body))
|
|
596
|
+
if section_count >= 5:
|
|
597
|
+
workflow_score += 2
|
|
598
|
+
workflow_notes.append(f"{section_count} sections")
|
|
599
|
+
elif section_count >= 3:
|
|
600
|
+
workflow_score += 1
|
|
601
|
+
workflow_notes.append(f"{section_count} sections (add more)")
|
|
602
|
+
if not workflow_notes:
|
|
603
|
+
workflow_notes.append("unclear workflow")
|
|
604
|
+
breakdown["workflow_clarity"] = (workflow_score, ", ".join(workflow_notes))
|
|
605
|
+
|
|
606
|
+
total = sum(v[0] for v in breakdown.values())
|
|
607
|
+
return {"score": total, "max": 25, "breakdown": breakdown}
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def score_utility(path: Path, body: str, fm: dict) -> dict:
|
|
611
|
+
"""Utility (20 pts max)."""
|
|
612
|
+
breakdown = {}
|
|
613
|
+
body_lower = body.lower()
|
|
614
|
+
|
|
615
|
+
# Problem Solving Power (8 pts)
|
|
616
|
+
problem_score = 0
|
|
617
|
+
problem_notes = []
|
|
618
|
+
if "## overview" in body_lower:
|
|
619
|
+
overview_match = re.search(r"## overview\s*\n(.*?)(?=\n##|\Z)", body, re.IGNORECASE | re.DOTALL)
|
|
620
|
+
if overview_match and len(overview_match.group(1).strip()) > 50:
|
|
621
|
+
problem_score += 4
|
|
622
|
+
problem_notes.append("has overview")
|
|
623
|
+
if "## prerequisites" in body_lower:
|
|
624
|
+
problem_score += 2
|
|
625
|
+
problem_notes.append("has prerequisites")
|
|
626
|
+
if "## output" in body_lower:
|
|
627
|
+
problem_score += 2
|
|
628
|
+
problem_notes.append("has output spec")
|
|
629
|
+
if not problem_notes:
|
|
630
|
+
problem_notes.append("unclear problem/solution")
|
|
631
|
+
breakdown["problem_solving"] = (problem_score, ", ".join(problem_notes))
|
|
632
|
+
|
|
633
|
+
# Degrees of Freedom (5 pts)
|
|
634
|
+
freedom_score = 0
|
|
635
|
+
freedom_notes = []
|
|
636
|
+
if re.search(r"(?i)(optional|configur|parameter|argument|flag|option)", body):
|
|
637
|
+
freedom_score += 2
|
|
638
|
+
freedom_notes.append("has options")
|
|
639
|
+
if re.search(r"(?i)(alternatively|or use|another approach|you can also)", body):
|
|
640
|
+
freedom_score += 2
|
|
641
|
+
freedom_notes.append("shows alternatives")
|
|
642
|
+
if re.search(r"(?i)(extend|customize|modify|adapt)", body):
|
|
643
|
+
freedom_score += 1
|
|
644
|
+
freedom_notes.append("extensible")
|
|
645
|
+
if not freedom_notes:
|
|
646
|
+
freedom_notes.append("rigid implementation")
|
|
647
|
+
breakdown["degrees_of_freedom"] = (freedom_score, ", ".join(freedom_notes))
|
|
648
|
+
|
|
649
|
+
# Feedback Loops (4 pts)
|
|
650
|
+
feedback_score = 0
|
|
651
|
+
feedback_notes = []
|
|
652
|
+
if "## error handling" in body_lower:
|
|
653
|
+
feedback_score += 2
|
|
654
|
+
feedback_notes.append("has error handling")
|
|
655
|
+
if re.search(r"(?i)(validate|verify|check|test|confirm)", body):
|
|
656
|
+
feedback_score += 1
|
|
657
|
+
feedback_notes.append("has validation")
|
|
658
|
+
if re.search(r"(?i)(troubleshoot|debug|diagnose|fix)", body):
|
|
659
|
+
feedback_score += 1
|
|
660
|
+
feedback_notes.append("has troubleshooting")
|
|
661
|
+
if not feedback_notes:
|
|
662
|
+
feedback_notes.append("no feedback mechanisms")
|
|
663
|
+
breakdown["feedback_loops"] = (feedback_score, ", ".join(feedback_notes))
|
|
664
|
+
|
|
665
|
+
# Examples & Templates (3 pts)
|
|
666
|
+
examples_score = 0
|
|
667
|
+
examples_notes = []
|
|
668
|
+
if "## examples" in body_lower or "**example" in body_lower:
|
|
669
|
+
examples_score += 2
|
|
670
|
+
examples_notes.append("has examples")
|
|
671
|
+
if "```" in body:
|
|
672
|
+
code_blocks = len(re.findall(r"```", body)) // 2
|
|
673
|
+
if code_blocks >= 2:
|
|
674
|
+
examples_score += 1
|
|
675
|
+
examples_notes.append(f"{code_blocks} code blocks")
|
|
676
|
+
if not examples_notes:
|
|
677
|
+
examples_notes.append("no examples")
|
|
678
|
+
breakdown["examples"] = (examples_score, ", ".join(examples_notes))
|
|
679
|
+
|
|
680
|
+
total = sum(v[0] for v in breakdown.values())
|
|
681
|
+
return {"score": total, "max": 20, "breakdown": breakdown}
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def score_spec_compliance(path: Path, body: str, fm: dict) -> dict:
|
|
685
|
+
"""Spec Compliance (15 pts max)."""
|
|
686
|
+
breakdown = {}
|
|
687
|
+
name = str(fm.get("name", ""))
|
|
688
|
+
desc = str(fm.get("description", ""))
|
|
689
|
+
|
|
690
|
+
# Frontmatter Validity (5 pts)
|
|
691
|
+
fm_score = 5
|
|
692
|
+
fm_notes = []
|
|
693
|
+
required = {"name", "description", "allowed-tools", "version", "author", "license"}
|
|
694
|
+
missing = required - set(fm.keys())
|
|
695
|
+
if missing:
|
|
696
|
+
fm_score -= min(len(missing), 4)
|
|
697
|
+
fm_notes.append(f"missing: {', '.join(sorted(missing))}")
|
|
698
|
+
if not fm_notes:
|
|
699
|
+
fm_notes.append("valid frontmatter")
|
|
700
|
+
breakdown["frontmatter_validity"] = (max(0, fm_score), ", ".join(fm_notes))
|
|
701
|
+
|
|
702
|
+
# Name Conventions (4 pts)
|
|
703
|
+
name_score = 4
|
|
704
|
+
name_notes = []
|
|
705
|
+
if not re.match(r"^[a-z0-9][a-z0-9-]*[a-z0-9]$", name) and len(name) > 1:
|
|
706
|
+
name_score -= 2
|
|
707
|
+
name_notes.append("not kebab-case")
|
|
708
|
+
if len(name) > 64:
|
|
709
|
+
name_score -= 1
|
|
710
|
+
name_notes.append("name too long")
|
|
711
|
+
if name != path.parent.name:
|
|
712
|
+
name_score -= 1
|
|
713
|
+
name_notes.append("name/folder mismatch")
|
|
714
|
+
if not name_notes:
|
|
715
|
+
name_notes.append("proper naming")
|
|
716
|
+
breakdown["name_conventions"] = (max(0, name_score), ", ".join(name_notes))
|
|
717
|
+
|
|
718
|
+
# Description Quality (4 pts)
|
|
719
|
+
desc_score = 4
|
|
720
|
+
desc_notes = []
|
|
721
|
+
if len(desc) < 50:
|
|
722
|
+
desc_score -= 2
|
|
723
|
+
desc_notes.append("too short")
|
|
724
|
+
if len(desc) > 1024:
|
|
725
|
+
desc_score -= 2
|
|
726
|
+
desc_notes.append("too long")
|
|
727
|
+
desc_lower = desc.lower()
|
|
728
|
+
if "i can" in desc_lower or "i will" in desc_lower:
|
|
729
|
+
desc_score -= 1
|
|
730
|
+
desc_notes.append("uses first person")
|
|
731
|
+
if "you can" in desc_lower or "you should" in desc_lower:
|
|
732
|
+
desc_score -= 1
|
|
733
|
+
desc_notes.append("uses second person")
|
|
734
|
+
if not desc_notes:
|
|
735
|
+
desc_notes.append("good description")
|
|
736
|
+
breakdown["description_quality"] = (max(0, desc_score), ", ".join(desc_notes))
|
|
737
|
+
|
|
738
|
+
# Optional Fields (2 pts)
|
|
739
|
+
opt_score = 2
|
|
740
|
+
opt_notes = []
|
|
741
|
+
if "model" in fm:
|
|
742
|
+
model = fm["model"]
|
|
743
|
+
if model not in ["inherit", "sonnet", "haiku", "opus"] and not str(model).startswith("claude-"):
|
|
744
|
+
opt_score -= 1
|
|
745
|
+
opt_notes.append("invalid model value")
|
|
746
|
+
if not opt_notes:
|
|
747
|
+
opt_notes.append("optional fields ok")
|
|
748
|
+
breakdown["optional_fields"] = (opt_score, ", ".join(opt_notes))
|
|
749
|
+
|
|
750
|
+
total = sum(v[0] for v in breakdown.values())
|
|
751
|
+
return {"score": total, "max": 15, "breakdown": breakdown}
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def score_writing_style(path: Path, body: str, fm: dict) -> dict:
|
|
755
|
+
"""Writing Style (10 pts max)."""
|
|
756
|
+
breakdown = {}
|
|
757
|
+
|
|
758
|
+
# Voice & Tense (4 pts)
|
|
759
|
+
voice_score = 4
|
|
760
|
+
voice_notes = []
|
|
761
|
+
imperative_verbs = ["create", "use", "run", "execute", "configure", "set", "add", "remove", "check", "verify"]
|
|
762
|
+
has_imperative = any(re.search(rf"(?m)^\s*\d+\.\s*{v}", body, re.IGNORECASE) for v in imperative_verbs)
|
|
763
|
+
if not has_imperative:
|
|
764
|
+
voice_score -= 2
|
|
765
|
+
voice_notes.append("use imperative voice")
|
|
766
|
+
if not voice_notes:
|
|
767
|
+
voice_notes.append("good voice")
|
|
768
|
+
breakdown["voice_tense"] = (voice_score, ", ".join(voice_notes))
|
|
769
|
+
|
|
770
|
+
# Objectivity (3 pts)
|
|
771
|
+
obj_score = 3
|
|
772
|
+
obj_notes = []
|
|
773
|
+
body_lower = body.lower()
|
|
774
|
+
if "you should" in body_lower or "you can" in body_lower or "you will" in body_lower:
|
|
775
|
+
obj_score -= 1
|
|
776
|
+
obj_notes.append("has second person")
|
|
777
|
+
if " i " in body_lower or "i can" in body_lower or "i'll" in body_lower:
|
|
778
|
+
obj_score -= 1
|
|
779
|
+
obj_notes.append("has first person")
|
|
780
|
+
if not obj_notes:
|
|
781
|
+
obj_notes.append("objective")
|
|
782
|
+
breakdown["objectivity"] = (max(0, obj_score), ", ".join(obj_notes))
|
|
783
|
+
|
|
784
|
+
# Conciseness (3 pts)
|
|
785
|
+
conc_score = 3
|
|
786
|
+
conc_notes = []
|
|
787
|
+
word_count = len(body.split())
|
|
788
|
+
lines = len(body.splitlines())
|
|
789
|
+
if word_count > 3000:
|
|
790
|
+
conc_score -= 2
|
|
791
|
+
conc_notes.append(f"verbose ({word_count} words)")
|
|
792
|
+
elif word_count > 2000:
|
|
793
|
+
conc_score -= 1
|
|
794
|
+
conc_notes.append(f"lengthy ({word_count} words)")
|
|
795
|
+
if lines > 400:
|
|
796
|
+
conc_score -= 1
|
|
797
|
+
conc_notes.append(f"many lines ({lines})")
|
|
798
|
+
if not conc_notes:
|
|
799
|
+
conc_notes.append("concise")
|
|
800
|
+
breakdown["conciseness"] = (max(0, conc_score), ", ".join(conc_notes))
|
|
801
|
+
|
|
802
|
+
total = sum(v[0] for v in breakdown.values())
|
|
803
|
+
return {"score": total, "max": 10, "breakdown": breakdown}
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def calculate_modifiers(path: Path, body: str, fm: dict) -> dict:
|
|
807
|
+
"""Modifiers (-5 to +5 pts)."""
|
|
808
|
+
modifiers = {}
|
|
809
|
+
name = str(fm.get("name", ""))
|
|
810
|
+
desc = str(fm.get("description", ""))
|
|
811
|
+
lines = len(body.splitlines())
|
|
812
|
+
|
|
813
|
+
# Bonuses
|
|
814
|
+
if name.endswith("ing") or any(name.endswith(f"-{s}") for s in ["ing"]):
|
|
815
|
+
modifiers["gerund_name"] = (+1, "gerund-style name")
|
|
816
|
+
|
|
817
|
+
sections = len(re.findall(r"(?m)^##\s+", body))
|
|
818
|
+
if sections >= 7:
|
|
819
|
+
modifiers["grep_friendly"] = (+1, "grep-friendly structure")
|
|
820
|
+
|
|
821
|
+
example_count = len(re.findall(r"(?i)\*\*example[:\s]", body))
|
|
822
|
+
if example_count >= 3:
|
|
823
|
+
modifiers["exemplary_examples"] = (+2, f"{example_count} labeled examples")
|
|
824
|
+
|
|
825
|
+
if "## resources" in body.lower():
|
|
826
|
+
external_links = len(re.findall(r"\[.*?\]\(https?://", body))
|
|
827
|
+
if external_links >= 2:
|
|
828
|
+
modifiers["external_resources"] = (+1, f"{external_links} external links")
|
|
829
|
+
|
|
830
|
+
# Penalties
|
|
831
|
+
desc_lower = desc.lower()
|
|
832
|
+
if "i can" in desc_lower or "i will" in desc_lower or "you can" in desc_lower or "you should" in desc_lower:
|
|
833
|
+
modifiers["person_in_desc"] = (-2, "first/second person in description")
|
|
834
|
+
|
|
835
|
+
has_toc = bool(re.search(r"(?mi)^##?\s*(table of contents|contents|toc)\b", body))
|
|
836
|
+
if lines > 150 and not has_toc:
|
|
837
|
+
modifiers["missing_toc"] = (-2, "long file needs TOC")
|
|
838
|
+
|
|
839
|
+
if "<" in body and ">" in body and re.search(r"<[a-z]+>", body):
|
|
840
|
+
modifiers["xml_tags"] = (-1, "XML-like tags in body")
|
|
841
|
+
|
|
842
|
+
total = sum(v[0] for v in modifiers.values())
|
|
843
|
+
total = max(-5, min(5, total))
|
|
844
|
+
return {"score": total, "max_bonus": 5, "max_penalty": -5, "items": modifiers}
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def grade_skill(path: Path, body: str, fm: dict) -> dict:
|
|
848
|
+
"""Calculate Intent Solutions 100-point grade."""
|
|
849
|
+
pda = score_progressive_disclosure(path, body, fm)
|
|
850
|
+
ease = score_ease_of_use(path, body, fm)
|
|
851
|
+
utility = score_utility(path, body, fm)
|
|
852
|
+
spec = score_spec_compliance(path, body, fm)
|
|
853
|
+
style = score_writing_style(path, body, fm)
|
|
854
|
+
mods = calculate_modifiers(path, body, fm)
|
|
855
|
+
|
|
856
|
+
base_score = pda["score"] + ease["score"] + utility["score"] + spec["score"] + style["score"]
|
|
857
|
+
total_score = max(0, min(100, base_score + mods["score"]))
|
|
858
|
+
|
|
859
|
+
if total_score >= 90:
|
|
860
|
+
grade = "A"
|
|
861
|
+
elif total_score >= 80:
|
|
862
|
+
grade = "B"
|
|
863
|
+
elif total_score >= 70:
|
|
864
|
+
grade = "C"
|
|
865
|
+
elif total_score >= 60:
|
|
866
|
+
grade = "D"
|
|
867
|
+
else:
|
|
868
|
+
grade = "F"
|
|
869
|
+
|
|
870
|
+
# Generate improvement suggestions
|
|
871
|
+
improvements = []
|
|
872
|
+
desc_lower = str(fm.get("description", "")).lower()
|
|
873
|
+
if "use when" not in desc_lower:
|
|
874
|
+
improvements.append('Add "Use when" to description (+3 pts)')
|
|
875
|
+
if "trigger with" not in desc_lower:
|
|
876
|
+
improvements.append('Add "Trigger with" to description (+3 pts)')
|
|
877
|
+
body_lower = body.lower()
|
|
878
|
+
if "## overview" not in body_lower:
|
|
879
|
+
improvements.append("Add ## Overview section (+4 pts)")
|
|
880
|
+
if "## prerequisites" not in body_lower:
|
|
881
|
+
improvements.append("Add ## Prerequisites section (+2 pts)")
|
|
882
|
+
if "## output" not in body_lower:
|
|
883
|
+
improvements.append("Add ## Output section (+2 pts)")
|
|
884
|
+
if "## error handling" not in body_lower:
|
|
885
|
+
improvements.append("Add ## Error Handling section (+2 pts)")
|
|
886
|
+
if "## examples" not in body_lower:
|
|
887
|
+
improvements.append("Add ## Examples section (+2 pts)")
|
|
888
|
+
if not fm.get("version"):
|
|
889
|
+
improvements.append("Add version field (+2 pts)")
|
|
890
|
+
if not fm.get("author"):
|
|
891
|
+
improvements.append("Add author field (+1 pt)")
|
|
892
|
+
if not fm.get("allowed-tools"):
|
|
893
|
+
improvements.append("Add allowed-tools field (+2 pts)")
|
|
894
|
+
|
|
895
|
+
return {
|
|
896
|
+
"score": total_score,
|
|
897
|
+
"grade": grade,
|
|
898
|
+
"improvements": improvements,
|
|
899
|
+
"breakdown": {
|
|
900
|
+
"progressive_disclosure": pda,
|
|
901
|
+
"ease_of_use": ease,
|
|
902
|
+
"utility": utility,
|
|
903
|
+
"spec_compliance": spec,
|
|
904
|
+
"writing_style": style,
|
|
905
|
+
"modifiers": mods,
|
|
906
|
+
},
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def validate_skill(path: Path, enterprise: bool = False) -> Dict[str, Any]:
|
|
911
|
+
"""Validate a SKILL.md file."""
|
|
912
|
+
try:
|
|
913
|
+
content = path.read_text(encoding="utf-8")
|
|
914
|
+
except Exception as e:
|
|
915
|
+
return {"fatal": f"Cannot read file: {e}"}
|
|
916
|
+
|
|
917
|
+
try:
|
|
918
|
+
fm, body = parse_frontmatter(content)
|
|
919
|
+
except Exception as e:
|
|
920
|
+
return {"fatal": str(e)}
|
|
921
|
+
|
|
922
|
+
errors: List[str] = []
|
|
923
|
+
warnings: List[str] = []
|
|
924
|
+
info: List[str] = []
|
|
925
|
+
|
|
926
|
+
name_e, name_w = validate_name(fm, path, enterprise)
|
|
927
|
+
errors.extend(name_e)
|
|
928
|
+
warnings.extend(name_w)
|
|
929
|
+
|
|
930
|
+
desc_e, desc_w = validate_description(fm, enterprise)
|
|
931
|
+
errors.extend(desc_e)
|
|
932
|
+
warnings.extend(desc_w)
|
|
933
|
+
|
|
934
|
+
tools_e, tools_w = validate_tools(fm, enterprise)
|
|
935
|
+
errors.extend(tools_e)
|
|
936
|
+
warnings.extend(tools_w)
|
|
937
|
+
|
|
938
|
+
opt_e, opt_w = validate_optional_fields(fm)
|
|
939
|
+
errors.extend(opt_e)
|
|
940
|
+
warnings.extend(opt_w)
|
|
941
|
+
|
|
942
|
+
if enterprise:
|
|
943
|
+
meta_e, meta_w = validate_enterprise_metadata(fm)
|
|
944
|
+
errors.extend(meta_e)
|
|
945
|
+
warnings.extend(meta_w)
|
|
946
|
+
|
|
947
|
+
body_e, body_w, body_i = validate_body(body, path, enterprise)
|
|
948
|
+
errors.extend(body_e)
|
|
949
|
+
warnings.extend(body_w)
|
|
950
|
+
info.extend(body_i)
|
|
951
|
+
|
|
952
|
+
resource_errors = validate_resources(body, path)
|
|
953
|
+
errors.extend(resource_errors)
|
|
954
|
+
|
|
955
|
+
word_count = len(body.split())
|
|
956
|
+
line_count = len(body.splitlines())
|
|
957
|
+
token_estimate = int(word_count * 1.3)
|
|
958
|
+
disclosure_score = calculate_disclosure_score(fm, body, path)
|
|
959
|
+
|
|
960
|
+
return {
|
|
961
|
+
"valid": len(errors) == 0,
|
|
962
|
+
"tier": "Enterprise" if enterprise else "Standard",
|
|
963
|
+
"errors": errors,
|
|
964
|
+
"warnings": warnings,
|
|
965
|
+
"info": info,
|
|
966
|
+
"stats": {
|
|
967
|
+
"word_count": word_count,
|
|
968
|
+
"line_count": line_count,
|
|
969
|
+
"token_estimate": token_estimate,
|
|
970
|
+
"disclosure_score": disclosure_score,
|
|
971
|
+
},
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def print_result(result: Dict[str, Any], path: Path) -> None:
|
|
976
|
+
"""Print validation result in human-readable format."""
|
|
977
|
+
if "fatal" in result:
|
|
978
|
+
print(f"FATAL: {result['fatal']}")
|
|
979
|
+
sys.exit(1)
|
|
980
|
+
|
|
981
|
+
tier = result["tier"]
|
|
982
|
+
stats = result["stats"]
|
|
983
|
+
|
|
984
|
+
print(f"\n{'=' * 60}")
|
|
985
|
+
print(f"SKILL VALIDATION: {path.name} [{tier} tier]")
|
|
986
|
+
print(f"{'=' * 60}\n")
|
|
987
|
+
|
|
988
|
+
if result["errors"]:
|
|
989
|
+
print("ERRORS:")
|
|
990
|
+
for e in result["errors"]:
|
|
991
|
+
print(f" x {e}")
|
|
992
|
+
print()
|
|
993
|
+
|
|
994
|
+
if result["warnings"]:
|
|
995
|
+
print("WARNINGS:")
|
|
996
|
+
for w in result["warnings"]:
|
|
997
|
+
print(f" ! {w}")
|
|
998
|
+
print()
|
|
999
|
+
|
|
1000
|
+
if result["info"]:
|
|
1001
|
+
print("INFO:")
|
|
1002
|
+
for i in result["info"]:
|
|
1003
|
+
print(f" - {i}")
|
|
1004
|
+
print()
|
|
1005
|
+
|
|
1006
|
+
print(f"Stats:")
|
|
1007
|
+
print(f" Words: {stats['word_count']}")
|
|
1008
|
+
print(f" Lines: {stats['line_count']}")
|
|
1009
|
+
print(f" Tokens (est.): {stats['token_estimate']}")
|
|
1010
|
+
print(f" Disclosure Score: {stats['disclosure_score']}/6")
|
|
1011
|
+
print()
|
|
1012
|
+
|
|
1013
|
+
if result["valid"]:
|
|
1014
|
+
print("PASSED")
|
|
1015
|
+
else:
|
|
1016
|
+
print(f"FAILED ({len(result['errors'])} errors)")
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def print_grade(grade_result: Dict[str, Any], path: Path) -> None:
|
|
1020
|
+
"""Print 100-point grade report."""
|
|
1021
|
+
bd = grade_result["breakdown"]
|
|
1022
|
+
|
|
1023
|
+
print(f"\n{'=' * 60}")
|
|
1024
|
+
print(f"SKILL GRADE: {path.name}")
|
|
1025
|
+
print(f"{'=' * 60}\n")
|
|
1026
|
+
|
|
1027
|
+
print(f"Grade: {grade_result['grade']} ({grade_result['score']}/100)\n")
|
|
1028
|
+
|
|
1029
|
+
# Print each pillar
|
|
1030
|
+
pillars = [
|
|
1031
|
+
("Progressive Disclosure", "progressive_disclosure"),
|
|
1032
|
+
("Ease of Use", "ease_of_use"),
|
|
1033
|
+
("Utility", "utility"),
|
|
1034
|
+
("Spec Compliance", "spec_compliance"),
|
|
1035
|
+
("Writing Style", "writing_style"),
|
|
1036
|
+
]
|
|
1037
|
+
|
|
1038
|
+
for label, key in pillars:
|
|
1039
|
+
pillar = bd[key]
|
|
1040
|
+
sub_scores = ", ".join(f"{k}={v[0]}" for k, v in pillar["breakdown"].items())
|
|
1041
|
+
print(f" {label + ':':28s} {pillar['score']:2d}/{pillar['max']} [{sub_scores}]")
|
|
1042
|
+
|
|
1043
|
+
# Modifiers
|
|
1044
|
+
mods = bd["modifiers"]
|
|
1045
|
+
mod_items = ", ".join(f"{k}={v[0]:+d}" for k, v in mods["items"].items()) if mods["items"] else "none"
|
|
1046
|
+
sign = "+" if mods["score"] >= 0 else ""
|
|
1047
|
+
print(f" {'Modifiers:':28s} {sign}{mods['score']} [{mod_items}]")
|
|
1048
|
+
print()
|
|
1049
|
+
|
|
1050
|
+
# Improvements
|
|
1051
|
+
if grade_result["improvements"]:
|
|
1052
|
+
print("Improvements:")
|
|
1053
|
+
for imp in grade_result["improvements"]:
|
|
1054
|
+
print(f" - {imp}")
|
|
1055
|
+
print()
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def main():
|
|
1059
|
+
parser = argparse.ArgumentParser(
|
|
1060
|
+
description="Validate SKILL.md files (Standard tier by default)"
|
|
1061
|
+
)
|
|
1062
|
+
parser.add_argument("path", help="Path to SKILL.md file")
|
|
1063
|
+
parser.add_argument(
|
|
1064
|
+
"--standard",
|
|
1065
|
+
action="store_true",
|
|
1066
|
+
help="Use Standard tier - AgentSkills.io minimum (default, kept for compatibility)",
|
|
1067
|
+
)
|
|
1068
|
+
parser.add_argument(
|
|
1069
|
+
"--enterprise",
|
|
1070
|
+
action="store_true",
|
|
1071
|
+
help="Use Enterprise tier: Standard + identity fields, scoped tools, sections, disclosure",
|
|
1072
|
+
)
|
|
1073
|
+
parser.add_argument("--grade", action="store_true", help="Run 100-point grading rubric")
|
|
1074
|
+
parser.add_argument("--json", action="store_true", help="JSON output")
|
|
1075
|
+
args = parser.parse_args()
|
|
1076
|
+
|
|
1077
|
+
path = Path(args.path).resolve()
|
|
1078
|
+
if not path.exists():
|
|
1079
|
+
print(f"Error: {path} not found", file=sys.stderr)
|
|
1080
|
+
sys.exit(1)
|
|
1081
|
+
|
|
1082
|
+
enterprise = args.enterprise and not args.standard
|
|
1083
|
+
|
|
1084
|
+
if args.grade:
|
|
1085
|
+
try:
|
|
1086
|
+
content = path.read_text(encoding="utf-8")
|
|
1087
|
+
fm, body = parse_frontmatter(content)
|
|
1088
|
+
except Exception as e:
|
|
1089
|
+
print(f"FATAL: {e}", file=sys.stderr)
|
|
1090
|
+
sys.exit(1)
|
|
1091
|
+
|
|
1092
|
+
grade_result = grade_skill(path, body, fm)
|
|
1093
|
+
|
|
1094
|
+
if args.json:
|
|
1095
|
+
# Serialize breakdown for JSON (convert tuples to dicts)
|
|
1096
|
+
output = {
|
|
1097
|
+
"score": grade_result["score"],
|
|
1098
|
+
"grade": grade_result["grade"],
|
|
1099
|
+
"improvements": grade_result["improvements"],
|
|
1100
|
+
"breakdown": {},
|
|
1101
|
+
}
|
|
1102
|
+
for pillar_key, pillar_data in grade_result["breakdown"].items():
|
|
1103
|
+
if pillar_key == "modifiers":
|
|
1104
|
+
output["breakdown"]["modifiers"] = {
|
|
1105
|
+
"score": pillar_data["score"],
|
|
1106
|
+
"items": {k: {"points": v[0], "reason": v[1]} for k, v in pillar_data["items"].items()},
|
|
1107
|
+
}
|
|
1108
|
+
else:
|
|
1109
|
+
output["breakdown"][pillar_key] = {
|
|
1110
|
+
"score": pillar_data["score"],
|
|
1111
|
+
"max": pillar_data["max"],
|
|
1112
|
+
"breakdown": {k: {"points": v[0], "reason": v[1]} for k, v in pillar_data["breakdown"].items()},
|
|
1113
|
+
}
|
|
1114
|
+
print(json.dumps(output, indent=2))
|
|
1115
|
+
else:
|
|
1116
|
+
print_grade(grade_result, path)
|
|
1117
|
+
|
|
1118
|
+
sys.exit(0 if grade_result["score"] >= 60 else 1)
|
|
1119
|
+
|
|
1120
|
+
# Standard validation
|
|
1121
|
+
result = validate_skill(path, enterprise=enterprise)
|
|
1122
|
+
|
|
1123
|
+
if args.json:
|
|
1124
|
+
print(json.dumps(result, indent=2))
|
|
1125
|
+
else:
|
|
1126
|
+
print_result(result, path)
|
|
1127
|
+
|
|
1128
|
+
sys.exit(0 if result.get("valid", False) else 1)
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
if __name__ == "__main__":
|
|
1132
|
+
main()
|