@cleocode/skills 2026.5.96 → 2026.5.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/ct-skill-validator/SKILL.md +68 -29
- package/skills/ct-skill-validator/evals/{evals.json → quality_evals.json} +1 -1
- package/skills/ct-skill-validator/references/validation-rules.md +77 -26
- package/skills/ct-skill-validator/scripts/_skill_finder.py +211 -0
- package/skills/ct-skill-validator/scripts/audit_body.py +10 -5
- package/skills/ct-skill-validator/scripts/check_depth.py +109 -6
- package/skills/ct-skill-validator/scripts/generate_validation_report.py +1 -1
- package/skills/ct-skill-validator/scripts/run_quality_eval.py +110 -0
- package/skills/ct-skill-validator/scripts/validate.py +87 -9
- package/skills/ct-skill-validator/scripts/__init__.py +0 -0
- package/skills/ct-skill-validator/scripts/__pycache__/audit_body.cpython-314.pyc +0 -0
- package/skills/ct-skill-validator/scripts/__pycache__/check_ecosystem.cpython-314.pyc +0 -0
- package/skills/ct-skill-validator/scripts/__pycache__/generate_validation_report.cpython-314.pyc +0 -0
- package/skills/ct-skill-validator/scripts/__pycache__/validate.cpython-314.pyc +0 -0
- package/skills/ct-skill-validator/scripts/check_manifest.py +0 -172
- /package/skills/ct-skill-validator/evals/{eval_set.json → trigger_queries.json} +0 -0
package/package.json
CHANGED
|
@@ -34,14 +34,18 @@ python ${CLAUDE_SKILL_DIR}/scripts/validate.py <skill-dir> --json
|
|
|
34
34
|
# Deep body quality audit (optional, run alongside validate.py):
|
|
35
35
|
python ${CLAUDE_SKILL_DIR}/scripts/audit_body.py <skill-dir>
|
|
36
36
|
|
|
37
|
-
# Manifest alignment check:
|
|
38
|
-
python
|
|
37
|
+
# Manifest alignment check: bundled into validate.py Tier 4. Use:
|
|
38
|
+
# python validate.py <skill-dir> --manifest <manifest.json> --dispatch-config <dispatch-config.json>
|
|
39
39
|
|
|
40
40
|
# Progressive-disclosure depth check (T9684 — CI gate):
|
|
41
41
|
python ${CLAUDE_SKILL_DIR}/scripts/check_depth.py <skill-dir>
|
|
42
42
|
|
|
43
43
|
# Repo-wide depth sweep:
|
|
44
44
|
python ${CLAUDE_SKILL_DIR}/scripts/check_depth.py <repo-root> --all
|
|
45
|
+
|
|
46
|
+
# Allowlist audit (CI / cron — exit 1 on findings):
|
|
47
|
+
python ${CLAUDE_SKILL_DIR}/scripts/check_depth.py <skill-dir> --audit-allowlist
|
|
48
|
+
python ${CLAUDE_SKILL_DIR}/scripts/check_depth.py <skill-dir> --audit-allowlist --json
|
|
45
49
|
```
|
|
46
50
|
|
|
47
51
|
**Depth rule (T9684):** A skill PASSES when ANY of:
|
|
@@ -54,6 +58,13 @@ Pre-existing stubs are allowlisted with follow-up task IDs in
|
|
|
54
58
|
`scripts/check_depth.py::ALLOWLIST`. Gold-standard skills:
|
|
55
59
|
`ct-orchestrator` (9 refs) and `ct-skill-creator` (7 refs).
|
|
56
60
|
|
|
61
|
+
**Allowlist hygiene:** every entry carries `last_reviewed: YYYY-MM-DD HH:MM:SS`.
|
|
62
|
+
`check_depth.py` runs a silent background audit on every invocation and emits
|
|
63
|
+
WARNs to stderr for malformed or stale (> 30 days) entries. Use
|
|
64
|
+
`--audit-allowlist` for an explicit pass that exits 1 on any finding —
|
|
65
|
+
suitable for a CI cron job. The threshold is tunable via
|
|
66
|
+
`ALLOWLIST_STALE_DAYS` at the top of `check_depth.py`.
|
|
67
|
+
|
|
57
68
|
The depth check runs on every PR touching `packages/skills/skills/**`
|
|
58
69
|
via `.github/workflows/skills-depth-check.yml`.
|
|
59
70
|
|
|
@@ -119,40 +130,68 @@ Repeat until verdict is `PASS` or `PASS_WITH_WARNINGS`. WARN is acceptable; ERRO
|
|
|
119
130
|
## Phase 3: Quality A/B Eval
|
|
120
131
|
|
|
121
132
|
Tests whether the skill actually improves agent output quality vs. no skill context.
|
|
122
|
-
|
|
133
|
+
Phase 3 is **delegated** — `ct-skill-validator` does static analysis; runtime
|
|
134
|
+
quality evals live in a dedicated skill (`skill-evaluator` preferred,
|
|
135
|
+
`ct-skill-creator` as legacy fallback).
|
|
136
|
+
|
|
137
|
+
> **Scope boundary:** `ct-skill-validator` is *static* — it checks structure,
|
|
138
|
+
> frontmatter, body, manifest, depth, ecosystem fit. For deep runtime A/B
|
|
139
|
+
> benchmarking, regression detection, and auto-improvement, the dispatcher
|
|
140
|
+
> below routes to `skill-evaluator`, which owns that workflow end-to-end.
|
|
141
|
+
|
|
142
|
+
The two eval files in `evals/` serve different purposes:
|
|
143
|
+
- `evals/trigger_queries.json` — trigger queries (does the description activate correctly?)
|
|
144
|
+
- `evals/quality_evals.json` — output-quality scenarios (does the validator produce the right report?)
|
|
145
|
+
|
|
146
|
+
### Dispatch (no hardcoded cross-skill paths)
|
|
147
|
+
|
|
148
|
+
`scripts/run_quality_eval.py` uses `_skill_finder.py` to dynamically locate
|
|
149
|
+
the eval skill at runtime. It searches:
|
|
150
|
+
|
|
151
|
+
1. `$SKILL_FINDER_PATH` (colon-separated override)
|
|
152
|
+
2. Direct sibling of this skill
|
|
153
|
+
3. `<this-skill>/../../skills/<name>/` (CLEO / awesome-skills layouts)
|
|
154
|
+
4. Walk-up ancestors + their project-shaped children (cross-project)
|
|
155
|
+
5. `~/.claude/skills/<name>/`
|
|
156
|
+
|
|
157
|
+
Show what would be used (without running anything):
|
|
158
|
+
```bash
|
|
159
|
+
python ${CLAUDE_SKILL_DIR}/scripts/run_quality_eval.py --list
|
|
160
|
+
```
|
|
123
161
|
|
|
124
162
|
**Trigger accuracy** — does the skill description trigger correctly?
|
|
125
163
|
```bash
|
|
126
|
-
python ${CLAUDE_SKILL_DIR}
|
|
127
|
-
--
|
|
128
|
-
--skill-path ${CLAUDE_SKILL_DIR}
|
|
164
|
+
python ${CLAUDE_SKILL_DIR}/scripts/run_quality_eval.py <skill-dir> \
|
|
165
|
+
--trigger --evals ${CLAUDE_SKILL_DIR}/evals/trigger_queries.json
|
|
129
166
|
```
|
|
130
167
|
|
|
131
|
-
**
|
|
168
|
+
**Quality eval** (with/without skill A/B + grading + blind comparison):
|
|
132
169
|
```bash
|
|
133
|
-
python ${CLAUDE_SKILL_DIR}
|
|
134
|
-
--
|
|
135
|
-
--
|
|
136
|
-
--model claude-sonnet-4-6 \
|
|
137
|
-
--max-iterations 5
|
|
170
|
+
python ${CLAUDE_SKILL_DIR}/scripts/run_quality_eval.py <skill-dir> \
|
|
171
|
+
--runs 3 --executor api \
|
|
172
|
+
--evals ${CLAUDE_SKILL_DIR}/evals/quality_evals.json
|
|
138
173
|
```
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
174
|
+
|
|
175
|
+
When `skill-evaluator` is the resolved target, the wrapper drives its full
|
|
176
|
+
loop: generate → run → grade → aggregate → analyze → detect-regression →
|
|
177
|
+
propose. See `skill-evaluator/SKILL.md` for the workflow it actually
|
|
178
|
+
executes.
|
|
179
|
+
|
|
180
|
+
When `ct-skill-creator` is the resolved fallback, the wrapper invokes its
|
|
181
|
+
`run_eval.py` with the same arguments translated to its CLI shape.
|
|
182
|
+
|
|
183
|
+
### Manual A/B (if you want to drive runs yourself)
|
|
184
|
+
|
|
185
|
+
If you need direct control of how runs are spawned (e.g. inside a real
|
|
186
|
+
Claude Code session with subagent isolation), invoke the resolved eval
|
|
187
|
+
skill's scripts directly — locate them with:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
EVAL_SKILL=$(python ${CLAUDE_SKILL_DIR}/scripts/_skill_finder.py skill-evaluator)
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
then drive that skill's documented workflow without any further hardcoded
|
|
194
|
+
paths in this file.
|
|
156
195
|
|
|
157
196
|
---
|
|
158
197
|
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"prompt": "Run the manifest alignment check for ct-skill-validator against the CLEO manifest",
|
|
43
43
|
"expected_output": "Manifest alignment results showing whether ct-skill-validator is registered correctly in manifest.json and dispatch-config.json",
|
|
44
44
|
"expectations": [
|
|
45
|
-
"Claude passes --manifest to validate.py
|
|
45
|
+
"Claude passes --manifest to validate.py (Tier 4 check)",
|
|
46
46
|
"The manifest.json path is correctly resolved",
|
|
47
47
|
"The output shows Tier 4 CLEO Integration results",
|
|
48
48
|
"Claude reports whether the skill is found in manifest.json"
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
# CLEO Skill Validator
|
|
1
|
+
# CLEO Skill Validator — Validation Rules
|
|
2
2
|
|
|
3
3
|
Complete rule reference for the 5-tier validation system.
|
|
4
4
|
|
|
5
5
|
## Overview
|
|
6
6
|
|
|
7
|
-
The CLEO Skill Validator
|
|
7
|
+
The CLEO Skill Validator enforces compliance across five tiers of increasing depth:
|
|
8
8
|
|
|
9
9
|
1. **Structure** — Does the skill have the required files and valid frontmatter?
|
|
10
10
|
2. **Frontmatter Quality** — Are all frontmatter fields correct, well-formed, and non-contradictory?
|
|
@@ -16,42 +16,79 @@ Tiers 1-3 run on every validation. Tiers 4-5 are opt-in via CLI flags.
|
|
|
16
16
|
|
|
17
17
|
## Allowed vs Forbidden Fields
|
|
18
18
|
|
|
19
|
-
###
|
|
19
|
+
### Allowed in SKILL.md frontmatter
|
|
20
|
+
|
|
21
|
+
Two groups: **agentskills.io spec fields** (the open standard) and
|
|
22
|
+
**Claude Code harness extensions** (honored by the runtime but not part of
|
|
23
|
+
the open spec).
|
|
24
|
+
|
|
25
|
+
#### From the agentskills.io spec
|
|
20
26
|
|
|
21
27
|
| Field | Type | Required | Description |
|
|
22
28
|
|-------|------|----------|-------------|
|
|
23
|
-
| `name` | string | Yes | Skill identifier, hyphen-case, max 64 chars |
|
|
29
|
+
| `name` | string | Yes | Skill identifier, hyphen-case, max 64 chars, must match parent directory name |
|
|
24
30
|
| `description` | string | Yes | What the skill does and when to use it, max 1024 chars |
|
|
31
|
+
| `license` | string | No | License name or reference to a bundled LICENSE file |
|
|
32
|
+
| `compatibility` | string | No | Environment requirements (max 500 chars). Only include when the skill needs specific runtime, packages, or network access |
|
|
33
|
+
| `metadata` | dict | No | Map from string keys to string values for additional metadata not defined by the spec |
|
|
34
|
+
| `allowed-tools` | string or list | No | Tools pre-approved without per-use prompts (experimental in spec) |
|
|
35
|
+
|
|
36
|
+
##### Recommended `metadata` sub-keys (string values per spec)
|
|
37
|
+
|
|
38
|
+
The agentskills.io spec defines `metadata` as "a map from string keys to string
|
|
39
|
+
values". Use it for authorship and version info that the spec doesn't define
|
|
40
|
+
fields for:
|
|
41
|
+
|
|
42
|
+
| Sub-key | Convention | Example |
|
|
43
|
+
|---------|-----------|---------|
|
|
44
|
+
| `author` | Author name or org | `author: example-org` |
|
|
45
|
+
| `version` | Skill version (always quoted as string) | `version: "1.0.0"` |
|
|
46
|
+
| `last_updated` | ISO timestamp `YYYY-MM-DD HH:MM:SS` (always quoted) | `last_updated: "2026-05-21 14:00:18"` |
|
|
47
|
+
| `related` | Related skills | `related: skill-creator, skill-evaluator` |
|
|
48
|
+
| `spec` | Spec the skill claims to follow | `spec: https://agentskills.io/specification.md` |
|
|
49
|
+
|
|
50
|
+
The validator emits a WARN when `metadata` is present without any of
|
|
51
|
+
`author`, `version`, or `last_updated`. Numeric values like `version: 1.0` are
|
|
52
|
+
flagged — the spec requires string values, so quote them: `version: "1.0"`.
|
|
53
|
+
|
|
54
|
+
#### Claude Code harness extensions
|
|
55
|
+
|
|
56
|
+
These are honored by the Claude Code runtime but are NOT part of the
|
|
57
|
+
agentskills.io open spec. Skills targeting other agent runtimes should
|
|
58
|
+
either omit them or document the dependency in `compatibility`.
|
|
59
|
+
|
|
60
|
+
| Field | Type | Required | Description |
|
|
61
|
+
|-------|------|----------|-------------|
|
|
25
62
|
| `argument-hint` | string | No | Shown in autocomplete, max 100 chars |
|
|
26
63
|
| `disable-model-invocation` | boolean | No | Prevent model from auto-invoking |
|
|
27
64
|
| `user-invocable` | boolean | No | Whether skill appears as slash command |
|
|
28
|
-
| `allowed-tools` | string or list | No | Tools pre-approved without per-use prompts |
|
|
29
65
|
| `model` | string | No | Override model for this skill |
|
|
30
66
|
| `context` | string | No | Must be "fork" if present |
|
|
31
67
|
| `agent` | string | No | Subagent type (Explore, Plan, etc.) |
|
|
32
68
|
| `hooks` | dict | No | Skill-scoped lifecycle hooks |
|
|
33
|
-
| `license` | string | No | License identifier |
|
|
34
69
|
|
|
35
|
-
###
|
|
70
|
+
### CLEO-only fields (forbidden in SKILL.md; belong in `manifest-entry.json`)
|
|
71
|
+
|
|
72
|
+
These fields hold CLEO-specific structured data that the Claude runtime
|
|
73
|
+
doesn't read. They live in `manifest-entry.json` so they don't bloat
|
|
74
|
+
the SKILL.md frontmatter or violate the agentskills.io spec.
|
|
36
75
|
|
|
37
76
|
| Field | Destination |
|
|
38
77
|
|-------|-------------|
|
|
39
|
-
| `version` | manifest.json |
|
|
40
|
-
| `tier` | manifest.json |
|
|
41
|
-
| `core` | manifest.json |
|
|
42
|
-
| `category` | manifest.json |
|
|
43
|
-
| `protocol` | manifest.json |
|
|
44
|
-
| `dependencies` | manifest.json |
|
|
45
|
-
| `sharedResources` | manifest.json |
|
|
46
|
-
| `
|
|
47
|
-
| `
|
|
48
|
-
| `
|
|
49
|
-
| `
|
|
50
|
-
| `
|
|
51
|
-
| `
|
|
52
|
-
| `
|
|
53
|
-
| `mvi_scope` | manifest.json |
|
|
54
|
-
| `requires_tiers` | manifest.json |
|
|
78
|
+
| `version` | manifest-entry.json (or `metadata.version` in SKILL.md) |
|
|
79
|
+
| `tier` | manifest-entry.json |
|
|
80
|
+
| `core` | manifest-entry.json |
|
|
81
|
+
| `category` | manifest-entry.json |
|
|
82
|
+
| `protocol` | manifest-entry.json |
|
|
83
|
+
| `dependencies` | manifest-entry.json |
|
|
84
|
+
| `sharedResources` | manifest-entry.json |
|
|
85
|
+
| `token_budget` | manifest-entry.json |
|
|
86
|
+
| `capabilities` | manifest-entry.json |
|
|
87
|
+
| `constraints` | manifest-entry.json |
|
|
88
|
+
| `tags` | manifest-entry.json |
|
|
89
|
+
| `triggers` | manifest-entry.json |
|
|
90
|
+
| `mvi_scope` | manifest-entry.json |
|
|
91
|
+
| `requires_tiers` | manifest-entry.json |
|
|
55
92
|
|
|
56
93
|
## Tier 1 — Structure Rules
|
|
57
94
|
|
|
@@ -94,14 +131,21 @@ Tiers 1-3 run on every validation. Tiers 4-5 are opt-in via CLI flags.
|
|
|
94
131
|
| T2-024 | `model` is a string if present | ERROR | Use model ID string |
|
|
95
132
|
| T2-025 | `agent` is a string if present | ERROR | Use agent type string |
|
|
96
133
|
| T2-026 | `hooks` is a dict if present | ERROR | Use key: value structure |
|
|
134
|
+
| T2-027 | `compatibility` is a string if present | ERROR | Use a plain string value |
|
|
135
|
+
| T2-028 | `compatibility` is 500 characters or fewer (agentskills.io spec) | ERROR | Shorten or move detail to `references/` |
|
|
136
|
+
| T2-029 | `metadata` is a dict if present (agentskills.io spec) | ERROR | Use key: value structure |
|
|
137
|
+
| T2-030 | `metadata` keys are all strings (agentskills.io spec) | ERROR | Quote non-string keys |
|
|
138
|
+
| T2-031 | `metadata` values are all strings (agentskills.io spec) | WARN | Quote numeric versions: `version: "1.0"` |
|
|
139
|
+
| T2-032 | `metadata` includes at least one of: author, version, last_updated | WARN | Add recommended traceability keys |
|
|
140
|
+
| T2-033 | `metadata.last_updated` (and `metadata.last_reviewed` if present) match `YYYY-MM-DD HH:MM:SS` | WARN | Use precise timestamp format, e.g. `"2026-05-21 14:00:18"` |
|
|
97
141
|
|
|
98
142
|
## Tier 3 — Body Quality Rules
|
|
99
143
|
|
|
100
144
|
| Rule ID | Check | Severity | Fix |
|
|
101
145
|
|---------|-------|----------|-----|
|
|
102
146
|
| T3-001 | Body is present (non-empty content after frontmatter) | WARN | Add content below the closing `---` |
|
|
103
|
-
| T3-002 | Body is under 600 lines | ERROR | Split into sub-documents or trim |
|
|
104
|
-
| T3-003 | Body is under
|
|
147
|
+
| T3-002 | Body is under 600 lines (hard cap) | ERROR | Split into sub-documents or trim |
|
|
148
|
+
| T3-003 | Body is under 500 lines (agentskills.io spec recommendation) | WARN | Move detail to `references/` for progressive disclosure |
|
|
105
149
|
| T3-004 | No placeholder text (`[Required:`, `TODO`, `REPLACE`, `[Add content`, `FIXME`, `TBD`) | WARN (per match) | Replace placeholders with real content |
|
|
106
150
|
| T3-005 | Bodies over 200 lines have `## ` section headers | WARN | Add section structure for readability |
|
|
107
151
|
| T3-006 | File references (`references/`, `scripts/`) point to existing files | WARN | Create the referenced file or fix the path |
|
|
@@ -156,9 +200,16 @@ Tiers 1-3 run on every validation. Tiers 4-5 are opt-in via CLI flags.
|
|
|
156
200
|
| T2-024 | 2 | `model` is string | ERROR |
|
|
157
201
|
| T2-025 | 2 | `agent` is string | ERROR |
|
|
158
202
|
| T2-026 | 2 | `hooks` is dict | ERROR |
|
|
203
|
+
| T2-027 | 2 | `compatibility` is string | ERROR |
|
|
204
|
+
| T2-028 | 2 | `compatibility` max 500 chars | ERROR |
|
|
205
|
+
| T2-029 | 2 | `metadata` is dict | ERROR |
|
|
206
|
+
| T2-030 | 2 | `metadata` keys are strings | ERROR |
|
|
207
|
+
| T2-031 | 2 | `metadata` values are strings | WARN |
|
|
208
|
+
| T2-032 | 2 | `metadata` has author/version/last_updated | WARN |
|
|
209
|
+
| T2-033 | 2 | `metadata` timestamp keys match `YYYY-MM-DD HH:MM:SS` | WARN |
|
|
159
210
|
| T3-001 | 3 | Body present | WARN |
|
|
160
211
|
| T3-002 | 3 | Body under 600 lines | ERROR |
|
|
161
|
-
| T3-003 | 3 | Body under
|
|
212
|
+
| T3-003 | 3 | Body under 500 lines (spec) | WARN |
|
|
162
213
|
| T3-004 | 3 | No placeholder text | WARN |
|
|
163
214
|
| T3-005 | 3 | Section headers in long bodies | WARN |
|
|
164
215
|
| T3-006 | 3 | File references exist | WARN |
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Dynamic skill resolution — finds a sibling skill directory by name without
|
|
3
|
+
any hardcoded paths.
|
|
4
|
+
|
|
5
|
+
Search order (first match wins):
|
|
6
|
+
|
|
7
|
+
1. $SKILL_FINDER_PATH environment variable — colon-separated override paths.
|
|
8
|
+
Each entry is treated as a directory that may contain `<name>/SKILL.md`
|
|
9
|
+
OR may itself be the skill directory if its basename matches.
|
|
10
|
+
|
|
11
|
+
2. `~/.claude/skill-finder-paths.txt` — user config; newline-separated root
|
|
12
|
+
directories to search. Useful when the install lives in `~/.claude/skills/`
|
|
13
|
+
but the project skills live under `/mnt/projects/<repo>/skills/`.
|
|
14
|
+
Lines starting with `#` are ignored.
|
|
15
|
+
|
|
16
|
+
3. Sibling of the calling skill — `<this-skill>/../<name>/SKILL.md`.
|
|
17
|
+
Most common in awesome-skills layouts where every skill is a peer.
|
|
18
|
+
|
|
19
|
+
4. Two-up + `skills/<name>/` — `<this-skill>/../../skills/<name>/SKILL.md`.
|
|
20
|
+
Matches cleocode `packages/skills/skills/...` and `repo/skills/...`.
|
|
21
|
+
|
|
22
|
+
5. Walk up from the calling skill looking for a `skills/<name>/SKILL.md`
|
|
23
|
+
on the ancestor chain AND its project-shaped children (depth-limited).
|
|
24
|
+
|
|
25
|
+
6. `~/.claude/skills/<name>/SKILL.md` — installed Claude Code skill.
|
|
26
|
+
|
|
27
|
+
The caller is determined from `Path(__file__).resolve()` — so a skill
|
|
28
|
+
running its own script can find a peer without knowing absolute paths.
|
|
29
|
+
|
|
30
|
+
Use:
|
|
31
|
+
from _skill_finder import find_skill
|
|
32
|
+
evaluator = find_skill("skill-evaluator")
|
|
33
|
+
if evaluator is None:
|
|
34
|
+
sys.exit("skill-evaluator not found")
|
|
35
|
+
|
|
36
|
+
CLI:
|
|
37
|
+
python _skill_finder.py <skill-name> # prints path, exits 1 if not found
|
|
38
|
+
python _skill_finder.py <skill-name> --json
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import argparse
|
|
44
|
+
import json
|
|
45
|
+
import os
|
|
46
|
+
import sys
|
|
47
|
+
from pathlib import Path
|
|
48
|
+
from typing import Iterator
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
MAX_WALK_DEPTH = 6
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _candidates(name: str, caller_skill_dir: Path) -> Iterator[Path]:
|
|
55
|
+
"""Yield candidate paths in priority order. Each may or may not exist."""
|
|
56
|
+
|
|
57
|
+
# 1. Explicit override via env var (colon-separated paths)
|
|
58
|
+
env = os.environ.get("SKILL_FINDER_PATH", "")
|
|
59
|
+
for entry in (e for e in env.split(":") if e):
|
|
60
|
+
p = Path(entry).expanduser()
|
|
61
|
+
# If the entry IS the skill directory, accept it; else treat as parent
|
|
62
|
+
if p.is_dir() and p.name == name and (p / "SKILL.md").exists():
|
|
63
|
+
yield p
|
|
64
|
+
continue
|
|
65
|
+
yield p / name
|
|
66
|
+
# Also probe common "skills/" subdirs under each user-configured root
|
|
67
|
+
yield p / "skills" / name
|
|
68
|
+
yield p / "packages" / "skills" / "skills" / name
|
|
69
|
+
|
|
70
|
+
# 2. User config file: ~/.claude/skill-finder-paths.txt
|
|
71
|
+
cfg = Path.home() / ".claude" / "skill-finder-paths.txt"
|
|
72
|
+
if cfg.exists():
|
|
73
|
+
try:
|
|
74
|
+
for line in cfg.read_text(encoding="utf-8").splitlines():
|
|
75
|
+
line = line.strip()
|
|
76
|
+
if not line or line.startswith("#"):
|
|
77
|
+
continue
|
|
78
|
+
root = Path(line).expanduser()
|
|
79
|
+
yield root / name
|
|
80
|
+
yield root / "skills" / name
|
|
81
|
+
yield root / "packages" / "skills" / "skills" / name
|
|
82
|
+
# Also peek into project-shaped children of the root
|
|
83
|
+
if root.is_dir():
|
|
84
|
+
try:
|
|
85
|
+
for child in root.iterdir():
|
|
86
|
+
if not child.is_dir() or child.name.startswith("."):
|
|
87
|
+
continue
|
|
88
|
+
if (child / "skills").is_dir():
|
|
89
|
+
yield child / "skills" / name
|
|
90
|
+
if (child / "packages" / "skills" / "skills").is_dir():
|
|
91
|
+
yield child / "packages" / "skills" / "skills" / name
|
|
92
|
+
except PermissionError:
|
|
93
|
+
pass
|
|
94
|
+
except (OSError, UnicodeDecodeError):
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
# 3. Direct sibling of the caller
|
|
98
|
+
yield caller_skill_dir.parent / name
|
|
99
|
+
|
|
100
|
+
# 4. Two-up + skills/<name>/
|
|
101
|
+
yield caller_skill_dir.parent.parent / "skills" / name
|
|
102
|
+
|
|
103
|
+
# 4. Walk up looking for skills/<name>/ on the ancestor chain.
|
|
104
|
+
# At each ancestor `cur`, probe:
|
|
105
|
+
# - cur/skills/<name> (standard layout)
|
|
106
|
+
# - cur/packages/skills/skills/<name> (CLEO layout)
|
|
107
|
+
# AND iterate the *children* of cur (which are potential project
|
|
108
|
+
# roots) and probe:
|
|
109
|
+
# - <child>/skills/<name>
|
|
110
|
+
# - <child>/packages/skills/skills/<name>
|
|
111
|
+
# This finds skills under sibling project roots — e.g. when the
|
|
112
|
+
# caller is in /mnt/projects/cleocode/.../ct-skill-validator/ and
|
|
113
|
+
# the target is in /mnt/projects/proxmox/skills/<name>/. The walk
|
|
114
|
+
# eventually reaches the common ancestor /mnt/projects/ where both
|
|
115
|
+
# cleocode and proxmox are children. Bounded by MAX_SIBLINGS per
|
|
116
|
+
# level to keep this fast on populated roots.
|
|
117
|
+
cur = caller_skill_dir.parent
|
|
118
|
+
for _ in range(MAX_WALK_DEPTH):
|
|
119
|
+
yield cur / "skills" / name
|
|
120
|
+
yield cur / "packages" / "skills" / "skills" / name
|
|
121
|
+
# Iterate children, pre-filtering to project-shaped dirs only.
|
|
122
|
+
# A project-shaped dir is one that has a `skills/` or
|
|
123
|
+
# `packages/skills/skills/` subdir on disk. The pre-filter is one
|
|
124
|
+
# extra stat per child but eliminates the vast majority of irrelevant
|
|
125
|
+
# candidates before they enter the search, keeping a fully-populated
|
|
126
|
+
# ancestor (e.g. 200+ peer projects) sub-second.
|
|
127
|
+
if cur.is_dir():
|
|
128
|
+
try:
|
|
129
|
+
for child in cur.iterdir():
|
|
130
|
+
if not child.is_dir() or child.name.startswith("."):
|
|
131
|
+
continue
|
|
132
|
+
if (child / "skills").is_dir():
|
|
133
|
+
yield child / "skills" / name
|
|
134
|
+
if (child / "packages" / "skills" / "skills").is_dir():
|
|
135
|
+
yield child / "packages" / "skills" / "skills" / name
|
|
136
|
+
except PermissionError:
|
|
137
|
+
pass
|
|
138
|
+
parent = cur.parent
|
|
139
|
+
if parent == cur:
|
|
140
|
+
break
|
|
141
|
+
cur = parent
|
|
142
|
+
|
|
143
|
+
# 5. Installed Claude Code skill
|
|
144
|
+
yield Path.home() / ".claude" / "skills" / name
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def caller_skill_dir() -> Path:
|
|
148
|
+
"""Resolve the directory of the skill that invoked this helper.
|
|
149
|
+
|
|
150
|
+
Assumes this file lives at `<skill>/scripts/_skill_finder.py`. If moved,
|
|
151
|
+
the caller can pass `caller_skill_dir` to `find_skill()` explicitly.
|
|
152
|
+
"""
|
|
153
|
+
return Path(__file__).resolve().parent.parent
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def find_skill(name: str, *, caller: Path | None = None) -> Path | None:
|
|
157
|
+
"""Return the resolved path of the named skill, or None if not found.
|
|
158
|
+
|
|
159
|
+
A skill is considered found if `<candidate>/SKILL.md` exists.
|
|
160
|
+
"""
|
|
161
|
+
caller_dir = (caller or caller_skill_dir()).resolve()
|
|
162
|
+
seen: set[Path] = set()
|
|
163
|
+
for cand in _candidates(name, caller_dir):
|
|
164
|
+
try:
|
|
165
|
+
resolved = cand.resolve()
|
|
166
|
+
except (OSError, RuntimeError):
|
|
167
|
+
continue
|
|
168
|
+
if resolved in seen:
|
|
169
|
+
continue
|
|
170
|
+
seen.add(resolved)
|
|
171
|
+
if (resolved / "SKILL.md").exists():
|
|
172
|
+
return resolved
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def find_first(names: list[str], *, caller: Path | None = None) -> tuple[str, Path] | None:
|
|
177
|
+
"""Return the (name, path) of the first found skill from a preference list.
|
|
178
|
+
|
|
179
|
+
Lets callers say "prefer skill-evaluator, fall back to ct-skill-creator"
|
|
180
|
+
without hardcoding either path.
|
|
181
|
+
"""
|
|
182
|
+
for n in names:
|
|
183
|
+
p = find_skill(n, caller=caller)
|
|
184
|
+
if p is not None:
|
|
185
|
+
return n, p
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def main() -> int:
|
|
190
|
+
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
191
|
+
ap.add_argument("name", help="skill name to resolve")
|
|
192
|
+
ap.add_argument("--caller", help="override the calling-skill directory")
|
|
193
|
+
ap.add_argument("--json", action="store_true", help="emit JSON instead of bare path")
|
|
194
|
+
args = ap.parse_args()
|
|
195
|
+
|
|
196
|
+
caller = Path(args.caller).expanduser().resolve() if args.caller else None
|
|
197
|
+
path = find_skill(args.name, caller=caller)
|
|
198
|
+
|
|
199
|
+
if args.json:
|
|
200
|
+
print(json.dumps({"name": args.name, "found": path is not None,
|
|
201
|
+
"path": str(path) if path else None}, indent=2))
|
|
202
|
+
else:
|
|
203
|
+
if path is None:
|
|
204
|
+
print(f"error: skill '{args.name}' not found on search path", file=sys.stderr)
|
|
205
|
+
return 1
|
|
206
|
+
print(str(path))
|
|
207
|
+
return 0 if path is not None else 1
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
if __name__ == "__main__":
|
|
211
|
+
sys.exit(main())
|
|
@@ -60,9 +60,13 @@ def audit_body(skill_path):
|
|
|
60
60
|
total_lines = len(body_lines)
|
|
61
61
|
|
|
62
62
|
# ── Section analysis ────────────────────────────────────────────────
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
# Strip fenced code blocks first — '#' inside bash blocks are comments,
|
|
64
|
+
# not markdown headings, and matching them produces false positives.
|
|
65
|
+
body_no_fences = re.sub(r"```[\s\S]*?```", "", body)
|
|
66
|
+
body_no_fences_lines = body_no_fences.split("\n")
|
|
67
|
+
h1_headers = re.findall(r"^# .+", body_no_fences, re.MULTILINE)
|
|
68
|
+
h2_headers = re.findall(r"^## .+", body_no_fences, re.MULTILINE)
|
|
69
|
+
h3_headers = re.findall(r"^### .+", body_no_fences, re.MULTILINE)
|
|
66
70
|
total_sections = len(h2_headers) + len(h3_headers)
|
|
67
71
|
|
|
68
72
|
if len(h1_headers) > 1:
|
|
@@ -70,7 +74,7 @@ def audit_body(skill_path):
|
|
|
70
74
|
|
|
71
75
|
first_h2_line = None
|
|
72
76
|
first_h3_line = None
|
|
73
|
-
for i, line in enumerate(
|
|
77
|
+
for i, line in enumerate(body_no_fences_lines):
|
|
74
78
|
if first_h2_line is None and line.startswith("## "):
|
|
75
79
|
first_h2_line = i
|
|
76
80
|
if first_h3_line is None and line.startswith("### "):
|
|
@@ -146,7 +150,8 @@ def audit_body(skill_path):
|
|
|
146
150
|
ok("placeholder-scan", "No placeholder text found")
|
|
147
151
|
|
|
148
152
|
# ── Duplicate headings ──────────────────────────────────────────────
|
|
149
|
-
|
|
153
|
+
# Reuse code-fence-stripped body for the same reason as section analysis.
|
|
154
|
+
all_headings = re.findall(r"^(#{1,6} .+)", body_no_fences, re.MULTILINE)
|
|
150
155
|
seen: dict[str, bool] = {}
|
|
151
156
|
dup_found = False
|
|
152
157
|
for heading in all_headings:
|
|
@@ -33,6 +33,7 @@ import sys
|
|
|
33
33
|
import re
|
|
34
34
|
import json
|
|
35
35
|
import argparse
|
|
36
|
+
import datetime
|
|
36
37
|
from pathlib import Path
|
|
37
38
|
|
|
38
39
|
|
|
@@ -42,14 +43,28 @@ MIN_BODY_LINES = 100
|
|
|
42
43
|
MIN_REF_FILES = 3
|
|
43
44
|
GOLD_STANDARDS = ("ct-orchestrator", "ct-skill-creator")
|
|
44
45
|
|
|
46
|
+
# Cadence for the allowlist audit — entries older than this are flagged stale.
|
|
47
|
+
ALLOWLIST_STALE_DAYS = 30
|
|
48
|
+
LAST_REVIEWED_RE = re.compile(r"last_reviewed:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
|
|
49
|
+
|
|
45
50
|
# Allowlist — pre-existing stub skills exempted at T9567 (E-SKILLS-DEPTH-BACKFILL).
|
|
46
|
-
# Each entry MUST have a follow-up task ID. Remove the entry once that task
|
|
47
|
-
# a depth backfill. New entries require owner approval — do not add
|
|
51
|
+
# Each entry MUST have a follow-up task ID. Remove the entry once that task
|
|
52
|
+
# lands a depth backfill. New entries require owner approval — do not add
|
|
53
|
+
# silently.
|
|
54
|
+
#
|
|
55
|
+
# AUDIT CADENCE: review every release cycle (or every 30 days, whichever
|
|
56
|
+
# comes first). Each entry below carries `last_reviewed` — if that timestamp
|
|
57
|
+
# is older than the cadence, run `python check_depth.py <skill-dir>` for each
|
|
58
|
+
# allowlisted skill, decide keep / remove, and bump `last_reviewed` to a
|
|
59
|
+
# fresh `date '+%Y-%m-%d %H:%M:%S'` value. See ALLOWLIST_STALE_DAYS above
|
|
60
|
+
# for the actual threshold the audit enforces.
|
|
61
|
+
#
|
|
62
|
+
# Format: skill-name -> "task-id: rationale | last_reviewed: YYYY-MM-DD HH:MM:SS"
|
|
48
63
|
ALLOWLIST: dict[str, str] = {
|
|
49
|
-
"ct-codebase-mapper": "T9567-followup: pre-existing; depth-backfill deferred",
|
|
50
|
-
"ct-master-tac":
|
|
51
|
-
"ct-memory":
|
|
52
|
-
"ct-stickynote":
|
|
64
|
+
"ct-codebase-mapper": "T9567-followup: pre-existing; depth-backfill deferred | last_reviewed: 2026-05-21 14:00:18",
|
|
65
|
+
"ct-master-tac": "T9567-followup: pre-existing; depth-backfill deferred | last_reviewed: 2026-05-21 14:00:18",
|
|
66
|
+
"ct-memory": "T9567-followup: pre-existing; depth-backfill deferred | last_reviewed: 2026-05-21 14:00:18",
|
|
67
|
+
"ct-stickynote": "T9567-followup: ephemeral note skill; minimal-by-design | last_reviewed: 2026-05-21 14:00:18",
|
|
53
68
|
}
|
|
54
69
|
|
|
55
70
|
|
|
@@ -218,6 +233,60 @@ def _print_report(report: dict) -> None:
|
|
|
218
233
|
print(f" * {r}")
|
|
219
234
|
|
|
220
235
|
|
|
236
|
+
def audit_allowlist(
|
|
237
|
+
*,
|
|
238
|
+
now: datetime.datetime | None = None,
|
|
239
|
+
stale_days: int = ALLOWLIST_STALE_DAYS,
|
|
240
|
+
) -> list[dict]:
|
|
241
|
+
"""Audit the ALLOWLIST for malformed or stale `last_reviewed` stamps.
|
|
242
|
+
|
|
243
|
+
Returns a list of finding dicts: each has `skill`, `severity` (WARN),
|
|
244
|
+
`message`, and (when parseable) `age_days`. An empty list means every
|
|
245
|
+
allowlist entry has a well-formed, fresh stamp.
|
|
246
|
+
|
|
247
|
+
`last_reviewed:` must match `YYYY-MM-DD HH:MM:SS`. Stamps older than
|
|
248
|
+
`stale_days` are flagged for re-audit.
|
|
249
|
+
"""
|
|
250
|
+
now = now or datetime.datetime.now()
|
|
251
|
+
findings: list[dict] = []
|
|
252
|
+
for skill, rationale in ALLOWLIST.items():
|
|
253
|
+
m = LAST_REVIEWED_RE.search(rationale)
|
|
254
|
+
if not m:
|
|
255
|
+
findings.append({
|
|
256
|
+
"skill": skill, "severity": "WARN",
|
|
257
|
+
"message": "missing or malformed 'last_reviewed: YYYY-MM-DD HH:MM:SS' stamp",
|
|
258
|
+
})
|
|
259
|
+
continue
|
|
260
|
+
stamp = m.group(1)
|
|
261
|
+
try:
|
|
262
|
+
ts = datetime.datetime.strptime(stamp, "%Y-%m-%d %H:%M:%S")
|
|
263
|
+
except ValueError as e:
|
|
264
|
+
findings.append({
|
|
265
|
+
"skill": skill, "severity": "WARN",
|
|
266
|
+
"message": f"invalid timestamp '{stamp}': {e}",
|
|
267
|
+
})
|
|
268
|
+
continue
|
|
269
|
+
age_days = (now - ts).days
|
|
270
|
+
if age_days > stale_days:
|
|
271
|
+
findings.append({
|
|
272
|
+
"skill": skill, "severity": "WARN", "age_days": age_days,
|
|
273
|
+
"message": (
|
|
274
|
+
f"stale: last_reviewed was {age_days}d ago "
|
|
275
|
+
f"(cadence: {stale_days}d). Audit and bump the stamp."
|
|
276
|
+
),
|
|
277
|
+
})
|
|
278
|
+
return findings
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _print_allowlist_audit(findings: list[dict], *, stream=sys.stderr) -> None:
|
|
282
|
+
if not findings:
|
|
283
|
+
return
|
|
284
|
+
print("=== allowlist audit ===", file=stream)
|
|
285
|
+
for f in findings:
|
|
286
|
+
print(f" ⚠️ {f['skill']}: {f['message']}", file=stream)
|
|
287
|
+
print(file=stream)
|
|
288
|
+
|
|
289
|
+
|
|
221
290
|
def walk_all_skills(root: Path) -> list[Path]:
|
|
222
291
|
"""Find all skill directories under packages/skills/skills/.
|
|
223
292
|
Skips manifest.json, _shared/, and any dir without SKILL.md."""
|
|
@@ -246,8 +315,42 @@ def main() -> int:
|
|
|
246
315
|
help="Walk every skill under packages/skills/skills/",
|
|
247
316
|
)
|
|
248
317
|
parser.add_argument("--json", action="store_true", help="Output JSON instead of text")
|
|
318
|
+
parser.add_argument(
|
|
319
|
+
"--audit-allowlist", action="store_true",
|
|
320
|
+
help=(
|
|
321
|
+
"Audit ALLOWLIST entries for malformed or stale `last_reviewed` "
|
|
322
|
+
"stamps and exit. Exits 1 if any finding is reported."
|
|
323
|
+
),
|
|
324
|
+
)
|
|
249
325
|
args = parser.parse_args()
|
|
250
326
|
|
|
327
|
+
# Standalone audit mode — for CI / cron use.
|
|
328
|
+
if args.audit_allowlist:
|
|
329
|
+
findings = audit_allowlist()
|
|
330
|
+
if args.json:
|
|
331
|
+
print(json.dumps({
|
|
332
|
+
"stale_days_cadence": ALLOWLIST_STALE_DAYS,
|
|
333
|
+
"findings": findings,
|
|
334
|
+
"passed": len(findings) == 0,
|
|
335
|
+
}, indent=2))
|
|
336
|
+
else:
|
|
337
|
+
if findings:
|
|
338
|
+
_print_allowlist_audit(findings, stream=sys.stdout)
|
|
339
|
+
print(f"=== SUMMARY ===\nFindings: {len(findings)}\nResult: FAIL",
|
|
340
|
+
file=sys.stdout)
|
|
341
|
+
else:
|
|
342
|
+
print("=== allowlist audit ===", file=sys.stdout)
|
|
343
|
+
print(f" ✅ all {len(ALLOWLIST)} entries have fresh stamps "
|
|
344
|
+
f"(cadence: {ALLOWLIST_STALE_DAYS}d)", file=sys.stdout)
|
|
345
|
+
print(f"\n=== SUMMARY ===\nFindings: 0\nResult: PASS",
|
|
346
|
+
file=sys.stdout)
|
|
347
|
+
return 1 if findings else 0
|
|
348
|
+
|
|
349
|
+
# Background audit — runs on every invocation, silent when clean, emits
|
|
350
|
+
# to stderr so --json output on stdout stays parseable.
|
|
351
|
+
if not args.json:
|
|
352
|
+
_print_allowlist_audit(audit_allowlist())
|
|
353
|
+
|
|
251
354
|
arg_path = Path(args.skill_dir).resolve()
|
|
252
355
|
manifest = Path(args.manifest).resolve() if args.manifest else None
|
|
253
356
|
|
|
@@ -254,7 +254,7 @@ def generate_html(
|
|
|
254
254
|
|
|
255
255
|
eco_html = _ecosystem_section(ecosystem) if ecosystem else '<div class="tier-section pending"><div class="tier-header">CLEO Ecosystem Compliance — Not yet run</div><div class="no-issues">Run: python check_ecosystem.py <skill-dir> | ecosystem-checker agent | save to ecosystem-check.json</div></div>'
|
|
256
256
|
|
|
257
|
-
grading_html = _grading_section(grading) if grading else '<div class="tier-section pending"><div class="tier-header" style="background:#3a2a5a">Quality Eval — Grading not yet run</div><div class="no-issues">Run
|
|
257
|
+
grading_html = _grading_section(grading) if grading else '<div class="tier-section pending"><div class="tier-header" style="background:#3a2a5a">Quality Eval — Grading not yet run</div><div class="no-issues">Run: <code>python scripts/run_quality_eval.py <skill-dir></code> (dispatches dynamically to skill-evaluator), then pass --grading grading.json</div></div>'
|
|
258
258
|
|
|
259
259
|
comparison_html = _comparison_section(comparison) if comparison else ""
|
|
260
260
|
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Phase 3 dispatcher — delegates runtime quality eval to a dedicated skill.
|
|
3
|
+
|
|
4
|
+
Prefers `skill-evaluator` (the dedicated quality-eval skill) and falls back
|
|
5
|
+
to `ct-skill-creator` (legacy eval infrastructure) when skill-evaluator
|
|
6
|
+
isn't found. Uses `_skill_finder.py` to resolve the target dynamically —
|
|
7
|
+
no hardcoded cross-skill paths.
|
|
8
|
+
|
|
9
|
+
Usage:
|
|
10
|
+
run_quality_eval.py <skill-dir> # full quality eval
|
|
11
|
+
run_quality_eval.py <skill-dir> --trigger # trigger-accuracy only
|
|
12
|
+
run_quality_eval.py <skill-dir> --runs 3 --executor api
|
|
13
|
+
run_quality_eval.py --list # show what's reachable
|
|
14
|
+
|
|
15
|
+
Exit codes:
|
|
16
|
+
0 — eval ran (or was prepared, in --executor print mode)
|
|
17
|
+
1 — target eval skill not found on the search path
|
|
18
|
+
2 — eval script inside the target skill exited non-zero
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
29
|
+
from _skill_finder import find_first # noqa: E402
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# Preference order — first found wins. Lets the user opt-in to a different
|
|
33
|
+
# eval skill via $SKILL_FINDER_PATH without code changes.
|
|
34
|
+
PREFERENCE = ["skill-evaluator", "ct-skill-creator"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main() -> int:
|
|
38
|
+
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
39
|
+
ap.add_argument("skill_dir", nargs="?", help="path to the skill being evaluated")
|
|
40
|
+
ap.add_argument("--trigger", action="store_true",
|
|
41
|
+
help="run trigger-accuracy eval only (description_eval.py)")
|
|
42
|
+
ap.add_argument("--runs", type=int, default=3, help="repeated runs per case")
|
|
43
|
+
ap.add_argument("--executor", default=None,
|
|
44
|
+
help="executor for the eval (passes through to the target script)")
|
|
45
|
+
ap.add_argument("--list", action="store_true",
|
|
46
|
+
help="show what eval skill would be used and exit")
|
|
47
|
+
ap.add_argument("--evals", default=None, help="explicit evals.json path")
|
|
48
|
+
args, extra = ap.parse_known_args()
|
|
49
|
+
|
|
50
|
+
resolved = find_first(PREFERENCE)
|
|
51
|
+
if resolved is None:
|
|
52
|
+
print(
|
|
53
|
+
f"error: none of {PREFERENCE} were found on the search path. "
|
|
54
|
+
f"Set SKILL_FINDER_PATH or install one of them.",
|
|
55
|
+
file=sys.stderr,
|
|
56
|
+
)
|
|
57
|
+
return 1
|
|
58
|
+
eval_skill_name, eval_skill_path = resolved
|
|
59
|
+
|
|
60
|
+
if args.list:
|
|
61
|
+
print(f"will use: {eval_skill_name} at {eval_skill_path}")
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
if not args.skill_dir:
|
|
65
|
+
ap.error("skill_dir is required unless --list is given")
|
|
66
|
+
|
|
67
|
+
target_skill = Path(args.skill_dir).expanduser().resolve()
|
|
68
|
+
if not (target_skill / "SKILL.md").exists():
|
|
69
|
+
print(f"error: '{args.skill_dir}' is not a skill directory (no SKILL.md)",
|
|
70
|
+
file=sys.stderr)
|
|
71
|
+
return 1
|
|
72
|
+
|
|
73
|
+
# Pick the right script per target eval skill
|
|
74
|
+
if eval_skill_name == "skill-evaluator":
|
|
75
|
+
if args.trigger:
|
|
76
|
+
script = eval_skill_path / "scripts" / "description_eval.py"
|
|
77
|
+
cmd = ["python3", str(script), "--skill", str(target_skill), "--runs", str(args.runs)]
|
|
78
|
+
else:
|
|
79
|
+
script = eval_skill_path / "scripts" / "run_eval.py"
|
|
80
|
+
cmd = ["python3", str(script), "--skill", str(target_skill), "--runs", str(args.runs)]
|
|
81
|
+
if args.evals:
|
|
82
|
+
cmd += ["--evals", args.evals]
|
|
83
|
+
if args.executor:
|
|
84
|
+
cmd += ["--executor", args.executor]
|
|
85
|
+
else:
|
|
86
|
+
# ct-skill-creator legacy paths
|
|
87
|
+
if args.trigger:
|
|
88
|
+
script = eval_skill_path / "scripts" / "run_eval.py"
|
|
89
|
+
cmd = ["python3", str(script), "--skill-path", str(target_skill)]
|
|
90
|
+
if args.evals:
|
|
91
|
+
cmd += ["--eval-set", args.evals]
|
|
92
|
+
else:
|
|
93
|
+
script = eval_skill_path / "scripts" / "run_eval.py"
|
|
94
|
+
cmd = ["python3", str(script), "--skill-path", str(target_skill)]
|
|
95
|
+
if args.evals:
|
|
96
|
+
cmd += ["--eval-set", args.evals]
|
|
97
|
+
|
|
98
|
+
if not script.exists():
|
|
99
|
+
print(f"error: expected script not found: {script}", file=sys.stderr)
|
|
100
|
+
return 1
|
|
101
|
+
|
|
102
|
+
cmd += extra # pass any additional flags straight through
|
|
103
|
+
print(f"[run_quality_eval] dispatching to {eval_skill_name}: {' '.join(cmd)}",
|
|
104
|
+
file=sys.stderr)
|
|
105
|
+
rc = subprocess.run(cmd).returncode
|
|
106
|
+
return 0 if rc == 0 else 2
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
if __name__ == "__main__":
|
|
110
|
+
sys.exit(main())
|
|
@@ -17,16 +17,44 @@ import yaml
|
|
|
17
17
|
import argparse
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
# Frontmatter fields allowed directly in SKILL.md.
|
|
21
|
+
#
|
|
22
|
+
# Sources (in order of authority):
|
|
23
|
+
# 1. agentskills.io spec — name, description, license, compatibility,
|
|
24
|
+
# metadata, allowed-tools
|
|
25
|
+
# https://agentskills.io/specification.md
|
|
26
|
+
# 2. Claude Code harness extensions — argument-hint, disable-model-invocation,
|
|
27
|
+
# user-invocable, model, context, agent, hooks
|
|
28
|
+
# (honored by the runtime but not part of the open spec)
|
|
29
|
+
#
|
|
30
|
+
# Anything in CLEO_ONLY is reserved for manifest-entry.json — the validator
|
|
31
|
+
# rejects those at SKILL.md top level.
|
|
32
|
+
#
|
|
33
|
+
# Per-spec author conventions for `metadata` (sub-keys, all strings):
|
|
34
|
+
# author, version, last_updated, related, spec
|
|
35
|
+
RECOMMENDED_METADATA_KEYS = {"author", "version", "last_updated"}
|
|
36
|
+
|
|
37
|
+
# Timestamp keys inside `metadata` whose value should match the precision
|
|
38
|
+
# convention. The agentskills.io spec doesn't pin a format, but we enforce
|
|
39
|
+
# YYYY-MM-DD HH:MM:SS for both `last_updated` (metadata convention) and
|
|
40
|
+
# `last_reviewed` (audit/allowlist convention) so audit trails stay precise.
|
|
41
|
+
TIMESTAMP_METADATA_KEYS = ("last_updated", "last_reviewed")
|
|
42
|
+
TIMESTAMP_FORMAT_RE = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$")
|
|
43
|
+
|
|
44
|
+
SPEC_FRONTMATTER = {
|
|
45
|
+
"name", "description", "license", "compatibility", "metadata", "allowed-tools",
|
|
24
46
|
}
|
|
47
|
+
HARNESS_EXTENSIONS = {
|
|
48
|
+
"argument-hint", "disable-model-invocation", "user-invocable",
|
|
49
|
+
"model", "context", "agent", "hooks",
|
|
50
|
+
}
|
|
51
|
+
ALLOWED_FRONTMATTER = SPEC_FRONTMATTER | HARNESS_EXTENSIONS
|
|
52
|
+
|
|
25
53
|
CLEO_ONLY = {
|
|
26
54
|
"version", "tier", "core", "category", "protocol",
|
|
27
|
-
"dependencies", "sharedResources",
|
|
55
|
+
"dependencies", "sharedResources",
|
|
28
56
|
"token_budget", "capabilities", "constraints",
|
|
29
|
-
"
|
|
57
|
+
"tags", "triggers", "mvi_scope", "requires_tiers",
|
|
30
58
|
}
|
|
31
59
|
|
|
32
60
|
MANIFEST_REQUIRED_FIELDS = [
|
|
@@ -213,6 +241,53 @@ def validate_skill(skill_path, manifest_path=None, dispatch_config_path=None, pr
|
|
|
213
241
|
if hooks_val is not None and not isinstance(hooks_val, dict):
|
|
214
242
|
error(tier, "'hooks' must be a dict")
|
|
215
243
|
|
|
244
|
+
# compatibility checks (agentskills.io spec: max 500 chars)
|
|
245
|
+
compat_val = frontmatter.get("compatibility")
|
|
246
|
+
if compat_val is not None:
|
|
247
|
+
if not isinstance(compat_val, str):
|
|
248
|
+
error(tier, "'compatibility' must be a string")
|
|
249
|
+
elif len(compat_val) > 500:
|
|
250
|
+
error(tier, f"'compatibility' exceeds 500 characters (got: {len(compat_val)})")
|
|
251
|
+
else:
|
|
252
|
+
ok(tier, "'compatibility' is valid")
|
|
253
|
+
|
|
254
|
+
# metadata checks (agentskills.io spec: map from string keys to string values)
|
|
255
|
+
metadata_val = frontmatter.get("metadata")
|
|
256
|
+
if metadata_val is not None:
|
|
257
|
+
if not isinstance(metadata_val, dict):
|
|
258
|
+
error(tier, "'metadata' must be a dict (map from string keys to string values)")
|
|
259
|
+
else:
|
|
260
|
+
non_string_keys = [k for k in metadata_val if not isinstance(k, str)]
|
|
261
|
+
if non_string_keys:
|
|
262
|
+
error(tier, f"'metadata' keys must all be strings (got non-string: {non_string_keys[:3]})")
|
|
263
|
+
non_string_vals = [k for k, v in metadata_val.items() if not isinstance(v, str)]
|
|
264
|
+
if non_string_vals:
|
|
265
|
+
warn(tier, (
|
|
266
|
+
f"'metadata' values should be strings per agentskills.io spec; "
|
|
267
|
+
f"non-string keys: {non_string_vals[:3]} (quote numeric versions: \"1.0\" not 1.0)"
|
|
268
|
+
))
|
|
269
|
+
present_recommended = RECOMMENDED_METADATA_KEYS & set(metadata_val.keys())
|
|
270
|
+
if not present_recommended:
|
|
271
|
+
warn(tier, (
|
|
272
|
+
"'metadata' present but contains none of the recommended keys "
|
|
273
|
+
f"({', '.join(sorted(RECOMMENDED_METADATA_KEYS))}); consider adding for traceability"
|
|
274
|
+
))
|
|
275
|
+
else:
|
|
276
|
+
ok(tier, f"'metadata' has recommended key(s): {', '.join(sorted(present_recommended))}")
|
|
277
|
+
# Timestamp format check on convention keys (precision: YYYY-MM-DD HH:MM:SS).
|
|
278
|
+
# The spec is silent on format; this is our audit-precision convention.
|
|
279
|
+
for ts_key in TIMESTAMP_METADATA_KEYS:
|
|
280
|
+
ts_val = metadata_val.get(ts_key)
|
|
281
|
+
if ts_val is None or not isinstance(ts_val, str):
|
|
282
|
+
continue
|
|
283
|
+
if not TIMESTAMP_FORMAT_RE.match(ts_val):
|
|
284
|
+
warn(tier, (
|
|
285
|
+
f"'metadata.{ts_key}' should match 'YYYY-MM-DD HH:MM:SS' "
|
|
286
|
+
f"(got: {ts_val[:40]!r}); use a value like \"2026-05-21 14:00:18\""
|
|
287
|
+
))
|
|
288
|
+
else:
|
|
289
|
+
ok(tier, f"'metadata.{ts_key}' has valid timestamp format")
|
|
290
|
+
|
|
216
291
|
# ── Tier 3 — Body Quality ───────────────────────────────────────────
|
|
217
292
|
tier = 3
|
|
218
293
|
|
|
@@ -228,10 +303,13 @@ def validate_skill(skill_path, manifest_path=None, dispatch_config_path=None, pr
|
|
|
228
303
|
body_lines = body.split("\n")
|
|
229
304
|
line_count = len(body_lines)
|
|
230
305
|
|
|
306
|
+
# Thresholds aligned with agentskills.io spec recommendation
|
|
307
|
+
# ("Keep your main SKILL.md under 500 lines"). 600 is the hard cap;
|
|
308
|
+
# 500 is the soft cap from the spec.
|
|
231
309
|
if line_count >= 600:
|
|
232
|
-
error(tier, f"Body is too long: {line_count} lines (
|
|
233
|
-
elif line_count >=
|
|
234
|
-
warn(tier, f"Body
|
|
310
|
+
error(tier, f"Body is too long: {line_count} lines (hard cap 600)")
|
|
311
|
+
elif line_count >= 500:
|
|
312
|
+
warn(tier, f"Body exceeds spec recommendation: {line_count} lines (keep under 500)")
|
|
235
313
|
else:
|
|
236
314
|
ok(tier, f"Body length OK ({line_count} lines)")
|
|
237
315
|
|
|
File without changes
|
|
Binary file
|
|
Binary file
|
package/skills/ct-skill-validator/scripts/__pycache__/generate_validation_report.cpython-314.pyc
DELETED
|
Binary file
|
|
Binary file
|
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""
|
|
3
|
-
CLEO manifest alignment checker.
|
|
4
|
-
Usage: check_manifest.py <skill-directory> <manifest-json> [--dispatch-config dispatch-config.json]
|
|
5
|
-
"""
|
|
6
|
-
import sys
|
|
7
|
-
import json
|
|
8
|
-
import re
|
|
9
|
-
import yaml
|
|
10
|
-
import argparse
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
|
|
13
|
-
MANIFEST_REQUIRED_FIELDS = [
|
|
14
|
-
"name", "version", "description", "path", "status",
|
|
15
|
-
"tier", "token_budget", "capabilities", "constraints",
|
|
16
|
-
]
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def check_manifest(skill_path, manifest_path, dispatch_config_path=None):
|
|
20
|
-
"""Check manifest alignment for a skill."""
|
|
21
|
-
skill_dir = Path(skill_path).resolve()
|
|
22
|
-
skill_name = skill_dir.name
|
|
23
|
-
manifest_file = Path(manifest_path).resolve()
|
|
24
|
-
errors = 0
|
|
25
|
-
warnings = 0
|
|
26
|
-
|
|
27
|
-
def error(msg):
|
|
28
|
-
nonlocal errors
|
|
29
|
-
errors += 1
|
|
30
|
-
print(f" \u274c ERROR: {msg}")
|
|
31
|
-
|
|
32
|
-
def warn(msg):
|
|
33
|
-
nonlocal warnings
|
|
34
|
-
warnings += 1
|
|
35
|
-
print(f" \u26a0\ufe0f WARN: {msg}")
|
|
36
|
-
|
|
37
|
-
def ok(msg):
|
|
38
|
-
print(f" \u2705 {msg}")
|
|
39
|
-
|
|
40
|
-
print(f"\n=== CLEO Manifest Check: {skill_name} ===\n")
|
|
41
|
-
|
|
42
|
-
# ── Read SKILL.md frontmatter ───────────────────────────────────────
|
|
43
|
-
print("--- SKILL.md ---")
|
|
44
|
-
skill_md = skill_dir / "SKILL.md"
|
|
45
|
-
if not skill_md.exists():
|
|
46
|
-
error("SKILL.md does not exist")
|
|
47
|
-
_print_summary(errors, warnings)
|
|
48
|
-
return errors
|
|
49
|
-
|
|
50
|
-
raw_content = skill_md.read_text(encoding="utf-8")
|
|
51
|
-
fm_match = re.match(r"^---\n(.*?)\n---", raw_content, re.DOTALL)
|
|
52
|
-
if not fm_match:
|
|
53
|
-
error("Could not extract frontmatter from SKILL.md")
|
|
54
|
-
_print_summary(errors, warnings)
|
|
55
|
-
return errors
|
|
56
|
-
|
|
57
|
-
try:
|
|
58
|
-
frontmatter = yaml.safe_load(fm_match.group(1))
|
|
59
|
-
except yaml.YAMLError as e:
|
|
60
|
-
error(f"Frontmatter YAML parse error: {e}")
|
|
61
|
-
_print_summary(errors, warnings)
|
|
62
|
-
return errors
|
|
63
|
-
|
|
64
|
-
if not isinstance(frontmatter, dict):
|
|
65
|
-
error("Frontmatter is not a dict")
|
|
66
|
-
_print_summary(errors, warnings)
|
|
67
|
-
return errors
|
|
68
|
-
|
|
69
|
-
fm_name = frontmatter.get("name", skill_name)
|
|
70
|
-
ok(f"SKILL.md frontmatter read (name: '{fm_name}')")
|
|
71
|
-
|
|
72
|
-
# ── Read manifest.json ──────────────────────────────────────────────
|
|
73
|
-
print("\n--- Manifest ---")
|
|
74
|
-
if not manifest_file.exists():
|
|
75
|
-
error(f"Manifest file not found: {manifest_path}")
|
|
76
|
-
_print_summary(errors, warnings)
|
|
77
|
-
return errors
|
|
78
|
-
|
|
79
|
-
try:
|
|
80
|
-
manifest_data = json.loads(manifest_file.read_text(encoding="utf-8"))
|
|
81
|
-
except json.JSONDecodeError as e:
|
|
82
|
-
error(f"Manifest is not valid JSON: {e}")
|
|
83
|
-
_print_summary(errors, warnings)
|
|
84
|
-
return errors
|
|
85
|
-
|
|
86
|
-
ok("Manifest parsed successfully")
|
|
87
|
-
|
|
88
|
-
skills_list = manifest_data.get("skills", [])
|
|
89
|
-
matching = [s for s in skills_list if s.get("name") == fm_name]
|
|
90
|
-
|
|
91
|
-
if not matching:
|
|
92
|
-
error(f"Skill '{fm_name}' not found in manifest.json skills[] array")
|
|
93
|
-
_print_summary(errors, warnings)
|
|
94
|
-
return errors
|
|
95
|
-
|
|
96
|
-
ok(f"Skill '{fm_name}' found in manifest.json")
|
|
97
|
-
entry = matching[0]
|
|
98
|
-
|
|
99
|
-
# Check required fields
|
|
100
|
-
print("\n--- Required Fields ---")
|
|
101
|
-
missing_fields = []
|
|
102
|
-
for field in MANIFEST_REQUIRED_FIELDS:
|
|
103
|
-
if field not in entry:
|
|
104
|
-
warn(f"Missing required field: '{field}'")
|
|
105
|
-
missing_fields.append(field)
|
|
106
|
-
else:
|
|
107
|
-
ok(f"'{field}' present")
|
|
108
|
-
|
|
109
|
-
# ── Dispatch config check ───────────────────────────────────────────
|
|
110
|
-
if dispatch_config_path:
|
|
111
|
-
print("\n--- Dispatch Config ---")
|
|
112
|
-
dc_file = Path(dispatch_config_path).resolve()
|
|
113
|
-
if not dc_file.exists():
|
|
114
|
-
error(f"Dispatch config not found: {dispatch_config_path}")
|
|
115
|
-
else:
|
|
116
|
-
try:
|
|
117
|
-
dc_data = json.loads(dc_file.read_text(encoding="utf-8"))
|
|
118
|
-
except json.JSONDecodeError as e:
|
|
119
|
-
error(f"Dispatch config is not valid JSON: {e}")
|
|
120
|
-
dc_data = None
|
|
121
|
-
|
|
122
|
-
if dc_data is not None:
|
|
123
|
-
overrides = dc_data.get("skill_overrides", {})
|
|
124
|
-
if fm_name not in overrides:
|
|
125
|
-
warn(f"Skill '{fm_name}' not found in dispatch-config.json skill_overrides")
|
|
126
|
-
else:
|
|
127
|
-
ok(f"Skill '{fm_name}' found in dispatch-config.json")
|
|
128
|
-
|
|
129
|
-
_print_summary(errors, warnings)
|
|
130
|
-
return errors
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _print_summary(errors, warnings):
|
|
134
|
-
"""Print the check summary."""
|
|
135
|
-
print(f"\n=== SUMMARY ===")
|
|
136
|
-
print(f"Errors: {errors}")
|
|
137
|
-
print(f"Warnings: {warnings}")
|
|
138
|
-
|
|
139
|
-
if errors > 0:
|
|
140
|
-
print(f"Result: FAIL")
|
|
141
|
-
elif warnings > 0:
|
|
142
|
-
print(f"Result: PASS (with warnings)")
|
|
143
|
-
else:
|
|
144
|
-
print(f"Result: PASS")
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
def main():
|
|
148
|
-
parser = argparse.ArgumentParser(
|
|
149
|
-
description="CLEO manifest alignment checker"
|
|
150
|
-
)
|
|
151
|
-
parser.add_argument("skill_dir", help="Path to the skill directory")
|
|
152
|
-
parser.add_argument("manifest", help="Path to manifest.json")
|
|
153
|
-
parser.add_argument("--dispatch-config", help="Path to dispatch-config.json")
|
|
154
|
-
|
|
155
|
-
args = parser.parse_args()
|
|
156
|
-
|
|
157
|
-
skill_path = Path(args.skill_dir).resolve()
|
|
158
|
-
if not skill_path.is_dir():
|
|
159
|
-
print(f"Error: '{args.skill_dir}' is not a directory", file=sys.stderr)
|
|
160
|
-
sys.exit(1)
|
|
161
|
-
|
|
162
|
-
error_count = check_manifest(
|
|
163
|
-
skill_path,
|
|
164
|
-
args.manifest,
|
|
165
|
-
dispatch_config_path=args.dispatch_config,
|
|
166
|
-
)
|
|
167
|
-
|
|
168
|
-
sys.exit(1 if error_count > 0 else 0)
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
if __name__ == "__main__":
|
|
172
|
-
main()
|
|
File without changes
|