bmad-module-skill-forge 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -176
- package/docs/agents.md +1 -1
- package/docs/concepts.md +96 -0
- package/docs/examples.md +39 -0
- package/docs/getting-started.md +15 -1
- package/docs/{architecture.md → how-it-works.md} +106 -5
- package/docs/index.md +14 -41
- package/docs/workflows.md +1 -1
- package/package.json +2 -2
- package/src/knowledge/agentskills-spec.md +25 -0
- package/src/knowledge/manual-section-integrity.md +8 -0
- package/src/knowledge/overview.md +2 -1
- package/src/knowledge/provenance-tracking.md +13 -0
- package/src/knowledge/skf-knowledge-index.csv +3 -2
- package/src/knowledge/skill-lifecycle.md +2 -2
- package/src/knowledge/split-body-strategy.md +41 -0
- package/src/module.yaml +2 -2
- package/src/workflows/analyze-source/data/skill-brief-schema.md +2 -0
- package/src/workflows/analyze-source/data/unit-detection-heuristics.md +26 -0
- package/src/workflows/analyze-source/steps-c/step-02-scan-project.md +1 -1
- package/src/workflows/analyze-source/steps-c/step-03-identify-units.md +1 -1
- package/src/workflows/analyze-source/steps-c/step-04-map-and-detect.md +5 -4
- package/src/workflows/analyze-source/steps-c/step-05-recommend.md +1 -1
- package/src/workflows/analyze-source/steps-c/step-06-generate-briefs.md +1 -1
- package/src/workflows/audit-skill/steps-c/step-01-init.md +2 -0
- package/src/workflows/audit-skill/steps-c/step-03-structural-diff.md +14 -0
- package/src/workflows/brief-skill/data/scope-templates.md +4 -0
- package/src/workflows/brief-skill/data/skill-brief-schema.md +4 -0
- package/src/workflows/brief-skill/steps-c/step-03-scope-definition.md +12 -0
- package/src/workflows/create-skill/data/compile-assembly-rules.md +43 -2
- package/src/workflows/create-skill/data/extraction-patterns-tracing.md +110 -0
- package/src/workflows/create-skill/data/extraction-patterns.md +12 -81
- package/src/workflows/create-skill/data/skill-sections.md +31 -5
- package/src/workflows/create-skill/data/source-resolution-protocols.md +9 -1
- package/src/workflows/create-skill/data/tier-degradation-rules.md +46 -0
- package/src/workflows/create-skill/steps-c/step-03-extract.md +24 -2
- package/src/workflows/create-skill/steps-c/step-05-compile.md +8 -4
- package/src/workflows/create-skill/steps-c/step-06-validate.md +12 -10
- package/src/workflows/create-skill/steps-c/step-07-generate-artifacts.md +14 -1
- package/src/workflows/create-stack-skill/steps-c/step-06-compile-stack.md +2 -2
- package/src/workflows/create-stack-skill/steps-c/step-08-validate.md +1 -1
- package/src/workflows/export-skill/data/snippet-format.md +1 -1
- package/src/workflows/export-skill/steps-c/step-02-package.md +7 -3
- package/src/workflows/export-skill/steps-c/step-03-generate-snippet.md +3 -1
- package/src/workflows/export-skill/workflow.md +1 -1
- package/src/workflows/quick-skill/steps-c/step-04-compile.md +3 -2
- package/src/workflows/test-skill/data/scoring-rules.md +9 -0
- package/src/workflows/test-skill/data/source-access-protocol.md +8 -0
- package/src/workflows/test-skill/steps-c/step-03-coverage-check.md +3 -1
- package/src/workflows/test-skill/steps-c/step-04-coherence-check.md +8 -5
- package/src/workflows/test-skill/steps-c/step-04b-external-validators.md +11 -1
- package/src/workflows/test-skill/steps-c/step-05-score.md +5 -4
- package/src/workflows/test-skill/steps-c/step-06-report.md +27 -0
- package/src/workflows/update-skill/data/tier-degradation-rules.md +46 -0
- package/src/workflows/update-skill/steps-c/step-02-detect-changes.md +11 -0
- package/src/workflows/update-skill/steps-c/step-03-re-extract.md +1 -0
- package/src/workflows/update-skill/steps-c/step-04-merge.md +8 -0
- package/src/workflows/update-skill/steps-c/step-05-validate.md +1 -1
- package/src/workflows/update-skill/steps-c/step-06-write.md +6 -1
- package/tools/cli/lib/ui.js +1 -1
|
@@ -47,5 +47,5 @@
|
|
|
47
47
|
- **integrations line**: Co-import patterns from metadata.json `integrations` for stack skills
|
|
48
48
|
- If fewer exports than the limit, list all available
|
|
49
49
|
- If no exports data available, omit the api line
|
|
50
|
-
- Section anchors must be verified against actual SKILL.md headings during generation
|
|
50
|
+
- Section anchors must be verified against actual SKILL.md headings during generation. For split-body skills (where `references/` exists and `## Full` headings are stubs), if a heading is missing from SKILL.md, rewrite the anchor to point to the reference file path (preferred). Omit the anchor line only if the heading cannot be found in either SKILL.md or `references/*.md`
|
|
51
51
|
- Skill path is relative to project root
|
|
@@ -59,9 +59,11 @@ Verify the skill directory contains the expected agentskills.io package layout:
|
|
|
59
59
|
├── SKILL.md ← Required: Active skill document
|
|
60
60
|
├── metadata.json ← Required: Machine-readable metadata
|
|
61
61
|
├── context-snippet.md ← Will be generated/updated in step-03
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
├── references/ ← Optional: Progressive disclosure
|
|
63
|
+
│ ├── {function-a}.md
|
|
64
|
+
│ └── {function-b}.md
|
|
65
|
+
├── scripts/ ← Optional: Executable automation
|
|
66
|
+
└── assets/ ← Optional: Templates, schemas, configs
|
|
65
67
|
```
|
|
66
68
|
|
|
67
69
|
**Check each component:**
|
|
@@ -75,6 +77,8 @@ Verify the skill directory contains the expected agentskills.io package layout:
|
|
|
75
77
|
- `generation_date` (ISO date string)
|
|
76
78
|
- `confidence_tier` ("Quick", "Forge", or "Deep")
|
|
77
79
|
3. `references/` — If exists, check at least one .md file present
|
|
80
|
+
4. `scripts/` — If exists, verify at least one file present and each file referenced in SKILL.md Section 7b exists. Warn for orphaned scripts (present but not referenced).
|
|
81
|
+
5. `assets/` — If exists, verify at least one file present and each file referenced in SKILL.md Section 7b exists. Warn for orphaned assets (present but not referenced).
|
|
78
82
|
|
|
79
83
|
### 2. Validate Metadata Completeness
|
|
80
84
|
|
|
@@ -73,7 +73,8 @@ Load {snippetFormatData} and read the format template for the skill type.
|
|
|
73
73
|
1. Read metadata.json for `version`, `exports` array
|
|
74
74
|
2. Select top exports (up to 10 for Deep tier, 5 otherwise). Append `()` to function names.
|
|
75
75
|
3. Read SKILL.md to extract: heading slugs for `#quick-start` and `#key-types`, inline summary of key types (~10 words)
|
|
76
|
-
4.
|
|
76
|
+
4. **Anchor verification (split-body awareness):** For each section anchor (`#quick-start`, `#key-types`), verify the heading exists in SKILL.md. If a `references/` directory exists and `## Full` headings in SKILL.md are absent or stubs (indicating split-body, not a stack skill's structural references), rewrite the anchor to point to the reference file path (e.g., `references/{file}.md#key-types`). If the heading cannot be resolved in either location, omit that anchor line from the snippet.
|
|
77
|
+
5. Derive gotchas from: T2-future annotations in evidence report (breaking changes), async requirements, version-specific behavior. If no gotchas, omit the gotchas line.
|
|
77
78
|
|
|
78
79
|
Generate:
|
|
79
80
|
```
|
|
@@ -157,6 +158,7 @@ ONLY WHEN snippet generation is complete (or skipped due to passive_context opt-
|
|
|
157
158
|
|
|
158
159
|
- Snippet format loaded from {snippetFormatData}
|
|
159
160
|
- Content generated matching exact Vercel-aligned format
|
|
161
|
+
- Section anchors verified against SKILL.md headings (split-body anchors rewritten or omitted)
|
|
160
162
|
- Token count estimated and within target
|
|
161
163
|
- File written (or previewed in dry-run)
|
|
162
164
|
- Passive context opt-out correctly handled (skip when disabled)
|
|
@@ -49,7 +49,7 @@ description: Package for distribution + platform-aware context injection
|
|
|
49
49
|
Load and read full config from {project-root}/_bmad/skf/config.yaml and resolve:
|
|
50
50
|
|
|
51
51
|
- `project_name`, `output_folder`, `user_name`, `communication_language`, `document_output_language`
|
|
52
|
-
- `skills_output_folder`, `forge_data_folder`
|
|
52
|
+
- `skills_output_folder`, `forge_data_folder`, `sidecar_path`
|
|
53
53
|
|
|
54
54
|
### 2. First Step Execution
|
|
55
55
|
|
|
@@ -79,8 +79,8 @@ description: >
|
|
|
79
79
|
```
|
|
80
80
|
|
|
81
81
|
**Frontmatter rules:**
|
|
82
|
-
- `name`: lowercase alphanumeric + hyphens only, must match the skill output directory name
|
|
83
|
-
- `description`: non-empty, max 1024 chars, optimized for agent discovery
|
|
82
|
+
- `name`: lowercase alphanumeric + hyphens only, must match the skill output directory name. Prefer gerund form (`processing-pdfs`) for clarity.
|
|
83
|
+
- `description`: non-empty, max 1024 chars, optimized for agent discovery. MUST use third-person voice ("Processes..." not "I can..." or "You can...").
|
|
84
84
|
- No other frontmatter fields — only `name` and `description` for community skills
|
|
85
85
|
|
|
86
86
|
**Required sections (after frontmatter):**
|
|
@@ -93,6 +93,7 @@ description: >
|
|
|
93
93
|
- **Configuration:** If configuration options were found in source
|
|
94
94
|
- **Dependencies:** Key dependencies from manifest
|
|
95
95
|
- **Notes:** Caveats, limitations, extraction confidence level
|
|
96
|
+
- **Scripts & Assets Note** (if source contains `scripts/`, `bin/`, `assets/`, `templates/`, or `schemas/` directories): "This package may include scripts and assets. Run create-skill for full extraction with provenance tracking."
|
|
96
97
|
|
|
97
98
|
**If confidence is low:**
|
|
98
99
|
- Include a note: "This skill was generated with limited source data. Consider running create-skill for a more thorough compilation."
|
|
@@ -27,6 +27,10 @@ When running in naive mode (no coherence category):
|
|
|
27
27
|
|
|
28
28
|
When neither skill-check nor tessl is available, redistribute the 10% external validation weight proportionally to the other active categories. When only one tool is available, use that tool's score as the external validation score.
|
|
29
29
|
|
|
30
|
+
## tessl and Split-Body Interaction
|
|
31
|
+
|
|
32
|
+
tessl evaluates SKILL.md body content only — it does not read `references/*.md` files. After split-body extraction, the tessl content score will drop significantly (e.g., 65% to 38%) because Tier 2 content is no longer inline. This is expected behavior and does not reflect actual content quality. When reporting scores for a split-body skill, note: "tessl content score reflects post-split inline content only. Use the pre-split tessl score as the content quality baseline."
|
|
33
|
+
|
|
30
34
|
## Tier-Dependent Scoring
|
|
31
35
|
|
|
32
36
|
### Quick Tier (no tools)
|
|
@@ -78,5 +82,10 @@ If no integration patterns exist, combined coherence equals reference validity.
|
|
|
78
82
|
| High | Signature mismatch between source and SKILL.md |
|
|
79
83
|
| Medium | Missing type or interface documentation |
|
|
80
84
|
| Medium | Migration section present/absent mismatch with T2-future annotation data (Deep tier) |
|
|
85
|
+
| Medium | Script/asset directory exists but no Scripts & Assets section in SKILL.md |
|
|
86
|
+
| Medium | Scripts & Assets section references file not found in scripts/ or assets/ directory |
|
|
87
|
+
| Low | Script/asset file present without provenance entry in provenance-map.json file_entries |
|
|
81
88
|
| Low | Missing optional metadata or examples |
|
|
89
|
+
| Low | Description trigger optimization recommended (third-person voice, negative triggers, or keyword coverage gaps) |
|
|
82
90
|
| Info | Style suggestions, non-blocking observations |
|
|
91
|
+
| Info | Discovery testing not performed — realistic prompt testing recommended before export |
|
|
@@ -23,6 +23,8 @@ Check if `{source_path}` (from metadata.json `source_root`) exists on disk. If y
|
|
|
23
23
|
**State 2 — Local absent, provenance-map exists:**
|
|
24
24
|
Check `{forge_data_folder}/{skill_name}/provenance-map.json`. If present, use it as the baseline export inventory — each entry contains structured fields: `export_name`, `export_type`, `params[]`, `return_type`, `source_file`, `source_line`, `confidence`, and `ast_node_type`. Cross-reference against SKILL.md documented exports for name-matching and param-by-param coverage. Signature verification compares SKILL.md's documented params/return types against provenance-map entries directly. If remote reading tools are available (zread, deepwiki, gh API, or similar), supplement by reading the entry point file for live signature verification. Set `analysis_confidence: provenance-map`.
|
|
25
25
|
|
|
26
|
+
**State 2 limitations:** Signature verification at State 2 is **string comparison only**, not semantic. Provenance-map stores parameters as flat string arrays (e.g., `["data: Union[BinaryIO, list, str]"]`), so `str` vs `String` or `list` vs `List[Any]` would be treated as mismatches even when semantically equivalent. For full type-aware verification (handling type aliases, generic equivalence), State 1 (local source) with AST re-parsing is required. When the SKILL.md was compiled from the same provenance-map (typical for create-then-test flows), strings match exactly and this limitation has no practical effect.
|
|
27
|
+
|
|
26
28
|
**State 3 — No provenance-map, metadata exports exist (quick-skill path):**
|
|
27
29
|
If no provenance-map.json exists (typical for quick-skill output), fall back to `metadata.json`'s `exports[]` array for the export name list. Coverage check becomes a self-consistency comparison: are all names in `exports[]` documented in SKILL.md with description, parameters, and return type? Signatures cannot be verified. If remote reading tools are available, supplement by reading the entry point for live export comparison. Set `analysis_confidence: metadata-only`.
|
|
28
30
|
|
|
@@ -35,3 +37,9 @@ If none of the above succeed, fall through to docs-only mode (section 0 already
|
|
|
35
37
|
Set `analysis_confidence` in context for use in Section 2 analysis depth, step-05 output, and step-05 scoring.
|
|
36
38
|
|
|
37
39
|
**Confidence tier mapping:** `full` = T1, `provenance-map` = T1, `metadata-only` = T1-low, `remote-only` = T1-low, `docs-only` = T3. This aligns with the T1/T1-low/T2/T3 scale used across all SKF workflows.
|
|
40
|
+
|
|
41
|
+
**Degradation notice rules:** When `analysis_confidence` is `provenance-map`, check the `confidence` field of provenance-map entries before emitting a degradation recommendation:
|
|
42
|
+
|
|
43
|
+
- **All/most entries T1 (AST-verified):** The provenance-map data is already at highest confidence. Do NOT recommend re-running with a local clone — it would produce identical results. Use: "Resolved via: provenance-map (T1 AST-verified at compilation time). Local clone not required — provenance data is already at highest confidence."
|
|
44
|
+
- **Mixed T1/T1-low entries:** Report the breakdown. Recommend local clone only for the T1-low entries: "Resolved via: provenance-map ({n} T1, {m} T1-low). Re-run with local clone to upgrade T1-low entries to full AST verification."
|
|
45
|
+
- **All/most entries T1-low or lower:** Keep the standard recommendation: "Re-run with local clone for full AST-backed verification."
|
|
@@ -85,6 +85,8 @@ Read SKILL.md and extract all documented items:
|
|
|
85
85
|
|
|
86
86
|
Build the **documented inventory** — a list of everything the SKILL.md claims the source provides.
|
|
87
87
|
|
|
88
|
+
**Split-body traversal:** If a `references/` directory exists alongside SKILL.md and SKILL.md's `## Full` headings are absent or stubs (not a stack skill's structural references), extend the documented inventory scan to include all `references/*.md` files. After split-body, Tier 2 content (Full API Reference, Full Type Definitions) lives in reference files — the inventory must reflect the full skill content regardless of where it resides.
|
|
89
|
+
|
|
88
90
|
### 2. Analyze Source Code (Tier-Dependent)
|
|
89
91
|
|
|
90
92
|
Start from the package entry point (see 0b) and identify the public API surface. Then analyze those exports at the appropriate tier depth.
|
|
@@ -223,7 +225,7 @@ ONLY WHEN all source files have been analyzed, the Coverage Analysis section has
|
|
|
223
225
|
|
|
224
226
|
### ✅ SUCCESS:
|
|
225
227
|
|
|
226
|
-
- All source files analyzed at appropriate tier depth
|
|
228
|
+
- All source files analyzed at appropriate tier depth; split-body references/ traversed when present
|
|
227
229
|
- Every finding has file:line citation (Forge/Deep) or file-level reference (Quick)
|
|
228
230
|
- Per-export status table complete
|
|
229
231
|
- Category scores calculated per scoring rules
|
|
@@ -72,6 +72,7 @@ Perform lightweight structural checks:
|
|
|
72
72
|
- Section headers are properly formatted
|
|
73
73
|
- Code examples have language annotations
|
|
74
74
|
- No broken markdown (unclosed code blocks, malformed tables)
|
|
75
|
+
- If `scripts/` or `assets/` directory exists alongside SKILL.md, a "Scripts & Assets" section (Section 7b) should be present
|
|
75
76
|
|
|
76
77
|
**Internal consistency:**
|
|
77
78
|
- Exports referenced in usage examples match exports listed in exports section
|
|
@@ -101,6 +102,7 @@ Scan SKILL.md for all cross-references:
|
|
|
101
102
|
- Skill references (`See SKILL.md for {other-skill}`, `Integrates with {package}`)
|
|
102
103
|
- Type imports (`import { Type } from './module'`)
|
|
103
104
|
- Integration pattern references (middleware chains, plugin hooks, shared state)
|
|
105
|
+
- Script/asset references (`scripts/{file}`, `assets/{file}`) in SKILL.md body
|
|
104
106
|
|
|
105
107
|
Launch a subprocess to grep/regex SKILL.md for reference patterns and return all found references with line numbers as structured JSON (`references_found[]` with line, type, target fields). If subprocess unavailable, scan in main thread.
|
|
106
108
|
|
|
@@ -114,6 +116,7 @@ DO NOT BE LAZY — For EACH reference found, launch a subprocess that:
|
|
|
114
116
|
- Type imports: type is actually exported from the referenced module
|
|
115
117
|
- Skill references: referenced skill exists in skills output folder
|
|
116
118
|
- Integration patterns: documented pattern matches actual implementation
|
|
119
|
+
- Script/asset references: verify the referenced file exists in the skill's `scripts/` or `assets/` directory
|
|
117
120
|
3. Returns structured validation JSON per reference (reference, line, target_exists, type_match, signature_match, issues[])
|
|
118
121
|
|
|
119
122
|
If subprocess unavailable, validate each reference in main thread.
|
|
@@ -147,7 +150,7 @@ Build integration completeness findings:
|
|
|
147
150
|
|
|
148
151
|
**Only execute if forge tier is Deep.** Skip silently for Quick/Forge tiers.
|
|
149
152
|
|
|
150
|
-
Check whether SKILL.md contains a "Migration & Deprecation Warnings" section (Section 4b). Then check the skill's `evidence-report.md` (at `
|
|
153
|
+
Check whether SKILL.md contains a "Migration & Deprecation Warnings" section (Section 4b). Then check the skill's `evidence-report.md` (at `{forge_data_folder}/{skill_name}/evidence-report.md`) for T2-future annotation counts.
|
|
151
154
|
|
|
152
155
|
- **If T2-future annotations > 0 AND Section 4b is absent:** Flag as Medium severity gap: "Migration section missing — T2-future annotations exist but Section 4b is not present in SKILL.md Tier 1."
|
|
153
156
|
- **If T2-future annotations = 0 AND Section 4b is present:** Flag as Medium severity gap: "Migration section unexpected — Section 4b is present but no T2-future annotations were produced."
|
|
@@ -168,7 +171,7 @@ Basic structural validation of **{skill_name}**:
|
|
|
168
171
|
- {N} structural issues found
|
|
169
172
|
- Coherence category not scored (weight redistributed to coverage)
|
|
170
173
|
|
|
171
|
-
**Proceeding to
|
|
174
|
+
**Proceeding to external validation...**"
|
|
172
175
|
|
|
173
176
|
**For Contextual Mode:**
|
|
174
177
|
"**Coherence check complete (contextual mode).**
|
|
@@ -180,11 +183,11 @@ Reference validation of **{skill_name}**:
|
|
|
180
183
|
|
|
181
184
|
**{N} issues found** — details in Coherence Analysis section.
|
|
182
185
|
|
|
183
|
-
**Proceeding to
|
|
186
|
+
**Proceeding to external validation...**"
|
|
184
187
|
|
|
185
188
|
### 8. Auto-Proceed
|
|
186
189
|
|
|
187
|
-
Display: "**Proceeding to
|
|
190
|
+
Display: "**Proceeding to external validation...**"
|
|
188
191
|
|
|
189
192
|
#### Menu Handling Logic:
|
|
190
193
|
|
|
@@ -197,7 +200,7 @@ Display: "**Proceeding to scoring...**"
|
|
|
197
200
|
|
|
198
201
|
## CRITICAL STEP COMPLETION NOTE
|
|
199
202
|
|
|
200
|
-
ONLY WHEN coherence analysis is complete (naive structural or contextual full validation), the Coherence Analysis section has been appended to {outputFile}, and coherence scores (if contextual) have been calculated, will you then load and read fully `{nextStepFile}` to execute
|
|
203
|
+
ONLY WHEN coherence analysis is complete (naive structural or contextual full validation), the Coherence Analysis section has been appended to {outputFile}, and coherence scores (if contextual) have been calculated, will you then load and read fully `{nextStepFile}` to execute external validation.
|
|
201
204
|
|
|
202
205
|
---
|
|
203
206
|
|
|
@@ -56,6 +56,16 @@ Run external validation tools (`skill-check` and `tessl`) against the skill dire
|
|
|
56
56
|
|
|
57
57
|
Read {outputFile} frontmatter to get the skill directory path (`skillDir`).
|
|
58
58
|
|
|
59
|
+
### 1b. Check for Recent Validation Results (Auto-Reuse)
|
|
60
|
+
|
|
61
|
+
Before running external validators, check if `{forge_data_folder}/{skill_name}/evidence-report.md` contains validation results (a `## Validation Results` section with quality scores).
|
|
62
|
+
|
|
63
|
+
**Staleness check:** Compare the evidence report's generation date against SKILL.md's last-modified timestamp (or `metadata.json` `generation_date`). If SKILL.md was modified after the evidence report was generated (e.g., after update-skill), the cached results are stale — skip auto-reuse and proceed to section 2 for a fresh run.
|
|
64
|
+
|
|
65
|
+
If recent, non-stale results exist (from a create-skill run that just completed), auto-reuse them — skip re-running validators and use the existing scores. Record: "External validation: reused from create-skill evidence report." Skip to section 5 (append results).
|
|
66
|
+
|
|
67
|
+
If no evidence report exists, it contains no validation section, or results are stale, proceed to section 2 (fresh run).
|
|
68
|
+
|
|
59
69
|
### 2. Run skill-check
|
|
60
70
|
|
|
61
71
|
**Check availability:**
|
|
@@ -112,7 +122,7 @@ Store in context: `tessl_description_score`, `tessl_content_score`, `tessl_avera
|
|
|
112
122
|
|
|
113
123
|
**If tessl content score < 70%:** Flag a warning:
|
|
114
124
|
|
|
115
|
-
"**Content quality warning:** tessl scored content at {score}%. This often indicates SKILL.md lacks inline actionable content (e.g., after split-body).
|
|
125
|
+
"**Content quality warning:** tessl scored content at {score}%. This often indicates SKILL.md lacks inline actionable content (e.g., after split-body). If this is a split-body skill, the score drop is expected — tessl evaluates only SKILL.md body, not `references/*.md` (see scoring-rules.md). Consider using selective split to keep actionable content inline."
|
|
116
126
|
|
|
117
127
|
**If tessl fails entirely:** Record `tessl_score: N/A`, log warning, continue.
|
|
118
128
|
|
|
@@ -5,6 +5,7 @@ description: 'Calculate completeness score from coverage and coherence findings'
|
|
|
5
5
|
nextStepFile: './step-06-report.md'
|
|
6
6
|
outputFile: '{forge_data_folder}/{skill_name}/test-report-{skill_name}.md'
|
|
7
7
|
scoringRulesFile: '../data/scoring-rules.md'
|
|
8
|
+
sourceAccessProtocol: '../data/source-access-protocol.md'
|
|
8
9
|
---
|
|
9
10
|
|
|
10
11
|
# Step 5: Score
|
|
@@ -136,14 +137,14 @@ Append the **Completeness Score** section to `{outputFile}`:
|
|
|
136
137
|
**Analysis Confidence:** {full | provenance-map | metadata-only | remote-only | docs-only}
|
|
137
138
|
```
|
|
138
139
|
|
|
139
|
-
If `analysis_confidence` is not `full`, append a degradation notice
|
|
140
|
+
If `analysis_confidence` is not `full`, append a degradation notice. **The notice must be confidence-aware** — see the degradation notice rules in `{sourceAccessProtocol}`:
|
|
140
141
|
|
|
141
142
|
```markdown
|
|
142
143
|
### Access Degradation Notice
|
|
143
144
|
|
|
144
|
-
**Resolved via:** {analysis_confidence}
|
|
145
|
-
**Impact:** {describe limitation — e.g., "Signature checks limited to name-matching. Source file:line citations from provenance-map, not live AST."}
|
|
146
|
-
**Recommendation:**
|
|
145
|
+
**Resolved via:** {analysis_confidence} {confidence breakdown if provenance-map, e.g., "(T1 AST-verified at compilation time)" or "(12 T1, 3 T1-low)"}
|
|
146
|
+
**Impact:** {describe limitation — e.g., "Signature checks limited to name-matching. Source file:line citations from provenance-map, not live AST." — or "Provenance data is at highest confidence; no limitation." for all-T1 provenance-map}
|
|
147
|
+
**Recommendation:** {confidence-dependent — see {sourceAccessProtocol} degradation notice rules. Do NOT recommend local clone when provenance-map entries are already T1.}
|
|
147
148
|
```
|
|
148
149
|
|
|
149
150
|
### 7. Update Output Frontmatter
|
|
@@ -83,8 +83,13 @@ Load `{scoringRulesFile}` for gap severity classification:
|
|
|
83
83
|
| **High** | Signature mismatch between source and SKILL.md |
|
|
84
84
|
| **Medium** | Missing type or interface documentation |
|
|
85
85
|
| **Medium** | Migration section present/absent mismatch with T2-future annotation data (Deep tier) |
|
|
86
|
+
| **Medium** | Script/asset directory exists but no Scripts & Assets section in SKILL.md |
|
|
87
|
+
| **Medium** | Scripts & Assets section references file not found in scripts/ or assets/ directory |
|
|
88
|
+
| **Low** | Script/asset file present without provenance entry in provenance-map.json file_entries |
|
|
86
89
|
| **Low** | Missing optional metadata or examples |
|
|
90
|
+
| **Low** | Description trigger optimization recommended (third-person voice, negative triggers, or keyword coverage gaps) |
|
|
87
91
|
| **Info** | Style suggestions, non-blocking observations |
|
|
92
|
+
| **Info** | Discovery testing not performed — realistic prompt testing recommended before export |
|
|
88
93
|
|
|
89
94
|
### 3. Classify and Order Gaps
|
|
90
95
|
|
|
@@ -98,6 +103,25 @@ Load the Gap Report section format from `{outputFormatsFile}`. Count gaps by sev
|
|
|
98
103
|
|
|
99
104
|
If no gaps found, append a clean pass message recommending **export-skill** workflow.
|
|
100
105
|
|
|
106
|
+
### 4b. Discovery and Description Quality Recommendations
|
|
107
|
+
|
|
108
|
+
After gap enumeration, append a **Discovery Quality** subsection to the gap report. Use the `Gap Entry Format` from `{outputFormatsFile}` for any Low/Info entries. The prose recommendations below are appended after the gap entries:
|
|
109
|
+
|
|
110
|
+
**Description optimization:** If tessl `description_score` (from step 04b) is below 90%, or if skill-check flagged description issues, recommend description improvements:
|
|
111
|
+
- Check that the description uses third-person voice consistently
|
|
112
|
+
- Check for specific trigger keywords that match how users would phrase requests
|
|
113
|
+
- Check for negative triggers ("NOT for: ...") to prevent false matches
|
|
114
|
+
- Check for alternative skill references for excluded use cases
|
|
115
|
+
|
|
116
|
+
**Discovery testing recommendation:** Regardless of pass/fail, always append:
|
|
117
|
+
|
|
118
|
+
"**Discovery testing recommended.** Before export, test the skill with 3-5 realistic prompts phrased the way real users actually talk — with casual language, typos, incomplete context, and implicit references. A skill tested only with clean prompts may fail to trigger in production. Example realistic prompt patterns:
|
|
119
|
+
- Vague: 'can you help me with this csv file my boss sent'
|
|
120
|
+
- Implicit: 'why did revenue drop last quarter'
|
|
121
|
+
- Abbreviated: 'run the {skill-name} thing on this data'"
|
|
122
|
+
|
|
123
|
+
Record discovery testing status as Info-level in the gap table. This is advisory — it does not affect the score.
|
|
124
|
+
|
|
101
125
|
### 5. Finalize Output Document
|
|
102
126
|
|
|
103
127
|
Update `{outputFile}` frontmatter:
|
|
@@ -132,6 +156,8 @@ Update `{outputFile}` frontmatter:
|
|
|
132
156
|
|
|
133
157
|
---
|
|
134
158
|
|
|
159
|
+
**See Discovery Quality section in the report for description optimization and realistic prompt testing recommendations.**
|
|
160
|
+
|
|
135
161
|
**Test report finalized.**"
|
|
136
162
|
|
|
137
163
|
### 7. Present MENU OPTIONS
|
|
@@ -163,6 +189,7 @@ This is the final step of the test-skill workflow. When the user selects C, the
|
|
|
163
189
|
- Gaps classified by severity using scoring rules
|
|
164
190
|
- Gaps ordered by severity (Critical first)
|
|
165
191
|
- Every gap has a specific, actionable remediation suggestion
|
|
192
|
+
- Discovery quality section appended with description optimization and testing recommendations
|
|
166
193
|
- Remediation summary with counts and effort estimates
|
|
167
194
|
- Output document finalized with all stepsCompleted
|
|
168
195
|
- Correct next workflow recommended (export-skill or update-skill)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Tier Degradation Rules
|
|
2
|
+
|
|
3
|
+
## Remote Source at Forge/Deep Tier
|
|
4
|
+
|
|
5
|
+
When `source_repo` is a remote URL (GitHub URL or owner/repo format) and the tier is Forge or Deep:
|
|
6
|
+
|
|
7
|
+
- **ast-grep requires local files** — it cannot operate on remote URLs
|
|
8
|
+
|
|
9
|
+
**Ephemeral clone strategy (preferred):**
|
|
10
|
+
|
|
11
|
+
1. Check `git` availability (`git --version`). `git` is effectively guaranteed at Deep tier (via `gh` dependency) but NOT guaranteed at Forge tier.
|
|
12
|
+
2. If `git` is available: perform an ephemeral shallow clone to a system temp path (`{system_temp}/skf-ephemeral-{skill-name}-{timestamp}/`).
|
|
13
|
+
3. For create-skill: use `--depth 1 --single-branch --filter=blob:none`; if `include_patterns` are specified, apply mode selection: if `exclude_patterns` are absent, use cone mode (convert include_patterns to directory roots; use `--skip-checks` for individual file paths); if `exclude_patterns` are present, use `--no-cone` mode (pass gitignore-style patterns with `!`-prefixed excludes — includes first, then negations). See source-resolution-protocols.md for the full conversion rules.
|
|
14
|
+
4. For update-skill: use sparse-checkout with `--skip-checks` scoped to the changed files from the change manifest only (file paths require `--skip-checks`). No `--branch` flag — uses the remote default branch (must match the branch used during original create-skill run).
|
|
15
|
+
5. If clone succeeds: use the local clone path for AST extraction. All results are T1 with `[AST:...]` citations.
|
|
16
|
+
6. Cleanup: delete the temp directory after extraction inventory is built and all data is in context. The clone never persists beyond the extraction step.
|
|
17
|
+
|
|
18
|
+
**Fallback (clone fails or `git` unavailable):**
|
|
19
|
+
|
|
20
|
+
- The extraction step MUST warn the user explicitly before degrading
|
|
21
|
+
- **create-skill:** Warning must include actionable guidance — clone locally and update `source_repo` in the brief to the local path
|
|
22
|
+
- **update-skill:** Warning must include actionable guidance — clone locally, re-run [CS] Create Skill with the local path to regenerate provenance data, then re-run the update
|
|
23
|
+
- Extraction proceeds using Quick tier strategy (source reading via gh_bridge)
|
|
24
|
+
- All results labeled T1-low with `[SRC:...]` citations
|
|
25
|
+
- The degradation reason is recorded in the evidence report
|
|
26
|
+
|
|
27
|
+
Silent degradation is **forbidden**. The user must always know when AST extraction was skipped and why.
|
|
28
|
+
|
|
29
|
+
## AST Tool Unavailable at Forge/Deep Tier
|
|
30
|
+
|
|
31
|
+
When the tier is Forge or Deep but ast-grep is not functional:
|
|
32
|
+
|
|
33
|
+
- The extraction step MUST warn the user explicitly before degrading
|
|
34
|
+
- Warning must include actionable guidance: run [SF] Setup Forge to detect tools
|
|
35
|
+
- Extraction proceeds using Quick tier strategy
|
|
36
|
+
- All results labeled T1-low
|
|
37
|
+
- The degradation reason is recorded in the evidence report
|
|
38
|
+
|
|
39
|
+
## Per-File AST Failure
|
|
40
|
+
|
|
41
|
+
When ast-grep fails on an individual file (parse error, unsupported syntax):
|
|
42
|
+
|
|
43
|
+
- Fall back to source reading for **that file only**
|
|
44
|
+
- Other files continue with AST extraction
|
|
45
|
+
- The affected file's results are labeled T1-low; unaffected files retain T1
|
|
46
|
+
- Log a warning noting which file degraded and why
|
|
@@ -112,6 +112,14 @@ Launch subprocesses in parallel that compare source state against provenance map
|
|
|
112
112
|
- Cross-reference deleted files/exports with added files/exports
|
|
113
113
|
- If content similarity > 80%: classify as RENAMED instead of deleted+added
|
|
114
114
|
|
|
115
|
+
**Category D — Script/asset file changes:**
|
|
116
|
+
- Compare `file_entries` from provenance-map.json against current source files
|
|
117
|
+
- For each file_entry: compute current SHA-256 content hash, compare against stored hash
|
|
118
|
+
- Files with changed hashes → MODIFIED_FILE
|
|
119
|
+
- Files in provenance but missing from source → DELETED_FILE
|
|
120
|
+
- Files in source matching detection patterns (scripts/, bin/, assets/, templates/) but not in provenance → NEW_FILE
|
|
121
|
+
- Files in `scripts/[MANUAL]/` or `assets/[MANUAL]/` → SKIP (user-authored, preserved)
|
|
122
|
+
|
|
115
123
|
Aggregate all subprocess results into a unified change manifest.
|
|
116
124
|
|
|
117
125
|
**If degraded mode (no provenance map):**
|
|
@@ -136,6 +144,9 @@ Change Manifest:
|
|
|
136
144
|
exports_renamed: [count]
|
|
137
145
|
exports_moved: [count]
|
|
138
146
|
|
|
147
|
+
scripts_modified, scripts_added, scripts_deleted: {counts}
|
|
148
|
+
assets_modified, assets_added, assets_deleted: {counts}
|
|
149
|
+
|
|
139
150
|
Per-file detail:
|
|
140
151
|
{file_path}:
|
|
141
152
|
status: MODIFIED|ADDED|DELETED|MOVED
|
|
@@ -4,6 +4,7 @@ description: 'Tier-aware AST extraction on changed files only, producing fresh e
|
|
|
4
4
|
|
|
5
5
|
nextStepFile: './step-04-merge.md'
|
|
6
6
|
extractionPatternsData: '../../create-skill/data/extraction-patterns.md'
|
|
7
|
+
tierDegradationRulesData: '../data/tier-degradation-rules.md'
|
|
7
8
|
remoteSourceResolutionData: '../data/remote-source-resolution.md'
|
|
8
9
|
---
|
|
9
10
|
|
|
@@ -100,6 +100,14 @@ Follow the merge priority order from {mergeConflictRulesFile}:
|
|
|
100
100
|
- Place before any [MANUAL] blocks at section boundary
|
|
101
101
|
- No conflicts expected (new content, no existing [MANUAL])
|
|
102
102
|
|
|
103
|
+
**Priority 6 — Process script/asset file changes (from Category D in change manifest):**
|
|
104
|
+
- MODIFIED_FILE: queue file for re-copy from source, update `file_entries` content_hash
|
|
105
|
+
- DELETED_FILE: queue file for removal from `scripts/` or `assets/`, remove from `file_entries`
|
|
106
|
+
- NEW_FILE: queue file for copy from source, add to `file_entries`
|
|
107
|
+
- Files in `scripts/[MANUAL]/` or `assets/[MANUAL]/` are never modified (user-authored)
|
|
108
|
+
- Update Section 7b manifest table to reflect changes
|
|
109
|
+
- Update `metadata.json` `scripts[]`/`assets[]` arrays and `stats.scripts_count`/`stats.assets_count`
|
|
110
|
+
|
|
103
111
|
### 4. Check for Conflicts
|
|
104
112
|
|
|
105
113
|
Scan all merge operations for flagged conflicts:
|
|
@@ -73,7 +73,7 @@ Launch subprocesses in parallel for each validation category, aggregating result
|
|
|
73
73
|
|
|
74
74
|
**If available**, run: `npx skill-check check <skill-dir> --fix --format json --no-security-scan`
|
|
75
75
|
|
|
76
|
-
Parse JSON output for quality score (0-100), auto-fixed issues, and remaining diagnostics. If `body.max_lines` reported,
|
|
76
|
+
Parse JSON output for quality score (0-100), auto-fixed issues, and remaining diagnostics. If `body.max_lines` reported, prefer selective split: extract only the largest Tier 2 section(s) to `references/`, keeping all Tier 1 content inline (see `knowledge/split-body-strategy.md`). Fall back to `npx skill-check split-body <skill-dir> --write` only if selective split is not feasible. After any split, verify `#quick-start` and `#key-types` anchors still resolve in SKILL.md.
|
|
77
77
|
|
|
78
78
|
**If unavailable**, perform manual check: validate merged SKILL.md structure, verify required sections (exports, usage patterns, conventions), verify export entries have name/type/signature/file:line reference, flag missing sections.
|
|
79
79
|
|
|
@@ -96,6 +96,11 @@ Write to `{forge_data_folder}/{skill_name}/provenance-map.json`:
|
|
|
96
96
|
**For new exports:**
|
|
97
97
|
- Add new entry with full structured fields: `export_name`, `export_type`, `params[]`, `return_type`, `source_file`, `source_line`, `confidence`, `extraction_method`, `ast_node_type`
|
|
98
98
|
|
|
99
|
+
**For script/asset file changes (if `file_entries` exists):**
|
|
100
|
+
- MODIFIED_FILE: copy updated file to `scripts/` or `assets/`, update `content_hash` in `file_entries`
|
|
101
|
+
- DELETED_FILE: remove file from `scripts/` or `assets/`, remove entry from `file_entries`
|
|
102
|
+
- NEW_FILE: copy file to `scripts/` or `assets/`, add entry to `file_entries` with `file_name`, `file_type`, `source_file`, `confidence: "T1-low"`, `extraction_method: "file-copy"`, `content_hash`
|
|
103
|
+
|
|
99
104
|
**Add update operation metadata:**
|
|
100
105
|
```json
|
|
101
106
|
{
|
|
@@ -177,7 +182,7 @@ External tool checks deferred from step-05 now run against the written files:
|
|
|
177
182
|
|
|
178
183
|
**If skill-check available:**
|
|
179
184
|
- Run: `npx skill-check check {skills_output_folder}/{skill_name} --fix --format json --no-security-scan`
|
|
180
|
-
- If `body.max_lines` reported,
|
|
185
|
+
- If `body.max_lines` reported, prefer selective split (see `knowledge/split-body-strategy.md`). Fall back to `npx skill-check split-body {skills_output_folder}/{skill_name} --write` if not feasible. Verify anchors resolve after split.
|
|
181
186
|
- Run: `npx skill-check diff` if original version was preserved
|
|
182
187
|
- Run: `npx skill-check check {skills_output_folder}/{skill_name} --format json` for security scan
|
|
183
188
|
|
package/tools/cli/lib/ui.js
CHANGED
|
@@ -61,7 +61,7 @@ class UI {
|
|
|
61
61
|
console.log(row(chalk.white.bold('Skill Forge') + chalk.dim(` v${version}`)));
|
|
62
62
|
console.log(row(chalk.dim('Agent Skill Compiler') + ' '.repeat(15) + brand.spark('⚒')));
|
|
63
63
|
console.log(mid);
|
|
64
|
-
console.log(row(chalk.dim('
|
|
64
|
+
console.log(row(chalk.dim('Code · Docs · Discourse → Verified agent skills')));
|
|
65
65
|
console.log(bottom);
|
|
66
66
|
console.log();
|
|
67
67
|
|