bmad-module-skill-forge 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +0 -8
- package/docs/_data/pinned.yaml +1 -1
- package/docs/_internal/STABILITY.md +1 -1
- package/docs/architecture.md +2 -2
- package/docs/examples.md +1 -1
- package/docs/workflows.md +5 -5
- package/package.json +2 -2
- package/src/README.md +1 -1
- package/src/knowledge/ccc-bridge.md +12 -12
- package/src/knowledge/qmd-registry.md +5 -5
- package/src/shared/health-check.md +5 -5
- package/src/shared/references/description-guard-protocol.md +100 -0
- package/src/shared/references/output-contract-schema.md +1 -1
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +6 -6
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +9 -3
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +149 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +22 -2
- package/src/shared/scripts/skf-build-change-manifest.py +420 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +321 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +357 -0
- package/src/shared/scripts/skf-description-guard.py +359 -0
- package/src/shared/scripts/skf-detect-language.py +2 -2
- package/src/shared/scripts/skf-detect-scripts-assets.py +613 -0
- package/src/shared/scripts/skf-detect-tools.py +97 -8
- package/src/shared/scripts/skf-detect-workspaces.py +1 -1
- package/src/shared/scripts/skf-disqualify-candidates.py +576 -0
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +3 -3
- package/src/shared/scripts/skf-emit-result-envelope.py +93 -9
- package/src/shared/scripts/skf-enumerate-stack-skills.py +514 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +11 -11
- package/src/shared/scripts/skf-hash-content.py +284 -0
- package/src/shared/scripts/skf-load-provenance.py +295 -0
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +45 -1
- package/src/shared/scripts/skf-pair-intersect.py +250 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +433 -0
- package/src/shared/scripts/skf-qmd-classify-collections.py +42 -8
- package/src/shared/scripts/skf-recommend-scope-type.py +2 -2
- package/src/shared/scripts/skf-render-quick-metadata.py +2 -2
- package/src/shared/scripts/skf-resolve-authoritative-files.py +504 -0
- package/src/shared/scripts/skf-scan-manifests.py +738 -0
- package/src/shared/scripts/skf-scan-skill-md-structure.py +375 -0
- package/src/shared/scripts/skf-update-active-symlink.py +315 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +5 -5
- package/src/shared/scripts/skf-validate-brief-schema.py +348 -0
- package/src/shared/scripts/skf-write-skill-brief.py +61 -5
- package/src/skf-analyze-source/SKILL.md +69 -16
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/customize.toml +55 -0
- package/src/skf-analyze-source/{steps-c/step-01b-continue.md → references/continue.md} +12 -12
- package/src/skf-analyze-source/{steps-c/step-06-generate-briefs.md → references/generate-briefs.md} +29 -10
- package/src/skf-analyze-source/{steps-c/step-07-health-check.md → references/health-check.md} +5 -3
- package/src/skf-analyze-source/{steps-c/step-03-identify-units.md → references/identify-units.md} +44 -14
- package/src/skf-analyze-source/{steps-c/step-01-init.md → references/init.md} +27 -6
- package/src/skf-analyze-source/{steps-c/step-04-map-and-detect.md → references/map-and-detect.md} +52 -25
- package/src/skf-analyze-source/{steps-c/step-05-recommend.md → references/recommend.md} +15 -8
- package/src/skf-analyze-source/{steps-c/step-02-scan-project.md → references/scan-project.md} +12 -10
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +2 -0
- package/src/skf-analyze-source/templates/analysis-report-template.md +6 -6
- package/src/skf-audit-skill/SKILL.md +66 -15
- package/src/skf-audit-skill/assets/drift-report-template.md +6 -5
- package/src/skf-audit-skill/customize.toml +49 -0
- package/src/skf-audit-skill/{steps-c/step-07-health-check.md → references/health-check.md} +5 -3
- package/src/skf-audit-skill/references/init.md +305 -0
- package/src/skf-audit-skill/{steps-c/step-02-re-index.md → references/re-index.md} +11 -11
- package/src/skf-audit-skill/{steps-c/step-06-report.md → references/report.md} +25 -7
- package/src/skf-audit-skill/{steps-c/step-04-semantic-diff.md → references/semantic-diff.md} +6 -6
- package/src/skf-audit-skill/{steps-c/step-05-severity-classify.md → references/severity-classify.md} +5 -5
- package/src/skf-audit-skill/references/severity-rules.md +2 -0
- package/src/skf-audit-skill/{steps-c/step-03-structural-diff.md → references/structural-diff.md} +29 -16
- package/src/skf-brief-skill/SKILL.md +48 -18
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +32 -2
- package/src/skf-brief-skill/customize.toml +44 -0
- package/src/skf-brief-skill/{steps-c/step-02-analyze-target.md → references/analyze-target.md} +5 -5
- package/src/skf-brief-skill/{steps-c/step-04-confirm-brief.md → references/confirm-brief.md} +13 -10
- package/src/skf-brief-skill/references/draft-checkpoint.md +4 -4
- package/src/skf-brief-skill/{steps-c/step-01-gather-intent.md → references/gather-intent.md} +24 -24
- package/src/skf-brief-skill/references/headless-args.md +9 -9
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +1 -1
- package/src/skf-brief-skill/{steps-c/step-06-health-check.md → references/health-check.md} +4 -2
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +2 -2
- package/src/skf-brief-skill/references/qmd-collection-registration.md +1 -1
- package/src/skf-brief-skill/{steps-c/step-03-scope-definition.md → references/scope-definition.md} +19 -16
- package/src/skf-brief-skill/references/version-resolution.md +2 -2
- package/src/skf-brief-skill/{steps-c/step-05-write-brief.md → references/write-brief.md} +11 -10
- package/src/skf-create-skill/SKILL.md +25 -19
- package/src/skf-create-skill/assets/compile-assembly-rules.md +3 -3
- package/src/skf-create-skill/assets/skill-sections.md +5 -5
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +11 -11
- package/src/skf-create-skill/customize.toml +35 -0
- package/src/skf-create-skill/references/authoritative-files-protocol.md +142 -0
- package/src/skf-create-skill/{steps-c/step-05-compile.md → references/compile.md} +16 -16
- package/src/skf-create-skill/{steps-c/step-03d-component-extraction.md → references/component-extraction.md} +16 -16
- package/src/skf-create-skill/{steps-c/step-02-ecosystem-check.md → references/ecosystem-check.md} +4 -4
- package/src/skf-create-skill/{steps-c/step-04-enrich.md → references/enrich.md} +3 -3
- package/src/skf-create-skill/{steps-c/step-03-extract.md → references/extract.md} +52 -120
- package/src/skf-create-skill/references/extraction-patterns.md +8 -8
- package/src/skf-create-skill/{steps-c/step-07-generate-artifacts.md → references/generate-artifacts.md} +17 -9
- package/src/skf-create-skill/{steps-c/step-09-health-check.md → references/health-check.md} +4 -2
- package/src/skf-create-skill/{steps-c/step-01-load-brief.md → references/load-brief.md} +40 -32
- package/src/skf-create-skill/{steps-c/step-08-report.md → references/report.md} +8 -8
- package/src/skf-create-skill/references/source-resolution-protocols.md +6 -6
- package/src/skf-create-skill/{steps-c/sub/step-02b-ccc-discover.md → references/sub/ccc-discover.md} +5 -5
- package/src/skf-create-skill/{steps-c/sub/step-03c-fetch-docs.md → references/sub/fetch-docs.md} +28 -12
- package/src/skf-create-skill/{steps-c/sub/step-03b-fetch-temporal.md → references/sub/fetch-temporal.md} +81 -50
- package/src/skf-create-skill/{steps-c/step-06-validate.md → references/validate.md} +46 -26
- package/src/skf-create-stack-skill/SKILL.md +76 -17
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +102 -0
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +1 -1
- package/src/skf-create-stack-skill/customize.toml +46 -0
- package/src/skf-create-stack-skill/{steps-c/step-06-compile-stack.md → references/compile-stack.md} +6 -5
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +3 -1
- package/src/skf-create-stack-skill/{steps-c/step-05-detect-integrations.md → references/detect-integrations.md} +32 -11
- package/src/skf-create-stack-skill/{steps-c/step-02-detect-manifests.md → references/detect-manifests.md} +28 -28
- package/src/skf-create-stack-skill/{steps-c/step-07-generate-output.md → references/generate-output.md} +11 -85
- package/src/skf-create-stack-skill/{steps-c/step-10-health-check.md → references/health-check.md} +4 -2
- package/src/skf-create-stack-skill/{steps-c/step-01-init.md → references/init.md} +70 -4
- package/src/skf-create-stack-skill/references/integration-patterns.md +2 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +2 -0
- package/src/skf-create-stack-skill/{steps-c/step-04-parallel-extract.md → references/parallel-extract.md} +40 -17
- package/src/skf-create-stack-skill/{steps-c/step-03-rank-and-confirm.md → references/rank-and-confirm.md} +5 -4
- package/src/skf-create-stack-skill/{steps-c/step-09-report.md → references/report.md} +3 -3
- package/src/skf-create-stack-skill/{steps-c/step-08-validate.md → references/validate.md} +6 -6
- package/src/skf-drop-skill/SKILL.md +84 -12
- package/src/skf-drop-skill/customize.toml +57 -0
- package/src/skf-drop-skill/{steps-c/step-02-execute.md → references/execute.md} +49 -34
- package/src/skf-drop-skill/{steps-c/step-04-health-check.md → references/health-check.md} +6 -4
- package/src/skf-drop-skill/{steps-c/step-03-report.md → references/report.md} +13 -5
- package/src/skf-drop-skill/{steps-c/step-01-select.md → references/select.md} +63 -15
- package/src/skf-export-skill/SKILL.md +85 -16
- package/src/skf-export-skill/assets/managed-section-format.md +3 -3
- package/src/skf-export-skill/assets/snippet-format.md +2 -2
- package/src/skf-export-skill/customize.toml +49 -0
- package/src/skf-export-skill/{steps-c/step-03-generate-snippet.md → references/generate-snippet.md} +18 -12
- package/src/skf-export-skill/{steps-c/step-07-health-check.md → references/health-check.md} +6 -4
- package/src/skf-export-skill/{steps-c/step-01-load-skill.md → references/load-skill.md} +21 -51
- package/src/skf-export-skill/references/manifest-rebuild.md +68 -0
- package/src/skf-export-skill/references/multi-skill-mode.md +38 -0
- package/src/skf-export-skill/references/orphan-context-detection.md +75 -0
- package/src/skf-export-skill/references/orphan-row-detection.md +102 -0
- package/src/skf-export-skill/{steps-c/step-02-package.md → references/package.md} +7 -7
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +74 -0
- package/src/skf-export-skill/{steps-c/step-06-summary.md → references/summary.md} +30 -6
- package/src/skf-export-skill/{steps-c/step-05-token-report.md → references/token-report.md} +5 -5
- package/src/skf-export-skill/{steps-c/step-04-update-context.md → references/update-context.md} +116 -91
- package/src/skf-quick-skill/SKILL.md +63 -123
- package/src/skf-quick-skill/customize.toml +44 -0
- package/src/skf-quick-skill/references/batch-mode.md +102 -0
- package/src/skf-quick-skill/{steps-c/step-04-compile.md → references/compile.md} +10 -10
- package/src/skf-quick-skill/{steps-c/step-02-ecosystem-check.md → references/ecosystem-check.md} +5 -5
- package/src/skf-quick-skill/{steps-c/step-06-finalize.md → references/finalize.md} +7 -7
- package/src/skf-quick-skill/{steps-c/step-07-health-check.md → references/health-check.md} +3 -3
- package/src/skf-quick-skill/{steps-c/step-03-quick-extract.md → references/quick-extract.md} +5 -5
- package/src/skf-quick-skill/references/registry-resolution.md +2 -0
- package/src/skf-quick-skill/{steps-c/step-01-resolve-target.md → references/resolve-target.md} +14 -10
- package/src/skf-quick-skill/{steps-c/step-05-write-and-validate.md → references/write-and-validate.md} +4 -4
- package/src/skf-refine-architecture/SKILL.md +86 -16
- package/src/skf-refine-architecture/customize.toml +49 -0
- package/src/skf-refine-architecture/{steps-c/step-05-compile.md → references/compile.md} +4 -4
- package/src/skf-refine-architecture/{steps-c/step-02-gap-analysis.md → references/gap-analysis.md} +5 -5
- package/src/skf-refine-architecture/{steps-c/step-07-health-check.md → references/health-check.md} +6 -4
- package/src/skf-refine-architecture/{steps-c/step-04-improvements.md → references/improvements.md} +5 -5
- package/src/skf-refine-architecture/references/init.md +144 -0
- package/src/skf-refine-architecture/{steps-c/step-03-issue-detection.md → references/issue-detection.md} +5 -5
- package/src/skf-refine-architecture/references/refinement-rules.md +2 -0
- package/src/skf-refine-architecture/{steps-c/step-06-report.md → references/report.md} +14 -5
- package/src/skf-rename-skill/SKILL.md +82 -12
- package/src/skf-rename-skill/customize.toml +52 -0
- package/src/skf-rename-skill/{steps-c/step-02-execute.md → references/execute.md} +94 -99
- package/src/skf-rename-skill/references/health-check.md +30 -0
- package/src/skf-rename-skill/references/rebuild-context.md +110 -0
- package/src/skf-rename-skill/{steps-c/step-03-report.md → references/report.md} +13 -5
- package/src/skf-rename-skill/{steps-c/step-01-select.md → references/select.md} +59 -20
- package/src/skf-setup/SKILL.md +35 -35
- package/src/skf-setup/customize.toml +33 -0
- package/src/skf-setup/{steps-c/step-03-auto-index.md → references/auto-index.md} +10 -18
- package/src/skf-setup/{steps-c/step-01b-ccc-index.md → references/ccc-index.md} +14 -22
- package/src/skf-setup/{steps-c/step-01-detect-and-tier.md → references/detect-and-tier.md} +23 -10
- package/src/skf-setup/{steps-c/step-05-health-check.md → references/health-check.md} +1 -1
- package/src/skf-setup/{steps-c/step-04-report.md → references/report.md} +21 -19
- package/src/skf-setup/references/tier-rules.md +1 -1
- package/src/skf-setup/{steps-c/step-02-write-config.md → references/write-config.md} +14 -13
- package/src/skf-test-skill/SKILL.md +77 -15
- package/src/skf-test-skill/customize.toml +54 -0
- package/src/skf-test-skill/{steps-c/step-04-coherence-check.md → references/coherence-check.md} +28 -71
- package/src/skf-test-skill/{steps-c/step-03-coverage-check.md → references/coverage-check.md} +16 -39
- package/src/skf-test-skill/{steps-c/step-02-detect-mode.md → references/detect-mode.md} +4 -26
- package/src/skf-test-skill/{steps-c/step-04b-external-validators.md → references/external-validators.md} +6 -28
- package/src/skf-test-skill/references/health-check.md +14 -0
- package/src/skf-test-skill/{steps-c/step-01-init.md → references/init.md} +21 -37
- package/src/skf-test-skill/references/migration-section-rules.md +4 -2
- package/src/skf-test-skill/{steps-c/step-06-report.md → references/report.md} +41 -51
- package/src/skf-test-skill/{steps-c/step-05-score.md → references/score.md} +26 -43
- package/src/skf-test-skill/references/scoring-rules.md +8 -6
- package/src/skf-test-skill/references/source-access-protocol.md +7 -5
- package/src/skf-test-skill/scripts/compute-score.py +88 -17
- package/src/skf-test-skill/templates/test-report-template.md +13 -13
- package/src/skf-update-skill/SKILL.md +23 -14
- package/src/skf-update-skill/customize.toml +44 -0
- package/src/skf-update-skill/{steps-c/step-02-detect-changes.md → references/detect-changes.md} +176 -68
- package/src/skf-update-skill/references/health-check.md +32 -0
- package/src/skf-update-skill/{steps-c/step-01-init.md → references/init.md} +56 -8
- package/src/skf-update-skill/references/manual-section-rules.md +4 -0
- package/src/skf-update-skill/references/merge-conflict-rules.md +4 -0
- package/src/skf-update-skill/{steps-c/step-04-merge.md → references/merge.md} +8 -8
- package/src/skf-update-skill/{steps-c/step-03-re-extract.md → references/re-extract.md} +53 -44
- package/src/skf-update-skill/references/remote-source-resolution.md +5 -1
- package/src/skf-update-skill/references/report.md +214 -0
- package/src/skf-update-skill/{steps-c/step-05-validate.md → references/validate.md} +5 -5
- package/src/skf-update-skill/{steps-c/step-06-write.md → references/write.md} +73 -51
- package/src/skf-verify-stack/SKILL.md +89 -17
- package/src/skf-verify-stack/assets/feasibility-report-template.md +4 -4
- package/src/skf-verify-stack/customize.toml +50 -0
- package/src/skf-verify-stack/references/coverage-patterns.md +2 -2
- package/src/skf-verify-stack/{steps-c/step-02-coverage.md → references/coverage.md} +7 -7
- package/src/skf-verify-stack/{steps-c/step-07-health-check.md → references/health-check.md} +7 -5
- package/src/skf-verify-stack/references/init.md +166 -0
- package/src/skf-verify-stack/references/integration-verification-rules.md +1 -1
- package/src/skf-verify-stack/{steps-c/step-03-integrations.md → references/integrations.md} +12 -12
- package/src/skf-verify-stack/{steps-c/step-06-report.md → references/report.md} +22 -41
- package/src/skf-verify-stack/{steps-c/step-04-requirements.md → references/requirements.md} +7 -7
- package/src/skf-verify-stack/{steps-c/step-05-synthesize.md → references/synthesize.md} +8 -8
- package/tools/validate-docs-drift.js +1 -1
- package/src/skf-audit-skill/steps-c/step-01-init.md +0 -221
- package/src/skf-refine-architecture/steps-c/step-01-init.md +0 -136
- package/src/skf-rename-skill/steps-c/step-04-health-check.md +0 -22
- package/src/skf-test-skill/steps-c/step-07-health-check.md +0 -25
- package/src/skf-update-skill/steps-c/step-07-report.md +0 -148
- package/src/skf-update-skill/steps-c/step-08-health-check.md +0 -22
- package/src/skf-verify-stack/steps-c/step-01-init.md +0 -178
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
---
|
|
2
|
-
nextStepFile: '
|
|
2
|
+
nextStepFile: 'health-check.md'
|
|
3
3
|
|
|
4
4
|
outputFile: '{forge_version}/test-report-{skill_name}-{run_id}.md'
|
|
5
5
|
scoringRulesFile: 'references/scoring-rules.md'
|
|
6
6
|
outputFormatsFile: 'assets/output-section-formats.md'
|
|
7
7
|
# outputContractSchema and healthCheck resolve relative to the SKF module root
|
|
8
|
-
# (`_bmad/skf/` when installed, `src/` during
|
|
9
|
-
# this step file. Both paths are probed in
|
|
8
|
+
# (`{project-root}/_bmad/skf/` when installed, `{project-root}/src/` during
|
|
9
|
+
# development), NOT relative to this step file. Both paths are probed in
|
|
10
|
+
# order; HALT if neither exists.
|
|
10
11
|
outputContractSchema: 'shared/references/output-contract-schema.md'
|
|
11
12
|
healthCheckProbeOrder:
|
|
12
13
|
- '{project-root}/_bmad/skf/shared/health-check.md'
|
|
@@ -16,22 +17,13 @@ atomicWriteProbeOrder:
|
|
|
16
17
|
- '{project-root}/src/shared/scripts/skf-atomic-write.py'
|
|
17
18
|
---
|
|
18
19
|
|
|
20
|
+
<!-- Config: communicate in {communication_language}. Test report prose in {document_output_language}. -->
|
|
21
|
+
|
|
19
22
|
# Step 6: Gap Report
|
|
20
23
|
|
|
21
24
|
## STEP GOAL:
|
|
22
25
|
|
|
23
|
-
Generate a detailed gap report listing every issue found during coverage and coherence analysis, assign severity to each gap, provide specific actionable remediation suggestions, and finalize the test report document. This
|
|
24
|
-
|
|
25
|
-
## Rules
|
|
26
|
-
|
|
27
|
-
- Focus on gap enumeration, severity classification, and remediation — do not recalculate scores
|
|
28
|
-
- Remediation suggestions reference specific files, exports, and line numbers
|
|
29
|
-
- Gaps are ordered by severity (Critical > High > Medium > Low > Info)
|
|
30
|
-
- Chains to the local health-check step via `{nextStepFile}` after completion — the user-facing report is NOT the terminal step
|
|
31
|
-
|
|
32
|
-
## MANDATORY SEQUENCE
|
|
33
|
-
|
|
34
|
-
**CRITICAL:** Follow this sequence exactly. Do not skip, reorder, or improvise unless user explicitly requests a change.
|
|
26
|
+
Generate a detailed gap report listing every issue found during coverage and coherence analysis, assign severity to each gap, provide specific actionable remediation suggestions, and finalize the test report document. Do not recalculate scores — that ran in step 5. This step chains to the local health-check step via `{nextStepFile}` after completion; the user-facing report is NOT the terminal step.
|
|
35
27
|
|
|
36
28
|
### 1. Collect All Issues
|
|
37
29
|
|
|
@@ -74,7 +66,7 @@ Load `{scoringRulesFile}` for gap severity classification:
|
|
|
74
66
|
|
|
75
67
|
Load `{outputFormatsFile}` for gap entry format and remediation quality rules.
|
|
76
68
|
|
|
77
|
-
For each issue, assign severity from `{scoringRulesFile}` and generate a specific remediation following the quality rules in `{outputFormatsFile}`. Order gaps by severity: Critical → High → Medium → Low → Info.
|
|
69
|
+
For each issue, assign severity from `{scoringRulesFile}` and generate a specific remediation following the quality rules in `{outputFormatsFile}`. Remediation suggestions MUST reference specific files, exports, and line numbers. Order gaps by severity: Critical → High → Medium → Low → Info.
|
|
78
70
|
|
|
79
71
|
### 4. Generate Remediation Summary and Append Gap Report
|
|
80
72
|
|
|
@@ -82,7 +74,9 @@ Load the Gap Report section format from `{outputFormatsFile}`. Count gaps by sev
|
|
|
82
74
|
|
|
83
75
|
If no gaps found, append a clean pass message recommending **export-skill** workflow.
|
|
84
76
|
|
|
85
|
-
### 4b. Discovery Testing (MANDATORY
|
|
77
|
+
### 4b. Discovery Testing (MANDATORY)
|
|
78
|
+
|
|
79
|
+
**`--no-discovery` flag bypass (precedes the precondition check).** If `no_discovery: true` is set in workflow context (from §1 of `init.md` — `--no-discovery` flag on invocation), record an Info-severity note in the Discovery Quality subsection: `discovery — skipped: --no-discovery flag set`, log the bypass, and SKIP §4b.1–§4b.3. Proceed to §4b.4 (description optimization) only if tessl/skill-check flagged description issues; otherwise skip directly to §4c.
|
|
86
80
|
|
|
87
81
|
After gap enumeration, perform minimum-viable discovery testing. This is a **Medium-weight** check contributing to the Discovery Quality subsection — no longer advisory boilerplate.
|
|
88
82
|
|
|
@@ -93,7 +87,7 @@ Count the skills in `{skillsOutputFolder}`: `ls -1d {skillsOutputFolder}/*/ 2>/d
|
|
|
93
87
|
- If `catalog_size < 2`: **skip §4b.1–§4b.3**. Record an Info-severity note in the Discovery Quality subsection: `discovery — skipped: catalog size N={catalog_size}, requires ≥2 candidates for meaningful routing`. The routing test is vacuous with one candidate (any prompt returns the sole skill); reporting `3/3 PASS` under those conditions inflates the Discovery score and masks genuinely bad description triggers. Proceed to §4b.4 (description optimization) if tessl/skill-check flagged description issues; otherwise skip to §4c.
|
|
94
88
|
- If `catalog_size >= 2`: continue with §4b.1 as written.
|
|
95
89
|
|
|
96
|
-
Optional escape hatch: the workflow accepts `--discovery-catalog=all` to broaden the candidate pool to
|
|
90
|
+
Optional escape hatch: the workflow accepts `--discovery-catalog=all` to broaden the candidate pool to `{project-root}/.claude/skills/` or `{project-root}/_bmad/agents/` for single-skill repos where the repo-local catalog is trivially too small. When the flag is set, rebuild `catalog_size` from the broader pool before the precondition check.
|
|
97
91
|
|
|
98
92
|
**4b.1 Extract realistic prompts from the skill under test:**
|
|
99
93
|
|
|
@@ -137,12 +131,12 @@ Realistic prompt patterns for synthesis (§4b.1 fallback):
|
|
|
137
131
|
- Implicit: "why did {metric} drop last {period}"
|
|
138
132
|
- Abbreviated: "run the {keyword} thing on this data"
|
|
139
133
|
|
|
140
|
-
### 4c. Result Contract (atomic write
|
|
134
|
+
### 4c. Result Contract (atomic write)
|
|
141
135
|
|
|
142
136
|
**Resolve `{atomicWriteHelper}`:** probe `{atomicWriteProbeOrder}`. HALT if neither candidate exists — the contract is a downstream-consumer protocol and must never be written non-atomically.
|
|
143
137
|
|
|
144
138
|
Write the result contract per `{outputContractSchema}`:
|
|
145
|
-
- Per-run record: `{forge_version}/skf-test-skill-result-{run_id}.json` (the `{run_id}` set in step
|
|
139
|
+
- Per-run record: `{forge_version}/skf-test-skill-result-{run_id}.json` (the `{run_id}` set in step 1 §6a — already carries UTC timestamp + PID + random suffix, so no same-second collision).
|
|
146
140
|
- Latest copy: `{forge_version}/skf-test-skill-result-latest.json` (stable path for pipeline consumers — copy, not symlink).
|
|
147
141
|
|
|
148
142
|
Both writes MUST go through the atomic writer so partial writes are never observable:
|
|
@@ -155,29 +149,37 @@ cat payload.json | python3 {atomicWriteHelper} write --target {forge_version}/sk
|
|
|
155
149
|
|
|
156
150
|
Payload contents:
|
|
157
151
|
- `outputs[]` — include the test report path at `{outputFile}` with its `{run_id}` suffix
|
|
158
|
-
- `summary` — `score`, `threshold`, `result` (`"PASS"`, `"PASS_WITH_DRIFT"`, `"FAIL"`, or **`"INCONCLUSIVE"`**), `testMode` (naive/contextual), `activeCategories[]`, `inconclusiveReasons[]` (when present). `PASS_WITH_DRIFT` is set when the workflow observed workspace drift and the user passed `--allow-workspace-drift` — see step
|
|
152
|
+
- `summary` — `score`, `threshold`, `result` (`"PASS"`, `"PASS_WITH_DRIFT"`, `"FAIL"`, or **`"INCONCLUSIVE"`**), `testMode` (naive/contextual), `activeCategories[]`, `inconclusiveReasons[]` (when present). `PASS_WITH_DRIFT` is set when the workflow observed workspace drift and the user passed `--allow-workspace-drift` — see step 5 §5 drift override. Downstream consumers MUST treat `PASS_WITH_DRIFT` as a non-exportable result: re-run against the pinned commit before export.
|
|
159
153
|
- `runId` — the workflow's `{run_id}` for downstream correlation
|
|
160
154
|
- `healthCheckDispatched` — boolean, set by §7 after the dispatch decision
|
|
161
155
|
|
|
162
|
-
The `{forge_version}/.test-skill.lock` acquired in step
|
|
156
|
+
The `{forge_version}/.test-skill.lock` acquired in step 1 §6b remains held until the end of this step — it guards against concurrent latest-file overwrites.
|
|
157
|
+
|
|
158
|
+
**Post-finalization hook.** If `{onCompleteCommand}` (resolved in SKILL.md On Activation §3 from `workflow.on_complete` scalar) is non-empty, invoke it as:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
{onCompleteCommand} --result-path={forge_version}/skf-test-skill-result-{run_id}.json
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Run it with a bounded timeout (default 60s). On success: log Info note "on_complete — invoked: {command}" and continue. On non-zero exit, timeout, or any failure: append the failure reason to `workflow_warnings[]` (e.g. `on_complete — failed (exit {N}): {stderr_first_line}`) and continue. **The hook must never fail the workflow** — its purpose is integration glue (notify a CI router, post to a queue, archive the result) and any failure there is orthogonal to the test verdict. If `{onCompleteCommand}` is empty, this hook is a no-op (no log entry needed).
|
|
163
165
|
|
|
164
166
|
### 5. Finalize Output Document — Enforce Step Completeness
|
|
165
167
|
|
|
166
|
-
**
|
|
168
|
+
**Incremental step tracking:** read `stepsCompleted` from the output frontmatter. The expected set is the canonical chain:
|
|
167
169
|
|
|
168
170
|
```
|
|
169
|
-
['
|
|
170
|
-
'
|
|
171
|
-
'
|
|
172
|
-
'
|
|
173
|
-
'
|
|
174
|
-
'
|
|
175
|
-
'
|
|
171
|
+
['init',
|
|
172
|
+
'detect-mode',
|
|
173
|
+
'coverage-check',
|
|
174
|
+
'coherence-check',
|
|
175
|
+
'external-validators',
|
|
176
|
+
'score',
|
|
177
|
+
'report']
|
|
176
178
|
```
|
|
177
179
|
|
|
178
|
-
If any expected entry is missing, HALT with "step completeness violation — missing {list}; workflow state is inconsistent, do not finalize the report". Only append `'
|
|
180
|
+
If any expected entry is missing, HALT with "step completeness violation — missing {list}; workflow state is inconsistent, do not finalize the report". Only append `'report'` and write back after the check passes.
|
|
179
181
|
|
|
180
|
-
**
|
|
182
|
+
**Section anchor presence check (companion to stepsCompleted).** The
|
|
181
183
|
report template ships six canonical H2 anchors — one per populating step. An
|
|
182
184
|
off-sequence run (e.g. a step wrote its section into the wrong anchor, or a
|
|
183
185
|
subagent truncated the file) can leave `stepsCompleted` intact while a section
|
|
@@ -194,12 +196,12 @@ return ≥1 match:
|
|
|
194
196
|
```
|
|
195
197
|
|
|
196
198
|
On any miss, HALT with "report anchor missing: {anchor} — section was not
|
|
197
|
-
appended by its owning step". Do NOT append `'
|
|
199
|
+
appended by its owning step". Do NOT append `'report'` and do NOT
|
|
198
200
|
write the result contract. The template in `templates/test-report-template.md`
|
|
199
201
|
declares these anchors as TBD placeholders; a miss means a step silently
|
|
200
202
|
skipped its append.
|
|
201
203
|
|
|
202
|
-
**INCONCLUSIVE as gate:** if `testResult == 'inconclusive'` (from step
|
|
204
|
+
**INCONCLUSIVE as gate:** if `testResult == 'inconclusive'` (from step 5), the report final presentation (§6) and result contract (§4c) have already been written with that verdict. Do NOT auto-map INCONCLUSIVE to PASS or FAIL. Recommend `manual-review`. The step must still complete (health-check runs unconditionally) — INCONCLUSIVE is a report-time signal, not a workflow abort.
|
|
203
205
|
|
|
204
206
|
### 6. Present Final Report
|
|
205
207
|
|
|
@@ -240,7 +242,7 @@ skipped its append.
|
|
|
240
242
|
|
|
241
243
|
**Test report finalized.**"
|
|
242
244
|
|
|
243
|
-
### 6b. Headless Exit Status
|
|
245
|
+
### 6b. Headless Exit Status
|
|
244
246
|
|
|
245
247
|
If `{headless_mode}`:
|
|
246
248
|
- `testResult: 'pass'` → `exit 0`
|
|
@@ -252,7 +254,9 @@ Non-headless mode always drops to the menu in §7.
|
|
|
252
254
|
|
|
253
255
|
### 7. Health-Check Dispatch + MENU OPTIONS
|
|
254
256
|
|
|
255
|
-
|
|
257
|
+
**`--no-health-check` flag bypass (precedes the health-check resolution).** If `no_health_check: true` is set in workflow context (from §1 of `init.md` — `--no-health-check` flag on invocation), set `health_check_dispatched: false` in the output report frontmatter and mirror `healthCheckDispatched: false` into the result contract written in §4c (re-write atomically via `{atomicWriteHelper}`). Log Info note "health-check — skipped: --no-health-check flag set" and EXIT THE WORKFLOW: in `{headless_mode}`, exit with the code already determined in §6b; non-headless, simply terminate after the §6 presentation. Do NOT resolve `{healthCheckFile}`, do NOT display the menu, do NOT chain to `{nextStepFile}`. The §6b exit branch is the terminal state under this flag.
|
|
258
|
+
|
|
259
|
+
Resolve `{healthCheckFile}`: probe `{healthCheckProbeOrder}` in order. **HALT** if neither candidate exists — the health-check is the true terminal step; without it the workflow cannot complete honestly:
|
|
256
260
|
|
|
257
261
|
```
|
|
258
262
|
Error: cannot locate shared/health-check.md at either of:
|
|
@@ -272,19 +276,5 @@ Also mirror the boolean into the `healthCheckDispatched` field of the result con
|
|
|
272
276
|
|
|
273
277
|
Display: "**Test complete.** [C] Finish"
|
|
274
278
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
- IF C: Load, read the full file, and execute `{nextStepFile}` — the health-check step is the true terminal step of this workflow.
|
|
278
|
-
- IF Any other: help user respond, then redisplay menu
|
|
279
|
-
|
|
280
|
-
#### EXECUTION RULES:
|
|
281
|
-
|
|
282
|
-
- ALWAYS halt and wait for user input after presenting menu
|
|
283
|
-
- **GATE [default: C]** — If `{headless_mode}`: auto-proceed with [C] Continue, log: "headless: auto-continue past report menu"; set `health_check_dispatched: true` in frontmatter before chaining
|
|
284
|
-
- C triggers the health check, which is the true workflow exit
|
|
285
|
-
- User may ask questions about the report before finishing
|
|
286
|
-
|
|
287
|
-
## CRITICAL STEP COMPLETION NOTE
|
|
288
|
-
|
|
289
|
-
When the user selects C, this step chains to the local health-check step (`{nextStepFile}`), which in turn delegates to `shared/health-check.md`. After the health check completes, the test-skill workflow is fully done. The test report document at `{outputFile}` contains the full analysis: Test Summary, Coverage Analysis, Coherence Analysis, Completeness Score, and Gap Report.
|
|
279
|
+
On [C] (or auto-proceed in `{headless_mode}` — log: "headless: auto-continue past report menu"): set `health_check_dispatched: true` in frontmatter, then load and execute `{nextStepFile}` (the local health-check dispatcher). The test report document at `{outputFile}` contains the full analysis: Test Summary, Coverage Analysis, Coherence Analysis, Completeness Score, and Gap Report.
|
|
290
280
|
|
|
@@ -1,36 +1,34 @@
|
|
|
1
1
|
---
|
|
2
|
-
nextStepFile: '
|
|
2
|
+
nextStepFile: 'report.md'
|
|
3
3
|
outputFile: '{forge_version}/test-report-{skill_name}-{run_id}.md'
|
|
4
4
|
scoringRulesFile: 'references/scoring-rules.md'
|
|
5
5
|
sourceAccessProtocol: 'references/source-access-protocol.md'
|
|
6
6
|
scoringScript: 'scripts/compute-score.py'
|
|
7
7
|
---
|
|
8
8
|
|
|
9
|
+
<!-- Config: communicate in {communication_language}. -->
|
|
10
|
+
|
|
9
11
|
# Step 5: Score
|
|
10
12
|
|
|
11
13
|
## STEP GOAL:
|
|
12
14
|
|
|
13
15
|
Calculate the overall completeness score by aggregating coverage, coherence, and external validation category scores with the appropriate weight distribution (naive or contextual), apply the pass/fail threshold, and determine the test result.
|
|
14
16
|
|
|
15
|
-
## Rules
|
|
16
|
-
|
|
17
|
-
- Focus only on score calculation — do not generate remediation suggestions (Step 06)
|
|
18
|
-
- Score must be deterministic — same inputs always produce same output
|
|
19
|
-
- Show the math: category scores, weights, weighted contributions, total
|
|
20
|
-
|
|
21
|
-
## MANDATORY SEQUENCE
|
|
22
|
-
|
|
23
|
-
**CRITICAL:** Follow this sequence exactly. Do not skip, reorder, or improvise unless user explicitly requests a change.
|
|
24
|
-
|
|
25
17
|
### 1. Load Scoring Rules
|
|
26
18
|
|
|
27
19
|
Load `{scoringRulesFile}` to get:
|
|
28
20
|
- Category weights (naive vs contextual distribution)
|
|
29
|
-
- Default threshold (80%)
|
|
30
21
|
- Tier-dependent scoring adjustments
|
|
31
|
-
- Any custom threshold override from workflow input
|
|
32
22
|
|
|
33
|
-
**
|
|
23
|
+
**Resolve the pass threshold (precedence: CLI > scalar > bundled fallback):**
|
|
24
|
+
|
|
25
|
+
1. If the workflow received `--threshold=<N>` on invocation, use that integer as `effective_threshold` (CLI wins).
|
|
26
|
+
2. Else if the resolved `{defaultThreshold}` workflow-context variable (from SKILL.md On Activation §3 — `workflow.default_threshold` scalar, default `80`) is set, use it as `effective_threshold`.
|
|
27
|
+
3. Else fall back to `80` (the bundled default — this branch should be unreachable when SKILL.md resolution ran correctly, but keeps the step robust if customize.toml resolution failed silently).
|
|
28
|
+
|
|
29
|
+
Pass `effective_threshold` into the scoring-input JSON's `threshold` field in §3a (the compute-score.py script already honors this field). The CLI flag and the scalar feed the same downstream field; the script does not need to know which layer supplied the value.
|
|
30
|
+
|
|
31
|
+
**Docs-only mode check:** If the Coverage Analysis section in `{outputFile}` notes docs-only mode (set by step 3 for skills with all `[EXT:...]` citations and no local source), apply Quick-tier weight redistribution: Signature Accuracy and Type Coverage are not scored, their weights (22% + 14%) are redistributed proportionally to remaining active categories. Coverage score is based on documentation completeness rather than source coverage (as calculated by step 3).
|
|
34
32
|
|
|
35
33
|
### 2. Read Category Scores from Output
|
|
36
34
|
|
|
@@ -53,9 +51,9 @@ Read `{outputFile}` and extract the category scores calculated in previous steps
|
|
|
53
51
|
|
|
54
52
|
**Read testMode from {outputFile} frontmatter.**
|
|
55
53
|
|
|
56
|
-
#### 2b. Apply
|
|
54
|
+
#### 2b. Apply State 2 Undercount Deduction (pre-script)
|
|
57
55
|
|
|
58
|
-
If `analysis_confidence == 'provenance-map'` (State 2) AND step
|
|
56
|
+
If `analysis_confidence == 'provenance-map'` (State 2) AND step 3 recorded a provenance vs metadata divergence > 5% (see §4b in step 3), apply a 10-point deduction to `exportCoverage` BEFORE building the scoring input:
|
|
59
57
|
|
|
60
58
|
```
|
|
61
59
|
exportCoverage_adjusted = max(0, exportCoverage - 10)
|
|
@@ -81,7 +79,7 @@ Build a JSON object from the data gathered in steps 1-2:
|
|
|
81
79
|
"coherence": "{combined_coherence_percentage or null if naive mode}",
|
|
82
80
|
"externalValidation": "{external_validation_score or null if N/A}"
|
|
83
81
|
},
|
|
84
|
-
"threshold": "{
|
|
82
|
+
"threshold": "{effective_threshold from §1 — CLI --threshold wins, then workflow.default_threshold scalar, then 80}"
|
|
85
83
|
}
|
|
86
84
|
```
|
|
87
85
|
|
|
@@ -90,10 +88,10 @@ Build a JSON object from the data gathered in steps 1-2:
|
|
|
90
88
|
#### 3b. Run the Scoring Script
|
|
91
89
|
|
|
92
90
|
```bash
|
|
93
|
-
|
|
91
|
+
echo '<JSON>' | uv run {scoringScript} --stdin
|
|
94
92
|
```
|
|
95
93
|
|
|
96
|
-
Where `{scoringScript}` is the path resolved from the frontmatter variable (relative to the skill root, i.e., the skf-test-skill/ directory).
|
|
94
|
+
Where `{scoringScript}` is the path resolved from the frontmatter variable (relative to the skill root, i.e., the skf-test-skill/ directory). The script also accepts the JSON as a positional argument (`uv run {scoringScript} '<JSON>'`) or via `--json-input '<JSON>'`; `--stdin` is preferred since it avoids shell-quote escaping of nested JSON.
|
|
97
95
|
|
|
98
96
|
Parse the JSON output. The script returns:
|
|
99
97
|
- `weights` — final redistributed weights per category
|
|
@@ -127,9 +125,9 @@ Report: "**Note:** Scoring script unavailable — calculated manually per scorin
|
|
|
127
125
|
|
|
128
126
|
The scoring script returns `totalScore` and a preliminary `result`. Two caps may override the script's PASS into FAIL — applied in this order, and ONLY if the script did NOT return INCONCLUSIVE (floor always takes precedence):
|
|
129
127
|
|
|
130
|
-
**Cap 1 — Tooling degraded
|
|
128
|
+
**Cap 1 — Tooling degraded:** if `analysisConfidence == "degraded"` or `toolingStatus` indicates a missing helper (`python3-missing`, `frontmatter-validator-missing`), set `effective_score = min(totalScore, threshold - 1)` and record `scoring_notes: tooling degraded — capped below threshold until helper restored`.
|
|
131
129
|
|
|
132
|
-
**Cap 2 — Docs-only without external validators
|
|
130
|
+
**Cap 2 — Docs-only without external validators:** if `docsOnly == true` AND `externalValidation` score was null (neither skill-check nor tessl ran), set `effective_score = min(effective_score, threshold - 1)` and record `scoring_notes: docs-only without external validators — capped below threshold`.
|
|
133
131
|
|
|
134
132
|
If either cap fired and `result` was `PASS`, change it to `FAIL`. If `result` was already `INCONCLUSIVE`, leave it — the evidence-floor verdict is never overridden by a cap.
|
|
135
133
|
|
|
@@ -147,7 +145,7 @@ IF score < threshold → FAIL
|
|
|
147
145
|
- `active_categories < 2` (after all redistribution), OR
|
|
148
146
|
- `tier == "Quick"` AND Export Coverage is the sole scoring contributor
|
|
149
147
|
|
|
150
|
-
**Tooling-degraded cap
|
|
148
|
+
**Tooling-degraded cap:** If the `analysisConfidence` in output frontmatter is `degraded` (python3 missing, frontmatter validator missing, or other degraded state flagged in step 1), the step MUST cap the score at `threshold - 1` BEFORE the PASS/FAIL comparison. This forces a deterministic FAIL until tooling is restored. Do NOT override an INCONCLUSIVE result with the cap — INCONCLUSIVE remains the verdict.
|
|
151
149
|
|
|
152
150
|
### 5. Determine Next Workflow Recommendation
|
|
153
151
|
|
|
@@ -155,12 +153,12 @@ Based on test result:
|
|
|
155
153
|
|
|
156
154
|
**IF PASS:**
|
|
157
155
|
- `nextWorkflow: 'export-skill'` — skill is ready for export
|
|
158
|
-
- **
|
|
159
|
-
`allow_workspace_drift: true` (set in step
|
|
156
|
+
- **Drift override:** if workflow context carries
|
|
157
|
+
`allow_workspace_drift: true` (set in step 1 §5b when the user passed
|
|
160
158
|
`--allow-workspace-drift` AND the workspace HEAD did not match
|
|
161
159
|
`metadata.source_commit`), the PASS is a **conditional PASS**:
|
|
162
160
|
- Write `testResult: 'pass-with-drift'` to the output frontmatter instead of
|
|
163
|
-
bare `'pass'`. The result contract (§4c of step
|
|
161
|
+
bare `'pass'`. The result contract (§4c of step 6) mirrors the same
|
|
164
162
|
value.
|
|
165
163
|
- Override `nextWorkflow` to `'update-skill'` — **refuse to recommend
|
|
166
164
|
`export-skill`**. The drift override weakens the workflow's strongest
|
|
@@ -221,12 +219,12 @@ If `analysis_confidence` is not `full`, append a degradation notice. **The notic
|
|
|
221
219
|
### 7. Update Output Frontmatter
|
|
222
220
|
|
|
223
221
|
Update `{outputFile}` frontmatter:
|
|
224
|
-
- `testResult: '{pass|pass-with-drift|fail|inconclusive}'` (lowercase; mirrors script `result`, with `pass-with-drift` substituted for `pass` when `allow_workspace_drift` was set and drift was observed — see §5
|
|
222
|
+
- `testResult: '{pass|pass-with-drift|fail|inconclusive}'` (lowercase; mirrors script `result`, with `pass-with-drift` substituted for `pass` when `allow_workspace_drift` was set and drift was observed — see §5 drift override)
|
|
225
223
|
- `score: '{total}%'`
|
|
226
224
|
- `threshold: '{threshold}%'`
|
|
227
225
|
- `analysisConfidence: '{full|degraded|provenance-map|metadata-only|remote-only|docs-only}'`
|
|
228
226
|
- `nextWorkflow: '{export-skill|update-skill|manual-review}'`
|
|
229
|
-
- Append `'
|
|
227
|
+
- Append `'score'` to `stepsCompleted`
|
|
230
228
|
|
|
231
229
|
### 8. Report Score
|
|
232
230
|
|
|
@@ -247,20 +245,5 @@ Update `{outputFile}` frontmatter:
|
|
|
247
245
|
|
|
248
246
|
**Proceeding to gap report...**"
|
|
249
247
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
Display: "**Proceeding to gap report...**"
|
|
253
|
-
|
|
254
|
-
#### Menu Handling Logic:
|
|
255
|
-
|
|
256
|
-
- After score is calculated and frontmatter updated, immediately load, read entire file, then execute {nextStepFile}
|
|
257
|
-
|
|
258
|
-
#### EXECUTION RULES:
|
|
259
|
-
|
|
260
|
-
- This is an auto-proceed scoring step with no user choices
|
|
261
|
-
- Proceed directly to next step after score is determined
|
|
262
|
-
|
|
263
|
-
## CRITICAL STEP COMPLETION NOTE
|
|
264
|
-
|
|
265
|
-
ONLY WHEN the score is calculated, pass/fail is determined, the Completeness Score section is appended to {outputFile}, and frontmatter is updated with testResult and score, will you then load and read fully `{nextStepFile}` to execute gap report generation.
|
|
248
|
+
Update stepsCompleted, then load and execute {nextStepFile}.
|
|
266
249
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
<!-- Config: communicate in {communication_language}. -->
|
|
2
|
+
|
|
1
3
|
# Scoring Rules
|
|
2
4
|
|
|
3
5
|
## Default Threshold
|
|
@@ -44,7 +46,7 @@ tessl evaluates SKILL.md body content only — it does not read `references/*.md
|
|
|
44
46
|
|
|
45
47
|
### Docs-Only Mode (all [EXT:...] citations, any tier)
|
|
46
48
|
|
|
47
|
-
When `docs_only_mode: true` is set by step
|
|
49
|
+
When `docs_only_mode: true` is set by step 3 (indicating a skill where all SKILL.md citations are `[EXT:...]` format with no local source code):
|
|
48
50
|
|
|
49
51
|
- **Signature Accuracy:** Not scored (no source to compare against)
|
|
50
52
|
- **Type Coverage:** Not scored (no source to compare against)
|
|
@@ -54,7 +56,7 @@ When `docs_only_mode: true` is set by step-03 (indicating a skill where all SKIL
|
|
|
54
56
|
|
|
55
57
|
This is functionally identical to Quick tier weight redistribution but with a different coverage denominator (self-consistency instead of source comparison).
|
|
56
58
|
|
|
57
|
-
**
|
|
59
|
+
**External-validator requirement for docs-only:** docs-only mode removes two categories (Signature Accuracy, Type Coverage) from scoring. If External Validation is ALSO unavailable, the evidence base collapses to Coverage alone (naive) or Coverage + Coherence (contextual) — which in the naive/Quick case trips the minimum-evidence floor (INCONCLUSIVE). To keep docs-only skills gradable when external validators are present but still deterministic when they are missing: **when `docsOnly: true` AND `externalValidation is null`, step 5 MUST cap `totalScore` at `threshold - 1` (forcing FAIL) before the INCONCLUSIVE floor is evaluated.** This prevents a docs-only skill from PASSing with only one or two redistributed categories carrying all the weight. Implement in step 5 §4 as a pre-compare cap, recorded in the report as `scoring_notes: docs-only without external validators — capped below threshold`.
|
|
58
60
|
|
|
59
61
|
### Stack Skills (Any Tier)
|
|
60
62
|
|
|
@@ -64,7 +66,7 @@ When `metadata.json.skill_type == "stack"` (set `stackSkill: true` in the scorin
|
|
|
64
66
|
- **Type Coverage:** N/A — same rationale; the type surface belongs to the external libraries.
|
|
65
67
|
- **Weight redistribution:** Same as Quick tier / docs-only / State 2 — Signature Accuracy (22%) and Type Coverage (14%) weights redistributed proportionally to remaining active categories (Export Coverage, Coherence, External Validation).
|
|
66
68
|
- **Applies regardless of detected tier** (Quick, Forge, Forge+, Deep) and is independent of `docsOnly` and `state2`. A stack skill can also be docs-only or State 2; the skip reasons combine additively (e.g. `"stack skill (external type surface) + State 2 (provenance-map)"`).
|
|
67
|
-
- **Detection:** step
|
|
69
|
+
- **Detection:** step 5 reads `metadata.json.skill_type` from the skill package. If the value is `"stack"`, set `stackSkill: true` in the scoring input JSON.
|
|
68
70
|
|
|
69
71
|
Equivalence-class note: stack skills with `docsOnly:false` / `state2:false` map to the same equivalence class as State 2 contextual rows (class B) or State 2 naive rows (class D) — the redistribution math is identical; only the `skipReasons` string changes.
|
|
70
72
|
|
|
@@ -80,7 +82,7 @@ When source is not locally available and analysis resolves to State 2 (provenanc
|
|
|
80
82
|
|
|
81
83
|
Note: When provenance-map entries are predominantly T1 (AST-verified at compilation time), the coverage and name-matching data is already at highest confidence. The N/A categories reflect the inability to re-verify at test time, not low-quality extraction data.
|
|
82
84
|
|
|
83
|
-
**
|
|
85
|
+
**State 2 undercount risk acknowledgement:** provenance-map is a cached extraction snapshot — if the source has evolved since extraction, public API adds/removes will NOT surface in Export Coverage (denominator is frozen to the provenance-map union). When `state2: true` AND step 3 records any provenance vs metadata divergence (e.g. union > either source by >5%), apply a flat **10% deduction** to `exportCoverage` before calling the scoring script, AND set `analysis_confidence: provenance-map` (already set) with a report note: `scoring_notes: State 2 undercount risk acknowledged — 10% deduction applied to Export Coverage`. Rationale: the skill cannot be reliably scored on a frozen denominator when the cache is known to disagree with its own metadata; prefer understating over overstating.
|
|
84
86
|
|
|
85
87
|
### Forge Tier (ast-grep)
|
|
86
88
|
- Export Coverage: AST-backed export comparison
|
|
@@ -98,7 +100,7 @@ Note: When provenance-map entries are predominantly T1 (AST-verified at compilat
|
|
|
98
100
|
- Cross-repository reference verification
|
|
99
101
|
- QMD knowledge enrichment for coherence
|
|
100
102
|
- Full scoring formula with maximum depth
|
|
101
|
-
- **Migration & Deprecation Warnings section:** If T2-future annotations exist in the enrichment data, verify that Section 4b is present in SKILL.md Tier 1 and that each warning traces to a T2 provenance citation. If no T2-future annotations exist, Section 4b should normally be absent (not empty). Presence/absence mismatch is a Medium severity gap — with one Info-severity exception for historical-migration content (completed package renames, consolidated import paths, shipped API cutovers that remain load-bearing for training-data drift remediation). See `
|
|
103
|
+
- **Migration & Deprecation Warnings section:** If T2-future annotations exist in the enrichment data, verify that Section 4b is present in SKILL.md Tier 1 and that each warning traces to a T2 provenance citation. If no T2-future annotations exist, Section 4b should normally be absent (not empty). Presence/absence mismatch is a Medium severity gap — with one Info-severity exception for historical-migration content (completed package renames, consolidated import paths, shipped API cutovers that remain load-bearing for training-data drift remediation). See `references/coherence-check.md` §2b/§5b for the three-case rule.
|
|
102
104
|
|
|
103
105
|
## Redistribution Combinations Matrix (M3 — terminology + determinism)
|
|
104
106
|
|
|
@@ -179,7 +181,7 @@ Three-state gate — **PASS / FAIL / INCONCLUSIVE**. `INCONCLUSIVE` is not PASS
|
|
|
179
181
|
- `active_categories` = count of categories with a non-zero final weight *after* all redistribution (Quick tier, docs-only, State 2, external-validator-unavailable). Categories with a redistributed weight of 0 do not count as active, even if they received a score.
|
|
180
182
|
- **If `active_categories < 2`** → force `result: INCONCLUSIVE` with rationale `"insufficient evidence: only {N} active category"`. A single active category cannot cross-validate itself and a PASS would be a false signal.
|
|
181
183
|
- **If `tier == "Quick"` AND the sole active contributor is Export Coverage** → force `result: INCONCLUSIVE` with rationale `"Quick tier: Export Coverage alone is insufficient evidence — add a second active category by upgrading tier or enabling external validators"`. This catches the degenerate case where every signature/type/coherence/external category gets redistributed to 0 and Export Coverage is doing all the work.
|
|
182
|
-
- The floor is enforced by `scripts/compute-score.py`. The step
|
|
184
|
+
- The floor is enforced by `scripts/compute-score.py`. The step 5 scoring step reads `result` from the script output and writes it into the test report frontmatter unchanged.
|
|
183
185
|
|
|
184
186
|
- Otherwise:
|
|
185
187
|
- score >= threshold → PASS
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
<!-- Config: communicate in {communication_language}. -->
|
|
2
|
+
|
|
1
3
|
# Source Access Protocol
|
|
2
4
|
|
|
3
5
|
## Source API Surface Definition
|
|
@@ -14,7 +16,7 @@
|
|
|
14
16
|
|
|
15
17
|
**Resolution order:**
|
|
16
18
|
|
|
17
|
-
1. **Prefer `metadata.json.stats.effective_denominator`** when present. `skf-create-skill` step
|
|
19
|
+
1. **Prefer `metadata.json.stats.effective_denominator`** when present. `skf-create-skill` step 5 §4 writes this field for stratified-scope skills. When set, use it directly as the `exports_public_api` count for coverage scoring.
|
|
18
20
|
2. **Fall back to live re-derivation** when `effective_denominator` is absent (older skills, quick-tier output, or skills compiled before this rule existed). Read the brief's scope globs from `forge-data/{skill_name}/skill-brief.yaml`, resolve them against `source_path`, filter out files matching `scope.exclude`, and compute the source API surface as the **union of named exports across the matched files only**. The skill brief's `scope.notes` field should document the stratification strategy (e.g., "Tier A: fully documented; Tier B: deferred to references; Tier C: excluded") — when present, treat it as confirmation that the curated subset is by design, not a scope gap.
|
|
19
21
|
|
|
20
22
|
**Honor `scope.tier_a_include` when present.** When re-deriving, prefer the brief-level `scope.tier_a_include` narrow include list over the coarse `scope.include`. `tier_a_include` is an optional brief field that lists only the authoring surface the brief actually intends to document (tier A), letting the denominator match the brief's authoring-vs-installing intent even when `scope.include` uses coarse globs that also match internal infrastructure. When `tier_a_include` is present, resolve its globs (still filtered by `scope.exclude`), compute the union across those files, and use that count as the denominator. When absent, fall back to resolving `scope.include`.
|
|
@@ -38,7 +40,7 @@
|
|
|
38
40
|
|
|
39
41
|
**Confidence:** leave `analysis_confidence` unchanged (still `full` or `provenance-map` per the waterfall). Pattern-reference does not degrade confidence — the surface is smaller than a library barrel, not lower quality. Annotate the coverage report with: `Pattern-reference — denominator: {tier_a_include union | canonicalized provenance-map count} ({N} pattern surfaces)`.
|
|
40
42
|
|
|
41
|
-
**When this clause does NOT apply:** any repo with a non-empty barrel file, any monorepo (use the stratified-scope clause), or any single-package repo whose `scope.type` is explicitly `public-api` / `specific-modules` / `component-library` / `docs-only` (those scope types have their own denominator semantics). Also does NOT apply if `scope.type: "reference-app"` exists in the enum (pending upgrade in `skf-create-skill/
|
|
43
|
+
**When this clause does NOT apply:** any repo with a non-empty barrel file, any monorepo (use the stratified-scope clause), or any single-package repo whose `scope.type` is explicitly `public-api` / `specific-modules` / `component-library` / `docs-only` (those scope types have their own denominator semantics). Also does NOT apply if `scope.type: "reference-app"` exists in the enum (pending upgrade in `skf-create-skill/references/extract.md`) — in that case the brief speaks for itself and this clause's filesystem trigger is moot.
|
|
42
44
|
|
|
43
45
|
Internal module symbols are **excluded** from the coverage denominator unless they are explicitly documented in SKILL.md (in which case they count as documented extras, not missing coverage).
|
|
44
46
|
|
|
@@ -87,7 +89,7 @@ Check `{forge_data_folder}/{skill_name}/provenance-map.json`. If present AND con
|
|
|
87
89
|
- Build the coverage denominator from the **union** of provenance-map entry names and metadata.json `exports[]` names. Exports present in metadata but absent from provenance-map are counted as "missing documentation" in the coverage calculation.
|
|
88
90
|
- If metadata.json is unavailable or has no `exports[]` array, use provenance-map count alone with a note: "Coverage denominator is provenance-map only — may undercount if extraction was incomplete." If remote reading tools are available (zread, deepwiki, gh API, or similar), supplement by reading the entry point file for live signature verification. Set `analysis_confidence: provenance-map`.
|
|
89
91
|
|
|
90
|
-
**State 2 limitations:** Signature verification at State 2 is **string comparison only**, not semantic. Provenance-map stores parameters as flat string arrays (e.g., `["data: Union[BinaryIO, list, str]"]`), so `str` vs `String` or `list` vs `List[Any]` would be treated as mismatches even when semantically equivalent. For full type-aware verification (handling type aliases, generic equivalence), State 1 (local source) with AST re-parsing is required. When the SKILL.md was compiled from the same provenance-map (typical for create-then-test flows), most strings will match. However, enrichment (step
|
|
92
|
+
**State 2 limitations:** Signature verification at State 2 is **string comparison only**, not semantic. Provenance-map stores parameters as flat string arrays (e.g., `["data: Union[BinaryIO, list, str]"]`), so `str` vs `String` or `list` vs `List[Any]` would be treated as mismatches even when semantically equivalent. For full type-aware verification (handling type aliases, generic equivalence), State 1 (local source) with AST re-parsing is required. When the SKILL.md was compiled from the same provenance-map (typical for create-then-test flows), most strings will match. However, enrichment (step 4) and doc-fetching (step 3c) during compilation may alter parameter descriptions, add type annotations, or normalize signatures, causing mismatches even in create-then-test flows. Expect some string-level mismatches and treat them as compilation artifacts, not source drift signals, until signature fidelity is enforced by step 5's Signature Fidelity Rule (see `signature_source` field in provenance-map entries).
|
|
91
93
|
|
|
92
94
|
**State 3 — No provenance-map, metadata exports exist (quick-skill path):**
|
|
93
95
|
If no provenance-map.json exists (typical for quick-skill output), fall back to `metadata.json`'s `exports[]` array for the export name list. Coverage check becomes a self-consistency comparison: are all names in `exports[]` documented in SKILL.md with description, parameters, and return type? Signatures cannot be verified. If remote reading tools are available, supplement by reading the entry point for live export comparison. Set `analysis_confidence: metadata-only`.
|
|
@@ -96,9 +98,9 @@ If no provenance-map.json exists (typical for quick-skill output), fall back to
|
|
|
96
98
|
If neither provenance-map nor metadata exports provide a usable baseline, but remote reading tools (zread, deepwiki, gh API, or similar) are available and `source_repo` is set in metadata.json, read the entry point remotely to build the export inventory from scratch. Name-matching only — no AST. Set `analysis_confidence: remote-only`.
|
|
97
99
|
|
|
98
100
|
**State 5 — No source access at all:**
|
|
99
|
-
If none of the above succeed, fall through to docs-only mode (as defined in
|
|
101
|
+
If none of the above succeed, fall through to docs-only mode (as defined in coverage-check.md Section 0: pre-analysis source type detection). Set `analysis_confidence: docs-only`. Warn: "**No source access available.** Coverage check evaluates documentation self-consistency only. Re-run with local clone or remote access for source-backed verification."
|
|
100
102
|
|
|
101
|
-
Set `analysis_confidence` in context for use in Section 2 analysis depth, step
|
|
103
|
+
Set `analysis_confidence` in context for use in Section 2 analysis depth, step 5 output, and step 5 scoring.
|
|
102
104
|
|
|
103
105
|
**Confidence tier mapping:** `full` = T1, `provenance-map` = T1, `metadata-only` = T1-low, `remote-only` = T1-low, `docs-only` = T3. This aligns with the T1/T1-low/T2/T3 scale used across all SKF workflows.
|
|
104
106
|
|
|
@@ -8,11 +8,34 @@ Pure-function scoring script for the SKF test-skill workflow (step-05).
|
|
|
8
8
|
Implements the weight tables, skip conditions, and proportional redistribution
|
|
9
9
|
defined in scoring-rules.md.
|
|
10
10
|
|
|
11
|
-
CLI:
|
|
11
|
+
CLI usage:
|
|
12
|
+
uv run compute-score.py '<JSON>' # JSON literal as positional arg
|
|
13
|
+
uv run compute-score.py --json-input '<JSON>' # explicit flag form
|
|
14
|
+
cat input.json | uv run compute-score.py --stdin # piped input
|
|
15
|
+
|
|
16
|
+
Input schema (one object):
|
|
17
|
+
{
|
|
18
|
+
"mode": "contextual" | "naive",
|
|
19
|
+
"tier": "Quick" | "Forge" | "Forge+" | "Deep",
|
|
20
|
+
"scores": {
|
|
21
|
+
"exportCoverage": <0-100>,
|
|
22
|
+
"signatureAccuracy": <0-100>,
|
|
23
|
+
"typeCoverage": <0-100>,
|
|
24
|
+
"coherence": <0-100>,
|
|
25
|
+
"externalValidation": <0-100>
|
|
26
|
+
},
|
|
27
|
+
"threshold": <0-100, optional, default 80>,
|
|
28
|
+
"evidenceCount": <int, optional, used for INCONCLUSIVE floor>
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
Exit codes:
|
|
32
|
+
0 — score computed (verdict may be PASS/FAIL/INCONCLUSIVE)
|
|
33
|
+
1 — input error (bad JSON, missing fields, schema violation)
|
|
12
34
|
"""
|
|
13
35
|
|
|
14
36
|
from __future__ import annotations
|
|
15
37
|
|
|
38
|
+
import argparse
|
|
16
39
|
import json
|
|
17
40
|
import math
|
|
18
41
|
import sys
|
|
@@ -286,25 +309,73 @@ def compute_score(inp):
|
|
|
286
309
|
|
|
287
310
|
# --- CLI Entry Point ---
|
|
288
311
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
)
|
|
298
|
-
|
|
312
|
+
|
|
313
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
314
|
+
parser = argparse.ArgumentParser(
|
|
315
|
+
prog="compute-score",
|
|
316
|
+
description=(
|
|
317
|
+
"Deterministic Completeness Score calculator. Input is a single JSON "
|
|
318
|
+
"object describing mode, tier, scores, and optional threshold/evidence "
|
|
319
|
+
"count; output is the verdict + redistributed score breakdown."
|
|
320
|
+
),
|
|
321
|
+
epilog=(
|
|
322
|
+
"Example:\n"
|
|
323
|
+
" uv run compute-score.py "
|
|
324
|
+
"'{\"mode\":\"contextual\",\"tier\":\"Deep\","
|
|
325
|
+
"\"scores\":{\"exportCoverage\":92,\"signatureAccuracy\":85,"
|
|
326
|
+
"\"typeCoverage\":100,\"coherence\":80,\"externalValidation\":78}}'"
|
|
327
|
+
),
|
|
328
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
329
|
+
)
|
|
330
|
+
src = parser.add_mutually_exclusive_group()
|
|
331
|
+
src.add_argument(
|
|
332
|
+
"json_input",
|
|
333
|
+
nargs="?",
|
|
334
|
+
help="JSON object as a positional argument (single-quote it on the shell).",
|
|
335
|
+
)
|
|
336
|
+
src.add_argument(
|
|
337
|
+
"--json-input",
|
|
338
|
+
dest="json_input_flag",
|
|
339
|
+
help="JSON object passed via flag (overrides positional).",
|
|
340
|
+
)
|
|
341
|
+
src.add_argument(
|
|
342
|
+
"--stdin",
|
|
343
|
+
action="store_true",
|
|
344
|
+
help="Read the JSON object from stdin.",
|
|
345
|
+
)
|
|
346
|
+
return parser
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _resolve_input(args: argparse.Namespace) -> str:
|
|
350
|
+
if args.stdin:
|
|
351
|
+
return sys.stdin.read()
|
|
352
|
+
if args.json_input_flag is not None:
|
|
353
|
+
return args.json_input_flag
|
|
354
|
+
if args.json_input is not None:
|
|
355
|
+
return args.json_input
|
|
356
|
+
return ""
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def main(argv: list[str] | None = None) -> int:
|
|
360
|
+
parser = _build_parser()
|
|
361
|
+
args = parser.parse_args(argv)
|
|
362
|
+
raw = _resolve_input(args)
|
|
363
|
+
if not raw.strip():
|
|
364
|
+
parser.print_usage(file=sys.stderr)
|
|
365
|
+
print("error: no input provided (positional arg, --json-input, or --stdin)", file=sys.stderr)
|
|
366
|
+
return 1
|
|
299
367
|
|
|
300
368
|
try:
|
|
301
|
-
data = json.loads(
|
|
302
|
-
except json.JSONDecodeError:
|
|
303
|
-
print(
|
|
304
|
-
|
|
305
|
-
)
|
|
306
|
-
sys.exit(1)
|
|
369
|
+
data = json.loads(raw)
|
|
370
|
+
except json.JSONDecodeError as exc:
|
|
371
|
+
print(json.dumps(make_error(f"Invalid JSON: {exc.msg}"), indent=2))
|
|
372
|
+
return 1
|
|
307
373
|
|
|
308
374
|
result = compute_score(data)
|
|
309
375
|
print(json.dumps(result, indent=2))
|
|
376
|
+
return 0
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
if __name__ == "__main__":
|
|
380
|
+
raise SystemExit(main())
|
|
310
381
|
sys.exit(1 if "error" in result else 0)
|