bmad-module-skill-forge 4.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/agents.md +1 -1
- package/docs/architecture.md +5 -5
- package/docs/campaign.md +1 -1
- package/docs/concepts.md +4 -4
- package/docs/examples.md +32 -25
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/troubleshooting.md +9 -3
- package/docs/verifying-a-skill.md +42 -21
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +6 -6
- package/package.json +1 -1
- package/src/knowledge/tool-resolution.md +6 -6
- package/src/shared/references/headless-gate-convention.md +4 -1
- package/src/shared/references/output-contract-schema.md +3 -3
- package/src/shared/references/pipeline-contracts.md +2 -1
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +1 -1
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +11 -3
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-build-change-manifest.py +1180 -189
- package/src/shared/scripts/skf-classify-changed-files.py +15 -10
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +64 -20
- package/src/shared/scripts/skf-extract-public-api.py +472 -120
- package/src/shared/scripts/skf-extraction-inventory.py +10 -6
- package/src/shared/scripts/skf-extraction-snapshot.py +6 -6
- package/src/shared/scripts/skf-parse-gaps.py +136 -26
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +574 -45
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +3 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/references/generate-briefs.md +1 -1
- package/src/skf-analyze-source/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/SKILL.md +4 -3
- package/src/skf-audit-skill/references/re-index.md +2 -2
- package/src/skf-audit-skill/references/report.md +1 -1
- package/src/skf-brief-skill/SKILL.md +3 -2
- package/src/skf-brief-skill/assets/skill-brief-schema.md +2 -2
- package/src/skf-brief-skill/references/invocation-contract.md +2 -2
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-campaign/SKILL.md +3 -2
- package/src/skf-campaign/references/campaign-contracts.md +2 -0
- package/src/skf-campaign/references/step-05-skill-loop.md +8 -3
- package/src/skf-campaign/references/step-10-export.md +3 -2
- package/src/skf-create-skill/SKILL.md +4 -3
- package/src/skf-create-skill/assets/skill-sections.md +1 -1
- package/src/skf-create-skill/references/ccc-index-check.md +1 -1
- package/src/skf-create-skill/references/component-extraction.md +1 -1
- package/src/skf-create-skill/references/extract.md +3 -2
- package/src/skf-create-skill/references/extraction-patterns.md +3 -3
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +8 -3
- package/src/skf-create-skill/references/sub/ccc-discover.md +1 -1
- package/src/skf-create-skill/references/validate.md +1 -1
- package/src/skf-create-stack-skill/SKILL.md +3 -2
- package/src/skf-create-stack-skill/references/generate-output.md +1 -1
- package/src/skf-create-stack-skill/references/invocation-contract.md +1 -1
- package/src/skf-create-stack-skill/references/parallel-extract.md +2 -2
- package/src/skf-create-stack-skill/references/report.md +1 -1
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +3 -2
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +1 -1
- package/src/skf-export-skill/SKILL.md +3 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/invocation-contract.md +1 -1
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/summary.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +3 -2
- package/src/skf-forger/references/pipeline-mode.md +2 -2
- package/src/skf-quick-skill/SKILL.md +4 -3
- package/src/skf-quick-skill/references/finalize.md +1 -1
- package/src/skf-quick-skill/references/halt-contract.md +2 -2
- package/src/skf-quick-skill/references/write-and-validate.md +1 -1
- package/src/skf-refine-architecture/SKILL.md +8 -9
- package/src/skf-refine-architecture/customize.toml +1 -1
- package/src/skf-refine-architecture/references/report.md +1 -1
- package/src/skf-rename-skill/SKILL.md +3 -2
- package/src/skf-rename-skill/references/invocation-contract.md +2 -2
- package/src/skf-setup/SKILL.md +3 -2
- package/src/skf-setup/references/ccc-index.md +1 -1
- package/src/skf-setup/references/invocation-contract.md +1 -1
- package/src/skf-test-skill/SKILL.md +3 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +11 -8
- package/src/skf-test-skill/references/coverage-check-branches.md +81 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +3 -3
- package/src/skf-test-skill/references/coverage-check.md +60 -97
- package/src/skf-test-skill/references/init.md +2 -1
- package/src/skf-test-skill/references/inventory-contract.md +27 -0
- package/src/skf-test-skill/references/invocation-contract.md +3 -3
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +5 -5
- package/src/skf-test-skill/references/scoring-rules.md +8 -7
- package/src/skf-test-skill/references/signature-comparison-rules.md +72 -0
- package/src/skf-test-skill/references/source-access-protocol.md +14 -10
- package/src/skf-test-skill/references/step-hard-gate.md +1 -0
- package/src/skf-test-skill/scripts/build-result-context.py +20 -3
- package/src/skf-test-skill/scripts/gap-ledger.py +456 -176
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +516 -79
- package/src/skf-test-skill/scripts/locate-export-segments.py +685 -76
- package/src/skf-test-skill/scripts/reconcile-coverage.py +192 -39
- package/src/skf-test-skill/scripts/score-signatures.py +1205 -65
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +350 -35
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +3 -2
- package/src/skf-update-skill/references/detect-changes.md +1 -1
- package/src/skf-update-skill/references/gap-driven.md +54 -33
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +15 -10
- package/src/skf-update-skill/references/re-extract.md +3 -1
- package/src/skf-update-skill/references/report.md +6 -5
- package/src/skf-update-skill/references/write.md +42 -20
- package/src/skf-verify-stack/SKILL.md +4 -3
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -213,15 +213,15 @@ uv run {runLockHelper} acquire --lock "{forge_version}/.test-skill.lock" --owner
|
|
|
213
213
|
|
|
214
214
|
The run lock renewed here stays held until §7 releases it, so no other run publishes over this run's report or result files meanwhile.
|
|
215
215
|
|
|
216
|
-
**Build the result payload** before the report takes its public name. Every value in it comes from the run's own records: `{resultContextScript}` (it resolves relative to the skill root) reads the report frontmatter, the gap ledger and the scoring output, checks the verdict fields score.md §7 wrote (`testResult`, `score`, `threshold` and the fallback fields) against the scoring output, derives `verdict`, `exit_code` and `next_workflow` from `testResult`, and writes the shared emitter's payload, which names the report by its public name. Pass `--score` for a scored run (a run the hard gate blocked has no scoring output), `--no-health-check` when that flag was given, and `--warning` when SKILL.md On Activation step 3 kept a `{customization_resolver_unavailable}` reason: first write `customization_resolver_unavailable: {customization_resolver_unavailable}` to `{run_dir}/resolver-warning.txt` with a file write, never `echo` (the reason can hold quotes, backticks or `$( )`), and pass that file's text (the emitter adds the warnings a stage recorded in the run's sink):
|
|
216
|
+
**Build the result payload** before the report takes its public name. Every value in it comes from the run's own records: `{resultContextScript}` (it resolves relative to the skill root) reads the report frontmatter, the gap ledger and the scoring output, checks the verdict fields score.md §7 wrote (`testResult`, `score`, `threshold` and the fallback fields) against the scoring output, checks that `analysisConfidence` names a source access (`full`, `provenance-map`, `metadata-only`, `remote-only` or `docs-only`), a blocked run's included, derives `verdict`, `exit_code` and `next_workflow` from `testResult`, and writes the shared emitter's payload, which names the report by its public name. Pass `--score` for a scored run (a run the hard gate blocked has no scoring output), `--no-health-check` when that flag was given, and `--warning` when SKILL.md On Activation step 3 kept a `{customization_resolver_unavailable}` reason: first write `customization_resolver_unavailable: {customization_resolver_unavailable}` to `{run_dir}/resolver-warning.txt` with a file write, never `echo` (the reason can hold quotes, backticks or `$( )`), and pass that file's text (the emitter adds the warnings a stage recorded in the run's sink):
|
|
217
217
|
|
|
218
218
|
```bash
|
|
219
219
|
uv run {resultContextScript} --report "{report_file}" --report-path "{publishedReportFile}" --ledger "{ledgerFile}" [--score "{run_dir}/score.json"] [--no-health-check] [--warning "$(cat "{run_dir}/resolver-warning.txt")"] --output "{run_dir}/result-context.json"
|
|
220
220
|
```
|
|
221
221
|
|
|
222
|
-
Bind `{emit_target}` ← `target` from its JSON (`stderr` for a run the hard gate blocked, else `stdout`). Exit 2: correct the command and run it again. Exit 1: HALT with its `error` (`halt_reason: "helper-failed"`, phase `report:result-contract`); the report keeps its in-progress name, so export-skill never reads a verdict the scoring output did not settle.
|
|
222
|
+
Bind `{emit_target}` ← `target` from its JSON (`stderr` for a run the hard gate blocked, else `stdout`). Exit 2: correct the command and run it again. Exit 1: HALT with its `error` (`halt_reason: "helper-failed"`, phase `report:result-contract`), except when the `error` names `analysisConfidence`: then write `analysisConfidence: '{analysis_confidence}'`, the value source-access-protocol.md resolved (the Coverage Analysis section's **Source Access:** line records it), to the frontmatter and run it again, halting only when it exits 1 again; the report keeps its in-progress name, so export-skill never reads a verdict the scoring output did not settle.
|
|
223
223
|
|
|
224
|
-
**Publish the report.** Set `health_check_dispatched
|
|
224
|
+
**Publish the report.** Set `health_check_dispatched`, already in the `{outputFile}` frontmatter: `false` when `no_health_check` is true (the `--no-health-check` flag), else `true`, since the health check now runs with no menu before it. The checks passed, so the report takes its public name, the one export-skill and update-skill read, a blocked run's included. From `{project-root}`, run:
|
|
225
225
|
|
|
226
226
|
```bash
|
|
227
227
|
mv "{report_file}" "{publishedReportFile}"
|
|
@@ -235,7 +235,7 @@ and bind `{report_file}` ← `{publishedReportFile}`. If the move fails, HALT (`
|
|
|
235
235
|
uv run {emitEnvelopeHelper} emit --workflow skf-test-skill --run-dir "{run_dir}" --result-dir "{forge_version}" --target {emit_target} < "{run_dir}/result-context.json"
|
|
236
236
|
```
|
|
237
237
|
|
|
238
|
-
It writes `{forge_version}/skf-test-skill-result-{YYYYMMDD-HHmmss}.json` (UTC; it picks the name) and then its `skf-test-skill-result-latest.json` copy (
|
|
238
|
+
It writes `{forge_version}/skf-test-skill-result-{YYYYMMDD-HHmmss}.json` (UTC; it picks the name) and then its `skf-test-skill-result-latest.json` copy (the record of the last run whose emitter wrote this copy, at a stable path: a copy, not a symlink; a caller reads its own run's `SKF_TEST_RESULT_JSON` line), each atomically, so a partial write is never observable; stamps the timestamp, `run_id`, `headless_decisions` and `warnings` into both; and prints the `SKF_TEST_RESULT_JSON:` line. Bind `{result_line}` ← that line and `{result_path}` ← its `result_path`. A run the hard gate blocked writes the record too, as a FAIL (`status: "error"`, `verdict: "FAIL"`, `halt_reason: "hard-gate-blocked"`, `exit_code` 2), so after a blocked run `skf-test-skill-result-latest.json` holds its FAIL, never an earlier run's verdict. A result file that could not be written leaves `result_path` null and a `result_file_write_failed` warning in the line, and the run still finishes. If the emitter exits non-zero, run it once more; if it fails again, HALT (`halt_reason: "write-failed"`, phase `report:result-contract`) with its stderr: the run's result contract could not be written. `references/invocation-contract.md` lists the record's fields.
|
|
239
239
|
|
|
240
240
|
Once the result files are written, append `'report'` to `stepsCompleted` in the `{outputFile}` frontmatter (it stays out of the expected set above).
|
|
241
241
|
|
|
@@ -289,7 +289,7 @@ The report is final once §4c published it and wrote the result files: no later
|
|
|
289
289
|
**update-skill** — The skill scored above threshold, but `--allow-workspace-drift` was in effect: the test ran against workspace HEAD, not `metadata.source_commit`. A conditional PASS is not trustworthy enough to export. Re-sync the source tree to the pinned commit (or re-extract against current HEAD) and re-run test-skill without the drift override before exporting.
|
|
290
290
|
|
|
291
291
|
{IF FAIL after scoring:}
|
|
292
|
-
**update-skill `--from-test-report`**: This skill needs remediation. Review the Gap Report above, then run `@Ferris US {skill_name} --from-test-report`: Update Skill
|
|
292
|
+
**update-skill `--from-test-report`**: This skill needs remediation. Review the Gap Report above, then run `@Ferris US {skill_name} --from-test-report`: Update Skill repairs the gaps it routes, at the skill's pinned commit, and lists each other gap as not repaired, with the reason. A `stale-documentation` or `metadata-drift` gap needs what its remediation names: a person, Create Skill or the skill brief. Test it again afterwards.
|
|
293
293
|
|
|
294
294
|
{IF the hard gate blocked the run:}
|
|
295
295
|
**update-skill `--from-test-report`**: The hard gate stopped this run before scoring on {blocking_gaps} Critical or High gap(s), listed first in the Gap Report. Run `@Ferris US {skill_name} --from-test-report` to repair them, then test again.
|
|
@@ -18,12 +18,12 @@ This table is the single source of truth for classifying a gap. Every stage reco
|
|
|
18
18
|
|
|
19
19
|
| Severity | Category | Criteria |
|
|
20
20
|
|----------|-----------------------------|----------|
|
|
21
|
-
| Critical | `signature-mismatch` | Wrong signature: a documented signature that differs from the source in its parameters (name, type, order, optionality) or its return type (coverage-check §2) |
|
|
21
|
+
| Critical | `signature-mismatch` | Wrong signature: a documented signature that differs from the source in its parameters (name, type, order, optionality) or its return type, as signature-comparison-rules.md reads each form (a component by its props; an annotation either side leaves out is not compared), in any way but those the Low `signature-mismatch` row and the Info `observation` row grade, such as a renamed Python parameter (the script cannot see which parameters a caller must pass by position), an optional parameter left out of a form the skill does not label partial, or a mismatch with no `difference` (coverage-check §2 and §2b) |
|
|
22
22
|
| Critical | `fabricated-signature` | Fabricated signature: a documented export absent at the source line the skill cites for it. Decided only where ast-grep enumerated the exports at the pinned commit: the name is outside that surface, and the cited file is missing or defines no such name (coverage-check §2c) |
|
|
23
23
|
| Critical | `broken-reference` | Broken reference (contextual mode): a reference whose target does not exist, such as a file not at the stated path, a skill not in the skills output folder, a type no module declares, or a `scripts/` or `assets/` file the skill names (coherence-check §4) |
|
|
24
24
|
| High | `inaccurate-reference` | Inaccurate reference (contextual mode): the target exists but does not match, such as a type the cited module does not export, a signature that differs, or an integration pattern that does not match the implementation (coherence-check §4) |
|
|
25
25
|
| High | `reference-escape` | A reference whose real path leaves the skill, its source and, for a stack, the skills output folder (coherence-check §4) |
|
|
26
|
-
| High | `split-body-mismatch` |
|
|
26
|
+
| High | `split-body-mismatch` | Two files of the skill (the SKILL.md body or a `references/` file) document one export differently (coverage-check §1b) |
|
|
27
27
|
| High | `numerator-inflation` | A declared count (`stats.exports_documented`, or the provenance map's distinct entry names) equals the denominator while declared exports are absent from SKILL.md and `references/` (coverage-check §4b) |
|
|
28
28
|
| High | `structural` | Naive mode: a missing required section, an unbalanced code fence, or a documented async or sync behaviour its example contradicts (coherence-check §2.1, §2.2, §2.5) |
|
|
29
29
|
| High | `discovery` | Discovery test: at most 1 of 3 realistic prompts routed to the skill (report §4b.3) |
|
|
@@ -31,21 +31,22 @@ This table is the single source of truth for classifying a gap. Every stage reco
|
|
|
31
31
|
| Medium | `missing-type` | Missing type or interface documentation |
|
|
32
32
|
| Medium | `stale-documentation` | A documented export the enumerated source surface lacks that is not a fabricated signature and not a documented extra: classify-stale found no single file that declares it (an import, a re-export or an indented local does not count, nor does a module so named when the inventory gives the name a `function`, `hook`, `class`, `interface`, `enum` or `type` kind, as the Info `observation` row says) and the skill's citations on the lines that name it cite no single one of the files that do, it is a dotted name, the brief scoped its file out (`excluded.outsideScope`), or the run could not look: the source is not Python or TS/JS, classify-stale did not run (it needs the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map), or `surface.json` has no complete `extraction` (coverage-check §2c) |
|
|
33
33
|
| Medium | `migration-section` | Migration section present/absent mismatch with T2-future annotation data (Deep tier; Case 1 and Case 3 of `migration-section-rules.md`) |
|
|
34
|
-
| Medium | `metadata-drift` | Metadata drift: intra-cluster export counts diverge by more than 10% (barrel: `stats.exports_public_api` vs `exports[].length`; documented surface: `stats.exports_documented` vs the provenance-map named-export count), or denominator deflation: the re-derived source barrel exceeds `effective_denominator` by more than 25% and the brief has no `scope.tier_a_include` glob that counts a public name (coverage-check §4 and §4b) |
|
|
35
|
-
| Medium | `denominator-inflation` | Denominator inflation: the stratified-scope `scope.include` union exceeds the provenance-map entry count by more than 25% (the brief has no `scope.tier_a_include` glob that counts a public name) |
|
|
34
|
+
| Medium | `metadata-drift` | Metadata drift: intra-cluster export counts diverge by more than 10% (barrel: `stats.exports_public_api` vs `exports[].length`; documented surface: `stats.exports_documented` vs the provenance-map named-export count), or denominator deflation: the re-derived source barrel exceeds `effective_denominator` by more than 25% and the brief has no `scope.tier_a_include` glob that counts a public name, or denominator drift: on a stratified-scope, specific-modules or multi-entry skill, the brief's `scope.tier_a_include` set differs from `effective_denominator` by more than 25% either way, or counts a name against an `effective_denominator` of 0, a stale count to recount (coverage-check §4 and §4b) |
|
|
35
|
+
| Medium | `denominator-inflation` | Denominator inflation: the stratified-scope `scope.include` union exceeds the provenance-map entry count by more than 25% (the brief has no `scope.tier_a_include` glob that counts a public name). On a stratified-scope or multi-entry barrel over that union, the names it counts and the skill does not document are one more such gap beside it at the brief, listing at most 50 and deferring to its fix, instead of a `missing-export` gap each; Export Coverage still counts them |
|
|
36
36
|
| Medium | `brief-scope-stale` | Stale brief scope: a `scope.include` glob matches no file in the source tested, so the surface restores the root exports defined in a file no glob covers that the brief does not exclude, or a `scope.tier_a_include` glob matches no file in the source tested, so it counts no name (coverage-check §4) |
|
|
37
37
|
| Medium | `scripts-assets` | A `scripts/` or `assets/` directory exists but SKILL.md has no Scripts & Assets section (coherence-check §2.7 and §4) |
|
|
38
38
|
| Medium | `integration-pattern` | Incomplete integration pattern (contextual mode): one entry of coherence-check §5's `incomplete_patterns` |
|
|
39
39
|
| Medium | `structural` | Naive mode: an opening code fence without a language tag, an exported function or method no usage-family section or reference file names, or a table row whose column count differs from its header (coherence-check §2.3, §2.4, §2.6) |
|
|
40
40
|
| Medium | `discovery` | Discovery test: 2 of 3 realistic prompts routed to the skill (report §4b.3) |
|
|
41
|
+
| Low | `signature-mismatch` | A renamed parameter callers cannot see: a mismatch the signature judge tags `parameter-name` (defined in signature-comparison-rules.md, Difference tags) at the file the surface records for the export, the one the plan sent the check to or a candidate the judge compared with, when it is a `.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`, `.go`, `.rs`, `.java`, `.c`, `.h`, `.cc`, `.cpp`, `.cxx` or `.hpp` file, a language with no keyword arguments (not a Vue component, whose props callers set by name), as score-signatures.py checks. It counts as matching in Signature Accuracy (coverage-check §2 and §2b) |
|
|
41
42
|
| Low | `scripts-assets-provenance` | Script/asset file present without provenance entry in provenance-map.json file_entries |
|
|
42
|
-
| Low | `provenance-line` | Provenance line is not the definition of an export (`line-not-definition
|
|
43
|
+
| Low | `provenance-line` | Provenance line is not the definition of an export (`line-not-definition`), or its citation does not resolve: the pinned source lacks its file (`file-missing`) or the line is past the file's end, or below 1 (`line-out-of-bounds`) (coverage-check §4c) |
|
|
43
44
|
| Low | `migration-section` | Case 3 of `migration-section-rules.md` that the reviewer downgraded with an inline justification |
|
|
44
45
|
| Low | `metadata` | Missing optional metadata or examples |
|
|
45
46
|
| Low | `description` | Description trigger optimization recommended (third-person voice, negative triggers, or keyword coverage gaps), a Tessl Review description suggestion included |
|
|
46
47
|
| Low | `external-validator` | A skill-check diagnostic or a Tessl Review validation finding the run left unresolved (external-validators §5b) |
|
|
47
|
-
| Info | `observation` | Style suggestions, non-blocking observations, and a documented extra: a documented name outside the enumerated surface that is not a fabricated signature and that the source still declares at column 0 (a module so named counts, unless the inventory gives the name a `function`, `hook`, `class`, `interface`, `enum` or `type` kind, no provenance map entry of the name types it a `module` or `package` and the module is not a TS/JS file that is itself the export, with an `export default` or a `module.exports =`; among several declaring files, the one the skill cites on a line naming it), found only in a Python or TS/JS source where classify-stale ran (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and `surface.json` has a complete `extraction`: `Documented extra: {name}`, at that line, which update-skill never routes (coverage-check §2c) |
|
|
48
|
-
| Info | `provenance-unverified` | Provenance line not verified: the line-check rules found no definition line (coverage-check §4c) |
|
|
48
|
+
| Info | `observation` | Style suggestions, non-blocking observations, and a documented extra: a documented name outside the enumerated surface that is not a fabricated signature and that the source still declares at column 0 (a module so named counts, unless the inventory gives the name a `function`, `hook`, `class`, `interface`, `enum` or `type` kind, no provenance map entry of the name types it a `module` or `package` and the module is not a TS/JS file that is itself the export, with an `export default` or a `module.exports =`; among several declaring files, the one the skill cites on a line naming it), found only in a Python or TS/JS source where classify-stale ran (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and `surface.json` has a complete `extraction`: `Documented extra: {name}`, at that line, which update-skill never routes (coverage-check §2c). Also a partial signature: a mismatch the signature judge tags `omitted-optional` (defined in signature-comparison-rules.md, Difference tags) on a form the skill labels partial (simplified, key parameters, a `...` that stands in for omitted members, not rest or variadic syntax), which the inventory marks `signature_form: "partial"`, as score-signatures.py checks: `Partial signature: {name}`, at the declaration, counted as matching in Signature Accuracy (coverage-check §2 and §2b). Also a return annotation the body contradicts: a mismatch the signature judge tags `annotation-contradicted` (defined in signature-comparison-rules.md, Difference tags) whose documented signature states a return type and writes the source's parameters, and whose `evidence_line`, the body line that returns the documented type, comes after the declaration and before the next one and, with the local source, starts with `return` (or is a one-line declaration's own expression body), as score-signatures.py checks: `Annotation contradicted: {name}`, at the declaration, counted as matching in Signature Accuracy, with no change to the skill (the annotation is reported upstream) (coverage-check §2 and §2b) Also a documentation error a Deep-tier run sees that no script checks (an import path whose module does not export the name, a type member that differs from its declaration): `Not checked by script: {name}`, at the documenting line, written by hand (coverage-check §2 and §5b) |
|
|
49
|
+
| Info | `provenance-unverified` | Provenance line not verified: the line-check rules found no definition line, or the recorded line is not a line number (`line-invalid`) (coverage-check §4c) |
|
|
49
50
|
| Info | `migration-section` | Historical migration content in Section 4b with no T2-future annotation (Case 2 of `migration-section-rules.md`) |
|
|
50
51
|
| Info | `multi-denominator` | Multi-denominator reporting: barrel and documented-surface clusters diverge by design (>10% cross-cluster) |
|
|
51
52
|
| Info | `discovery` | Discovery testing not performed (`--no-discovery`, a catalog of fewer than 2 skills, or no subagents to route the prompts): realistic prompt testing recommended before export (report §4b) |
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# Signature Comparison Rules
|
|
2
|
+
|
|
3
|
+
Every signature judge compares by these rules: coverage-check.md §1's split-body cross-check, its §2 judge (or the main thread in its place), its §5b split-body Source (the Split-Body Sources section of coverage-check-branches.md), coverage-check-tiers.md's fallback per-file scan and coherence-check.md §4's reference judge (its `signature_match`). They decide what is a mismatch and how to tag one; each judge returns the JSON its step gives.
|
|
4
|
+
|
|
5
|
+
## Reading a form
|
|
6
|
+
|
|
7
|
+
A form claims only what it writes, and only what it claims is compared. Its parameter list claims each parameter it lists, in that order, and no other: a parameter it leaves out is one it says the export does not take (Difference tags say when that is `omitted-optional`). A type or a return type it does not write claims nothing (Annotations).
|
|
8
|
+
|
|
9
|
+
Three kinds of form do not claim a complete parameter list:
|
|
10
|
+
|
|
11
|
+
- A call or usage example passes values (literals, variables, `name=value`): `RecursiveSplitter()`, `embed(text, model=m)`, `<Canvas of={X} />`. It is compared as a call: each argument it passes must exist, by position or by name, and every required parameter must be passed; an optional parameter it leaves out is no difference. An empty call `X()` is one too, so it matches a source whose parameters are all optional. A component usage `<X a={x} b="y" />` reads as a list of bare names of the props it sets, and a call `f(a, b)` as its arguments in order, each the parameter at its position (or, written `name=value`, the one it names). A JSX spread (`{...props}`) or children can pass a required prop, and a call's `...args` a required parameter, so neither leaves one out. An entry's `usage_signature` written as a usage is read this way, and each usage in a list of usages is compared as a form of its own, under its entry's `form` index.
|
|
12
|
+
- An ellipsis (`...` or `…`) in a list stands for members it does not show and claims nothing about them: `(...)` and `(tasks, data, dataset, ...)` leave out no parameter.
|
|
13
|
+
- A list of bare names, with no type, default or `?` (`connect(host, port)`, `({ themes, defaultTheme })`), names parameters without claiming the list is complete: an optional member it leaves out is no difference, a required one is a mismatch.
|
|
14
|
+
|
|
15
|
+
A form that writes a type or a default for a parameter (`(date: Date) => string`) claims its list is complete: an optional parameter it leaves out is an `omitted-optional` difference (Difference tags).
|
|
16
|
+
|
|
17
|
+
Optionality is stated only by a default, a `?` (on the name, or as `?:`) or an optional marker: `optional` in a docstring-style list, a Required column, a Default cell that gives a value. A type never states it, and an empty Default cell states nothing. A list of parameters or props, or a table, that states optionality for none of its members states none, and optionality is not compared: `connect(host: str, port: int)` matches `def connect(host: str, port: int = 5432)`. Once a list states optionality for any member, each member it writes with no mark, unless it is rest or variadic, claims to be required; a Default cell and a destructuring default mark only their own member (Components). A default's value is not compared: it only marks its parameter optional.
|
|
18
|
+
|
|
19
|
+
A form that opens with text the skill states once for several names, then `; ` and the name's own members (inventory-contract.md), writes both: the members the shared text states are members that form writes, read before its own.
|
|
20
|
+
|
|
21
|
+
In a language without keyword arguments (TS/JS, Go, Rust, Java, C, C++), leaving out an optional parameter that comes before one the form writes shifts the caller's positions: that is a mismatch, not `omitted-optional` (`slice(str, end?)` against `(str, start?, end?)`).
|
|
22
|
+
|
|
23
|
+
The names of Python's `*args` and `**kwargs` are never compared (`**kwargs` against `**options` is no difference), and the parameters after `*` or `*args` are compared by name, not order. A positional-only `/` or keyword-only `*` marker the form leaves out is not compared, and one it adds where the source has none is no difference: calls that follow the form are valid.
|
|
24
|
+
|
|
25
|
+
The source's declaration decides: the documented form is compared with the parameters, the optionality and the annotations the signature declares, with one exception, a return annotation the body contradicts (Annotations).
|
|
26
|
+
|
|
27
|
+
In the split-body cross-check both forms are documented: each is read by these rules, a type, return type or type arguments one of them leaves out are not compared, and a parameter's optionality is compared only when both forms state it. An ellipsis claims nothing about the members it stands for, required ones included. A call, a list of bare names or a form its own file labels partial (each side's label is read from that side's file, not from the export's one `signature_form`) may leave out a parameter the other form does not state as required, unless leaving it out shifts a caller's positions (a language without keyword arguments, as above); any other form claims its list is complete, so leaving out a parameter the other writes is a mismatch (inventory-contract.md's examples). Each export's form is read alone: what the rows of other exports write, in the same table or file, states nothing about it, so `(action, sink, tracking_record, child_invalidation)` matches `child_invalidation: X | None = None` even when every other row of its table writes its defaults.
|
|
28
|
+
|
|
29
|
+
A name the skill documents in several forms (a brief usage in SKILL.md, a full props list in a reference file) comes with `alternatives`: coverage-check.md §2's check lists them after its `documented` form, and the fallback per-file scan's `documentedAlternatives` holds them by name. Compare the declaration with each form the check offers, each by its own reading rules, and report a mismatch for each form that does not match, with its index as `form` (0 is `documented`, 1 the first alternative), its `documented_sig` written as that form and its `difference` tagged against it. With `candidates`, a form matches when it matches any candidate, and its mismatch is reported at the closest candidate.
|
|
30
|
+
|
|
31
|
+
## Components
|
|
32
|
+
|
|
33
|
+
A component is an export the source declares as one: a function that takes one props object (plus a `ref`, for `forwardRef`) and returns JSX, an element or `null`, or a value the source declares as a component type, a wrapper's result, a class component, a `defineComponent` result or a Svelte component, in React, Preact, Solid, Vue or Svelte. A function with several positional parameters is a function, whatever it returns (`flexRender(Comp, props)`, `createPortal(children, container)`). A component's signature is its props `P`: a function component's props type, a class component's props type argument, a Vue component's `props` option or `defineProps` type or runtime object, or a Svelte component's props (`$props()`, or its `export let` declarations).
|
|
34
|
+
|
|
35
|
+
`FC<P>`, `React.FC<P>`, `FunctionComponent<P>`, `ComponentType<P>`, a `memo`, `forwardRef` or other higher-order component result over `P`, and `(props: P) => JSX.Element` (or `ReactElement`, `ReactNode` or `null`) are one contract, the props `P`: a component's return type and its wrapper are never compared. Callers set props by name, so their order is never compared.
|
|
36
|
+
|
|
37
|
+
A component's own props are the members written in its package: its interface or object type, object literals intersected into it, and members computed from its own code (`VariantProps<typeof buttonVariants>`). Members of a type from another package (`React.ComponentProps<"button">`, `HTMLAttributes<...>`, `ComponentProps<typeof Link>`) are inherited, as are the `ref` a `forwardRef` result takes and the optional `children` of `PropsWithChildren<P>`. A prop the form writes must exist on `P`, own or inherited. A component whose props type lives in a package outside the source tree (`const Tabs = TabsPrimitive.Root`) has its props left uncompared: only that it is a component is checked.
|
|
38
|
+
|
|
39
|
+
A prop the form writes that `P` lacks, or a type or an optionality it writes that the source contradicts, is a mismatch. So is a required own prop it leaves out, and a required inherited one (a `Link`'s `href`) left out of a full form. An optional own prop a typed props list or a table leaves out is an `omitted-optional` difference; one a list of bare names or a usage leaves out, and an optional inherited prop, is no difference. `| null` and `| undefined` on an optional prop are no difference (`VariantProps` adds both). A props type name the form writes that the source does not declare (`Button(props: ButtonProps)` against a source with no `ButtonProps`) is not compared by name, only the props the form lists. Values in a usage example (`variant="subtle"`) are not compared.
|
|
40
|
+
|
|
41
|
+
A props list states optionality as Reading a form says: `({ of, meta, story })`, or a list of names, states none, and in an object type literal (`{ of?: X; meta: M }`) that states it for any member, each member written with no mark claims to be required. A table states optionality by a Required column, a `?` on a prop's name or a Default cell that gives a value; its Type column states types, never optionality. A Default cell with a value marks only its own row optional and an empty one states nothing, so Create Skill's `| Prop | Type | Default | Description |` table never reads a prop with an empty Default cell as required. A destructuring default (`({ of, meta = {} })`) likewise marks only its own member, and the others state no optionality.
|
|
42
|
+
|
|
43
|
+
## Annotations
|
|
44
|
+
|
|
45
|
+
- A parameter written without a type is not compared on type: its name and order are, and its optionality as Reading a form says. Bare `*args` matches `*args: P.args`, and `**kwargs` matches `**kwargs: P.kwargs`.
|
|
46
|
+
- A return type the form does not state is not compared. A form states one when its entry has a non-blank `return_type` or the form writes one: after `->`, `=>`, `→` or `):`, or in its language's own position (Go's result list after the parameters, the type before the name in C, C++ and Java). A blank `return_type` (`""`) states none, so an entry with `params: ""` and no return type matches `def f() -> None`.
|
|
47
|
+
- A type the source leaves unannotated, on a parameter or a return, is not compared, except that a TS/JS parameter with a literal default has that literal's type: `pageSize = 20` is a `number`, so `pageSize?: string` is a mismatch. An unannotated return stays uncompared.
|
|
48
|
+
- A documented parameter type that names some of the members of the source's union is no mismatch, since a caller passing it makes a valid call (`schema: VectorSchemaProvider` against `VectorSchemaProvider | MultiVectorSchemaProvider`); a documented return type must cover every member the source can return.
|
|
49
|
+
- Type arguments the form leaves out are not compared: `dict` matches `dict[str, Any]`, and `list[dict]` matches `list[dict[str, Any]]`. Type arguments it writes are: `dict[str, int]` against `dict[str, str]` is a mismatch.
|
|
50
|
+
- A documented return type that matches what the body demonstrably returns, where the declared annotation says otherwise, is a mismatch tagged `annotation-contradicted` with `evidence_line`, the body line that returns that value: the skill documents what callers get, and the contradiction is the source's, which `score` reports without blocking. Against a declaration annotated `-> Callable[[], Callable[[], None]]` whose line 19 returns the function, `-> Callable[[], None]` is tagged with `evidence_line` 19. Tag it only when you can name that line: a body line that starts with `return`, or the declaration line itself when its own expression body follows its parameters there (`=> expr`, a Python `def ...: return x`). A Rust tail expression, a Kotlin or Scala expression body and a Ruby implicit return have no such line and cannot be tagged: there, as when you cannot name the line, the declaration decides. A judge whose result has no `difference` field (coherence-check.md §4's `signature_match`, coverage-check.md §5b's split-body Source) reads such a documented return type, at a line it can name, as agreeing with the source: a match for `signature_match`, and for §5b a form that agrees with the source. It holds for a return type alone, never a parameter's annotation.
|
|
51
|
+
|
|
52
|
+
## Equivalent spellings
|
|
53
|
+
|
|
54
|
+
Two spellings of one type are no mismatch:
|
|
55
|
+
|
|
56
|
+
- `Optional[X]`, `Union[X, None]` and `X | None`; `Union[A, B]` and `A | B`; a union's members in any order (`None | X`, `B | A`).
|
|
57
|
+
- `typing`'s `List`, `Dict`, `Tuple`, `Set`, `FrozenSet` and `Type` and the builtins `list`, `dict`, `tuple`, `set`, `frozenset` and `type`.
|
|
58
|
+
- A name with or without its `typing.`, `collections.abc.` or `React.` prefix: `React.ReactNode` and `ReactNode`.
|
|
59
|
+
- A quoted forward reference and the name it quotes: `"Node"` and `Node`.
|
|
60
|
+
- `Array<T>` and `T[]`; `ReadonlyArray<T>` and `readonly T[]`.
|
|
61
|
+
- A type alias the source declares and the type it names.
|
|
62
|
+
- Type variables renamed consistently: `(a: T) -> T` against `(a: U) -> U`, but not `(a: T, b: U)` against `(a: U, b: U)`.
|
|
63
|
+
- Type arguments the source leaves out are not compared either: the form's `dict[str, Any]` matches the source's bare `dict`.
|
|
64
|
+
- A union of callables that take the same parameters and one callable returning the union of their returns: `Callable[[A], X] | Callable[[A], Y]` and `(a: A) -> X | Y`.
|
|
65
|
+
|
|
66
|
+
## Overload sets
|
|
67
|
+
|
|
68
|
+
A declaration opening an overload set (Python `@overload` stubs, TS bodiless signatures) is read through its implementation: a documented signature matching the implementation or any overload is no mismatch, and so is one whose parameter types are, position by position, the unions of the overloads' types.
|
|
69
|
+
|
|
70
|
+
## Difference tags
|
|
71
|
+
|
|
72
|
+
A fact these rules leave uncompared is no difference. A mismatch carries `difference` when the two differ only in one of these ways, else no key: `parameter-name` (only parameter names differ: types, order, optionality and return type equal; a destructured property is part of a type), `omitted-optional` (the documented form leaves out only members the source marks optional) or `annotation-contradicted` (only the return type differs: `documented_sig` and `source_sig` write the same parameters, and the body returns the documented type at the mismatch's `evidence_line`, as Annotations says). A parameter or field with a default or a `?`, a TS/JS rest parameter, or a Python `*args` or `**kwargs`, is optional; a Rust `Option<T>` field a struct literal still requires is not. A renamed component prop is never `parameter-name`: callers set props by name, so it is a mismatch with no tag. Tag it in any language and form: `score` (coverage-check.md §2b) checks the facts. The example in coverage-check.md §2, tagged `omitted-optional`, stays Critical against a full form.
|
|
@@ -11,36 +11,40 @@
|
|
|
11
11
|
- **Go:** exported identifiers (capitalized) from the package's public-facing files
|
|
12
12
|
- **Rust:** items in `pub use` from `lib.rs` or `mod.rs`
|
|
13
13
|
|
|
14
|
-
The clauses below decide which part of that surface is the coverage denominator. `load-coverage-inputs.py surface` picks the clause from the brief (`{forge_data_folder}/{skill_name}/skill-brief.yaml`), `{source_is_monorepo}` (coverage-check.md §0b, passed as `--monorepo`), the extraction and `metadata.json`, and writes it to `surface.json` `denominator` with the branch, the name set, the rung it takes (coverage-check.md §2b) and the entry points' state it read (`barrel`), beside the counts each clause names (`sets`, `candidates` and `guards`). Three of its inputs are yours to pass as flags, the last two only where no file gives them:
|
|
14
|
+
The clauses below decide which part of that surface is the coverage denominator. `load-coverage-inputs.py surface` picks the clause from the brief (`{forge_data_folder}/{skill_name}/skill-brief.yaml`), `{source_is_monorepo}` (coverage-check.md §0b, passed as `--monorepo`), the extraction and `metadata.json`, and writes it to `surface.json` `denominator` with the branch, the name set, the rung it takes (coverage-check.md §2b) and the entry points' state it read (`barrel`), beside the counts each clause names (`sets`, `candidates` and `guards`). A workspace member counts toward `{source_is_monorepo}` only through the files the brief keeps: excluding its source files drops it, excluding only its manifest does not, and a workspace with no member left is no workspace. Three of its inputs are yours to pass as flags, the last two only where no file gives them:
|
|
15
15
|
|
|
16
16
|
- **Pattern-reference intent** (`--pattern-reference-intent`): `scope.notes` says the skill demonstrates an integration pattern rather than a library API ("Reference app, not a library", "pattern-reference", "embedded-pattern skill", "skill value is the … pattern"; the author's words are authoritative).
|
|
17
17
|
- **A curated monorepo subset** (`--curated-subset`), where the extraction has no `counts.curated_subset` (Quick tier, the fallback per-file scan, States 2 to 4): `{source_is_monorepo}` is true and the brief's `scope.include` lists a curated subset rather than the full workspace.
|
|
18
18
|
- **The entry points' state** (`--barrel`), where no complete extraction or quick output gives it (the fallback per-file scan when the runner listed entry points, Quick tier read by eye, State 4): `none` when no barrel file is at any recognized entry-point path (`__init__.py`, `index.ts`/`index.js`, `lib.rs`, `mod.rs`), `empty` when the primary entry point exports nothing (`export {};`, an empty `__init__.py`, a `lib.rs` with no `pub use`), else `barrel`.
|
|
19
19
|
|
|
20
|
-
The recipe runner already re-derives each barrel (multi-line re-export blocks, star re-exports and a Rust `pub use` chain included) and counts each name once,
|
|
20
|
+
The recipe runner already re-derives each barrel (multi-line re-export blocks, star re-exports and a Rust `pub use` chain included) and counts each name once, however many in-scope files define it, so a barrel that only re-exports adds nothing of its own. Each of those files gives the name a surface record of its own (the runner's `entry_point_diff.other_definitions` beside its `public` row), so a set holds the name when any of its files matches, and the signature plan can compare the declaration the skill cites. Without the runner's names (Quick tier, the fallback per-file scan, States 2 to 4) the script builds the `scope.include` and `tier_a_include` sets from the files the names carry and the brief's globs (the runner's JSON the fallback scan passes adds only its scope and stale globs), and there is no `subpaths` or `root` set (coverage-check.md §2b). Never re-count a barrel by hand. A `scope.tier_a_include` glob counts a name only in a file a `scope.include` glob also matches, and one that matches no file (`guards.staleScope.unmatchedTierAInclude`, a brief older than the source) counts none: when the tier A globs count no name, `surface.json` has no `tier_a_include` set. An annotation below takes each `{candidates.<field>}` count from `surface.json`, and a null one reads `absent`; a `*Files` count is the files that define the set's names (at Quick tier, word it as the files its names sit at). Each clause, by its `denominator.clause`, with what stays yours:
|
|
21
21
|
|
|
22
|
-
- **Empty-barrel packages (`empty-barrel`; copy-paste or subpath-only distribution):** the primary entry point is empty, so the denominator is the `scope.include` set. The brief's `scope.notes` should say the distribution model is by design; when it does, treat the empty barrel as intended rather than a bug. With no brief and an empty barrel, set `analysis_confidence: docs-only` and report that the source API surface could not be determined. Annotate: `Empty barrel, denominator: scope.include union ({
|
|
22
|
+
- **Empty-barrel packages (`empty-barrel`; copy-paste or subpath-only distribution):** the primary entry point is empty, so the denominator is the `scope.include` set. The brief's `scope.notes` should say the distribution model is by design; when it does, treat the empty barrel as intended rather than a bug. With no brief and an empty barrel, set `analysis_confidence: docs-only` and report that the source API surface could not be determined. Annotate: `Empty barrel, denominator: scope.include union ({candidates.scopeIncludeUnion} exports)`.
|
|
23
23
|
|
|
24
|
-
- **Stratified-scope monorepo packages (`stratified-scope`; curated subsets of multi-package repos):** the denominator reflects only the authored surface, not the monorepo's global export count: the `stats.effective_denominator` Create Skill stored, unless `guards.deflation.fires` (a Medium `metadata-drift` gap, coverage-check.md §4: a legitimate scope reduction goes in the brief, through `scope.tier_a_include` or `scope.exclude`, never into `metadata.stats`), then the `tier_a_include` and `scope.include` sets. With no surface read from the source (`guards.deflation.applicable` false), annotate the report `effective_denominator used unverified: no source access to re-derive the barrel`. When `guards.inflation.fires` (the coarse globs sweep in internal infrastructure, a Medium `denominator-inflation` gap) and `guards.umbrella.umbrella` is true
|
|
24
|
+
- **Stratified-scope monorepo packages (`stratified-scope`; curated subsets of multi-package repos):** the denominator reflects only the authored surface, not the monorepo's global export count: the `stats.effective_denominator` Create Skill stored, unless `guards.deflation.fires` (a Medium `metadata-drift` gap, coverage-check.md §4: a legitimate scope reduction goes in the brief, through `scope.tier_a_include` or `scope.exclude`, never into `metadata.stats`), then the `tier_a_include` set when the guard measured the stored count against it (`guards.deflation.basis` `tier_a_include`, in either direction: a stale count, which re-running Create Skill recounts, storing the runner's tier A count when `effective_denominator_decision.store` holds and `scope.notes` documents the stratification and otherwise dropping the stored count; on an incomplete extraction a set below the stored count leaves the stored count), else the `scope.include` set; with no stored count, the `tier_a_include` and `scope.include` sets. Never set `stats.effective_denominator` by hand: Update Skill's recount drops it. With no surface read from the source (`guards.deflation.applicable` false), annotate the report `effective_denominator used unverified: no source access to re-derive the barrel`. When `guards.inflation.fires` (the coarse globs sweep in internal infrastructure, a Medium `denominator-inflation` gap), the `scope.include` union is itself inflated: when the deflation guard fires too, that union stays the denominator and both gaps ask for a `scope.tier_a_include` list of the files that define the authored names (when `guards.umbrella.umbrella` is true, those files rather than the barrel). When the barrel branch counts that union, here or on the multi-entry ladder, the names it counts and the skill does not document are one more `denominator-inflation` gap beside the guard's at the brief, which defers to the guard's fix, not a `missing-export` gap each. Leave `analysis_confidence` unchanged. Annotate the report: `Stratified scope, denominator: {effective_denominator | tier_a_include union | scope.include union} ({candidates.tierAIncludeFiles | candidates.scopeIncludeFiles, by denominator.surfaceSet} files define its names, {candidates.tierAIncludeUnion | candidates.scopeIncludeUnion, by the same set} exports union)`; when `denominator.surfaceSet` is `all` no scope set gives a count, so both read `absent`. `scope.notes` should document the stratification (for example "Tier A: fully documented; Tier B: deferred to references; Tier C: excluded"); when it does, treat the curated subset as intended, not a scope gap.
|
|
25
25
|
|
|
26
26
|
- **Standard barrel (`standard-barrel`, the `all` set):** `full-library` skills, single-package repositories and briefs where the full monorepo is intentionally in scope. A `scope.include` glob that matches no file (a brief written for an older layout of the source) does not shrink that barrel: `guards.staleScope` restores the root exports defined in a file no glob covers, unless `scope.exclude` or a `scope-expansion` amendment whose latest action is `skipped` or `demoted-include` matches the file. The stale glob is a Medium `brief-scope-stale` gap (coverage-check.md §4), which also names each `scope.tier_a_include` glob that matches no file.
|
|
27
27
|
|
|
28
|
-
- **Pattern-reference apps (`pattern-reference`; non-library source):** a single-package repo that demonstrates an integration pattern rather than distributing a library API. The denominator is the `tier_a_include` set when `surface.json` has it, else the canonicalized provenance map, since `skf-create-skill`'s extraction already curated the map to the authored pattern surface. Leave `analysis_confidence` unchanged. Annotate: `Pattern-reference, denominator: {tier_a_include union | canonicalized provenance-map count} ({
|
|
28
|
+
- **Pattern-reference apps (`pattern-reference`; non-library source):** a single-package repo that demonstrates an integration pattern rather than distributing a library API. The denominator is the `tier_a_include` set when `surface.json` has it, else the canonicalized provenance map, since `skf-create-skill`'s extraction already curated the map to the authored pattern surface. Leave `analysis_confidence` unchanged. Annotate: `Pattern-reference, denominator: {tier_a_include union | canonicalized provenance-map count} ({candidates.tierAIncludeUnion | canonical.canonical, by the denominator} pattern surfaces)`, the canonicalized count being the `canonical.canonical` of the `surface.json` the `--fold` rerun wrote.
|
|
29
29
|
|
|
30
|
-
- **Single-crate curated subset (`specific-modules`):** a single-package repo whose brief carves a subset of the crate's public surface with `scope.include` and `scope.exclude`: the in-scope reachable barrel, or the `stats.effective_denominator` Create Skill stored for it, under the same deflation guard. Annotate: `Specific-modules subset, denominator: in-scope reachable barrel ({
|
|
30
|
+
- **Single-crate curated subset (`specific-modules`):** a single-package repo whose brief carves a subset of the crate's public surface with `scope.include` and `scope.exclude`: the in-scope reachable barrel, or the `stats.effective_denominator` Create Skill stored for it, under the same deflation guard. Annotate: `Specific-modules subset, denominator: in-scope reachable barrel ({candidates.scopeIncludeUnion} items from {candidates.scopeIncludeFiles} files, after scope.include/exclude filtering)`.
|
|
31
31
|
|
|
32
|
-
- **Multi-entry (exports-map) packages (`multi-entry`):** an in-scope `package.json` declares an `exports` map with more than one non-root subpath (`extraction.arms.multi_subpath_exports`). Installers reach every published subpath, so the denominator is the `subpaths` set (wildcard subpaths are left out: their surface is unbounded), or a stored count or `tier_a_include` set ahead of it. The root-barrel count (`candidates.rootBarrel`) is the secondary candidate in coverage-check.md §4
|
|
32
|
+
- **Multi-entry (exports-map) packages (`multi-entry`):** an in-scope `package.json` declares an `exports` map with more than one non-root subpath (`extraction.arms.multi_subpath_exports`). Installers reach every published subpath, so the denominator is the `subpaths` set (wildcard subpaths are left out: their surface is unbounded), or a stored count or `tier_a_include` set ahead of it, the stored count under the same deflation guard, whose fire counts the `tier_a_include` set when the surface has it, else the `scope.include` set. The root-barrel count (`candidates.rootBarrel`) is the secondary candidate in the Denominator Candidates block coverage-check.md §4 appends from coverage-check-branches.md. Annotate: `Multi-entry (exports-map), denominator: {effective_denominator | tier_a_include union | scope.include union | subpath union} ({candidates.subpathEntries} subpath entry points, {candidates.subpathUnion} exports union; root barrel: {candidates.rootBarrel})`.
|
|
33
33
|
|
|
34
|
-
Internal module symbols are **excluded** from the coverage denominator unless they are explicitly documented in SKILL.md (in which case they count as documented extras, not missing coverage): the surface holds only the names the entry points export, and the extraction's `internal` names never reach it. A documented extra reaches coverage-check.md §2c as a stale name. In a Python or TS/JS source, on a run where §2c runs `classify-stale` (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and whose `surface.json` came from a complete extraction, it is an Info `observation` at the line where `classify-stale` finds the source still declares it (`defined_at`, or among several declaring files the one the skill cites on a line naming it), which update-skill never routes. Otherwise, and for a name a brief scopes out (`excluded.outsideScope`), it stays a Medium `stale-documentation` gap.
|
|
34
|
+
Internal module symbols are **excluded** from the coverage denominator unless they are explicitly documented in SKILL.md (in which case they count as documented extras, not missing coverage): the surface holds only the names the entry points export, and the extraction's `internal` names never reach it. A documented extra reaches coverage-check.md §2c as a stale name. In a Python or TS/JS source, on a run where §2c runs `classify-stale` (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and whose `surface.json` came from a complete extraction, it is an Info `observation` at the line where `classify-stale` finds the source still declares it (`defined_at`, or among several declaring files the one the skill cites on a line naming it), which update-skill never routes. Otherwise, and for a name a brief scopes out (`excluded.outsideScope`), it stays a Medium `stale-documentation` gap. When the brief is what scopes the file out (a `scope.exclude` glob matches it, or no `scope.include` glob covers it), the gap names that file and line and the glob: to keep the documentation, narrow the exclude glob (removing it takes a `demoted-exclude` amendment) or widen `scope.include`; else remove that documentation.
|
|
35
35
|
|
|
36
36
|
This matches the extraction-patterns.md convention used during skill creation: coverage measures how well SKILL.md documents what users actually import, not the entire internal codebase.
|
|
37
37
|
|
|
38
38
|
### Provenance-map canonicalization
|
|
39
39
|
|
|
40
|
-
A provenance map may hold **bookkeeping variants** of one export, artifacts of how the library structures its registry (Storybook's component-plus-story decomposition, accessibility renderer shadowing, exact-match renderer disambiguation). Counting them as separate exports inflates the denominator with names that are structural duplicates of a documented base export. When a clause takes its denominator from the provenance map, pass `--fold` to `surface`: it folds `X_def`, `X_exact` and `a11y_X` back to `X`, and `<prefix>X` back to `X` for each renderer prefix you pass as `--fold-prefix` (`mobile_`, `web_`, `ssr_`, only folded when `X` is an entry too), and reports the fold summary in `canonical.summary` for the report.
|
|
40
|
+
A provenance map may hold **bookkeeping variants** of one export, artifacts of how the library structures its registry (Storybook's component-plus-story decomposition, accessibility renderer shadowing, exact-match renderer disambiguation). Counting them as separate exports inflates the denominator with names that are structural duplicates of a documented base export. When a clause takes its denominator from the provenance map, pass `--fold` to `surface`: it folds `X_def`, `X_exact` and `a11y_X` back to `X`, and `<prefix>X` back to `X` for each renderer prefix you pass as `--fold-prefix` (`mobile_`, `web_`, `ssr_`, only folded when `X` is an entry too), and reports the fold summary in `canonical.summary` for the report. Test-skill defines this fold convention: the base form is authoritative, and the variant is a sibling record. The provenance line check (coverage-check §4c) applies the same three rules, without `--fold-prefix` or `--keep`, when the cited file never names the variant: it checks the line against `X`.
|
|
41
41
|
|
|
42
42
|
A map whose raw count equals `stats.effective_denominator` is already canonical: `surface` folds nothing (`canonical.skipped`). A non-zero `canonical.residual` after folding (raw 222, canonical 215, `effective_denominator` 216) is an unexplained-reconciliation note; do not invent fold rules to close it. One judgment stays here: a variant the library's public surface genuinely distinguishes (an `a11y_Checkbox` documented and installed on its own) is a real export: pass it as `--keep`. Check SKILL.md for explicit documentation of the variant first; when in doubt, keep it and report both forms.
|
|
43
43
|
|
|
44
|
+
### What the Surface Holds
|
|
45
|
+
|
|
46
|
+
The surface holds the names the entry points export, less those defined in a file the brief scopes out (`excluded.outsideScope`); the recipes' internal matches are never part of it. A Python `__init__.py` nested in the folder of another package that exports a name is no barrel: its names (`excluded.nestedEntries`) leave only the `all` and `root` sets and the umbrella ratio, and `plan` still compares their signatures. When a `scope.include` glob matches no file (`guards.staleScope.fires`), the root exports defined in a file no glob covers and the brief does not exclude are back in it (`guards.staleScope.restored`). A stale `scope.tier_a_include` glob fires the guard too (`guards.staleScope.unmatchedTierAInclude`) and counts no name.
|
|
47
|
+
|
|
44
48
|
## Source Access Resolution
|
|
45
49
|
|
|
46
50
|
Before analysis, determine source access level. Walk through these states in order — use the first that succeeds:
|
|
@@ -64,7 +68,7 @@ If neither provenance-map nor metadata exports provide a usable baseline, but re
|
|
|
64
68
|
**State 5 — No source access at all:**
|
|
65
69
|
If none of the above succeed, fall through to docs-only mode (as defined in coverage-check.md Section 0: pre-analysis source type detection). Set `analysis_confidence: docs-only`. Warn: "**No source access available.** Coverage check evaluates documentation self-consistency only. Re-run with local clone or remote access for source-backed verification."
|
|
66
70
|
|
|
67
|
-
Set `analysis_confidence` in context for use in Section 2 analysis depth, step 5 output, and step 5 scoring. It names the source access only: tooling health is `toolingStatus`, which init.md records, and the two never overwrite each other.
|
|
71
|
+
Set `analysis_confidence` in context for use in Section 2 analysis depth, step 5 output, and step 5 scoring, and write it at once to the frontmatter of `{report_file}` (init.md §6c created it) as `analysisConfidence: '{analysis_confidence}'`, again each time a later rule changes it (an empty barrel with no brief makes it `docs-only`), so the hard gate and scoring find the value resolved here on disk, never one recalled after the context was compacted; step-hard-gate.md §3 and score.md §7 write the same value again. It names the source access only: tooling health is `toolingStatus`, which init.md records, and the two never overwrite each other.
|
|
68
72
|
|
|
69
73
|
**Confidence tier mapping:** `full` = T1, `provenance-map` = T1, `metadata-only` = T1-low, `remote-only` = T1-low, `docs-only` = T3. This aligns with the T1/T1-low/T2/T3 scale used across all SKF workflows.
|
|
70
74
|
|
|
@@ -51,6 +51,7 @@ The gate blocks the run, and scoring never runs. The run still ends through the
|
|
|
51
51
|
Update `{outputFile}` frontmatter:
|
|
52
52
|
- Set `hardGate: 'blocked'`
|
|
53
53
|
- Set `testResult: 'fail'`
|
|
54
|
+
- Set `analysisConfidence: '{analysis_confidence}'`: the source access coverage-check resolved, as score.md §7 writes it for a scored run
|
|
54
55
|
- Set `nextWorkflow: 'update-skill'`
|
|
55
56
|
- Append `'hard-gate'` to `stepsCompleted`
|
|
56
57
|
|
|
@@ -14,7 +14,10 @@ than the model retyping them from memory:
|
|
|
14
14
|
- the report frontmatter: skillName, runId, testMode, hardGate,
|
|
15
15
|
testResult, workspaceDrift, score, threshold, thresholdFallback,
|
|
16
16
|
originalThreshold and evidenceReportPath (the steps write the verdict
|
|
17
|
-
inputs there, so a compacted context cannot change them)
|
|
17
|
+
inputs there, so a compacted context cannot change them), and
|
|
18
|
+
analysisConfidence, which must be one of compute-score.py's
|
|
19
|
+
ANALYSIS_CONFIDENCE for a scored run and a blocked one alike (score.md
|
|
20
|
+
§7 writes it, or step-hard-gate.md §3 on a blocked run);
|
|
18
21
|
- the gap ledger: the gap counts by severity (gap-ledger.py's summary);
|
|
19
22
|
- the scoring script's output, which score.md saves: the verdict, score
|
|
20
23
|
and threshold it settled, activeCategories and inconclusiveReasons. A
|
|
@@ -82,8 +85,9 @@ takes: "stderr" for a blocked run, else "stdout".
|
|
|
82
85
|
Exit codes:
|
|
83
86
|
0 the payload was written
|
|
84
87
|
1 an input is missing or unreadable, the report's frontmatter holds no
|
|
85
|
-
settled verdict
|
|
86
|
-
|
|
88
|
+
settled verdict, an analysisConfidence that names no source access or
|
|
89
|
+
a verdict field the scoring output does not settle, or --output cannot
|
|
90
|
+
be written ({"status": "error", "error"} on stdout)
|
|
87
91
|
2 a usage error: no --score for a run the hard gate did not block (the
|
|
88
92
|
same JSON on stdout), or a flag argparse refuses (its usage on stderr)
|
|
89
93
|
"""
|
|
@@ -180,6 +184,11 @@ def _fallback_floor() -> int:
|
|
|
180
184
|
return _sibling("compute-score.py", "skf_compute_score_floor").FALLBACK_FLOOR
|
|
181
185
|
|
|
182
186
|
|
|
187
|
+
def _analysis_confidence() -> tuple[str, ...]:
|
|
188
|
+
"""The source access levels a report may record: compute-score.py's, which scores by them."""
|
|
189
|
+
return tuple(_sibling("compute-score.py", "skf_compute_score_confidence").ANALYSIS_CONFIDENCE)
|
|
190
|
+
|
|
191
|
+
|
|
183
192
|
def settled_test_result(scored: dict, workspace_drift: str | None) -> str | None:
|
|
184
193
|
"""The testResult the scoring output settles, or None when it holds no verdict.
|
|
185
194
|
|
|
@@ -241,6 +250,14 @@ def build(report: str, ledger: str, score: str | None = None, health_check: bool
|
|
|
241
250
|
except gl.LedgerError as exc:
|
|
242
251
|
raise ContextError(f"--ledger {ledger}: {exc}") from exc
|
|
243
252
|
|
|
253
|
+
# Every run records its source access, a blocked one included (step-hard-gate.md §3): a report missing it
|
|
254
|
+
# halts here, before report.md publishes it.
|
|
255
|
+
confidence = front.get("analysisConfidence")
|
|
256
|
+
levels = _analysis_confidence()
|
|
257
|
+
if confidence not in levels:
|
|
258
|
+
raise ContextError(f"--report {report} records no source access (analysisConfidence {confidence!r}, "
|
|
259
|
+
f"not one of: {', '.join(levels)})")
|
|
260
|
+
|
|
244
261
|
skill = front.get("skillName") or None
|
|
245
262
|
blocked = front.get("hardGate") == "blocked"
|
|
246
263
|
if blocked:
|