bmad-module-skill-forge 3.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +3 -3
  3. package/docs/_data/pinned.yaml +19 -13
  4. package/docs/architecture.md +2 -2
  5. package/docs/concepts.md +4 -4
  6. package/docs/examples.md +33 -26
  7. package/docs/getting-started.md +3 -3
  8. package/docs/how-it-works.md +2 -2
  9. package/docs/index.md +1 -1
  10. package/docs/skill-model.md +41 -39
  11. package/docs/troubleshooting.md +8 -0
  12. package/docs/verifying-a-skill.md +46 -21
  13. package/docs/why-skf.md +1 -1
  14. package/docs/workflows.md +5 -5
  15. package/package.json +1 -1
  16. package/src/knowledge/qmd-registry.md +1 -1
  17. package/src/knowledge/version-paths.md +1 -1
  18. package/src/shared/health-check.md +2 -50
  19. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
  20. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
  21. package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
  22. package/src/shared/scripts/skf-atomic-write.py +212 -59
  23. package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
  24. package/src/shared/scripts/skf-classify-changed-files.py +78 -0
  25. package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
  26. package/src/shared/scripts/skf-count-tokens.py +54 -10
  27. package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
  28. package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
  29. package/src/shared/scripts/skf-detect-workspaces.py +67 -0
  30. package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
  31. package/src/shared/scripts/skf-extract-public-api.py +1673 -189
  32. package/src/shared/scripts/skf-extraction-inventory.py +584 -46
  33. package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
  34. package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
  35. package/src/shared/scripts/skf-github-probe.py +47 -8
  36. package/src/shared/scripts/skf-hash-content.py +14 -8
  37. package/src/shared/scripts/skf-load-provenance.py +3 -1
  38. package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
  39. package/src/shared/scripts/skf-parse-gaps.py +135 -26
  40. package/src/shared/scripts/skf-preflight.py +15 -1
  41. package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
  42. package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
  43. package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
  44. package/src/shared/scripts/skf-resolve-customization.py +197 -0
  45. package/src/shared/scripts/skf-shard-body.py +239 -41
  46. package/src/shared/scripts/skf-structural-diff.py +133 -18
  47. package/src/shared/scripts/skf-update-active-symlink.py +115 -62
  48. package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
  49. package/src/shared/scripts/skf-validate-output.py +12 -3
  50. package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
  51. package/src/shared/scripts/skf-write-skill-brief.py +41 -8
  52. package/src/shared/tool-requirements.yaml +8 -2
  53. package/src/skf-analyze-source/SKILL.md +2 -2
  54. package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
  55. package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
  56. package/src/skf-audit-skill/SKILL.md +2 -2
  57. package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
  58. package/src/skf-audit-skill/references/headless-contract.md +1 -1
  59. package/src/skf-audit-skill/references/health-check.md +2 -0
  60. package/src/skf-audit-skill/references/init.md +2 -1
  61. package/src/skf-audit-skill/references/re-index.md +14 -9
  62. package/src/skf-audit-skill/references/report.md +2 -2
  63. package/src/skf-audit-skill/references/semantic-diff.md +15 -15
  64. package/src/skf-audit-skill/references/structural-diff.md +10 -5
  65. package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
  66. package/src/skf-brief-skill/SKILL.md +3 -3
  67. package/src/skf-brief-skill/assets/scope-templates.md +1 -1
  68. package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
  69. package/src/skf-brief-skill/references/analyze-target.md +6 -6
  70. package/src/skf-brief-skill/references/confirm-brief.md +1 -1
  71. package/src/skf-brief-skill/references/scope-definition.md +4 -4
  72. package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
  73. package/src/skf-brief-skill/references/version-resolution.md +5 -9
  74. package/src/skf-campaign/SKILL.md +2 -2
  75. package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
  76. package/src/skf-create-skill/SKILL.md +2 -2
  77. package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
  78. package/src/skf-create-skill/assets/skill-sections.md +7 -7
  79. package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
  80. package/src/skf-create-skill/references/compile.md +31 -60
  81. package/src/skf-create-skill/references/component-extraction.md +3 -3
  82. package/src/skf-create-skill/references/enrich.md +2 -2
  83. package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
  84. package/src/skf-create-skill/references/extract.md +23 -19
  85. package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
  86. package/src/skf-create-skill/references/extraction-patterns.md +10 -11
  87. package/src/skf-create-skill/references/generate-artifacts.md +1 -1
  88. package/src/skf-create-skill/references/report.md +21 -5
  89. package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
  90. package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
  91. package/src/skf-create-stack-skill/SKILL.md +2 -2
  92. package/src/skf-create-stack-skill/references/validate.md +1 -1
  93. package/src/skf-drop-skill/SKILL.md +2 -2
  94. package/src/skf-drop-skill/references/execute.md +5 -1
  95. package/src/skf-drop-skill/references/health-check.md +2 -0
  96. package/src/skf-drop-skill/references/invocation-contract.md +1 -1
  97. package/src/skf-drop-skill/references/report.md +2 -2
  98. package/src/skf-drop-skill/references/select.md +7 -3
  99. package/src/skf-export-skill/SKILL.md +2 -2
  100. package/src/skf-export-skill/assets/snippet-format.md +5 -5
  101. package/src/skf-export-skill/references/generate-snippet.md +22 -15
  102. package/src/skf-export-skill/references/load-skill.md +1 -1
  103. package/src/skf-export-skill/references/token-report.md +29 -9
  104. package/src/skf-forger/SKILL.md +2 -2
  105. package/src/skf-quick-skill/SKILL.md +2 -2
  106. package/src/skf-refine-architecture/SKILL.md +6 -8
  107. package/src/skf-rename-skill/SKILL.md +2 -2
  108. package/src/skf-setup/SKILL.md +2 -2
  109. package/src/skf-test-skill/SKILL.md +2 -2
  110. package/src/skf-test-skill/assets/output-section-formats.md +6 -4
  111. package/src/skf-test-skill/references/coherence-check.md +5 -4
  112. package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
  113. package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
  114. package/src/skf-test-skill/references/coverage-check.md +69 -111
  115. package/src/skf-test-skill/references/init.md +5 -6
  116. package/src/skf-test-skill/references/migration-section-rules.md +26 -14
  117. package/src/skf-test-skill/references/report.md +11 -9
  118. package/src/skf-test-skill/references/scoring-rules.md +8 -6
  119. package/src/skf-test-skill/references/source-access-protocol.md +15 -18
  120. package/src/skf-test-skill/scripts/build-result-context.py +114 -21
  121. package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
  122. package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
  123. package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
  124. package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
  125. package/src/skf-test-skill/scripts/score-signatures.py +356 -61
  126. package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
  127. package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
  128. package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
  129. package/src/skf-test-skill/templates/test-report-template.md +1 -0
  130. package/src/skf-update-skill/SKILL.md +2 -2
  131. package/src/skf-update-skill/references/detect-changes.md +32 -13
  132. package/src/skf-update-skill/references/gap-driven.md +37 -22
  133. package/src/skf-update-skill/references/init.md +6 -2
  134. package/src/skf-update-skill/references/invocation-contract.md +1 -1
  135. package/src/skf-update-skill/references/merge.md +17 -12
  136. package/src/skf-update-skill/references/re-extract.md +4 -2
  137. package/src/skf-update-skill/references/report.md +7 -4
  138. package/src/skf-update-skill/references/write.md +77 -36
  139. package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
  140. package/src/skf-verify-stack/SKILL.md +2 -2
  141. package/tools/cli/lib/tool-check.js +158 -2
@@ -65,7 +65,7 @@ Load `{outputFormatsFile}` for the Ledger Record Format and the remediation qual
65
65
  Every run that reaches this step, a blocked one included, ends through one sequence, and a check that fails stops it before anything is published:
66
66
 
67
67
  1. §4b records the discovery outcome in the ledger (a blocked run skips it).
68
- 2. §4c renders the Gap Report from the ledger, checks that every stage ran and wrote its section and that the health check is installed, then publishes the report, writes the result contract and runs the on_complete hook.
68
+ 2. §4c renders the Gap Report from the ledger, checks that every stage ran and wrote its section and that the health check is installed, then builds the result payload, publishes the report, writes the result contract and runs the on_complete hook.
69
69
  3. §6 presents the result, §6b keeps the headless envelope for the run's last line, and §7 releases the run lock and hands over to the health check.
70
70
 
71
71
  ### 4b. Discovery Testing
@@ -213,27 +213,29 @@ uv run {runLockHelper} acquire --lock "{forge_version}/.test-skill.lock" --owner
213
213
 
214
214
  The run lock renewed here stays held until §7 releases it, so no other run publishes over this run's report or result files meanwhile.
215
215
 
216
- **Publish the report.** Set `health_check_dispatched` in the `{outputFile}` frontmatter: `false` when `no_health_check` is true (the `--no-health-check` flag), else `true`, since the health check now runs with no menu before it. The checks passed, so the report takes its public name, the one export-skill and update-skill read, a blocked run's included. From `{project-root}`, run:
216
+ **Build the result payload** before the report takes its public name. Every value in it comes from the run's own records: `{resultContextScript}` (it resolves relative to the skill root) reads the report frontmatter, the gap ledger and the scoring output, checks the verdict fields score.md §7 wrote (`testResult`, `score`, `threshold` and the fallback fields) against the scoring output, derives `verdict`, `exit_code` and `next_workflow` from `testResult`, and writes the shared emitter's payload, which names the report by its public name. Pass `--score` for a scored run (a run the hard gate blocked has no scoring output), `--no-health-check` when that flag was given, and `--warning` when SKILL.md On Activation step 3 kept a `{customization_resolver_unavailable}` reason: first write `customization_resolver_unavailable: {customization_resolver_unavailable}` to `{run_dir}/resolver-warning.txt` with a file write, never `echo` (the reason can hold quotes, backticks or `$( )`), and pass that file's text (the emitter adds the warnings a stage recorded in the run's sink):
217
217
 
218
218
  ```bash
219
- mv "{report_file}" "{publishedReportFile}"
219
+ uv run {resultContextScript} --report "{report_file}" --report-path "{publishedReportFile}" --ledger "{ledgerFile}" [--score "{run_dir}/score.json"] [--no-health-check] [--warning "$(cat "{run_dir}/resolver-warning.txt")"] --output "{run_dir}/result-context.json"
220
220
  ```
221
221
 
222
- and bind `{report_file}` ← `{publishedReportFile}`. If the move fails, HALT (`halt_reason: "write-failed"`, phase `report:publish`, `"path": "{publishedReportFile}"`): no result file has been written, so the previous run's verdict still stands.
222
+ Bind `{emit_target}` ← `target` from its JSON (`stderr` for a run the hard gate blocked, else `stdout`). Exit 2: correct the command and run it again. Exit 1: HALT with its `error` (`halt_reason: "helper-failed"`, phase `report:result-contract`); the report keeps its in-progress name, so export-skill never reads a verdict the scoring output did not settle.
223
223
 
224
- **Write the result contract.** The shared emitter writes it, per `{outputContractSchema}`, and every value in it comes from the run's own records: `{resultContextScript}` (it resolves relative to the skill root) reads the report frontmatter, the gap ledger and the scoring output, derives `verdict`, `exit_code` and `next_workflow` from `testResult`, and writes the emitter's payload. Pass `--score` for a scored run (a run the hard gate blocked has no scoring output), `--no-health-check` when that flag was given, and `--warning` when SKILL.md On Activation step 3 kept a `{customization_resolver_unavailable}` reason: first write `customization_resolver_unavailable: {customization_resolver_unavailable}` to `{run_dir}/resolver-warning.txt` with a file write, never `echo` (the reason can hold quotes, backticks or `$( )`), and pass that file's text (the emitter adds the warnings a stage recorded in the run's sink):
224
+ **Publish the report.** Set `health_check_dispatched`, already in the `{outputFile}` frontmatter: `false` when `no_health_check` is true (the `--no-health-check` flag), else `true`, since the health check now runs with no menu before it. The checks passed, so the report takes its public name, the one export-skill and update-skill read, a blocked run's included. From `{project-root}`, run:
225
225
 
226
226
  ```bash
227
- uv run {resultContextScript} --report "{report_file}" --ledger "{ledgerFile}" [--score "{run_dir}/score.json"] [--no-health-check] [--warning "$(cat "{run_dir}/resolver-warning.txt")"] --output "{run_dir}/result-context.json"
227
+ mv "{report_file}" "{publishedReportFile}"
228
228
  ```
229
229
 
230
- Bind `{emit_target}` ← `target` from its JSON (`stderr` for a run the hard gate blocked, else `stdout`). Exit 1: HALT with its `error` (`halt_reason: "helper-failed"`, phase `report:result-contract`). Then, in every mode, run:
230
+ and bind `{report_file}` ← `{publishedReportFile}`. If the move fails, HALT (`halt_reason: "write-failed"`, phase `report:publish`, `"path": "{publishedReportFile}"`): no result file has been written, so the previous run's verdict still stands.
231
+
232
+ **Write the result contract.** The shared emitter writes it, per `{outputContractSchema}`. In every mode, run:
231
233
 
232
234
  ```bash
233
235
  uv run {emitEnvelopeHelper} emit --workflow skf-test-skill --run-dir "{run_dir}" --result-dir "{forge_version}" --target {emit_target} < "{run_dir}/result-context.json"
234
236
  ```
235
237
 
236
- It writes `{forge_version}/skf-test-skill-result-{YYYYMMDD-HHmmss}.json` (UTC; it picks the name) and then its `skf-test-skill-result-latest.json` copy (stable path for pipeline consumers: a copy, not a symlink), each atomically, so a partial write is never observable; stamps the timestamp, `run_id`, `headless_decisions` and `warnings` into both; and prints the `SKF_TEST_RESULT_JSON:` line. Bind `{result_line}` ← that line and `{result_path}` ← its `result_path`. A run the hard gate blocked writes the record too, as a FAIL (`status: "error"`, `verdict: "FAIL"`, `halt_reason: "hard-gate-blocked"`, `exit_code` 2), so `skf-test-skill-result-latest.json` never holds an earlier run's verdict. A result file that could not be written leaves `result_path` null and a `result_file_write_failed` warning in the line, and the run still finishes. If the emitter exits non-zero, run both commands once more; if it fails again, HALT (`halt_reason: "write-failed"`, phase `report:result-contract`) with its stderr: the run's result contract could not be written. `references/invocation-contract.md` lists the record's fields.
238
+ It writes `{forge_version}/skf-test-skill-result-{YYYYMMDD-HHmmss}.json` (UTC; it picks the name) and then its `skf-test-skill-result-latest.json` copy (stable path for pipeline consumers: a copy, not a symlink), each atomically, so a partial write is never observable; stamps the timestamp, `run_id`, `headless_decisions` and `warnings` into both; and prints the `SKF_TEST_RESULT_JSON:` line. Bind `{result_line}` ← that line and `{result_path}` ← its `result_path`. A run the hard gate blocked writes the record too, as a FAIL (`status: "error"`, `verdict: "FAIL"`, `halt_reason: "hard-gate-blocked"`, `exit_code` 2), so `skf-test-skill-result-latest.json` never holds an earlier run's verdict. A result file that could not be written leaves `result_path` null and a `result_file_write_failed` warning in the line, and the run still finishes. If the emitter exits non-zero, run it once more; if it fails again, HALT (`halt_reason: "write-failed"`, phase `report:result-contract`) with its stderr: the run's result contract could not be written. `references/invocation-contract.md` lists the record's fields.
237
239
 
238
240
  Once the result files are written, append `'report'` to `stepsCompleted` in the `{outputFile}` frontmatter (it stays out of the expected set above).
239
241
 
@@ -287,7 +289,7 @@ The report is final once §4c published it and wrote the result files: no later
287
289
  **update-skill** — The skill scored above threshold, but `--allow-workspace-drift` was in effect: the test ran against workspace HEAD, not `metadata.source_commit`. A conditional PASS is not trustworthy enough to export. Re-sync the source tree to the pinned commit (or re-extract against current HEAD) and re-run test-skill without the drift override before exporting.
288
290
 
289
291
  {IF FAIL after scoring:}
290
- **update-skill `--from-test-report`**: This skill needs remediation. Review the Gap Report above, then run `@Ferris US {skill_name} --from-test-report`: Update Skill reads this report's gaps and repairs the skill at its pinned commit. Test it again afterwards.
292
+ **update-skill `--from-test-report`**: This skill needs remediation. Review the Gap Report above, then run `@Ferris US {skill_name} --from-test-report`: Update Skill repairs the gaps it routes, at the skill's pinned commit, and lists each other gap as not repaired, with the reason. A `stale-documentation` or `metadata-drift` gap needs what its remediation names: a person, Create Skill or the skill brief. Test it again afterwards.
291
293
 
292
294
  {IF the hard gate blocked the run:}
293
295
  **update-skill `--from-test-report`**: The hard gate stopped this run before scoring on {blocking_gaps} Critical or High gap(s), listed first in the Gap Report. Run `@Ferris US {skill_name} --from-test-report` to repair them, then test again.
@@ -18,32 +18,34 @@ This table is the single source of truth for classifying a gap. Every stage reco
18
18
 
19
19
  | Severity | Category | Criteria |
20
20
  |----------|-----------------------------|----------|
21
- | Critical | `signature-mismatch` | Wrong signature: a documented signature that differs from the source in its parameters (name, type, order, optionality) or its return type (coverage-check §2) |
21
+ | Critical | `signature-mismatch` | Wrong signature: a documented signature that differs from the source in its parameters (name, type, order, optionality) or its return type, in any way but the two the Low `signature-mismatch` row and the Info `observation` row grade, such as a renamed Python parameter (the script cannot see which parameters a caller must pass by position), an optional parameter left out of a form the skill does not label partial, or a mismatch with no `difference` (coverage-check §2 and §2b) |
22
22
  | Critical | `fabricated-signature` | Fabricated signature: a documented export absent at the source line the skill cites for it. Decided only where ast-grep enumerated the exports at the pinned commit: the name is outside that surface, and the cited file is missing or defines no such name (coverage-check §2c) |
23
23
  | Critical | `broken-reference` | Broken reference (contextual mode): a reference whose target does not exist, such as a file not at the stated path, a skill not in the skills output folder, a type no module declares, or a `scripts/` or `assets/` file the skill names (coherence-check §4) |
24
24
  | High | `inaccurate-reference` | Inaccurate reference (contextual mode): the target exists but does not match, such as a type the cited module does not export, a signature that differs, or an integration pattern that does not match the implementation (coherence-check §4) |
25
25
  | High | `reference-escape` | A reference whose real path leaves the skill, its source and, for a stack, the skills output folder (coherence-check §4) |
26
26
  | High | `split-body-mismatch` | The SKILL.md body and a `references/` file document one export differently (coverage-check §1b) |
27
- | High | `numerator-inflation` | `stats.exports_documented` equals the denominator while declared exports are absent from SKILL.md and `references/` (coverage-check §4b) |
27
+ | High | `numerator-inflation` | A declared count (`stats.exports_documented`, or the provenance map's distinct entry names) equals the denominator while declared exports are absent from SKILL.md and `references/` (coverage-check §4b) |
28
28
  | High | `structural` | Naive mode: a missing required section, an unbalanced code fence, or a documented async or sync behaviour its example contradicts (coherence-check §2.1, §2.2, §2.5) |
29
29
  | High | `discovery` | Discovery test: at most 1 of 3 realistic prompts routed to the skill (report §4b.3) |
30
30
  | Medium | `missing-export` | Missing export: an exported function or class the skill does not document. It lowers Export Coverage, and the threshold decides the verdict |
31
31
  | Medium | `missing-type` | Missing type or interface documentation |
32
- | Medium | `stale-documentation` | A documented export the enumerated source surface lacks that is not a fabricated signature: its cited file defines it, or no cited line could be checked (coverage-check §2c) |
32
+ | Medium | `stale-documentation` | A documented export the enumerated source surface lacks that is not a fabricated signature and not a documented extra: classify-stale found no single file that declares it (an import, a re-export or an indented local does not count, nor does a module so named when the inventory gives the name a `function`, `hook`, `class`, `interface`, `enum` or `type` kind, as the Info `observation` row says) and the skill's citations on the lines that name it cite no single one of the files that do, it is a dotted name, the brief scoped its file out (`excluded.outsideScope`), or the run could not look: the source is not Python or TS/JS, classify-stale did not run (it needs the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map), or `surface.json` has no complete `extraction` (coverage-check §2c) |
33
33
  | Medium | `migration-section` | Migration section present/absent mismatch with T2-future annotation data (Deep tier; Case 1 and Case 3 of `migration-section-rules.md`) |
34
- | Medium | `metadata-drift` | Metadata drift: intra-cluster export counts diverge by more than 10% (barrel: `stats.exports_public_api` vs `exports[].length`; documented surface: `stats.exports_documented` vs the provenance-map named-export count), or denominator deflation: the re-derived source barrel exceeds `effective_denominator` by more than 25% and the brief has no `scope.tier_a_include` (coverage-check §4 and §4b) |
35
- | Medium | `denominator-inflation` | Denominator inflation: the stratified-scope `scope.include` union exceeds the provenance-map entry count by more than 25% (brief missing `scope.tier_a_include`) |
34
+ | Medium | `metadata-drift` | Metadata drift: intra-cluster export counts diverge by more than 10% (barrel: `stats.exports_public_api` vs `exports[].length`; documented surface: `stats.exports_documented` vs the provenance-map named-export count), or denominator deflation: the re-derived source barrel exceeds `effective_denominator` by more than 25% and the brief has no `scope.tier_a_include` glob that counts a public name, or denominator drift: on a stratified-scope, specific-modules or multi-entry skill, the brief's `scope.tier_a_include` set differs from `effective_denominator` by more than 25% either way, or counts a name against an `effective_denominator` of 0, a stale count to recount (coverage-check §4 and §4b) |
35
+ | Medium | `denominator-inflation` | Denominator inflation: the stratified-scope `scope.include` union exceeds the provenance-map entry count by more than 25% (the brief has no `scope.tier_a_include` glob that counts a public name). On a stratified-scope or multi-entry barrel over that union, the names it counts and the skill does not document are one more such gap beside it at the brief, listing at most 50 and deferring to its fix, instead of a `missing-export` gap each; Export Coverage still counts them |
36
+ | Medium | `brief-scope-stale` | Stale brief scope: a `scope.include` glob matches no file in the source tested, so the surface restores the root exports defined in a file no glob covers that the brief does not exclude, or a `scope.tier_a_include` glob matches no file in the source tested, so it counts no name (coverage-check §4) |
36
37
  | Medium | `scripts-assets` | A `scripts/` or `assets/` directory exists but SKILL.md has no Scripts & Assets section (coherence-check §2.7 and §4) |
37
38
  | Medium | `integration-pattern` | Incomplete integration pattern (contextual mode): one entry of coherence-check §5's `incomplete_patterns` |
38
39
  | Medium | `structural` | Naive mode: an opening code fence without a language tag, an exported function or method no usage-family section or reference file names, or a table row whose column count differs from its header (coherence-check §2.3, §2.4, §2.6) |
39
40
  | Medium | `discovery` | Discovery test: 2 of 3 realistic prompts routed to the skill (report §4b.3) |
41
+ | Low | `signature-mismatch` | A renamed parameter callers cannot see: a mismatch the signature judge tags `parameter-name` (only parameter names differ; types, order, optionality, defaults and return type are equal) at the file the surface records for the export, the one the plan sent the check to or a candidate the judge compared with, when it is a `.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`, `.go`, `.rs`, `.java`, `.c`, `.h`, `.cc`, `.cpp`, `.cxx` or `.hpp` file, a language with no keyword arguments (not a Vue component, whose props callers set by name), as score-signatures.py checks. It counts as matching in Signature Accuracy (coverage-check §2 and §2b) |
40
42
  | Low | `scripts-assets-provenance` | Script/asset file present without provenance entry in provenance-map.json file_entries |
41
43
  | Low | `provenance-line` | Provenance line is not the definition of an export (`line-not-definition`, coverage-check §4c) |
42
44
  | Low | `migration-section` | Case 3 of `migration-section-rules.md` that the reviewer downgraded with an inline justification |
43
45
  | Low | `metadata` | Missing optional metadata or examples |
44
46
  | Low | `description` | Description trigger optimization recommended (third-person voice, negative triggers, or keyword coverage gaps), a Tessl Review description suggestion included |
45
47
  | Low | `external-validator` | A skill-check diagnostic or a Tessl Review validation finding the run left unresolved (external-validators §5b) |
46
- | Info | `observation` | Style suggestions, non-blocking observations |
48
+ | Info | `observation` | Style suggestions, non-blocking observations, and a documented extra: a documented name outside the enumerated surface that is not a fabricated signature and that the source still declares at column 0 (a module so named counts, unless the inventory gives the name a `function`, `hook`, `class`, `interface`, `enum` or `type` kind, no provenance map entry of the name types it a `module` or `package` and the module is not a TS/JS file that is itself the export, with an `export default` or a `module.exports =`; among several declaring files, the one the skill cites on a line naming it), found only in a Python or TS/JS source where classify-stale ran (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and `surface.json` has a complete `extraction`: `Documented extra: {name}`, at that line, which update-skill never routes (coverage-check §2c). Also a partial signature: a mismatch the signature judge tags `omitted-optional` (the documented form leaves out only members the source marks optional) on a form the skill labels partial (simplified, key parameters, a `...` that stands in for omitted members, not rest or variadic syntax), which the inventory marks `signature_form: "partial"`, as score-signatures.py checks: `Partial signature: {name}`, at the declaration, counted as matching in Signature Accuracy (coverage-check §2 and §2b) |
47
49
  | Info | `provenance-unverified` | Provenance line not verified: the line-check rules found no definition line (coverage-check §4c) |
48
50
  | Info | `migration-section` | Historical migration content in Section 4b with no T2-future annotation (Case 2 of `migration-section-rules.md`) |
49
51
  | Info | `multi-denominator` | Multi-denominator reporting: barrel and documented-surface clusters diverge by design (>10% cross-cluster) |
@@ -6,43 +6,40 @@
6
6
 
7
7
  **Source API surface** = the package's top-level public exports. These are the symbols reachable from the primary entry point without importing internal modules:
8
8
 
9
- - **Python:** symbols exported in `__init__.py` (including re-exports) — exclude private (`_prefixed`) names
9
+ - **Python:** symbols exported in the package root's `__init__.py` (including re-exports), excluding private (`_prefixed`) names. A sub-package behind a folder with no `__init__.py` (a FastAPI `routers/__init__.py`, say) is not a second package root when the folder of another package root that exports a name holds it, unless `scope.include` or `scope.tier_a_include` lists that `__init__.py`: its names stay out of the `all` and `root` sets (`excluded.nestedEntries`), while the `scope.include` and `tier_a_include` sets still count them and their signatures are still compared
10
10
  - **TypeScript/JavaScript:** named exports from `index.ts` / `index.js` — exclude unexported locals
11
11
  - **Go:** exported identifiers (capitalized) from the package's public-facing files
12
12
  - **Rust:** items in `pub use` from `lib.rs` or `mod.rs`
13
13
 
14
- The clauses below decide which part of that surface is the coverage denominator. Choosing the clause is a judgment, made here from the brief (`{forge_data_folder}/{skill_name}/skill-brief.yaml`), `{source_is_monorepo}` (coverage-check.md §0b) and the extraction; every count a clause names comes from `load-coverage-inputs.py` (coverage-check.md §2), in `surface.json`: the name `sets`, the `candidates` and the `guards`. The recipe runner already re-derives each barrel (multi-line re-export blocks, star re-exports and a Rust `pub use` chain included) and counts each name once, at the file that defines it, so a barrel that only re-exports adds nothing of its own. Without an extraction (Quick tier, the fallback per-file scan, States 2 to 4) the script builds the `scope.include` and `tier_a_include` sets from the files the names carry and the brief's globs, and there is no `subpaths` or `root` set (coverage-check.md §2b). Never re-count a barrel by hand.
14
+ The clauses below decide which part of that surface is the coverage denominator. `load-coverage-inputs.py surface` picks the clause from the brief (`{forge_data_folder}/{skill_name}/skill-brief.yaml`), `{source_is_monorepo}` (coverage-check.md §0b, passed as `--monorepo`), the extraction and `metadata.json`, and writes it to `surface.json` `denominator` with the branch, the name set, the rung it takes (coverage-check.md §2b) and the entry points' state it read (`barrel`), beside the counts each clause names (`sets`, `candidates` and `guards`). A workspace member counts toward `{source_is_monorepo}` only through the files the brief keeps: excluding its source files drops it, excluding only its manifest does not, and a workspace with no member left is no workspace. Three of its inputs are yours to pass as flags, the last two only where no file gives them:
15
15
 
16
- - **Empty-barrel packages (copy-paste / subpath-only distribution):** the primary entry point is empty or re-exports nothing (`export {};`, an empty `__init__.py`, a `lib.rs` with no `pub use`). Do **not** compute coverage against the empty barrel: the denominator is the `scope.include` set, the union of named exports across the files the brief authorizes. The brief's `scope.notes` should say the distribution model is by design; when it does, treat the empty barrel as intended rather than a bug. With no brief and an empty barrel, set `analysis_confidence: docs-only` and report that the source API surface could not be determined.
16
+ - **Pattern-reference intent** (`--pattern-reference-intent`): `scope.notes` says the skill demonstrates an integration pattern rather than a library API ("Reference app, not a library", "pattern-reference", "embedded-pattern skill", "skill value is the … pattern"; the author's words are authoritative).
17
+ - **A curated monorepo subset** (`--curated-subset`), where the extraction has no `counts.curated_subset` (Quick tier, the fallback per-file scan, States 2 to 4): `{source_is_monorepo}` is true and the brief's `scope.include` lists a curated subset rather than the full workspace.
18
+ - **The entry points' state** (`--barrel`), where no complete extraction or quick output gives it (the fallback per-file scan when the runner listed entry points, Quick tier read by eye, State 4): `none` when no barrel file is at any recognized entry-point path (`__init__.py`, `index.ts`/`index.js`, `lib.rs`, `mod.rs`), `empty` when the primary entry point exports nothing (`export {};`, an empty `__init__.py`, a `lib.rs` with no `pub use`), else `barrel`.
17
19
 
18
- - **Stratified-scope monorepo packages (curated subsets of multi-package repos):** `{source_is_monorepo}` is true AND the brief's `scope.include` lists a curated subset rather than the full workspace. The denominator reflects only the authored surface, not the monorepo's global export count. Resolution order:
20
+ The recipe runner already re-derives each barrel (multi-line re-export blocks, star re-exports and a Rust `pub use` chain included) and counts each name once, however many in-scope files define it, so a barrel that only re-exports adds nothing of its own. Each of those files gives the name a surface record of its own (the runner's `entry_point_diff.other_definitions` beside its `public` row), so a set holds the name when any of its files matches, and the signature plan can compare the declaration the skill cites. Without the runner's names (Quick tier, the fallback per-file scan, States 2 to 4) the script builds the `scope.include` and `tier_a_include` sets from the files the names carry and the brief's globs (the runner's JSON the fallback scan passes adds only its scope and stale globs), and there is no `subpaths` or `root` set (coverage-check.md §2b). Never re-count a barrel by hand. A `scope.tier_a_include` glob counts a name only in a file a `scope.include` glob also matches, and one that matches no file (`guards.staleScope.unmatchedTierAInclude`, a brief older than the source) counts none: when the tier A globs count no name, `surface.json` has no `tier_a_include` set. Each clause, by its `denominator.clause`, with what stays yours:
19
21
 
20
- 1. `stats.effective_denominator` (`candidates.statsEffectiveDenominator`) when `metadata.json` has it: the scalar branch, **subject to the deflation guard**. `effective_denominator` is a stored count, and a gap-driven `skf-update-skill` run can lower it, legitimately (a rescope expressed in the brief) or by gaming (dropping in-scope exports and setting it to the documented total). `guards.deflation` re-derives the surface from `scope.include` less `scope.exclude`, and when the brief carries no `scope.tier_a_include` (the only legitimate narrowing mechanism) and the re-derived source barrel exceeds `effective_denominator` by more than `guards.thresholds.deflationPct` percent, `guards.deflation.fires` is true and the `scope.include` set is the denominator instead, with a Medium `metadata-drift` gap (coverage-check.md §4). A legitimate scope reduction is expressed in the brief, through `scope.tier_a_include` or by adding the removed exports to `scope.exclude` (the mechanism `skf-update-skill` gap-driven rescope uses), never by editing `metadata.stats` to equal the documented count. With no surface read from the source (`guards.deflation.applicable` false), the guard does not run: annotate the report `effective_denominator used unverified: no source access to re-derive the barrel`.
21
- 2. Otherwise the `tier_a_include` set when the brief has `scope.tier_a_include`, the narrow list of the surface the brief intends to document.
22
- 3. Otherwise the `scope.include` set, **subject to the inflation guard**: `guards.inflation` compares it with the provenance-map entry count, and when `guards.inflation.fires` (above `guards.thresholds.inflationPct` percent) the coarse globs sweep in internal infrastructure the brief did not mean to document, a Medium `denominator-inflation` gap pointing at `scope.tier_a_include`. When `guards.umbrella.umbrella` is true (the root barrel is mostly re-exports), recommend `stats.effective_denominator` as the remediation instead.
22
+ - **Empty-barrel packages (`empty-barrel`; copy-paste or subpath-only distribution):** the primary entry point is empty, so the denominator is the `scope.include` set. The brief's `scope.notes` should say the distribution model is by design; when it does, treat the empty barrel as intended rather than a bug. With no brief and an empty barrel, set `analysis_confidence: docs-only` and report that the source API surface could not be determined. Annotate: `Empty barrel, denominator: scope.include union ({N} exports)`.
23
23
 
24
- Leave `analysis_confidence` unchanged: stratified scope changes the denominator, not the confidence. Annotate the report: `Stratified scope, denominator: {effective_denominator | tier_a_include union | scope.include union} ({N} files matched, {M} exports union)`. `scope.notes` should document the stratification (for example "Tier A: fully documented; Tier B: deferred to references; Tier C: excluded"); when it does, treat the curated subset as intended, not a scope gap.
24
+ - **Stratified-scope monorepo packages (`stratified-scope`; curated subsets of multi-package repos):** the denominator reflects only the authored surface, not the monorepo's global export count: the `stats.effective_denominator` Create Skill stored, unless `guards.deflation.fires` (a Medium `metadata-drift` gap, coverage-check.md §4: a legitimate scope reduction goes in the brief, through `scope.tier_a_include` or `scope.exclude`, never into `metadata.stats`), then the `tier_a_include` set when the guard measured the stored count against it (`guards.deflation.basis` `tier_a_include`, in either direction: a stale count, which re-running Create Skill recounts, storing the runner's tier A count when `effective_denominator_decision.store` holds and `scope.notes` documents the stratification and otherwise dropping the stored count; on an incomplete extraction a set below the stored count leaves the stored count), else the `scope.include` set; with no stored count, the `tier_a_include` and `scope.include` sets. Never set `stats.effective_denominator` by hand: Update Skill's recount drops it. With no surface read from the source (`guards.deflation.applicable` false), annotate the report `effective_denominator used unverified: no source access to re-derive the barrel`. When `guards.inflation.fires` (the coarse globs sweep in internal infrastructure, a Medium `denominator-inflation` gap), the `scope.include` union is itself inflated: when the deflation guard fires too, that union stays the denominator and both gaps ask for a `scope.tier_a_include` list of the files that define the authored names (when `guards.umbrella.umbrella` is true, those files rather than the barrel). When the barrel branch counts that union, here or on the multi-entry ladder, the names it counts and the skill does not document are one more `denominator-inflation` gap beside the guard's at the brief, which defers to the guard's fix, not a `missing-export` gap each. Leave `analysis_confidence` unchanged. Annotate the report: `Stratified scope, denominator: {effective_denominator | tier_a_include union | scope.include union} ({N} files matched, {M} exports union)`. `scope.notes` should document the stratification (for example "Tier A: fully documented; Tier B: deferred to references; Tier C: excluded"); when it does, treat the curated subset as intended, not a scope gap.
25
25
 
26
- **When this clause does not apply:** `scope.type: "full-library"` skills, single-package repositories, or stratified briefs where the full monorepo is intentionally in scope: use the standard barrel (the `all` set), **unless** the repo is a pattern-reference app or publishes a multi-subpath `exports` map.
26
+ - **Standard barrel (`standard-barrel`, the `all` set):** `full-library` skills, single-package repositories and briefs where the full monorepo is intentionally in scope. A `scope.include` glob that matches no file (a brief written for an older layout of the source) does not shrink that barrel: `guards.staleScope` restores the root exports defined in a file no glob covers, unless `scope.exclude` or a `scope-expansion` amendment whose latest action is `skipped` or `demoted-include` matches the file. The stale glob is a Medium `brief-scope-stale` gap (coverage-check.md §4), which also names each `scope.tier_a_include` glob that matches no file.
27
27
 
28
- - **Pattern-reference apps (non-library source):** a single-package repo that demonstrates an integration pattern rather than distributing a library API. Either trigger fires: `scope.notes` flags pattern-reference intent ("Reference app, not a library", "pattern-reference", "embedded-pattern skill", "skill value is the … pattern"; the author's words are authoritative), or the source has no barrel file at any recognized entry-point path (`__init__.py`, `index.ts`/`index.js`, `lib.rs`, `mod.rs`), `{source_is_monorepo}` is false and no multi-subpath `exports` map exists. The denominator is the `tier_a_include` set when the brief has one, else the canonicalized provenance map: build `surface.json` from the map alone, with coverage-check.md §2's State 2 command less its `--metadata` and with `--fold` (canonicalization, below), since `skf-create-skill`'s extraction already curated the map to the authored pattern surface. Leave `analysis_confidence` unchanged. Annotate: `Pattern-reference, denominator: {tier_a_include union | canonicalized provenance-map count} ({N} pattern surfaces)`. It does not apply to a repo with a non-empty barrel, a monorepo, or a `scope.type` of `specific-modules`, `public-api` with a multi-subpath `exports` map, `component-library`, `docs-only` or `reference-app` (whose brief carries its own pattern-surface denominator).
28
+ - **Pattern-reference apps (`pattern-reference`; non-library source):** a single-package repo that demonstrates an integration pattern rather than distributing a library API. The denominator is the `tier_a_include` set when `surface.json` has it, else the canonicalized provenance map, since `skf-create-skill`'s extraction already curated the map to the authored pattern surface. Leave `analysis_confidence` unchanged. Annotate: `Pattern-reference, denominator: {tier_a_include union | canonicalized provenance-map count} ({N} pattern surfaces)`.
29
29
 
30
- - **Single-crate curated subset (`scope.type: "specific-modules"`):** a single-package repo whose brief carves a subset of the crate's public surface with `scope.include` and `scope.exclude`. The denominator is the in-scope reachable barrel, the `scope.include` set (`stats.effective_denominator` first when present, under the same deflation guard). Annotate: `Specific-modules subset, denominator: in-scope reachable barrel ({N} items from {M} modules, after scope.include/exclude filtering)`. It does not apply to a monorepo, a `full-library` scope or an empty barrel.
30
+ - **Single-crate curated subset (`specific-modules`):** a single-package repo whose brief carves a subset of the crate's public surface with `scope.include` and `scope.exclude`: the in-scope reachable barrel, or the `stats.effective_denominator` Create Skill stored for it, under the same deflation guard. Annotate: `Specific-modules subset, denominator: in-scope reachable barrel ({N} items from {M} modules, after scope.include/exclude filtering)`.
31
31
 
32
- - **Multi-entry (exports-map) packages:** an in-scope `package.json` declares an `exports` map with more than one non-root subpath and `{source_is_monorepo}` is false (`extraction.arms.multi_subpath_exports`), for `scope.type` `full-library` or `public-api`. Installers reach every published subpath, so the denominator is the `subpaths` set (wildcard subpaths are left out: their surface is unbounded), after the same ladder: `stats.effective_denominator` first (deflation guard), then the `tier_a_include` set. The root-barrel count (`candidates.rootBarrel`) is the secondary candidate in coverage-check.md §4's Denominator Candidates block. Annotate: `Multi-entry (exports-map), denominator: {effective_denominator | tier_a_include union | subpath union} ({N} subpaths resolved, {M} exports union; root barrel: {R})`. A root-only or wildcard-only map keeps the standard root barrel.
32
+ - **Multi-entry (exports-map) packages (`multi-entry`):** an in-scope `package.json` declares an `exports` map with more than one non-root subpath (`extraction.arms.multi_subpath_exports`). Installers reach every published subpath, so the denominator is the `subpaths` set (wildcard subpaths are left out: their surface is unbounded), or a stored count or `tier_a_include` set ahead of it, the stored count under the same deflation guard, whose fire counts the `tier_a_include` set when the surface has it, else the `scope.include` set. The root-barrel count (`candidates.rootBarrel`) is the secondary candidate in the Denominator Candidates block coverage-check.md §4 appends from coverage-check-branches.md. Annotate: `Multi-entry (exports-map), denominator: {effective_denominator | tier_a_include union | scope.include union | subpath union} ({N} subpaths resolved, {M} exports union; root barrel: {R})`.
33
33
 
34
- Internal module symbols are **excluded** from the coverage denominator unless they are explicitly documented in SKILL.md (in which case they count as documented extras, not missing coverage): the surface holds only the names the entry points export, and the extraction's `internal` names never reach it.
34
+ Internal module symbols are **excluded** from the coverage denominator unless they are explicitly documented in SKILL.md (in which case they count as documented extras, not missing coverage): the surface holds only the names the entry points export, and the extraction's `internal` names never reach it. A documented extra reaches coverage-check.md §2c as a stale name. In a Python or TS/JS source, on a run where §2c runs `classify-stale` (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and whose `surface.json` came from a complete extraction, it is an Info `observation` at the line where `classify-stale` finds the source still declares it (`defined_at`, or among several declaring files the one the skill cites on a line naming it), which update-skill never routes. Otherwise, and for a name a brief scopes out (`excluded.outsideScope`), it stays a Medium `stale-documentation` gap. When the brief is what scopes the file out (a `scope.exclude` glob matches it, or no `scope.include` glob covers it), the gap names that file and line and the glob: to keep the documentation, narrow the exclude glob (removing it takes a `demoted-exclude` amendment) or widen `scope.include`; else remove that documentation.
35
35
 
36
36
  This matches the extraction-patterns.md convention used during skill creation: coverage measures how well SKILL.md documents what users actually import, not the entire internal codebase.
37
37
 
38
38
  ### Provenance-map canonicalization
39
39
 
40
- A provenance map may hold **bookkeeping variants** of one export, artifacts of how the library structures its registry (Storybook's component-plus-story decomposition, accessibility renderer shadowing, exact-match renderer disambiguation). Counting them as separate exports inflates the denominator with names that are structural duplicates of a documented base export. When a clause takes its denominator from the provenance map, pass `--fold` to `surface`: it folds `X_def`, `X_exact` and `a11y_X` back to `X`, and `<prefix>X` back to `X` for each renderer prefix you pass as `--fold-prefix` (`mobile_`, `web_`, `ssr_`, only folded when `X` is an entry too), and reports the fold summary in `canonical.summary` for the report. This matches the convention `skf-create-skill` records in `metadata.json.stats.effective_denominator_source`: the base form is authoritative, the variant a sibling record.
40
+ A provenance map may hold **bookkeeping variants** of one export, artifacts of how the library structures its registry (Storybook's component-plus-story decomposition, accessibility renderer shadowing, exact-match renderer disambiguation). Counting them as separate exports inflates the denominator with names that are structural duplicates of a documented base export. When a clause takes its denominator from the provenance map, pass `--fold` to `surface`: it folds `X_def`, `X_exact` and `a11y_X` back to `X`, and `<prefix>X` back to `X` for each renderer prefix you pass as `--fold-prefix` (`mobile_`, `web_`, `ssr_`, only folded when `X` is an entry too), and reports the fold summary in `canonical.summary` for the report. Test-skill defines this fold convention: the base form is authoritative, and the variant is a sibling record.
41
41
 
42
- Two judgments stay here:
43
-
44
- - A variant the library's public surface genuinely distinguishes (an `a11y_Checkbox` documented and installed on its own) is a real export: pass it as `--keep`. Check SKILL.md for explicit documentation of the variant first; when in doubt, keep it and report both forms.
45
- - When `stats.effective_denominator` is present and the raw entry count matches it, the map is already canonical: do not fold. When drift remains after folding (raw 222, canonical 215, `effective_denominator` 216), record the residual as an unexplained-reconciliation note; do not invent fold rules to close it.
42
+ A map whose raw count equals `stats.effective_denominator` is already canonical: `surface` folds nothing (`canonical.skipped`). A non-zero `canonical.residual` after folding (raw 222, canonical 215, `effective_denominator` 216) is an unexplained-reconciliation note; do not invent fold rules to close it. One judgment stays here: a variant the library's public surface genuinely distinguishes (an `a11y_Checkbox` documented and installed on its own) is a real export: pass it as `--keep`. Check SKILL.md for explicit documentation of the variant first; when in doubt, keep it and report both forms.
46
43
 
47
44
  ## Source Access Resolution
48
45
 
@@ -12,13 +12,25 @@ follows from records the run already wrote, so this script builds it rather
12
12
  than the model retyping them from memory:
13
13
 
14
14
  - the report frontmatter: skillName, runId, testMode, hardGate,
15
- testResult, score, threshold, thresholdFallback, originalThreshold and
16
- evidenceReportPath (the steps write the verdict inputs there, so a
17
- compacted context cannot change them);
15
+ testResult, workspaceDrift, score, threshold, thresholdFallback,
16
+ originalThreshold and evidenceReportPath (the steps write the verdict
17
+ inputs there, so a compacted context cannot change them);
18
18
  - the gap ledger: the gap counts by severity (gap-ledger.py's summary);
19
- - the scoring script's output, which score.md saves: activeCategories and
20
- inconclusiveReasons. A run the hard gate blocked was never scored and
21
- has none.
19
+ - the scoring script's output, which score.md saves: the verdict, score
20
+ and threshold it settled, activeCategories and inconclusiveReasons. A
21
+ run the hard gate blocked was never scored and has none.
22
+
23
+ A scored run's verdict fields are checked against the scoring output
24
+ (`scored_fields`): testResult must be the verdict the output settled,
25
+ `effectiveResult` when the output has one, else `result`, in lower case,
26
+ with `pass` read as `pass-with-drift` when the frontmatter workspaceDrift
27
+ is `overridden`; score its `totalScore`; threshold the one the verdict was
28
+ settled against (compute-score.py's FALLBACK_FLOOR after a threshold
29
+ fallback, else its `threshold`); and thresholdFallback and
30
+ originalThreshold its override group's. A field copied from anything else
31
+ (`result` over a cap's FAIL, a bare pass under drift, the target threshold
32
+ of a run the fallback accepted) exits 1 and writes no payload, so
33
+ report.md halts before it publishes the report or writes a result file.
22
34
 
23
35
  The derived fields live here, in one place:
24
36
 
@@ -49,9 +61,14 @@ Payload (written to --output, UTF-8 JSON):
49
61
  "runId": <runId>, "healthCheckDispatched": <bool>}}
50
62
 
51
63
  CLI:
52
- uv run build-result-context.py --report <report.md> --ledger <ledger.json>
53
- [--score <scoring output>] [--no-health-check] [--warning <text>]...
54
- --output <result-context.json>
64
+ uv run build-result-context.py --report <report.md> [--report-path <path>]
65
+ --ledger <ledger.json> [--score <scoring output>] [--no-health-check]
66
+ [--warning <text>]... --output <result-context.json>
67
+
68
+ --score is required unless the frontmatter says the hard gate blocked the
69
+ run. --report-path is the path the payload names the report by (default:
70
+ --report): report.md builds the payload from the report before it gives
71
+ the report its public name, and passes that name here.
55
72
 
56
73
  --warning hands the emitter a warning the run kept for its result (such as
57
74
  `customization_resolver_unavailable: <reason>`); the emitter adds the ones
@@ -65,8 +82,10 @@ takes: "stderr" for a blocked run, else "stdout".
65
82
  Exit codes:
66
83
  0 the payload was written
67
84
  1 an input is missing or unreadable, the report's frontmatter holds no
68
- settled verdict, or --output cannot be written ({"status": "error",
69
- "error"} on stdout)
85
+ settled verdict or a verdict field the scoring output does not settle,
86
+ or --output cannot be written ({"status": "error", "error"} on stdout)
87
+ 2 a usage error: no --score for a run the hard gate did not block (the
88
+ same JSON on stdout), or a flag argparse refuses (its usage on stderr)
70
89
  """
71
90
 
72
91
  from __future__ import annotations
@@ -93,6 +112,10 @@ class ContextError(Exception):
93
112
  """An input the payload cannot be built from: exit 1."""
94
113
 
95
114
 
115
+ class UsageError(ContextError):
116
+ """A command the caller can correct and run again: exit 2."""
117
+
118
+
96
119
  def _sibling(name: str, module: str):
97
120
  """Import a script that sits beside this one (gap-ledger.py, hard-gate.py)."""
98
121
  path = Path(__file__).resolve().parent / name
@@ -152,9 +175,60 @@ def _read_json(path: str, label: str):
152
175
  raise ContextError(f"{label} {path} cannot be read: {exc}") from exc
153
176
 
154
177
 
178
+ def _fallback_floor() -> int:
179
+ """The threshold a fallback accepts a run at: compute-score.py's, which applies the fallback."""
180
+ return _sibling("compute-score.py", "skf_compute_score_floor").FALLBACK_FLOOR
181
+
182
+
183
+ def settled_test_result(scored: dict, workspace_drift: str | None) -> str | None:
184
+ """The testResult the scoring output settles, or None when it holds no verdict.
185
+
186
+ >>> settled_test_result({"result": "PASS"}, "ok")
187
+ 'pass'
188
+ >>> settled_test_result({"result": "PASS", "effectiveResult": "FAIL", "capReason": "Cap 1"}, "ok")
189
+ 'fail'
190
+ >>> settled_test_result({"result": "FAIL", "effectiveResult": "PASS"}, "overridden")
191
+ 'pass-with-drift'
192
+ >>> settled_test_result({"result": "INCONCLUSIVE"}, "overridden")
193
+ 'inconclusive'
194
+ >>> settled_test_result({"error": "bad flag", "code": "INVALID_INPUT"}, "ok") is None
195
+ True
196
+ """
197
+ verdict = scored.get("effectiveResult") or scored.get("result")
198
+ settled = verdict.lower() if isinstance(verdict, str) else None
199
+ if settled not in ("pass", "fail", "inconclusive"):
200
+ return None
201
+ return "pass-with-drift" if settled == "pass" and workspace_drift == "overridden" else settled
202
+
203
+
204
+ def scored_fields(scored: dict, workspace_drift: str | None) -> dict | None:
205
+ """The frontmatter verdict fields the scoring output settles, or None when it holds no verdict.
206
+
207
+ >>> scored_fields({"result": "PASS", "totalScore": 91.5, "threshold": 80}, "ok")
208
+ {'testResult': 'pass', 'score': 91.5, 'threshold': 80, 'thresholdFallback': False, 'originalThreshold': None}
209
+ >>> fell_back = {"result": "FAIL", "effectiveResult": "PASS", "capReason": None, "thresholdFallback": True,
210
+ ... "originalThreshold": 90, "totalScore": 85, "threshold": 90}
211
+ >>> scored_fields(fell_back, "ok")
212
+ {'testResult': 'pass', 'score': 85, 'threshold': 80, 'thresholdFallback': True, 'originalThreshold': 90}
213
+ """
214
+ test_result = settled_test_result(scored, workspace_drift)
215
+ if test_result is None:
216
+ return None
217
+ fallback = scored.get("thresholdFallback") is True
218
+ return {
219
+ "testResult": test_result,
220
+ "score": scored.get("totalScore"),
221
+ "threshold": _fallback_floor() if fallback else scored.get("threshold"),
222
+ "thresholdFallback": fallback,
223
+ "originalThreshold": scored.get("originalThreshold") if fallback else None,
224
+ }
225
+
226
+
155
227
  def build(report: str, ledger: str, score: str | None = None, health_check: bool = True,
156
- warnings: list[str] | None = None) -> dict:
157
- """The emitter payload for the run whose report and ledger are given."""
228
+ warnings: list[str] | None = None, report_path: str | None = None) -> dict:
229
+ """The emitter payload for the run whose report and ledger are given; `report_path`
230
+ (default `report`) is the path the payload names the report by."""
231
+ report_path = report_path or report
158
232
  try:
159
233
  front = read_frontmatter(Path(report).read_bytes().decode("utf-8-sig"))
160
234
  except (OSError, UnicodeDecodeError) as exc:
@@ -170,7 +244,7 @@ def build(report: str, ledger: str, score: str | None = None, health_check: bool
170
244
  skill = front.get("skillName") or None
171
245
  blocked = front.get("hardGate") == "blocked"
172
246
  if blocked:
173
- envelope = dict(_sibling("hard-gate.py", "skf_hard_gate").blocked_envelope(skill, report))
247
+ envelope = dict(_sibling("hard-gate.py", "skf_hard_gate").blocked_envelope(skill, report_path))
174
248
  result, active, inconclusive = "FAIL", [], None
175
249
  else:
176
250
  test_result = front.get("testResult", "")
@@ -183,14 +257,29 @@ def build(report: str, ledger: str, score: str | None = None, health_check: bool
183
257
  "verdict": verdict,
184
258
  "score": percent(front.get("score")),
185
259
  "threshold": percent(front.get("threshold")),
186
- "report_path": report,
260
+ "report_path": report_path,
187
261
  "next_workflow": next_workflow,
188
262
  "exit_code": exit_code,
189
263
  "halt_reason": None,
190
264
  }
191
- scored = _read_json(score, "--score") if score else {}
265
+ if not score:
266
+ raise UsageError("--score is required for a run the hard gate did not block: its verdict "
267
+ "fields are checked against the scoring output")
268
+ scored = _read_json(score, "--score")
192
269
  if not isinstance(scored, dict):
193
270
  raise ContextError(f"--score {score} is not a JSON object")
271
+ settled = scored_fields(scored, front.get("workspaceDrift"))
272
+ if settled is None:
273
+ raise ContextError(f"--score {score} holds no verdict (no effectiveResult or result)")
274
+ fell_back = front.get("thresholdFallback", "").lower() == "true"
275
+ written = {"testResult": test_result, "score": envelope["score"], "threshold": envelope["threshold"],
276
+ "thresholdFallback": fell_back,
277
+ "originalThreshold": percent(front.get("originalThreshold")) if fell_back else None}
278
+ wrong = [f"{key} {written[key]!r} where it settles {settled[key]!r}"
279
+ for key in settled if written[key] != settled[key]]
280
+ if wrong:
281
+ raise ContextError(f"--report {report} disagrees with the scoring output {score}: "
282
+ + "; ".join(wrong))
194
283
  active = scored.get("activeCategories", [])
195
284
  inconclusive = scored.get("inconclusiveReasons") if verdict == "INCONCLUSIVE" else None
196
285
 
@@ -216,7 +305,7 @@ def build(report: str, ledger: str, score: str | None = None, health_check: bool
216
305
  summary["hardGate"] = front.get("hardGate") or None
217
306
 
218
307
  contract = {"skill": WORKFLOW, **envelope,
219
- "outputs": [{"type": "report", "path": report}],
308
+ "outputs": [{"type": "report", "path": report_path}],
220
309
  "summary": summary,
221
310
  "runId": front.get("runId") or None,
222
311
  "healthCheckDispatched": health_check}
@@ -233,9 +322,12 @@ def _build_parser() -> argparse.ArgumentParser:
233
322
  description=("Build the shared emitter's payload for a finished test-skill run from its "
234
323
  "report frontmatter, its gap ledger and the scoring output."),
235
324
  )
236
- parser.add_argument("--report", required=True, help="the test report, at its published name")
325
+ parser.add_argument("--report", required=True, help="the test report to read")
326
+ parser.add_argument("--report-path",
327
+ help="the path the payload names the report by: its public name (default: --report)")
237
328
  parser.add_argument("--ledger", required=True, help="the run's gap ledger")
238
- parser.add_argument("--score", help="the scoring script's output (absent for a blocked run)")
329
+ parser.add_argument("--score", help="the scoring script's output (required unless the hard gate blocked "
330
+ "the run)")
239
331
  parser.add_argument("--no-health-check", action="store_true",
240
332
  help="the run was given --no-health-check (healthCheckDispatched false)")
241
333
  parser.add_argument("--warning", action="append", default=[],
@@ -247,13 +339,14 @@ def _build_parser() -> argparse.ArgumentParser:
247
339
  def main(argv: list[str] | None = None) -> int:
248
340
  args = _build_parser().parse_args(argv)
249
341
  try:
250
- payload = build(args.report, args.ledger, args.score, not args.no_health_check, args.warning)
342
+ payload = build(args.report, args.ledger, args.score, not args.no_health_check, args.warning,
343
+ args.report_path)
251
344
  out = Path(args.output)
252
345
  out.parent.mkdir(parents=True, exist_ok=True)
253
346
  out.write_bytes((json.dumps(payload, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
254
347
  except (ContextError, OSError) as exc:
255
348
  print(json.dumps({"status": "error", "error": str(exc)}))
256
- return 1
349
+ return 2 if isinstance(exc, UsageError) else 1
257
350
  blocked = payload["halt_reason"] is not None
258
351
  print(json.dumps({
259
352
  "status": payload["status"],