bmad-module-skill-forge 3.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +3 -3
- package/docs/_data/pinned.yaml +19 -13
- package/docs/architecture.md +2 -2
- package/docs/concepts.md +4 -4
- package/docs/examples.md +33 -26
- package/docs/getting-started.md +3 -3
- package/docs/how-it-works.md +2 -2
- package/docs/index.md +1 -1
- package/docs/skill-model.md +41 -39
- package/docs/troubleshooting.md +8 -0
- package/docs/verifying-a-skill.md +46 -21
- package/docs/why-skf.md +1 -1
- package/docs/workflows.md +5 -5
- package/package.json +1 -1
- package/src/knowledge/qmd-registry.md +1 -1
- package/src/knowledge/version-paths.md +1 -1
- package/src/shared/health-check.md +2 -50
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +2 -2
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +12 -4
- package/src/shared/scripts/schemas/skill-brief.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +212 -59
- package/src/shared/scripts/skf-build-change-manifest.py +1437 -178
- package/src/shared/scripts/skf-classify-changed-files.py +78 -0
- package/src/shared/scripts/skf-compare-file-hashes.py +266 -104
- package/src/shared/scripts/skf-count-tokens.py +54 -10
- package/src/shared/scripts/skf-derive-assembly-shape.py +18 -1
- package/src/shared/scripts/skf-detect-scripts-assets.py +196 -54
- package/src/shared/scripts/skf-detect-workspaces.py +67 -0
- package/src/shared/scripts/skf-emit-result-envelope.py +6 -5
- package/src/shared/scripts/skf-extract-public-api.py +1673 -189
- package/src/shared/scripts/skf-extraction-inventory.py +584 -46
- package/src/shared/scripts/skf-extraction-snapshot.py +158 -33
- package/src/shared/scripts/skf-forge-tier-rw.py +26 -3
- package/src/shared/scripts/skf-github-probe.py +47 -8
- package/src/shared/scripts/skf-hash-content.py +14 -8
- package/src/shared/scripts/skf-load-provenance.py +3 -1
- package/src/shared/scripts/skf-merge-doc-urls.py +142 -19
- package/src/shared/scripts/skf-parse-gaps.py +135 -26
- package/src/shared/scripts/skf-preflight.py +15 -1
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +25 -1
- package/src/shared/scripts/skf-render-metadata-stats.py +1 -1
- package/src/shared/scripts/skf-resolve-authoritative-files.py +29 -3
- package/src/shared/scripts/skf-resolve-customization.py +197 -0
- package/src/shared/scripts/skf-shard-body.py +239 -41
- package/src/shared/scripts/skf-structural-diff.py +133 -18
- package/src/shared/scripts/skf-update-active-symlink.py +115 -62
- package/src/shared/scripts/skf-validate-brief-schema.py +19 -1
- package/src/shared/scripts/skf-validate-output.py +12 -3
- package/src/shared/scripts/skf-verify-provenance-completeness.py +1439 -79
- package/src/shared/scripts/skf-write-skill-brief.py +41 -8
- package/src/shared/tool-requirements.yaml +8 -2
- package/src/skf-analyze-source/SKILL.md +2 -2
- package/src/skf-analyze-source/assets/skill-brief-schema.md +1 -1
- package/src/skf-analyze-source/references/map-unit-exports.md +1 -1
- package/src/skf-audit-skill/SKILL.md +2 -2
- package/src/skf-audit-skill/assets/drift-report-template.md +1 -0
- package/src/skf-audit-skill/references/headless-contract.md +1 -1
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +2 -1
- package/src/skf-audit-skill/references/re-index.md +14 -9
- package/src/skf-audit-skill/references/report.md +2 -2
- package/src/skf-audit-skill/references/semantic-diff.md +15 -15
- package/src/skf-audit-skill/references/structural-diff.md +10 -5
- package/src/skf-audit-skill/scripts/render-drift-tables.py +29 -2
- package/src/skf-brief-skill/SKILL.md +3 -3
- package/src/skf-brief-skill/assets/scope-templates.md +1 -1
- package/src/skf-brief-skill/assets/skill-brief-schema.md +4 -11
- package/src/skf-brief-skill/references/analyze-target.md +6 -6
- package/src/skf-brief-skill/references/confirm-brief.md +1 -1
- package/src/skf-brief-skill/references/scope-definition.md +4 -4
- package/src/skf-brief-skill/references/step-auto-brief.md +11 -29
- package/src/skf-brief-skill/references/version-resolution.md +5 -9
- package/src/skf-campaign/SKILL.md +2 -2
- package/src/skf-campaign/references/step-05-skill-loop.md +6 -1
- package/src/skf-create-skill/SKILL.md +2 -2
- package/src/skf-create-skill/assets/compile-assembly-rules.md +5 -3
- package/src/skf-create-skill/assets/skill-sections.md +7 -7
- package/src/skf-create-skill/references/authoritative-files-protocol.md +2 -23
- package/src/skf-create-skill/references/compile.md +31 -60
- package/src/skf-create-skill/references/component-extraction.md +3 -3
- package/src/skf-create-skill/references/enrich.md +2 -2
- package/src/skf-create-skill/references/entry-points-by-hand.md +24 -7
- package/src/skf-create-skill/references/extract.md +23 -19
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +9 -9
- package/src/skf-create-skill/references/extraction-patterns.md +10 -11
- package/src/skf-create-skill/references/generate-artifacts.md +1 -1
- package/src/skf-create-skill/references/report.md +21 -5
- package/src/skf-create-skill/references/source-resolution-protocols.md +12 -17
- package/src/skf-create-skill/references/sub/fetch-docs.md +1 -1
- package/src/skf-create-stack-skill/SKILL.md +2 -2
- package/src/skf-create-stack-skill/references/validate.md +1 -1
- package/src/skf-drop-skill/SKILL.md +2 -2
- package/src/skf-drop-skill/references/execute.md +5 -1
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +1 -1
- package/src/skf-drop-skill/references/report.md +2 -2
- package/src/skf-drop-skill/references/select.md +7 -3
- package/src/skf-export-skill/SKILL.md +2 -2
- package/src/skf-export-skill/assets/snippet-format.md +5 -5
- package/src/skf-export-skill/references/generate-snippet.md +22 -15
- package/src/skf-export-skill/references/load-skill.md +1 -1
- package/src/skf-export-skill/references/token-report.md +29 -9
- package/src/skf-forger/SKILL.md +2 -2
- package/src/skf-quick-skill/SKILL.md +2 -2
- package/src/skf-refine-architecture/SKILL.md +6 -8
- package/src/skf-rename-skill/SKILL.md +2 -2
- package/src/skf-setup/SKILL.md +2 -2
- package/src/skf-test-skill/SKILL.md +2 -2
- package/src/skf-test-skill/assets/output-section-formats.md +6 -4
- package/src/skf-test-skill/references/coherence-check.md +5 -4
- package/src/skf-test-skill/references/coverage-check-branches.md +61 -0
- package/src/skf-test-skill/references/coverage-check-tiers.md +5 -5
- package/src/skf-test-skill/references/coverage-check.md +69 -111
- package/src/skf-test-skill/references/init.md +5 -6
- package/src/skf-test-skill/references/migration-section-rules.md +26 -14
- package/src/skf-test-skill/references/report.md +11 -9
- package/src/skf-test-skill/references/scoring-rules.md +8 -6
- package/src/skf-test-skill/references/source-access-protocol.md +15 -18
- package/src/skf-test-skill/scripts/build-result-context.py +114 -21
- package/src/skf-test-skill/scripts/gap-ledger.py +580 -80
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +646 -107
- package/src/skf-test-skill/scripts/locate-export-segments.py +101 -20
- package/src/skf-test-skill/scripts/reconcile-coverage.py +450 -23
- package/src/skf-test-skill/scripts/score-signatures.py +356 -61
- package/src/skf-test-skill/scripts/stage-helper-payload.py +64 -13
- package/src/skf-test-skill/scripts/validate-inventory.py +108 -9
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +7 -5
- package/src/skf-test-skill/templates/test-report-template.md +1 -0
- package/src/skf-update-skill/SKILL.md +2 -2
- package/src/skf-update-skill/references/detect-changes.md +32 -13
- package/src/skf-update-skill/references/gap-driven.md +37 -22
- package/src/skf-update-skill/references/init.md +6 -2
- package/src/skf-update-skill/references/invocation-contract.md +1 -1
- package/src/skf-update-skill/references/merge.md +17 -12
- package/src/skf-update-skill/references/re-extract.md +4 -2
- package/src/skf-update-skill/references/report.md +7 -4
- package/src/skf-update-skill/references/write.md +77 -36
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +157 -42
- package/src/skf-verify-stack/SKILL.md +2 -2
- package/tools/cli/lib/tool-check.js +158 -2
|
@@ -65,7 +65,7 @@ Load `{outputFormatsFile}` for the Ledger Record Format and the remediation qual
|
|
|
65
65
|
Every run that reaches this step, a blocked one included, ends through one sequence, and a check that fails stops it before anything is published:
|
|
66
66
|
|
|
67
67
|
1. §4b records the discovery outcome in the ledger (a blocked run skips it).
|
|
68
|
-
2. §4c renders the Gap Report from the ledger, checks that every stage ran and wrote its section and that the health check is installed, then publishes the report, writes the result contract and runs the on_complete hook.
|
|
68
|
+
2. §4c renders the Gap Report from the ledger, checks that every stage ran and wrote its section and that the health check is installed, then builds the result payload, publishes the report, writes the result contract and runs the on_complete hook.
|
|
69
69
|
3. §6 presents the result, §6b keeps the headless envelope for the run's last line, and §7 releases the run lock and hands over to the health check.
|
|
70
70
|
|
|
71
71
|
### 4b. Discovery Testing
|
|
@@ -213,27 +213,29 @@ uv run {runLockHelper} acquire --lock "{forge_version}/.test-skill.lock" --owner
|
|
|
213
213
|
|
|
214
214
|
The run lock renewed here stays held until §7 releases it, so no other run publishes over this run's report or result files meanwhile.
|
|
215
215
|
|
|
216
|
-
**
|
|
216
|
+
**Build the result payload** before the report takes its public name. Every value in it comes from the run's own records: `{resultContextScript}` (it resolves relative to the skill root) reads the report frontmatter, the gap ledger and the scoring output, checks the verdict fields score.md §7 wrote (`testResult`, `score`, `threshold` and the fallback fields) against the scoring output, derives `verdict`, `exit_code` and `next_workflow` from `testResult`, and writes the shared emitter's payload, which names the report by its public name. Pass `--score` for a scored run (a run the hard gate blocked has no scoring output), `--no-health-check` when that flag was given, and `--warning` when SKILL.md On Activation step 3 kept a `{customization_resolver_unavailable}` reason: first write `customization_resolver_unavailable: {customization_resolver_unavailable}` to `{run_dir}/resolver-warning.txt` with a file write, never `echo` (the reason can hold quotes, backticks or `$( )`), and pass that file's text (the emitter adds the warnings a stage recorded in the run's sink):
|
|
217
217
|
|
|
218
218
|
```bash
|
|
219
|
-
|
|
219
|
+
uv run {resultContextScript} --report "{report_file}" --report-path "{publishedReportFile}" --ledger "{ledgerFile}" [--score "{run_dir}/score.json"] [--no-health-check] [--warning "$(cat "{run_dir}/resolver-warning.txt")"] --output "{run_dir}/result-context.json"
|
|
220
220
|
```
|
|
221
221
|
|
|
222
|
-
|
|
222
|
+
Bind `{emit_target}` ← `target` from its JSON (`stderr` for a run the hard gate blocked, else `stdout`). Exit 2: correct the command and run it again. Exit 1: HALT with its `error` (`halt_reason: "helper-failed"`, phase `report:result-contract`); the report keeps its in-progress name, so export-skill never reads a verdict the scoring output did not settle.
|
|
223
223
|
|
|
224
|
-
**
|
|
224
|
+
**Publish the report.** Set `health_check_dispatched`, already in the `{outputFile}` frontmatter: `false` when `no_health_check` is true (the `--no-health-check` flag), else `true`, since the health check now runs with no menu before it. The checks passed, so the report takes its public name, the one export-skill and update-skill read, a blocked run's included. From `{project-root}`, run:
|
|
225
225
|
|
|
226
226
|
```bash
|
|
227
|
-
|
|
227
|
+
mv "{report_file}" "{publishedReportFile}"
|
|
228
228
|
```
|
|
229
229
|
|
|
230
|
-
|
|
230
|
+
and bind `{report_file}` ← `{publishedReportFile}`. If the move fails, HALT (`halt_reason: "write-failed"`, phase `report:publish`, `"path": "{publishedReportFile}"`): no result file has been written, so the previous run's verdict still stands.
|
|
231
|
+
|
|
232
|
+
**Write the result contract.** The shared emitter writes it, per `{outputContractSchema}`. In every mode, run:
|
|
231
233
|
|
|
232
234
|
```bash
|
|
233
235
|
uv run {emitEnvelopeHelper} emit --workflow skf-test-skill --run-dir "{run_dir}" --result-dir "{forge_version}" --target {emit_target} < "{run_dir}/result-context.json"
|
|
234
236
|
```
|
|
235
237
|
|
|
236
|
-
It writes `{forge_version}/skf-test-skill-result-{YYYYMMDD-HHmmss}.json` (UTC; it picks the name) and then its `skf-test-skill-result-latest.json` copy (stable path for pipeline consumers: a copy, not a symlink), each atomically, so a partial write is never observable; stamps the timestamp, `run_id`, `headless_decisions` and `warnings` into both; and prints the `SKF_TEST_RESULT_JSON:` line. Bind `{result_line}` ← that line and `{result_path}` ← its `result_path`. A run the hard gate blocked writes the record too, as a FAIL (`status: "error"`, `verdict: "FAIL"`, `halt_reason: "hard-gate-blocked"`, `exit_code` 2), so `skf-test-skill-result-latest.json` never holds an earlier run's verdict. A result file that could not be written leaves `result_path` null and a `result_file_write_failed` warning in the line, and the run still finishes. If the emitter exits non-zero, run
|
|
238
|
+
It writes `{forge_version}/skf-test-skill-result-{YYYYMMDD-HHmmss}.json` (UTC; it picks the name) and then its `skf-test-skill-result-latest.json` copy (stable path for pipeline consumers: a copy, not a symlink), each atomically, so a partial write is never observable; stamps the timestamp, `run_id`, `headless_decisions` and `warnings` into both; and prints the `SKF_TEST_RESULT_JSON:` line. Bind `{result_line}` ← that line and `{result_path}` ← its `result_path`. A run the hard gate blocked writes the record too, as a FAIL (`status: "error"`, `verdict: "FAIL"`, `halt_reason: "hard-gate-blocked"`, `exit_code` 2), so `skf-test-skill-result-latest.json` never holds an earlier run's verdict. A result file that could not be written leaves `result_path` null and a `result_file_write_failed` warning in the line, and the run still finishes. If the emitter exits non-zero, run it once more; if it fails again, HALT (`halt_reason: "write-failed"`, phase `report:result-contract`) with its stderr: the run's result contract could not be written. `references/invocation-contract.md` lists the record's fields.
|
|
237
239
|
|
|
238
240
|
Once the result files are written, append `'report'` to `stepsCompleted` in the `{outputFile}` frontmatter (it stays out of the expected set above).
|
|
239
241
|
|
|
@@ -287,7 +289,7 @@ The report is final once §4c published it and wrote the result files: no later
|
|
|
287
289
|
**update-skill** — The skill scored above threshold, but `--allow-workspace-drift` was in effect: the test ran against workspace HEAD, not `metadata.source_commit`. A conditional PASS is not trustworthy enough to export. Re-sync the source tree to the pinned commit (or re-extract against current HEAD) and re-run test-skill without the drift override before exporting.
|
|
288
290
|
|
|
289
291
|
{IF FAIL after scoring:}
|
|
290
|
-
**update-skill `--from-test-report`**: This skill needs remediation. Review the Gap Report above, then run `@Ferris US {skill_name} --from-test-report`: Update Skill
|
|
292
|
+
**update-skill `--from-test-report`**: This skill needs remediation. Review the Gap Report above, then run `@Ferris US {skill_name} --from-test-report`: Update Skill repairs the gaps it routes, at the skill's pinned commit, and lists each other gap as not repaired, with the reason. A `stale-documentation` or `metadata-drift` gap needs what its remediation names: a person, Create Skill or the skill brief. Test it again afterwards.
|
|
291
293
|
|
|
292
294
|
{IF the hard gate blocked the run:}
|
|
293
295
|
**update-skill `--from-test-report`**: The hard gate stopped this run before scoring on {blocking_gaps} Critical or High gap(s), listed first in the Gap Report. Run `@Ferris US {skill_name} --from-test-report` to repair them, then test again.
|
|
@@ -18,32 +18,34 @@ This table is the single source of truth for classifying a gap. Every stage reco
|
|
|
18
18
|
|
|
19
19
|
| Severity | Category | Criteria |
|
|
20
20
|
|----------|-----------------------------|----------|
|
|
21
|
-
| Critical | `signature-mismatch` | Wrong signature: a documented signature that differs from the source in its parameters (name, type, order, optionality) or its return type (coverage-check §2) |
|
|
21
|
+
| Critical | `signature-mismatch` | Wrong signature: a documented signature that differs from the source in its parameters (name, type, order, optionality) or its return type, in any way but the two the Low `signature-mismatch` row and the Info `observation` row grade, such as a renamed Python parameter (the script cannot see which parameters a caller must pass by position), an optional parameter left out of a form the skill does not label partial, or a mismatch with no `difference` (coverage-check §2 and §2b) |
|
|
22
22
|
| Critical | `fabricated-signature` | Fabricated signature: a documented export absent at the source line the skill cites for it. Decided only where ast-grep enumerated the exports at the pinned commit: the name is outside that surface, and the cited file is missing or defines no such name (coverage-check §2c) |
|
|
23
23
|
| Critical | `broken-reference` | Broken reference (contextual mode): a reference whose target does not exist, such as a file not at the stated path, a skill not in the skills output folder, a type no module declares, or a `scripts/` or `assets/` file the skill names (coherence-check §4) |
|
|
24
24
|
| High | `inaccurate-reference` | Inaccurate reference (contextual mode): the target exists but does not match, such as a type the cited module does not export, a signature that differs, or an integration pattern that does not match the implementation (coherence-check §4) |
|
|
25
25
|
| High | `reference-escape` | A reference whose real path leaves the skill, its source and, for a stack, the skills output folder (coherence-check §4) |
|
|
26
26
|
| High | `split-body-mismatch` | The SKILL.md body and a `references/` file document one export differently (coverage-check §1b) |
|
|
27
|
-
| High | `numerator-inflation` | `stats.exports_documented
|
|
27
|
+
| High | `numerator-inflation` | A declared count (`stats.exports_documented`, or the provenance map's distinct entry names) equals the denominator while declared exports are absent from SKILL.md and `references/` (coverage-check §4b) |
|
|
28
28
|
| High | `structural` | Naive mode: a missing required section, an unbalanced code fence, or a documented async or sync behaviour its example contradicts (coherence-check §2.1, §2.2, §2.5) |
|
|
29
29
|
| High | `discovery` | Discovery test: at most 1 of 3 realistic prompts routed to the skill (report §4b.3) |
|
|
30
30
|
| Medium | `missing-export` | Missing export: an exported function or class the skill does not document. It lowers Export Coverage, and the threshold decides the verdict |
|
|
31
31
|
| Medium | `missing-type` | Missing type or interface documentation |
|
|
32
|
-
| Medium | `stale-documentation` | A documented export the enumerated source surface lacks that is not a fabricated signature:
|
|
32
|
+
| Medium | `stale-documentation` | A documented export the enumerated source surface lacks that is not a fabricated signature and not a documented extra: classify-stale found no single file that declares it (an import, a re-export or an indented local does not count, nor does a module so named when the inventory gives the name a `function`, `hook`, `class`, `interface`, `enum` or `type` kind, as the Info `observation` row says) and the skill's citations on the lines that name it cite no single one of the files that do, it is a dotted name, the brief scoped its file out (`excluded.outsideScope`), or the run could not look: the source is not Python or TS/JS, classify-stale did not run (it needs the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map), or `surface.json` has no complete `extraction` (coverage-check §2c) |
|
|
33
33
|
| Medium | `migration-section` | Migration section present/absent mismatch with T2-future annotation data (Deep tier; Case 1 and Case 3 of `migration-section-rules.md`) |
|
|
34
|
-
| Medium | `metadata-drift` | Metadata drift: intra-cluster export counts diverge by more than 10% (barrel: `stats.exports_public_api` vs `exports[].length`; documented surface: `stats.exports_documented` vs the provenance-map named-export count), or denominator deflation: the re-derived source barrel exceeds `effective_denominator` by more than 25% and the brief has no `scope.tier_a_include` (coverage-check §4 and §4b) |
|
|
35
|
-
| Medium | `denominator-inflation` | Denominator inflation: the stratified-scope `scope.include` union exceeds the provenance-map entry count by more than 25% (brief
|
|
34
|
+
| Medium | `metadata-drift` | Metadata drift: intra-cluster export counts diverge by more than 10% (barrel: `stats.exports_public_api` vs `exports[].length`; documented surface: `stats.exports_documented` vs the provenance-map named-export count), or denominator deflation: the re-derived source barrel exceeds `effective_denominator` by more than 25% and the brief has no `scope.tier_a_include` glob that counts a public name, or denominator drift: on a stratified-scope, specific-modules or multi-entry skill, the brief's `scope.tier_a_include` set differs from `effective_denominator` by more than 25% either way, or counts a name against an `effective_denominator` of 0, a stale count to recount (coverage-check §4 and §4b) |
|
|
35
|
+
| Medium | `denominator-inflation` | Denominator inflation: the stratified-scope `scope.include` union exceeds the provenance-map entry count by more than 25% (the brief has no `scope.tier_a_include` glob that counts a public name). On a stratified-scope or multi-entry barrel over that union, the names it counts and the skill does not document are one more such gap beside it at the brief, listing at most 50 and deferring to its fix, instead of a `missing-export` gap each; Export Coverage still counts them |
|
|
36
|
+
| Medium | `brief-scope-stale` | Stale brief scope: a `scope.include` glob matches no file in the source tested, so the surface restores the root exports defined in a file no glob covers that the brief does not exclude, or a `scope.tier_a_include` glob matches no file in the source tested, so it counts no name (coverage-check §4) |
|
|
36
37
|
| Medium | `scripts-assets` | A `scripts/` or `assets/` directory exists but SKILL.md has no Scripts & Assets section (coherence-check §2.7 and §4) |
|
|
37
38
|
| Medium | `integration-pattern` | Incomplete integration pattern (contextual mode): one entry of coherence-check §5's `incomplete_patterns` |
|
|
38
39
|
| Medium | `structural` | Naive mode: an opening code fence without a language tag, an exported function or method no usage-family section or reference file names, or a table row whose column count differs from its header (coherence-check §2.3, §2.4, §2.6) |
|
|
39
40
|
| Medium | `discovery` | Discovery test: 2 of 3 realistic prompts routed to the skill (report §4b.3) |
|
|
41
|
+
| Low | `signature-mismatch` | A renamed parameter callers cannot see: a mismatch the signature judge tags `parameter-name` (only parameter names differ; types, order, optionality, defaults and return type are equal) at the file the surface records for the export, the one the plan sent the check to or a candidate the judge compared with, when it is a `.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`, `.go`, `.rs`, `.java`, `.c`, `.h`, `.cc`, `.cpp`, `.cxx` or `.hpp` file, a language with no keyword arguments (not a Vue component, whose props callers set by name), as score-signatures.py checks. It counts as matching in Signature Accuracy (coverage-check §2 and §2b) |
|
|
40
42
|
| Low | `scripts-assets-provenance` | Script/asset file present without provenance entry in provenance-map.json file_entries |
|
|
41
43
|
| Low | `provenance-line` | Provenance line is not the definition of an export (`line-not-definition`, coverage-check §4c) |
|
|
42
44
|
| Low | `migration-section` | Case 3 of `migration-section-rules.md` that the reviewer downgraded with an inline justification |
|
|
43
45
|
| Low | `metadata` | Missing optional metadata or examples |
|
|
44
46
|
| Low | `description` | Description trigger optimization recommended (third-person voice, negative triggers, or keyword coverage gaps), a Tessl Review description suggestion included |
|
|
45
47
|
| Low | `external-validator` | A skill-check diagnostic or a Tessl Review validation finding the run left unresolved (external-validators §5b) |
|
|
46
|
-
| Info | `observation` | Style suggestions, non-blocking observations |
|
|
48
|
+
| Info | `observation` | Style suggestions, non-blocking observations, and a documented extra: a documented name outside the enumerated surface that is not a fabricated signature and that the source still declares at column 0 (a module so named counts, unless the inventory gives the name a `function`, `hook`, `class`, `interface`, `enum` or `type` kind, no provenance map entry of the name types it a `module` or `package` and the module is not a TS/JS file that is itself the export, with an `export default` or a `module.exports =`; among several declaring files, the one the skill cites on a line naming it), found only in a Python or TS/JS source where classify-stale ran (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and `surface.json` has a complete `extraction`: `Documented extra: {name}`, at that line, which update-skill never routes (coverage-check §2c). Also a partial signature: a mismatch the signature judge tags `omitted-optional` (the documented form leaves out only members the source marks optional) on a form the skill labels partial (simplified, key parameters, a `...` that stands in for omitted members, not rest or variadic syntax), which the inventory marks `signature_form: "partial"`, as score-signatures.py checks: `Partial signature: {name}`, at the declaration, counted as matching in Signature Accuracy (coverage-check §2 and §2b) |
|
|
47
49
|
| Info | `provenance-unverified` | Provenance line not verified: the line-check rules found no definition line (coverage-check §4c) |
|
|
48
50
|
| Info | `migration-section` | Historical migration content in Section 4b with no T2-future annotation (Case 2 of `migration-section-rules.md`) |
|
|
49
51
|
| Info | `multi-denominator` | Multi-denominator reporting: barrel and documented-surface clusters diverge by design (>10% cross-cluster) |
|
|
@@ -6,43 +6,40 @@
|
|
|
6
6
|
|
|
7
7
|
**Source API surface** = the package's top-level public exports. These are the symbols reachable from the primary entry point without importing internal modules:
|
|
8
8
|
|
|
9
|
-
- **Python:** symbols exported in `__init__.py` (including re-exports)
|
|
9
|
+
- **Python:** symbols exported in the package root's `__init__.py` (including re-exports), excluding private (`_prefixed`) names. A sub-package behind a folder with no `__init__.py` (a FastAPI `routers/__init__.py`, say) is not a second package root when the folder of another package root that exports a name holds it, unless `scope.include` or `scope.tier_a_include` lists that `__init__.py`: its names stay out of the `all` and `root` sets (`excluded.nestedEntries`), while the `scope.include` and `tier_a_include` sets still count them and their signatures are still compared
|
|
10
10
|
- **TypeScript/JavaScript:** named exports from `index.ts` / `index.js` — exclude unexported locals
|
|
11
11
|
- **Go:** exported identifiers (capitalized) from the package's public-facing files
|
|
12
12
|
- **Rust:** items in `pub use` from `lib.rs` or `mod.rs`
|
|
13
13
|
|
|
14
|
-
The clauses below decide which part of that surface is the coverage denominator.
|
|
14
|
+
The clauses below decide which part of that surface is the coverage denominator. `load-coverage-inputs.py surface` picks the clause from the brief (`{forge_data_folder}/{skill_name}/skill-brief.yaml`), `{source_is_monorepo}` (coverage-check.md §0b, passed as `--monorepo`), the extraction and `metadata.json`, and writes it to `surface.json` `denominator` with the branch, the name set, the rung it takes (coverage-check.md §2b) and the entry points' state it read (`barrel`), beside the counts each clause names (`sets`, `candidates` and `guards`). A workspace member counts toward `{source_is_monorepo}` only through the files the brief keeps: excluding its source files drops it, excluding only its manifest does not, and a workspace with no member left is no workspace. Three of its inputs are yours to pass as flags, the last two only where no file gives them:
|
|
15
15
|
|
|
16
|
-
- **
|
|
16
|
+
- **Pattern-reference intent** (`--pattern-reference-intent`): `scope.notes` says the skill demonstrates an integration pattern rather than a library API ("Reference app, not a library", "pattern-reference", "embedded-pattern skill", "skill value is the … pattern"; the author's words are authoritative).
|
|
17
|
+
- **A curated monorepo subset** (`--curated-subset`), where the extraction has no `counts.curated_subset` (Quick tier, the fallback per-file scan, States 2 to 4): `{source_is_monorepo}` is true and the brief's `scope.include` lists a curated subset rather than the full workspace.
|
|
18
|
+
- **The entry points' state** (`--barrel`), where no complete extraction or quick output gives it (the fallback per-file scan when the runner listed entry points, Quick tier read by eye, State 4): `none` when no barrel file is at any recognized entry-point path (`__init__.py`, `index.ts`/`index.js`, `lib.rs`, `mod.rs`), `empty` when the primary entry point exports nothing (`export {};`, an empty `__init__.py`, a `lib.rs` with no `pub use`), else `barrel`.
|
|
17
19
|
|
|
18
|
-
-
|
|
20
|
+
The recipe runner already re-derives each barrel (multi-line re-export blocks, star re-exports and a Rust `pub use` chain included) and counts each name once, however many in-scope files define it, so a barrel that only re-exports adds nothing of its own. Each of those files gives the name a surface record of its own (the runner's `entry_point_diff.other_definitions` beside its `public` row), so a set holds the name when any of its files matches, and the signature plan can compare the declaration the skill cites. Without the runner's names (Quick tier, the fallback per-file scan, States 2 to 4) the script builds the `scope.include` and `tier_a_include` sets from the files the names carry and the brief's globs (the runner's JSON the fallback scan passes adds only its scope and stale globs), and there is no `subpaths` or `root` set (coverage-check.md §2b). Never re-count a barrel by hand. A `scope.tier_a_include` glob counts a name only in a file a `scope.include` glob also matches, and one that matches no file (`guards.staleScope.unmatchedTierAInclude`, a brief older than the source) counts none: when the tier A globs count no name, `surface.json` has no `tier_a_include` set. Each clause, by its `denominator.clause`, with what stays yours:
|
|
19
21
|
|
|
20
|
-
|
|
21
|
-
2. Otherwise the `tier_a_include` set when the brief has `scope.tier_a_include`, the narrow list of the surface the brief intends to document.
|
|
22
|
-
3. Otherwise the `scope.include` set, **subject to the inflation guard**: `guards.inflation` compares it with the provenance-map entry count, and when `guards.inflation.fires` (above `guards.thresholds.inflationPct` percent) the coarse globs sweep in internal infrastructure the brief did not mean to document, a Medium `denominator-inflation` gap pointing at `scope.tier_a_include`. When `guards.umbrella.umbrella` is true (the root barrel is mostly re-exports), recommend `stats.effective_denominator` as the remediation instead.
|
|
22
|
+
- **Empty-barrel packages (`empty-barrel`; copy-paste or subpath-only distribution):** the primary entry point is empty, so the denominator is the `scope.include` set. The brief's `scope.notes` should say the distribution model is by design; when it does, treat the empty barrel as intended rather than a bug. With no brief and an empty barrel, set `analysis_confidence: docs-only` and report that the source API surface could not be determined. Annotate: `Empty barrel, denominator: scope.include union ({N} exports)`.
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
- **Stratified-scope monorepo packages (`stratified-scope`; curated subsets of multi-package repos):** the denominator reflects only the authored surface, not the monorepo's global export count: the `stats.effective_denominator` Create Skill stored, unless `guards.deflation.fires` (a Medium `metadata-drift` gap, coverage-check.md §4: a legitimate scope reduction goes in the brief, through `scope.tier_a_include` or `scope.exclude`, never into `metadata.stats`), then the `tier_a_include` set when the guard measured the stored count against it (`guards.deflation.basis` `tier_a_include`, in either direction: a stale count, which re-running Create Skill recounts, storing the runner's tier A count when `effective_denominator_decision.store` holds and `scope.notes` documents the stratification and otherwise dropping the stored count; on an incomplete extraction a set below the stored count leaves the stored count), else the `scope.include` set; with no stored count, the `tier_a_include` and `scope.include` sets. Never set `stats.effective_denominator` by hand: Update Skill's recount drops it. With no surface read from the source (`guards.deflation.applicable` false), annotate the report `effective_denominator used unverified: no source access to re-derive the barrel`. When `guards.inflation.fires` (the coarse globs sweep in internal infrastructure, a Medium `denominator-inflation` gap), the `scope.include` union is itself inflated: when the deflation guard fires too, that union stays the denominator and both gaps ask for a `scope.tier_a_include` list of the files that define the authored names (when `guards.umbrella.umbrella` is true, those files rather than the barrel). When the barrel branch counts that union, here or on the multi-entry ladder, the names it counts and the skill does not document are one more `denominator-inflation` gap beside the guard's at the brief, which defers to the guard's fix, not a `missing-export` gap each. Leave `analysis_confidence` unchanged. Annotate the report: `Stratified scope, denominator: {effective_denominator | tier_a_include union | scope.include union} ({N} files matched, {M} exports union)`. `scope.notes` should document the stratification (for example "Tier A: fully documented; Tier B: deferred to references; Tier C: excluded"); when it does, treat the curated subset as intended, not a scope gap.
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
- **Standard barrel (`standard-barrel`, the `all` set):** `full-library` skills, single-package repositories and briefs where the full monorepo is intentionally in scope. A `scope.include` glob that matches no file (a brief written for an older layout of the source) does not shrink that barrel: `guards.staleScope` restores the root exports defined in a file no glob covers, unless `scope.exclude` or a `scope-expansion` amendment whose latest action is `skipped` or `demoted-include` matches the file. The stale glob is a Medium `brief-scope-stale` gap (coverage-check.md §4), which also names each `scope.tier_a_include` glob that matches no file.
|
|
27
27
|
|
|
28
|
-
- **Pattern-reference apps (non-library source):** a single-package repo that demonstrates an integration pattern rather than distributing a library API.
|
|
28
|
+
- **Pattern-reference apps (`pattern-reference`; non-library source):** a single-package repo that demonstrates an integration pattern rather than distributing a library API. The denominator is the `tier_a_include` set when `surface.json` has it, else the canonicalized provenance map, since `skf-create-skill`'s extraction already curated the map to the authored pattern surface. Leave `analysis_confidence` unchanged. Annotate: `Pattern-reference, denominator: {tier_a_include union | canonicalized provenance-map count} ({N} pattern surfaces)`.
|
|
29
29
|
|
|
30
|
-
- **Single-crate curated subset (`
|
|
30
|
+
- **Single-crate curated subset (`specific-modules`):** a single-package repo whose brief carves a subset of the crate's public surface with `scope.include` and `scope.exclude`: the in-scope reachable barrel, or the `stats.effective_denominator` Create Skill stored for it, under the same deflation guard. Annotate: `Specific-modules subset, denominator: in-scope reachable barrel ({N} items from {M} modules, after scope.include/exclude filtering)`.
|
|
31
31
|
|
|
32
|
-
- **Multi-entry (exports-map) packages:** an in-scope `package.json` declares an `exports` map with more than one non-root subpath
|
|
32
|
+
- **Multi-entry (exports-map) packages (`multi-entry`):** an in-scope `package.json` declares an `exports` map with more than one non-root subpath (`extraction.arms.multi_subpath_exports`). Installers reach every published subpath, so the denominator is the `subpaths` set (wildcard subpaths are left out: their surface is unbounded), or a stored count or `tier_a_include` set ahead of it, the stored count under the same deflation guard, whose fire counts the `tier_a_include` set when the surface has it, else the `scope.include` set. The root-barrel count (`candidates.rootBarrel`) is the secondary candidate in the Denominator Candidates block coverage-check.md §4 appends from coverage-check-branches.md. Annotate: `Multi-entry (exports-map), denominator: {effective_denominator | tier_a_include union | scope.include union | subpath union} ({N} subpaths resolved, {M} exports union; root barrel: {R})`.
|
|
33
33
|
|
|
34
|
-
Internal module symbols are **excluded** from the coverage denominator unless they are explicitly documented in SKILL.md (in which case they count as documented extras, not missing coverage): the surface holds only the names the entry points export, and the extraction's `internal` names never reach it.
|
|
34
|
+
Internal module symbols are **excluded** from the coverage denominator unless they are explicitly documented in SKILL.md (in which case they count as documented extras, not missing coverage): the surface holds only the names the entry points export, and the extraction's `internal` names never reach it. A documented extra reaches coverage-check.md §2c as a stale name. In a Python or TS/JS source, on a run where §2c runs `classify-stale` (the Forge, Forge+ or Deep tier, `analysis_confidence` `full`, `workspaceDrift` not `overridden` and a bound provenance map) and whose `surface.json` came from a complete extraction, it is an Info `observation` at the line where `classify-stale` finds the source still declares it (`defined_at`, or among several declaring files the one the skill cites on a line naming it), which update-skill never routes. Otherwise, and for a name a brief scopes out (`excluded.outsideScope`), it stays a Medium `stale-documentation` gap. When the brief is what scopes the file out (a `scope.exclude` glob matches it, or no `scope.include` glob covers it), the gap names that file and line and the glob: to keep the documentation, narrow the exclude glob (removing it takes a `demoted-exclude` amendment) or widen `scope.include`; else remove that documentation.
|
|
35
35
|
|
|
36
36
|
This matches the extraction-patterns.md convention used during skill creation: coverage measures how well SKILL.md documents what users actually import, not the entire internal codebase.
|
|
37
37
|
|
|
38
38
|
### Provenance-map canonicalization
|
|
39
39
|
|
|
40
|
-
A provenance map may hold **bookkeeping variants** of one export, artifacts of how the library structures its registry (Storybook's component-plus-story decomposition, accessibility renderer shadowing, exact-match renderer disambiguation). Counting them as separate exports inflates the denominator with names that are structural duplicates of a documented base export. When a clause takes its denominator from the provenance map, pass `--fold` to `surface`: it folds `X_def`, `X_exact` and `a11y_X` back to `X`, and `<prefix>X` back to `X` for each renderer prefix you pass as `--fold-prefix` (`mobile_`, `web_`, `ssr_`, only folded when `X` is an entry too), and reports the fold summary in `canonical.summary` for the report.
|
|
40
|
+
A provenance map may hold **bookkeeping variants** of one export, artifacts of how the library structures its registry (Storybook's component-plus-story decomposition, accessibility renderer shadowing, exact-match renderer disambiguation). Counting them as separate exports inflates the denominator with names that are structural duplicates of a documented base export. When a clause takes its denominator from the provenance map, pass `--fold` to `surface`: it folds `X_def`, `X_exact` and `a11y_X` back to `X`, and `<prefix>X` back to `X` for each renderer prefix you pass as `--fold-prefix` (`mobile_`, `web_`, `ssr_`, only folded when `X` is an entry too), and reports the fold summary in `canonical.summary` for the report. Test-skill defines this fold convention: the base form is authoritative, and the variant is a sibling record.
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
- A variant the library's public surface genuinely distinguishes (an `a11y_Checkbox` documented and installed on its own) is a real export: pass it as `--keep`. Check SKILL.md for explicit documentation of the variant first; when in doubt, keep it and report both forms.
|
|
45
|
-
- When `stats.effective_denominator` is present and the raw entry count matches it, the map is already canonical: do not fold. When drift remains after folding (raw 222, canonical 215, `effective_denominator` 216), record the residual as an unexplained-reconciliation note; do not invent fold rules to close it.
|
|
42
|
+
A map whose raw count equals `stats.effective_denominator` is already canonical: `surface` folds nothing (`canonical.skipped`). A non-zero `canonical.residual` after folding (raw 222, canonical 215, `effective_denominator` 216) is an unexplained-reconciliation note; do not invent fold rules to close it. One judgment stays here: a variant the library's public surface genuinely distinguishes (an `a11y_Checkbox` documented and installed on its own) is a real export: pass it as `--keep`. Check SKILL.md for explicit documentation of the variant first; when in doubt, keep it and report both forms.
|
|
46
43
|
|
|
47
44
|
## Source Access Resolution
|
|
48
45
|
|
|
@@ -12,13 +12,25 @@ follows from records the run already wrote, so this script builds it rather
|
|
|
12
12
|
than the model retyping them from memory:
|
|
13
13
|
|
|
14
14
|
- the report frontmatter: skillName, runId, testMode, hardGate,
|
|
15
|
-
testResult, score, threshold, thresholdFallback,
|
|
16
|
-
evidenceReportPath (the steps write the verdict
|
|
17
|
-
compacted context cannot change them);
|
|
15
|
+
testResult, workspaceDrift, score, threshold, thresholdFallback,
|
|
16
|
+
originalThreshold and evidenceReportPath (the steps write the verdict
|
|
17
|
+
inputs there, so a compacted context cannot change them);
|
|
18
18
|
- the gap ledger: the gap counts by severity (gap-ledger.py's summary);
|
|
19
|
-
- the scoring script's output, which score.md saves:
|
|
20
|
-
|
|
21
|
-
has none.
|
|
19
|
+
- the scoring script's output, which score.md saves: the verdict, score
|
|
20
|
+
and threshold it settled, activeCategories and inconclusiveReasons. A
|
|
21
|
+
run the hard gate blocked was never scored and has none.
|
|
22
|
+
|
|
23
|
+
A scored run's verdict fields are checked against the scoring output
|
|
24
|
+
(`scored_fields`): testResult must be the verdict the output settled,
|
|
25
|
+
`effectiveResult` when the output has one, else `result`, in lower case,
|
|
26
|
+
with `pass` read as `pass-with-drift` when the frontmatter workspaceDrift
|
|
27
|
+
is `overridden`; score its `totalScore`; threshold the one the verdict was
|
|
28
|
+
settled against (compute-score.py's FALLBACK_FLOOR after a threshold
|
|
29
|
+
fallback, else its `threshold`); and thresholdFallback and
|
|
30
|
+
originalThreshold its override group's. A field copied from anything else
|
|
31
|
+
(`result` over a cap's FAIL, a bare pass under drift, the target threshold
|
|
32
|
+
of a run the fallback accepted) exits 1 and writes no payload, so
|
|
33
|
+
report.md halts before it publishes the report or writes a result file.
|
|
22
34
|
|
|
23
35
|
The derived fields live here, in one place:
|
|
24
36
|
|
|
@@ -49,9 +61,14 @@ Payload (written to --output, UTF-8 JSON):
|
|
|
49
61
|
"runId": <runId>, "healthCheckDispatched": <bool>}}
|
|
50
62
|
|
|
51
63
|
CLI:
|
|
52
|
-
uv run build-result-context.py --report <report.md> --
|
|
53
|
-
[--score <scoring output>] [--no-health-check]
|
|
54
|
-
--output <result-context.json>
|
|
64
|
+
uv run build-result-context.py --report <report.md> [--report-path <path>]
|
|
65
|
+
--ledger <ledger.json> [--score <scoring output>] [--no-health-check]
|
|
66
|
+
[--warning <text>]... --output <result-context.json>
|
|
67
|
+
|
|
68
|
+
--score is required unless the frontmatter says the hard gate blocked the
|
|
69
|
+
run. --report-path is the path the payload names the report by (default:
|
|
70
|
+
--report): report.md builds the payload from the report before it gives
|
|
71
|
+
the report its public name, and passes that name here.
|
|
55
72
|
|
|
56
73
|
--warning hands the emitter a warning the run kept for its result (such as
|
|
57
74
|
`customization_resolver_unavailable: <reason>`); the emitter adds the ones
|
|
@@ -65,8 +82,10 @@ takes: "stderr" for a blocked run, else "stdout".
|
|
|
65
82
|
Exit codes:
|
|
66
83
|
0 the payload was written
|
|
67
84
|
1 an input is missing or unreadable, the report's frontmatter holds no
|
|
68
|
-
settled verdict
|
|
69
|
-
"error"} on stdout)
|
|
85
|
+
settled verdict or a verdict field the scoring output does not settle,
|
|
86
|
+
or --output cannot be written ({"status": "error", "error"} on stdout)
|
|
87
|
+
2 a usage error: no --score for a run the hard gate did not block (the
|
|
88
|
+
same JSON on stdout), or a flag argparse refuses (its usage on stderr)
|
|
70
89
|
"""
|
|
71
90
|
|
|
72
91
|
from __future__ import annotations
|
|
@@ -93,6 +112,10 @@ class ContextError(Exception):
|
|
|
93
112
|
"""An input the payload cannot be built from: exit 1."""
|
|
94
113
|
|
|
95
114
|
|
|
115
|
+
class UsageError(ContextError):
|
|
116
|
+
"""A command the caller can correct and run again: exit 2."""
|
|
117
|
+
|
|
118
|
+
|
|
96
119
|
def _sibling(name: str, module: str):
|
|
97
120
|
"""Import a script that sits beside this one (gap-ledger.py, hard-gate.py)."""
|
|
98
121
|
path = Path(__file__).resolve().parent / name
|
|
@@ -152,9 +175,60 @@ def _read_json(path: str, label: str):
|
|
|
152
175
|
raise ContextError(f"{label} {path} cannot be read: {exc}") from exc
|
|
153
176
|
|
|
154
177
|
|
|
178
|
+
def _fallback_floor() -> int:
|
|
179
|
+
"""The threshold a fallback accepts a run at: compute-score.py's, which applies the fallback."""
|
|
180
|
+
return _sibling("compute-score.py", "skf_compute_score_floor").FALLBACK_FLOOR
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def settled_test_result(scored: dict, workspace_drift: str | None) -> str | None:
|
|
184
|
+
"""The testResult the scoring output settles, or None when it holds no verdict.
|
|
185
|
+
|
|
186
|
+
>>> settled_test_result({"result": "PASS"}, "ok")
|
|
187
|
+
'pass'
|
|
188
|
+
>>> settled_test_result({"result": "PASS", "effectiveResult": "FAIL", "capReason": "Cap 1"}, "ok")
|
|
189
|
+
'fail'
|
|
190
|
+
>>> settled_test_result({"result": "FAIL", "effectiveResult": "PASS"}, "overridden")
|
|
191
|
+
'pass-with-drift'
|
|
192
|
+
>>> settled_test_result({"result": "INCONCLUSIVE"}, "overridden")
|
|
193
|
+
'inconclusive'
|
|
194
|
+
>>> settled_test_result({"error": "bad flag", "code": "INVALID_INPUT"}, "ok") is None
|
|
195
|
+
True
|
|
196
|
+
"""
|
|
197
|
+
verdict = scored.get("effectiveResult") or scored.get("result")
|
|
198
|
+
settled = verdict.lower() if isinstance(verdict, str) else None
|
|
199
|
+
if settled not in ("pass", "fail", "inconclusive"):
|
|
200
|
+
return None
|
|
201
|
+
return "pass-with-drift" if settled == "pass" and workspace_drift == "overridden" else settled
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def scored_fields(scored: dict, workspace_drift: str | None) -> dict | None:
|
|
205
|
+
"""The frontmatter verdict fields the scoring output settles, or None when it holds no verdict.
|
|
206
|
+
|
|
207
|
+
>>> scored_fields({"result": "PASS", "totalScore": 91.5, "threshold": 80}, "ok")
|
|
208
|
+
{'testResult': 'pass', 'score': 91.5, 'threshold': 80, 'thresholdFallback': False, 'originalThreshold': None}
|
|
209
|
+
>>> fell_back = {"result": "FAIL", "effectiveResult": "PASS", "capReason": None, "thresholdFallback": True,
|
|
210
|
+
... "originalThreshold": 90, "totalScore": 85, "threshold": 90}
|
|
211
|
+
>>> scored_fields(fell_back, "ok")
|
|
212
|
+
{'testResult': 'pass', 'score': 85, 'threshold': 80, 'thresholdFallback': True, 'originalThreshold': 90}
|
|
213
|
+
"""
|
|
214
|
+
test_result = settled_test_result(scored, workspace_drift)
|
|
215
|
+
if test_result is None:
|
|
216
|
+
return None
|
|
217
|
+
fallback = scored.get("thresholdFallback") is True
|
|
218
|
+
return {
|
|
219
|
+
"testResult": test_result,
|
|
220
|
+
"score": scored.get("totalScore"),
|
|
221
|
+
"threshold": _fallback_floor() if fallback else scored.get("threshold"),
|
|
222
|
+
"thresholdFallback": fallback,
|
|
223
|
+
"originalThreshold": scored.get("originalThreshold") if fallback else None,
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
|
|
155
227
|
def build(report: str, ledger: str, score: str | None = None, health_check: bool = True,
|
|
156
|
-
warnings: list[str] | None = None) -> dict:
|
|
157
|
-
"""The emitter payload for the run whose report and ledger are given
|
|
228
|
+
warnings: list[str] | None = None, report_path: str | None = None) -> dict:
|
|
229
|
+
"""The emitter payload for the run whose report and ledger are given; `report_path`
|
|
230
|
+
(default `report`) is the path the payload names the report by."""
|
|
231
|
+
report_path = report_path or report
|
|
158
232
|
try:
|
|
159
233
|
front = read_frontmatter(Path(report).read_bytes().decode("utf-8-sig"))
|
|
160
234
|
except (OSError, UnicodeDecodeError) as exc:
|
|
@@ -170,7 +244,7 @@ def build(report: str, ledger: str, score: str | None = None, health_check: bool
|
|
|
170
244
|
skill = front.get("skillName") or None
|
|
171
245
|
blocked = front.get("hardGate") == "blocked"
|
|
172
246
|
if blocked:
|
|
173
|
-
envelope = dict(_sibling("hard-gate.py", "skf_hard_gate").blocked_envelope(skill,
|
|
247
|
+
envelope = dict(_sibling("hard-gate.py", "skf_hard_gate").blocked_envelope(skill, report_path))
|
|
174
248
|
result, active, inconclusive = "FAIL", [], None
|
|
175
249
|
else:
|
|
176
250
|
test_result = front.get("testResult", "")
|
|
@@ -183,14 +257,29 @@ def build(report: str, ledger: str, score: str | None = None, health_check: bool
|
|
|
183
257
|
"verdict": verdict,
|
|
184
258
|
"score": percent(front.get("score")),
|
|
185
259
|
"threshold": percent(front.get("threshold")),
|
|
186
|
-
"report_path":
|
|
260
|
+
"report_path": report_path,
|
|
187
261
|
"next_workflow": next_workflow,
|
|
188
262
|
"exit_code": exit_code,
|
|
189
263
|
"halt_reason": None,
|
|
190
264
|
}
|
|
191
|
-
|
|
265
|
+
if not score:
|
|
266
|
+
raise UsageError("--score is required for a run the hard gate did not block: its verdict "
|
|
267
|
+
"fields are checked against the scoring output")
|
|
268
|
+
scored = _read_json(score, "--score")
|
|
192
269
|
if not isinstance(scored, dict):
|
|
193
270
|
raise ContextError(f"--score {score} is not a JSON object")
|
|
271
|
+
settled = scored_fields(scored, front.get("workspaceDrift"))
|
|
272
|
+
if settled is None:
|
|
273
|
+
raise ContextError(f"--score {score} holds no verdict (no effectiveResult or result)")
|
|
274
|
+
fell_back = front.get("thresholdFallback", "").lower() == "true"
|
|
275
|
+
written = {"testResult": test_result, "score": envelope["score"], "threshold": envelope["threshold"],
|
|
276
|
+
"thresholdFallback": fell_back,
|
|
277
|
+
"originalThreshold": percent(front.get("originalThreshold")) if fell_back else None}
|
|
278
|
+
wrong = [f"{key} {written[key]!r} where it settles {settled[key]!r}"
|
|
279
|
+
for key in settled if written[key] != settled[key]]
|
|
280
|
+
if wrong:
|
|
281
|
+
raise ContextError(f"--report {report} disagrees with the scoring output {score}: "
|
|
282
|
+
+ "; ".join(wrong))
|
|
194
283
|
active = scored.get("activeCategories", [])
|
|
195
284
|
inconclusive = scored.get("inconclusiveReasons") if verdict == "INCONCLUSIVE" else None
|
|
196
285
|
|
|
@@ -216,7 +305,7 @@ def build(report: str, ledger: str, score: str | None = None, health_check: bool
|
|
|
216
305
|
summary["hardGate"] = front.get("hardGate") or None
|
|
217
306
|
|
|
218
307
|
contract = {"skill": WORKFLOW, **envelope,
|
|
219
|
-
"outputs": [{"type": "report", "path":
|
|
308
|
+
"outputs": [{"type": "report", "path": report_path}],
|
|
220
309
|
"summary": summary,
|
|
221
310
|
"runId": front.get("runId") or None,
|
|
222
311
|
"healthCheckDispatched": health_check}
|
|
@@ -233,9 +322,12 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
233
322
|
description=("Build the shared emitter's payload for a finished test-skill run from its "
|
|
234
323
|
"report frontmatter, its gap ledger and the scoring output."),
|
|
235
324
|
)
|
|
236
|
-
parser.add_argument("--report", required=True, help="the test report
|
|
325
|
+
parser.add_argument("--report", required=True, help="the test report to read")
|
|
326
|
+
parser.add_argument("--report-path",
|
|
327
|
+
help="the path the payload names the report by: its public name (default: --report)")
|
|
237
328
|
parser.add_argument("--ledger", required=True, help="the run's gap ledger")
|
|
238
|
-
parser.add_argument("--score", help="the scoring script's output (
|
|
329
|
+
parser.add_argument("--score", help="the scoring script's output (required unless the hard gate blocked "
|
|
330
|
+
"the run)")
|
|
239
331
|
parser.add_argument("--no-health-check", action="store_true",
|
|
240
332
|
help="the run was given --no-health-check (healthCheckDispatched false)")
|
|
241
333
|
parser.add_argument("--warning", action="append", default=[],
|
|
@@ -247,13 +339,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
247
339
|
def main(argv: list[str] | None = None) -> int:
|
|
248
340
|
args = _build_parser().parse_args(argv)
|
|
249
341
|
try:
|
|
250
|
-
payload = build(args.report, args.ledger, args.score, not args.no_health_check, args.warning
|
|
342
|
+
payload = build(args.report, args.ledger, args.score, not args.no_health_check, args.warning,
|
|
343
|
+
args.report_path)
|
|
251
344
|
out = Path(args.output)
|
|
252
345
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
253
346
|
out.write_bytes((json.dumps(payload, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
|
|
254
347
|
except (ContextError, OSError) as exc:
|
|
255
348
|
print(json.dumps({"status": "error", "error": str(exc)}))
|
|
256
|
-
return 1
|
|
349
|
+
return 2 if isinstance(exc, UsageError) else 1
|
|
257
350
|
blocked = payload["halt_reason"] is not None
|
|
258
351
|
print(json.dumps({
|
|
259
352
|
"status": payload["status"],
|