bmad-module-skill-forge 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +51 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +135 -18
- package/docs/verifying-a-skill.md +83 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +2 -2
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +32 -12
- package/src/shared/references/description-guard-protocol.md +14 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +30 -14
- package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +694 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
- package/src/shared/scripts/skf-description-guard.py +117 -9
- package/src/shared/scripts/skf-detect-docs.py +892 -4
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
- package/src/shared/scripts/skf-detect-tools.py +215 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +4084 -85
- package/src/shared/scripts/skf-extraction-inventory.py +746 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +661 -0
- package/src/shared/scripts/skf-hash-content.py +166 -12
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +159 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +161 -27
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +670 -119
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +5 -5
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
- package/src/shared/scripts/skf-write-skill-brief.py +367 -19
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/init.md +148 -210
- package/src/skf-audit-skill/references/re-index.md +96 -108
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +27 -18
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +75 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
- package/src/skf-brief-skill/SKILL.md +36 -25
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +17 -24
- package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +142 -125
- package/src/skf-brief-skill/references/confirm-brief.md +10 -8
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +3 -2
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
- package/src/skf-create-skill/assets/skill-sections.md +17 -6
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +107 -29
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
- package/src/skf-create-skill/references/extract.md +200 -161
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
- package/src/skf-create-skill/references/extraction-patterns.md +584 -196
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +28 -59
- package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +55 -18
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -57
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +79 -146
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +118 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -89
- package/src/skf-setup/references/detect-and-tier.md +58 -66
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +217 -254
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +163 -88
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +162 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +41 -155
- package/src/skf-test-skill/references/source-access-protocol.md +19 -92
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +287 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
- package/src/skf-test-skill/scripts/score-signatures.py +459 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +233 -222
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +250 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +180 -47
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +100 -35
- package/src/skf-update-skill/references/write.md +331 -149
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -645
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -0,0 +1,1384 @@
|
|
|
1
|
+
# /// script
|
|
2
|
+
# requires-python = ">=3.11"
|
|
3
|
+
# dependencies = []
|
|
4
|
+
# ///
|
|
5
|
+
"""SKF Parse Gaps: read a test report's gaps as JSON for update-skill.
|
|
6
|
+
|
|
7
|
+
skf-update-skill's gap-driven mode (`--from-test-report`) turns each gap of a
|
|
8
|
+
test report into a change-manifest entry: its severity, category, source
|
|
9
|
+
citation, remediation text and the source paths the remediation names.
|
|
10
|
+
Reading them by eye from the Gap Report, extracting paths by file extension
|
|
11
|
+
and checking that a path stays inside the source tree is fiddly work that
|
|
12
|
+
drifts between runs. This helper does it once.
|
|
13
|
+
|
|
14
|
+
Where the gaps come from:
|
|
15
|
+
|
|
16
|
+
1. The gap ledger, when there is one: `test-findings-{run_id}.json` beside
|
|
17
|
+
the report (the run id is the report's `runId` frontmatter, else the one
|
|
18
|
+
in its file name), or the file --ledger names. test-skill's stages write
|
|
19
|
+
it (gap-ledger.py) and the Gap Report is rendered from it. A ledger
|
|
20
|
+
whose `schema_version` is not 1 is refused (LEDGER_INVALID) rather than
|
|
21
|
+
read with the wrong field meanings.
|
|
22
|
+
2. Otherwise the rendered `## Gap Report` section of the report, for
|
|
23
|
+
reports written before the ledger existed: each `### GAP-{NNN}: {title}`
|
|
24
|
+
entry with its `**Severity:**`, `**Category:**`, `**Source:**`,
|
|
25
|
+
`**Export:**`, `**Issue:**` and `**Remediation:**` lines. Every
|
|
26
|
+
`## Gap Report` section is read, so a report that holds the section
|
|
27
|
+
twice (a placeholder and an appended copy) still yields its gaps; an id
|
|
28
|
+
seen twice is kept once.
|
|
29
|
+
|
|
30
|
+
Subcommands:
|
|
31
|
+
|
|
32
|
+
parse [--report <test-report.md>] [--ledger <test-findings.json>]
|
|
33
|
+
[--source-root <dir>] [--ext <extension> ...]
|
|
34
|
+
[--provenance-map <provenance-map.json>]
|
|
35
|
+
|
|
36
|
+
paths --source-root <dir> [--ext <extension> ...]
|
|
37
|
+
[--provenance-map <provenance-map.json>] <path> [<path> ...]
|
|
38
|
+
|
|
39
|
+
match --provenance-map <provenance-map.json> --name <export>
|
|
40
|
+
[--citation <file:line>]
|
|
41
|
+
|
|
42
|
+
translate --gaps <parse's output> --provenance-map <provenance-map.json>
|
|
43
|
+
[--judgments <answers.json>] [--read-only dry-run|detect-only]
|
|
44
|
+
[--date YYYY-MM-DD] -o <change-manifest.json>
|
|
45
|
+
|
|
46
|
+
`parse`: at least one of --report and --ledger. --ext (repeatable, `kt`
|
|
47
|
+
or `.vue`) adds a source file extension to the list below, and
|
|
48
|
+
--provenance-map adds the extension of each file its entries[]
|
|
49
|
+
source_file values name, for a skill whose source holds files of another
|
|
50
|
+
language, and each gap's `map_match` (see "Map match" below). Output
|
|
51
|
+
(stdout):
|
|
52
|
+
|
|
53
|
+
{
|
|
54
|
+
"status": "ok",
|
|
55
|
+
"read_from": "ledger" | "gap-report",
|
|
56
|
+
"report_path": "<path>" | null,
|
|
57
|
+
"ledger_path": "<path>" | null,
|
|
58
|
+
"source_root": "<real path>" | null,
|
|
59
|
+
"gap_count": N,
|
|
60
|
+
"counts": {"Critical": n, "High": n, "Medium": n, "Low": n, "Info": n},
|
|
61
|
+
"gaps": [ # Critical first, then by id
|
|
62
|
+
{
|
|
63
|
+
"id": "GAP-001",
|
|
64
|
+
"title": "...",
|
|
65
|
+
"severity": "Critical" | ... | null, # null: unreadable in the report
|
|
66
|
+
"category": "<slug>" | null, # null: a report older than the ledger
|
|
67
|
+
"category_group": "Coverage" | ... | null,
|
|
68
|
+
"source": "<Source text>" | null,
|
|
69
|
+
"source_citation": {"file": "...", "line": N} | null,
|
|
70
|
+
"export": "<name>" | null,
|
|
71
|
+
"issue": "..." | null,
|
|
72
|
+
"remediation": "..." | null,
|
|
73
|
+
"remediation_paths": ["..."], # path tokens as the remediation writes them
|
|
74
|
+
"resolved_paths": ["..."], # with --source-root only
|
|
75
|
+
"rejected_paths": [{"path", "reason"}], # with --source-root only
|
|
76
|
+
"map_match": {...} | null # with --provenance-map only
|
|
77
|
+
}
|
|
78
|
+
],
|
|
79
|
+
"resolved_files": ["..."], # with --source-root only
|
|
80
|
+
"warnings": ["..."]
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
`source_citation` is set only when the whole Source is a `file:line` pair
|
|
84
|
+
(`file:line-line`, `file:line:col` and `file#Lline` read the same).
|
|
85
|
+
|
|
86
|
+
Map match (update-skill gap-driven.md §4's lookup of a gap's export in the
|
|
87
|
+
provenance map): for a gap with an `export` (null for one with none),
|
|
88
|
+
|
|
89
|
+
{"status": "found" | "ambiguous" | "not-found",
|
|
90
|
+
"entry": {"source_file", "source_line", "export_type"} | null,
|
|
91
|
+
"candidates": [{"source_file", "source_line", "export_type"}, ...]}
|
|
92
|
+
|
|
93
|
+
`candidates` holds every entries[] row whose export_name is the export, in
|
|
94
|
+
the map's order. With a `source_citation`, only the rows whose source_file
|
|
95
|
+
is the citation's file count (both compared stripped, with backslashes read
|
|
96
|
+
as `/` and no leading `./`), and of several such rows the one whose
|
|
97
|
+
source_line is the citation's line. Without one, every candidate counts.
|
|
98
|
+
One row left is `found` (its `entry`), several are `ambiguous` (a spot-check
|
|
99
|
+
of the wrong one would move another export's line) and none is `not-found`.
|
|
100
|
+
`match` prints {"status": "ok", "name", "citation", "map_match"} for a name
|
|
101
|
+
a step takes from somewhere other than a gap's `export`, its citation given
|
|
102
|
+
as `file:line`.
|
|
103
|
+
|
|
104
|
+
`remediation_paths` holds, in order and once each, every token of the
|
|
105
|
+
Remediation text that names a source file (`.ts`, `.tsx`, `.js`, `.jsx`,
|
|
106
|
+
`.mjs`, `.cjs`, `.py`, `.rs`, `.go`, `.java`, `.kt`, `.kts`, `.cs`,
|
|
107
|
+
`.swift`, `.php`, `.rb`, `.c`, `.h`, `.cpp` and each --ext, with any
|
|
108
|
+
`:line` suffix dropped), every glob (`src/**/*.ts`), every token that ends
|
|
109
|
+
in `/`, and every code span that is a single path with a `/` and no file
|
|
110
|
+
extension (`` `packages/core/src` ``). A path may hold framework route
|
|
111
|
+
names such as `app/(shop)/[slug]/page.tsx`, `[id=integer]` or
|
|
112
|
+
`routes/posts.$slug.tsx`: each `(` and `[` must close within its own
|
|
113
|
+
folder name, which keeps code such as `Math.floor(x/2)` out. A Markdown
|
|
114
|
+
link `[text](target)` is read as its text and its target. URLs, scoped
|
|
115
|
+
package names (`@scope/name`), files with other extensions and, outside a
|
|
116
|
+
code span, a library name such as `Node.js` are not paths.
|
|
117
|
+
|
|
118
|
+
With --source-root, each path is also resolved under that folder:
|
|
119
|
+
a file is kept, a directory expands to the source files under it (not
|
|
120
|
+
under `.git`, `node_modules`, `__pycache__` or another folder starting
|
|
121
|
+
with `.`), and a glob expands under the root. Only `*` and `?` make a
|
|
122
|
+
path a glob: `**` spans folders, `*` and `?` stay within one name and
|
|
123
|
+
skip names that start with `.`, and `[` `]` are plain characters (a route
|
|
124
|
+
folder, not a character class). The glob walk follows no link and leaves
|
|
125
|
+
out `.git`, `node_modules`, `__pycache__` and folders starting with `.`
|
|
126
|
+
unless the glob names them; a folder the glob matches expands like a
|
|
127
|
+
directory. `resolved_paths` holds the files, relative to the root with
|
|
128
|
+
`/` separators and deduplicated (a link inside the root that points
|
|
129
|
+
inside it counts as its target). A path is rejected, with its reason in
|
|
130
|
+
`rejected_paths`, when it leaves the root (`outside-root`, also a glob
|
|
131
|
+
whose fixed folders do), when a link takes it outside
|
|
132
|
+
(`symlink-outside-root`), when it does not exist (`not-found`) or when a
|
|
133
|
+
directory or glob holds no file (`no-match`). `resolved_files` is the
|
|
134
|
+
union over all gaps, each file once.
|
|
135
|
+
|
|
136
|
+
`paths` resolves the paths it is given (each relative to the root, a
|
|
137
|
+
`:line` suffix dropped) the same way, for a path a step names itself
|
|
138
|
+
rather than a remediation text: {"status": "ok", "source_root",
|
|
139
|
+
"resolved_paths", "rejected_paths"}.
|
|
140
|
+
|
|
141
|
+
Translate (update-skill gap-driven.md §1): `translate` turns the gaps
|
|
142
|
+
`parse` wrote (--gaps) into the change manifest, written to -o as
|
|
143
|
+
{"mode": "gap-driven", "entries": [...]}. Each gap is routed by its
|
|
144
|
+
`category` through CHANGE_CATEGORIES (gap-driven.md §1's table), never by
|
|
145
|
+
its severity; a category the table does not hold is not routed. Each
|
|
146
|
+
entry, in this key order:
|
|
147
|
+
|
|
148
|
+
name the gap's `export`, else the answered `name`
|
|
149
|
+
gap_id, category the gap's `id`, its `category` (else the
|
|
150
|
+
answered one), `severity`
|
|
151
|
+
severity
|
|
152
|
+
source_citation the gap's, left out when it is null or names a
|
|
153
|
+
line inside the skill package (`SKILL.md` or a
|
|
154
|
+
`references/*.md` file, under any folder)
|
|
155
|
+
remediation_paths, resolved_paths, rejected_paths
|
|
156
|
+
the gap's ([] when parse had no --source-root);
|
|
157
|
+
a `provenance-completeness` gap with no
|
|
158
|
+
resolved path takes its resolved_paths from the
|
|
159
|
+
root check of the source file it cites (its
|
|
160
|
+
out-of-package citation, else the answered
|
|
161
|
+
`source_file`) under parse's source root, and
|
|
162
|
+
adds that check's refusals to its own
|
|
163
|
+
rejected_paths
|
|
164
|
+
change_category the table's, or DELETED_EXPORT for a rescope
|
|
165
|
+
remediation the gap's text, verbatim
|
|
166
|
+
provenance_completeness true, for a `provenance-completeness` gap only
|
|
167
|
+
rescope for a rescope only: {"amendment": {path,
|
|
168
|
+
action: "excluded", category:
|
|
169
|
+
"scope-expansion", reason (the remediation),
|
|
170
|
+
date, workflow: "skf-update-skill"},
|
|
171
|
+
"exclude": path}
|
|
172
|
+
map_match the `match` lookup of `name` with the kept
|
|
173
|
+
citation (null without a name)
|
|
174
|
+
|
|
175
|
+
A rescope (rule R1) is a `missing-export` or `missing-type` gap answered
|
|
176
|
+
`rescope: true`. Its path is the answered `source_file`, else the file of
|
|
177
|
+
its map entry when `map_match` is `found`, else its citation's file.
|
|
178
|
+
|
|
179
|
+
What it cannot decide it asks for. While any gap needs an answer it writes
|
|
180
|
+
nothing and prints {"status": "needs-judgment", "needs_judgment":
|
|
181
|
+
[{"gap_id", "title", "category", "issue", "remediation", "needs": [...]}]},
|
|
182
|
+
the gap's own text to judge from, each need one of:
|
|
183
|
+
|
|
184
|
+
category a gap with none (a report older than the ledger)
|
|
185
|
+
name an export-bearing gap with no `export` (with `category`:
|
|
186
|
+
any gap with neither)
|
|
187
|
+
rescope a `missing-export` or `missing-type` gap whose remediation
|
|
188
|
+
a person or a model wrote: any text but the one gap-ledger.py
|
|
189
|
+
generates for a missing export or type (`Document ... in
|
|
190
|
+
SKILL.md: ...`, `Find the N exports ...`), which names no
|
|
191
|
+
removal; whether it does is rule R1's judgment
|
|
192
|
+
source_file a rescope whose path is unknown, or a
|
|
193
|
+
`provenance-completeness` gap with no path to scan
|
|
194
|
+
|
|
195
|
+
--judgments answers them: a JSON object keyed by gap id, each value an
|
|
196
|
+
object of `category` (a slug, or null: not routed), `name` (a string, or
|
|
197
|
+
null: not routed), `rescope` (true or false; given for any `missing-export`
|
|
198
|
+
or `missing-type` gap, it is honoured) and `source_file` (a path relative
|
|
199
|
+
to the source root, or null). Other keys are left for gap-records. With
|
|
200
|
+
every need answered it writes the manifest and prints {"status":
|
|
201
|
+
"written", "output", "gap_count", "not_routed": [{"id", "title",
|
|
202
|
+
"category"}], "warnings": [...]}. `warnings` holds parse's warnings as
|
|
203
|
+
`test-report: <warning>`, `test-report: not routed: <id> (<category>)` for
|
|
204
|
+
each gap not routed, and, with --read-only, `proposed-amendment: excluded
|
|
205
|
+
<path> (scope-expansion); not written: --<mode>` for each rescope.
|
|
206
|
+
|
|
207
|
+
Exit codes:
|
|
208
|
+
0 the gaps (possibly none), the paths or the match were printed; the
|
|
209
|
+
manifest was written, or the answers it needs were printed
|
|
210
|
+
1 the report or ledger is missing or unreadable, or an option value is
|
|
211
|
+
unusable (no --report or --ledger, a --source-root that is not a
|
|
212
|
+
folder, an --ext that is not a file extension, a --provenance-map
|
|
213
|
+
that cannot be read as a JSON object, a --citation that is no
|
|
214
|
+
`file:line`, a --gaps file that is not parse's output, a --judgments
|
|
215
|
+
answer of the wrong type, an -o that cannot be written)
|
|
216
|
+
2 usage error (argparse: a missing or unknown argument, usage on stderr,
|
|
217
|
+
no JSON)
|
|
218
|
+
|
|
219
|
+
Errors print {"status": "error", "code", "error"} on stdout, with the codes
|
|
220
|
+
REPORT_MISSING, LEDGER_MISSING, LEDGER_INVALID and INVALID_INPUT.
|
|
221
|
+
"""
|
|
222
|
+
|
|
223
|
+
from __future__ import annotations
|
|
224
|
+
|
|
225
|
+
import argparse
|
|
226
|
+
import json
|
|
227
|
+
import os
|
|
228
|
+
import re
|
|
229
|
+
import sys
|
|
230
|
+
import time
|
|
231
|
+
from datetime import datetime, timezone
|
|
232
|
+
from pathlib import Path
|
|
233
|
+
|
|
234
|
+
SEVERITIES = ("Critical", "High", "Medium", "Low", "Info")
|
|
235
|
+
_SEVERITY_RANK = {s: i for i, s in enumerate(SEVERITIES)}
|
|
236
|
+
LEDGER_SCHEMA_VERSION = 1 # gap-ledger.py SCHEMA_VERSION
|
|
237
|
+
|
|
238
|
+
# The extraction languages (skf-detect-language.py _EXTENSION_TO_LANGUAGE)
|
|
239
|
+
# plus C and C++; --ext adds more.
|
|
240
|
+
SOURCE_EXTENSIONS = (
|
|
241
|
+
"ts", "tsx", "js", "jsx", "mjs", "cjs", "py", "rs", "go", "java",
|
|
242
|
+
"kt", "kts", "cs", "swift", "php", "rb", "c", "h", "cpp",
|
|
243
|
+
)
|
|
244
|
+
_SOURCE_SUFFIXES = frozenset(f".{ext}" for ext in SOURCE_EXTENSIONS)
|
|
245
|
+
_EXT_ARG = re.compile(r"\.?([A-Za-z][A-Za-z0-9]*)") # an --ext value, whole
|
|
246
|
+
SKIP_DIRS = frozenset({".git", "node_modules", "__pycache__"})
|
|
247
|
+
|
|
248
|
+
_RUN_ID_SAFE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]*$")
|
|
249
|
+
_RUN_ID_IN_NAME = re.compile(r"(\d{8}T\d{6}Z(?:-[A-Za-z0-9]+)*)\.md$")
|
|
250
|
+
_ID_RE = re.compile(r"^GAP-(\d+)$")
|
|
251
|
+
|
|
252
|
+
# Gap Report markdown.
|
|
253
|
+
_GAP_REPORT_H2 = re.compile(r"^##\s+Gap Report\s*$")
|
|
254
|
+
_SECTION_END = re.compile(r"^#{1,2}\s")
|
|
255
|
+
_HEADING = re.compile(r"^#{1,6}\s")
|
|
256
|
+
_GAP_HEADING = re.compile(
|
|
257
|
+
r"^#{2,6}\s+(?:\*\*)?(GAP-\d+)(?:\*\*)?\s*(?:[:.\u2013\u2014-]\s*)?(.*?)\s*$"
|
|
258
|
+
)
|
|
259
|
+
_FIELD = re.compile(
|
|
260
|
+
r"^\s*(?:[-*]\s+)?\*\*(severity|category|source|export|issue|remediation)\s*(?::\*\*|\*\*\s*:)\s*(.*)$",
|
|
261
|
+
re.IGNORECASE,
|
|
262
|
+
)
|
|
263
|
+
_MULTILINE_FIELDS = frozenset({"issue", "remediation"})
|
|
264
|
+
_FENCE = re.compile(r"^\s*(`{3,}|~{3,})")
|
|
265
|
+
_RULE = re.compile(r"^\s*(?:-{3,}|\*{3,}|_{3,})\s*$")
|
|
266
|
+
_CATEGORY_WITH_GROUP = re.compile(r"^([A-Za-z][A-Za-z &/-]*?)\s*\(\s*`?([a-z][a-z0-9-]*)`?\s*\)$")
|
|
267
|
+
_CATEGORY_SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
268
|
+
|
|
269
|
+
# Paths in remediation text. ( ) [ ] $ = + @ appear in framework route
|
|
270
|
+
# names: app/(shop)/[slug]/page.tsx, [id=integer], posts.$slug.tsx, @modal.
|
|
271
|
+
_CODE_SPAN = re.compile(r"(`+)(.+?)\1")
|
|
272
|
+
_LINE_SUFFIX = re.compile(r"(?::\d+(?:[-:]\d+)*|#L\d+(?:-L?\d+)?)$")
|
|
273
|
+
_PATH_TOKEN = re.compile(r"^[\w./*?\[\]()$=@+~-]+$")
|
|
274
|
+
_GLOB_CHARS = re.compile(r"[*?]")
|
|
275
|
+
_WORD_BREAK = re.compile(r"\s+|\]\(") # `](` joins a Markdown link's text to its target
|
|
276
|
+
_CLOSER = {"(": ")", "[": "]"}
|
|
277
|
+
_EXTENSION = re.compile(r"\.([A-Za-z][A-Za-z0-9]*)$")
|
|
278
|
+
_CITATION = re.compile(r"^(?P<file>[^\s`]+?)(?::(?P<line>\d+)(?:[-:]\d+)?|#L(?P<hline>\d+)(?:-L?\d+)?)$")
|
|
279
|
+
# Outside a code span, a bare `Name.js` with no folder is a library name
|
|
280
|
+
# (Node.js, Vue.js, D3.js), not a file.
|
|
281
|
+
_PRODUCT_NAME = re.compile(r"^(?:[A-Z][\w-]*|node|vue|next|nuxt|express|react|angular|ember|d3|three|chart|p5)\.js$")
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class ParseError(Exception):
|
|
285
|
+
def __init__(self, code: str, message: str) -> None:
|
|
286
|
+
super().__init__(message)
|
|
287
|
+
self.code = code
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
# --------------------------------------------------------------------------
|
|
291
|
+
# Small readers
|
|
292
|
+
# --------------------------------------------------------------------------
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def read_frontmatter(text: str) -> dict[str, str]:
|
|
296
|
+
"""Top-level scalar keys of a `---` frontmatter block, quotes removed."""
|
|
297
|
+
lines = text.lstrip("\ufeff").splitlines()
|
|
298
|
+
if not lines or lines[0].strip() != "---":
|
|
299
|
+
return {}
|
|
300
|
+
out: dict[str, str] = {}
|
|
301
|
+
for line in lines[1:]:
|
|
302
|
+
if line.strip() == "---":
|
|
303
|
+
return out
|
|
304
|
+
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*?)\s*$", line)
|
|
305
|
+
if m:
|
|
306
|
+
value = m.group(2)
|
|
307
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
|
|
308
|
+
value = value[1:-1]
|
|
309
|
+
out[m.group(1)] = value
|
|
310
|
+
return {}
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def strip_frontmatter(text: str) -> str:
|
|
314
|
+
lines = text.lstrip("\ufeff").splitlines()
|
|
315
|
+
if lines and lines[0].strip() == "---":
|
|
316
|
+
for i in range(1, len(lines)):
|
|
317
|
+
if lines[i].strip() == "---":
|
|
318
|
+
return "\n".join(lines[i + 1:])
|
|
319
|
+
return "\n".join(lines)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def normalize_severity(value: object) -> str | None:
|
|
323
|
+
"""The first severity word in a value, in title case.
|
|
324
|
+
|
|
325
|
+
>>> normalize_severity("**critical** (blocking)"), normalize_severity("n/a")
|
|
326
|
+
('Critical', None)
|
|
327
|
+
"""
|
|
328
|
+
if not isinstance(value, str):
|
|
329
|
+
return None
|
|
330
|
+
m = re.search(r"\b(critical|high|medium|low|info)\b", value, re.IGNORECASE)
|
|
331
|
+
return m.group(1).capitalize() if m else None
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _unquote(value: str) -> str:
|
|
335
|
+
value = value.strip()
|
|
336
|
+
if len(value) >= 2 and value[0] == "`" and value[-1] == "`":
|
|
337
|
+
value = value.strip("`").strip()
|
|
338
|
+
return value
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def source_citation(source: object) -> dict | None:
|
|
342
|
+
"""{file, line} when the whole Source is a file:line pair.
|
|
343
|
+
|
|
344
|
+
>>> source_citation("`src/utils.ts:42`")
|
|
345
|
+
{'file': 'src/utils.ts', 'line': 42}
|
|
346
|
+
>>> source_citation("src/a.py#L10-L12")
|
|
347
|
+
{'file': 'src/a.py', 'line': 10}
|
|
348
|
+
>>> source_citation("@storybook/addon-docs control primitives") is None
|
|
349
|
+
True
|
|
350
|
+
"""
|
|
351
|
+
if not isinstance(source, str):
|
|
352
|
+
return None
|
|
353
|
+
text = _unquote(source)
|
|
354
|
+
if "://" in text:
|
|
355
|
+
return None
|
|
356
|
+
m = _CITATION.match(text)
|
|
357
|
+
if not m:
|
|
358
|
+
return None
|
|
359
|
+
return {"file": m.group("file"), "line": int(m.group("line") or m.group("hline"))}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def parse_category(value: str | None) -> tuple[str | None, str | None]:
|
|
363
|
+
"""(category slug, group) from a Category line.
|
|
364
|
+
|
|
365
|
+
>>> parse_category("Coverage (missing-export)")
|
|
366
|
+
('missing-export', 'Coverage')
|
|
367
|
+
>>> parse_category("Coherence")
|
|
368
|
+
(None, 'Coherence')
|
|
369
|
+
>>> parse_category("broken-reference")
|
|
370
|
+
('broken-reference', None)
|
|
371
|
+
"""
|
|
372
|
+
if not value:
|
|
373
|
+
return None, None
|
|
374
|
+
text = _unquote(value)
|
|
375
|
+
m = _CATEGORY_WITH_GROUP.match(text)
|
|
376
|
+
if m:
|
|
377
|
+
return m.group(2), m.group(1).strip()
|
|
378
|
+
if _CATEGORY_SLUG.match(text):
|
|
379
|
+
return text, None
|
|
380
|
+
return None, text or None
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
# --------------------------------------------------------------------------
|
|
384
|
+
# Paths in remediation text
|
|
385
|
+
# --------------------------------------------------------------------------
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _brackets_close_per_name(token: str) -> bool:
|
|
389
|
+
"""True when each `/`-separated name closes the ( and [ it opens, and `=`
|
|
390
|
+
sits only inside [ ]: route names such as `(shop)`, `[[...slug]]` or
|
|
391
|
+
`[id=integer]`, not code such as `Math.floor(x/2)` or `key=value`.
|
|
392
|
+
|
|
393
|
+
>>> [_brackets_close_per_name(t) for t in ("app/(shop)/[id=int]/page.tsx", "Math.floor(x/2)", "a=b/c.ts")]
|
|
394
|
+
[True, False, False]
|
|
395
|
+
"""
|
|
396
|
+
for name in token.split("/"):
|
|
397
|
+
expected: list[str] = []
|
|
398
|
+
for ch in name:
|
|
399
|
+
if ch in _CLOSER:
|
|
400
|
+
expected.append(_CLOSER[ch])
|
|
401
|
+
elif ch in ")]":
|
|
402
|
+
if not expected or expected.pop() != ch:
|
|
403
|
+
return False
|
|
404
|
+
elif ch == "=" and "]" not in expected:
|
|
405
|
+
return False
|
|
406
|
+
if expected:
|
|
407
|
+
return False
|
|
408
|
+
return True
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _wraps(word: str, opener: str) -> bool:
|
|
412
|
+
"""True when the bracket that opens `word` is the one that ends it: `(src/a.ts)`."""
|
|
413
|
+
if not word.startswith(opener) or not word.endswith(_CLOSER[opener]):
|
|
414
|
+
return False
|
|
415
|
+
depth = 0
|
|
416
|
+
for index, ch in enumerate(word):
|
|
417
|
+
if ch == opener:
|
|
418
|
+
depth += 1
|
|
419
|
+
elif ch == _CLOSER[opener]:
|
|
420
|
+
depth -= 1
|
|
421
|
+
if depth == 0:
|
|
422
|
+
return index == len(word) - 1
|
|
423
|
+
return False
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _trim_word(word: str) -> str:
|
|
427
|
+
"""A word of prose without the punctuation around it.
|
|
428
|
+
|
|
429
|
+
A bracket the word closes itself stays, so a route folder keeps it:
|
|
430
|
+
`(shop)/page.tsx` keeps its `(`, `(see` and `src/a.ts),` lose theirs.
|
|
431
|
+
|
|
432
|
+
>>> [_trim_word(w) for w in ("(src/a.ts),", "app/(shop)/page.tsx).", "[slug]/page.tsx", "(see", "a.ts?")]
|
|
433
|
+
['src/a.ts', 'app/(shop)/page.tsx', '[slug]/page.tsx', 'see', 'a.ts']
|
|
434
|
+
"""
|
|
435
|
+
while True:
|
|
436
|
+
before = word
|
|
437
|
+
word = word.lstrip("{<'\"").rstrip("}>'\",;!.:")
|
|
438
|
+
if word.endswith("?") and not _GLOB_CHARS.search(word[:-1]):
|
|
439
|
+
word = word[:-1]
|
|
440
|
+
for opener, closer in _CLOSER.items():
|
|
441
|
+
if _wraps(word, opener):
|
|
442
|
+
word = word[1:-1]
|
|
443
|
+
if word.startswith(opener) and word.count(opener) > word.count(closer):
|
|
444
|
+
word = word[1:]
|
|
445
|
+
if word.endswith(closer) and word.count(closer) > word.count(opener):
|
|
446
|
+
word = word[:-1]
|
|
447
|
+
if word == before:
|
|
448
|
+
return word
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _path_kind(token: str, in_code: bool, suffixes: frozenset[str]) -> str | None:
|
|
452
|
+
"""'file', 'dir' or 'glob' when a token names a source path, else None."""
|
|
453
|
+
token = _LINE_SUFFIX.sub("", token)
|
|
454
|
+
if not token or "://" in token or not _PATH_TOKEN.match(token):
|
|
455
|
+
return None
|
|
456
|
+
if not re.search(r"[A-Za-z*]", token) or not _brackets_close_per_name(token):
|
|
457
|
+
return None
|
|
458
|
+
ext = _EXTENSION.search(token)
|
|
459
|
+
if token.startswith("@") and not (ext and f".{ext.group(1)}" in suffixes):
|
|
460
|
+
return None
|
|
461
|
+
if _GLOB_CHARS.search(token):
|
|
462
|
+
return "glob" if "/" in token or "*" in token else None
|
|
463
|
+
if token.endswith("/"):
|
|
464
|
+
return "dir"
|
|
465
|
+
if ext:
|
|
466
|
+
if f".{ext.group(1)}" not in suffixes:
|
|
467
|
+
return None
|
|
468
|
+
if not in_code and "/" not in token and _PRODUCT_NAME.match(token):
|
|
469
|
+
return None
|
|
470
|
+
return "file"
|
|
471
|
+
if in_code and "/" in token and token.strip("./"):
|
|
472
|
+
return "dir"
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def remediation_paths(text: str | None, suffixes: frozenset[str] = _SOURCE_SUFFIXES) -> list[str]:
|
|
477
|
+
"""Path tokens of a remediation text, in order, each once.
|
|
478
|
+
|
|
479
|
+
>>> remediation_paths("Update `SKILL.md` line 78 to match source at `src/utils.ts:42`.")
|
|
480
|
+
['src/utils.ts']
|
|
481
|
+
>>> remediation_paths("Scan packages/core/src/ and `src/**/*.tsx`; see references/api.md and/or https://x.io/a.ts")
|
|
482
|
+
['packages/core/src/', 'src/**/*.tsx']
|
|
483
|
+
>>> remediation_paths("Re-extract from `packages/hooks` (not `@scope/pkg`), as Node.js resolves lib/index.js.")
|
|
484
|
+
['packages/hooks', 'lib/index.js']
|
|
485
|
+
>>> remediation_paths("Read app/(shop)/[slug]/page.tsx, `routes/posts.$slug.tsx` (see [Store](src/Store.kt)).")
|
|
486
|
+
['app/(shop)/[slug]/page.tsx', 'routes/posts.$slug.tsx', 'src/Store.kt']
|
|
487
|
+
"""
|
|
488
|
+
if not text:
|
|
489
|
+
return []
|
|
490
|
+
found: list[str] = []
|
|
491
|
+
|
|
492
|
+
def add(token: str, in_code: bool) -> None:
|
|
493
|
+
if _path_kind(token, in_code, suffixes):
|
|
494
|
+
path = _LINE_SUFFIX.sub("", token)
|
|
495
|
+
if path not in found:
|
|
496
|
+
found.append(path)
|
|
497
|
+
|
|
498
|
+
def add_words(bare: str) -> None:
|
|
499
|
+
for word in _WORD_BREAK.split(bare):
|
|
500
|
+
word = _trim_word(word)
|
|
501
|
+
if word:
|
|
502
|
+
add(word, False)
|
|
503
|
+
|
|
504
|
+
position = 0
|
|
505
|
+
for m in _CODE_SPAN.finditer(text):
|
|
506
|
+
add_words(text[position:m.start()])
|
|
507
|
+
span = m.group(2).strip()
|
|
508
|
+
if span and not any(c.isspace() for c in span):
|
|
509
|
+
add(span, True)
|
|
510
|
+
position = m.end()
|
|
511
|
+
add_words(text[position:])
|
|
512
|
+
return found
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _inside(path: Path, root: Path) -> bool:
|
|
516
|
+
return path == root or root in path.parents
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _source_files_under(folder: Path, suffixes: frozenset[str]) -> list[Path]:
|
|
520
|
+
out: list[Path] = []
|
|
521
|
+
for dirpath, dirnames, filenames in os.walk(folder, followlinks=False):
|
|
522
|
+
dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS and not d.startswith("."))
|
|
523
|
+
for name in sorted(filenames):
|
|
524
|
+
if Path(name).suffix in suffixes:
|
|
525
|
+
out.append(Path(dirpath) / name)
|
|
526
|
+
return out
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _glob_regex(pattern: str, base: str = "") -> re.Pattern[str]:
|
|
530
|
+
"""A glob over `/` paths as a regex, below the folder `base` (taken as written).
|
|
531
|
+
|
|
532
|
+
`**` as a whole name spans any number of folders; `*` and `?` stay within
|
|
533
|
+
one name and, as in glob, skip a name that starts with `.`; every other
|
|
534
|
+
character is literal, `[` and `]` included, so `app/[slug]/*.tsx` names
|
|
535
|
+
the route folder instead of a character class.
|
|
536
|
+
|
|
537
|
+
>>> [bool(_glob_regex("**/*.ts", "src").fullmatch(p)) for p in ("src/a.ts", "src/x/y/a.ts", "src/.a.ts", "lib/a.ts")]
|
|
538
|
+
[True, True, False, False]
|
|
539
|
+
>>> bool(_glob_regex("app/[slug]/*.tsx").fullmatch("app/[slug]/page.tsx"))
|
|
540
|
+
True
|
|
541
|
+
"""
|
|
542
|
+
name = r"(?!\.)[^/]+"
|
|
543
|
+
parts: list[str] = [re.escape(f"{base}/")] if base else []
|
|
544
|
+
names = pattern.split("/")
|
|
545
|
+
for index, part in enumerate(names):
|
|
546
|
+
last = index == len(names) - 1
|
|
547
|
+
if part == "**":
|
|
548
|
+
parts.append(f"{name}(?:/{name})*" if last else f"(?:{name}/)*")
|
|
549
|
+
continue
|
|
550
|
+
regex = r"(?!\.)" if part[:1] in ("*", "?") else ""
|
|
551
|
+
for ch in part:
|
|
552
|
+
regex += "[^/]*" if ch == "*" else "[^/]" if ch == "?" else re.escape(ch)
|
|
553
|
+
parts.append(regex if last else regex + "/")
|
|
554
|
+
return re.compile("".join(parts), re.IGNORECASE if os.name == "nt" else 0)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _glob_walk(start: Path, root: Path, pattern: str, dirs_only: bool) -> list[str]:
|
|
558
|
+
"""Paths under `start` that match `pattern`, relative to `root`, sorted.
|
|
559
|
+
|
|
560
|
+
A walk that follows no link (a link in a folder is listed, never entered)
|
|
561
|
+
and never enters `.git`, `node_modules`, `__pycache__` or a folder
|
|
562
|
+
starting with `.`, unless the pattern names it. Without `**` it goes no
|
|
563
|
+
deeper than the pattern has names below `start`.
|
|
564
|
+
"""
|
|
565
|
+
base = start.relative_to(root).as_posix()
|
|
566
|
+
regex = _glob_regex(pattern, "" if base == "." else base)
|
|
567
|
+
names = pattern.split("/")
|
|
568
|
+
named = {n for n in names if not _GLOB_CHARS.search(n)}
|
|
569
|
+
depth_limit = None if "**" in names else len(names)
|
|
570
|
+
found: list[str] = []
|
|
571
|
+
for dirpath, dirnames, filenames in os.walk(start, followlinks=False):
|
|
572
|
+
here = Path(dirpath)
|
|
573
|
+
dirnames[:] = sorted(
|
|
574
|
+
d for d in dirnames if (d not in SKIP_DIRS and not d.startswith(".")) or d in named
|
|
575
|
+
)
|
|
576
|
+
for entry in (dirnames if dirs_only else dirnames + filenames):
|
|
577
|
+
rel = (here / entry).relative_to(root).as_posix()
|
|
578
|
+
if regex.fullmatch(rel):
|
|
579
|
+
found.append(rel)
|
|
580
|
+
if depth_limit is not None and len(here.relative_to(start).parts) + 1 >= depth_limit:
|
|
581
|
+
dirnames[:] = []
|
|
582
|
+
return sorted(found)
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def resolve_paths(
|
|
586
|
+
tokens: list[str], root: Path, suffixes: frozenset[str] = _SOURCE_SUFFIXES
|
|
587
|
+
) -> tuple[list[str], list[dict]]:
|
|
588
|
+
"""Resolve path tokens under a real root: (files relative to it, rejected)."""
|
|
589
|
+
resolved: list[str] = []
|
|
590
|
+
rejected: list[dict] = []
|
|
591
|
+
refusals = 0 # every rejection, counted before the dedupe
|
|
592
|
+
|
|
593
|
+
def keep(real: Path) -> None:
|
|
594
|
+
rel = real.relative_to(root).as_posix()
|
|
595
|
+
if rel not in resolved:
|
|
596
|
+
resolved.append(rel)
|
|
597
|
+
|
|
598
|
+
def reject(shown: str, reason: str) -> None:
|
|
599
|
+
nonlocal refusals
|
|
600
|
+
refusals += 1
|
|
601
|
+
entry = {"path": shown, "reason": reason}
|
|
602
|
+
if entry not in rejected:
|
|
603
|
+
rejected.append(entry)
|
|
604
|
+
|
|
605
|
+
def check(lexical_text: str, shown: str) -> Path | None:
|
|
606
|
+
lexical = Path(os.path.normpath(os.path.join(root, lexical_text)))
|
|
607
|
+
if not _inside(lexical, root):
|
|
608
|
+
reject(shown, "outside-root")
|
|
609
|
+
return None
|
|
610
|
+
real = Path(os.path.realpath(lexical))
|
|
611
|
+
if not _inside(real, root):
|
|
612
|
+
reject(shown, "symlink-outside-root")
|
|
613
|
+
return None
|
|
614
|
+
return real
|
|
615
|
+
|
|
616
|
+
def take(real: Path) -> int:
|
|
617
|
+
"""Keep a file, or the source files under a folder; return how many."""
|
|
618
|
+
if real.is_file():
|
|
619
|
+
keep(real)
|
|
620
|
+
return 1
|
|
621
|
+
if real.is_dir():
|
|
622
|
+
kept = 0
|
|
623
|
+
for file in _source_files_under(real, suffixes):
|
|
624
|
+
inner = check(str(file), file.relative_to(root).as_posix())
|
|
625
|
+
if inner is not None and inner.is_file():
|
|
626
|
+
keep(inner)
|
|
627
|
+
kept += 1
|
|
628
|
+
return kept
|
|
629
|
+
return 0
|
|
630
|
+
|
|
631
|
+
def take_glob(token: str) -> int:
|
|
632
|
+
"""Keep what a glob matches under the root; return how many files."""
|
|
633
|
+
names = token.rstrip("/").split("/")
|
|
634
|
+
first = next(i for i, name in enumerate(names) if _GLOB_CHARS.search(name))
|
|
635
|
+
# The folders before the first wildcard are fixed: a glob they take
|
|
636
|
+
# out of the root is refused whole, and nothing outside is listed.
|
|
637
|
+
start = check("/".join(names[:first]) or ".", token)
|
|
638
|
+
if start is None or not start.is_dir():
|
|
639
|
+
return 0
|
|
640
|
+
kept = 0
|
|
641
|
+
for rel in _glob_walk(start, root, "/".join(names[first:]), token.endswith("/")):
|
|
642
|
+
real = check(rel, rel)
|
|
643
|
+
if real is not None:
|
|
644
|
+
kept += take(real)
|
|
645
|
+
return kept
|
|
646
|
+
|
|
647
|
+
for token in tokens:
|
|
648
|
+
refusals_before = refusals
|
|
649
|
+
if _GLOB_CHARS.search(token):
|
|
650
|
+
kept = take_glob(token)
|
|
651
|
+
else:
|
|
652
|
+
real = check(token, token)
|
|
653
|
+
if real is None:
|
|
654
|
+
continue
|
|
655
|
+
if not real.exists():
|
|
656
|
+
reject(token, "not-found")
|
|
657
|
+
continue
|
|
658
|
+
kept = take(real)
|
|
659
|
+
# A path whose every file was rejected already says why.
|
|
660
|
+
if kept == 0 and refusals == refusals_before:
|
|
661
|
+
reject(token, "no-match")
|
|
662
|
+
return resolved, rejected
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
# --------------------------------------------------------------------------
|
|
666
|
+
# Gap sources
|
|
667
|
+
# --------------------------------------------------------------------------
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _gap(
|
|
671
|
+
*,
|
|
672
|
+
gid: str,
|
|
673
|
+
title: str,
|
|
674
|
+
severity: str | None,
|
|
675
|
+
category: str | None,
|
|
676
|
+
group: str | None,
|
|
677
|
+
source: str | None,
|
|
678
|
+
export: str | None,
|
|
679
|
+
issue: str | None,
|
|
680
|
+
remediation: str | None,
|
|
681
|
+
) -> dict:
|
|
682
|
+
"""One gap; parse_gaps() adds its `remediation_paths`."""
|
|
683
|
+
return {
|
|
684
|
+
"id": gid,
|
|
685
|
+
"title": title,
|
|
686
|
+
"severity": severity,
|
|
687
|
+
"category": category,
|
|
688
|
+
"category_group": group,
|
|
689
|
+
"source": source,
|
|
690
|
+
"source_citation": source_citation(source),
|
|
691
|
+
"export": export,
|
|
692
|
+
"issue": issue,
|
|
693
|
+
"remediation": remediation,
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def gaps_from_ledger(path: Path) -> list[dict]:
|
|
698
|
+
"""The gaps of a ledger file. Raises ParseError."""
|
|
699
|
+
if not path.is_file():
|
|
700
|
+
raise ParseError("LEDGER_MISSING", f"no ledger at {path}")
|
|
701
|
+
try:
|
|
702
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
703
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
704
|
+
raise ParseError("LEDGER_INVALID", f"cannot read {path}: {exc}") from exc
|
|
705
|
+
if not isinstance(data, dict):
|
|
706
|
+
raise ParseError("LEDGER_INVALID", f"{path} is not a JSON object")
|
|
707
|
+
version = data.get("schema_version")
|
|
708
|
+
if version is not None and version != LEDGER_SCHEMA_VERSION:
|
|
709
|
+
raise ParseError(
|
|
710
|
+
"LEDGER_INVALID",
|
|
711
|
+
f"{path} has schema_version {version!r}; this helper reads version {LEDGER_SCHEMA_VERSION}",
|
|
712
|
+
)
|
|
713
|
+
records = data.get("records")
|
|
714
|
+
if not isinstance(records, list):
|
|
715
|
+
raise ParseError("LEDGER_INVALID", f"{path} has no records list")
|
|
716
|
+
gaps: list[dict] = []
|
|
717
|
+
for index, record in enumerate(records):
|
|
718
|
+
if not isinstance(record, dict) or not isinstance(record.get("id"), str):
|
|
719
|
+
raise ParseError("LEDGER_INVALID", f"record {index} of {path} has no id")
|
|
720
|
+
severity = record.get("severity")
|
|
721
|
+
if severity not in SEVERITIES:
|
|
722
|
+
raise ParseError("LEDGER_INVALID", f"{record['id']} has an unknown severity {severity!r}")
|
|
723
|
+
|
|
724
|
+
def text(key: str) -> str | None:
|
|
725
|
+
value = record.get(key)
|
|
726
|
+
return value if isinstance(value, str) and value else None
|
|
727
|
+
|
|
728
|
+
gaps.append(
|
|
729
|
+
_gap(
|
|
730
|
+
gid=record["id"],
|
|
731
|
+
title=text("title") or "",
|
|
732
|
+
severity=severity,
|
|
733
|
+
category=text("category"),
|
|
734
|
+
group=text("group"),
|
|
735
|
+
source=text("source"),
|
|
736
|
+
export=text("export"),
|
|
737
|
+
issue=text("issue"),
|
|
738
|
+
remediation=text("remediation"),
|
|
739
|
+
)
|
|
740
|
+
)
|
|
741
|
+
return gaps
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def gaps_from_markdown(text: str) -> tuple[list[dict], list[str]]:
|
|
745
|
+
"""The GAP entries of every `## Gap Report` section. Returns (gaps, warnings)."""
|
|
746
|
+
lines = strip_frontmatter(text).splitlines()
|
|
747
|
+
gaps: list[dict] = []
|
|
748
|
+
warnings: list[str] = []
|
|
749
|
+
seen: set[str] = set()
|
|
750
|
+
sections = 0
|
|
751
|
+
in_section = in_fence = False
|
|
752
|
+
current: dict | None = None
|
|
753
|
+
field: str | None = None
|
|
754
|
+
|
|
755
|
+
def close() -> None:
|
|
756
|
+
nonlocal current, field
|
|
757
|
+
if current is not None:
|
|
758
|
+
gid = current["id"]
|
|
759
|
+
if gid in seen:
|
|
760
|
+
warnings.append(f"{gid} appears more than once; the first entry is kept")
|
|
761
|
+
else:
|
|
762
|
+
seen.add(gid)
|
|
763
|
+
values = {k: "\n".join(v).strip() or None for k, v in current["fields"].items()}
|
|
764
|
+
category, group = parse_category(values.get("category"))
|
|
765
|
+
severity = normalize_severity(values.get("severity"))
|
|
766
|
+
if severity is None:
|
|
767
|
+
warnings.append(f"{gid} has no readable severity")
|
|
768
|
+
source = values.get("source")
|
|
769
|
+
gaps.append(
|
|
770
|
+
_gap(
|
|
771
|
+
gid=gid,
|
|
772
|
+
title=current["title"],
|
|
773
|
+
severity=severity,
|
|
774
|
+
category=category,
|
|
775
|
+
group=group,
|
|
776
|
+
source=_unquote(source) if source else None,
|
|
777
|
+
export=_unquote(values["export"]) if values.get("export") else None,
|
|
778
|
+
issue=values.get("issue"),
|
|
779
|
+
remediation=values.get("remediation"),
|
|
780
|
+
)
|
|
781
|
+
)
|
|
782
|
+
current, field = None, None
|
|
783
|
+
|
|
784
|
+
for line in lines:
|
|
785
|
+
if in_fence:
|
|
786
|
+
if _FENCE.match(line):
|
|
787
|
+
in_fence = False
|
|
788
|
+
if current is not None and field in _MULTILINE_FIELDS:
|
|
789
|
+
current["fields"][field].append(line)
|
|
790
|
+
continue
|
|
791
|
+
if _FENCE.match(line):
|
|
792
|
+
in_fence = True
|
|
793
|
+
if current is not None and field in _MULTILINE_FIELDS:
|
|
794
|
+
current["fields"][field].append(line)
|
|
795
|
+
continue
|
|
796
|
+
if _SECTION_END.match(line):
|
|
797
|
+
close()
|
|
798
|
+
in_section = bool(_GAP_REPORT_H2.match(line))
|
|
799
|
+
sections += in_section
|
|
800
|
+
continue
|
|
801
|
+
if not in_section:
|
|
802
|
+
continue
|
|
803
|
+
if _HEADING.match(line):
|
|
804
|
+
close()
|
|
805
|
+
m = _GAP_HEADING.match(line)
|
|
806
|
+
if m:
|
|
807
|
+
current = {"id": m.group(1), "title": m.group(2).strip().strip("*").strip(), "fields": {}}
|
|
808
|
+
continue
|
|
809
|
+
if current is None:
|
|
810
|
+
continue
|
|
811
|
+
m = _FIELD.match(line)
|
|
812
|
+
if m:
|
|
813
|
+
field = m.group(1).lower()
|
|
814
|
+
current["fields"][field] = [m.group(2)]
|
|
815
|
+
continue
|
|
816
|
+
if _RULE.match(line):
|
|
817
|
+
field = None
|
|
818
|
+
continue
|
|
819
|
+
if field in _MULTILINE_FIELDS:
|
|
820
|
+
current["fields"][field].append(line)
|
|
821
|
+
close()
|
|
822
|
+
if sections == 0:
|
|
823
|
+
warnings.append("the report has no ## Gap Report section")
|
|
824
|
+
elif not gaps:
|
|
825
|
+
warnings.append("the Gap Report section holds no GAP entries")
|
|
826
|
+
return gaps, warnings
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def _gap_order(gap: dict) -> tuple[int, int]:
|
|
830
|
+
m = _ID_RE.match(gap["id"])
|
|
831
|
+
return (_SEVERITY_RANK.get(gap["severity"], len(SEVERITIES)), int(m.group(1)) if m else 0)
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def sibling_ledger(report: Path, text: str) -> Path | None:
|
|
835
|
+
"""`test-findings-{run_id}.json` beside the report, when the run id is known."""
|
|
836
|
+
run_id = read_frontmatter(text).get("runId") or ""
|
|
837
|
+
if not run_id:
|
|
838
|
+
m = _RUN_ID_IN_NAME.search(report.name)
|
|
839
|
+
run_id = m.group(1) if m else ""
|
|
840
|
+
if not _RUN_ID_SAFE.match(run_id):
|
|
841
|
+
return None
|
|
842
|
+
return report.parent / f"test-findings-{run_id}.json"
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def _norm_file(path: object) -> str | None:
|
|
846
|
+
"""A file as the lookup compares it: stripped, backslashes read as `/`, no leading `./`."""
|
|
847
|
+
if not isinstance(path, str) or not path.strip():
|
|
848
|
+
return None
|
|
849
|
+
text = path.strip().replace("\\", "/")
|
|
850
|
+
while text.startswith("./"):
|
|
851
|
+
text = text[2:]
|
|
852
|
+
return text
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def _line_number(value: object) -> int | None:
|
|
856
|
+
if isinstance(value, bool):
|
|
857
|
+
return None
|
|
858
|
+
if isinstance(value, int):
|
|
859
|
+
return value
|
|
860
|
+
if isinstance(value, str) and value.strip().isdigit():
|
|
861
|
+
return int(value.strip())
|
|
862
|
+
return None
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def map_match(entries: list, name: str, citation: dict | None) -> dict:
|
|
866
|
+
"""The provenance map entry an export names, keyed on its citation (see "Map match" in the module docstring)."""
|
|
867
|
+
def view(entry: dict) -> dict:
|
|
868
|
+
return {key: entry.get(key) for key in ("source_file", "source_line", "export_type")}
|
|
869
|
+
|
|
870
|
+
named = [e for e in entries if isinstance(e, dict) and isinstance(e.get("export_name"), str)
|
|
871
|
+
and e["export_name"].strip() == name.strip()]
|
|
872
|
+
left = named
|
|
873
|
+
if citation:
|
|
874
|
+
cited = _norm_file(citation.get("file"))
|
|
875
|
+
left = [e for e in named if _norm_file(e.get("source_file")) == cited]
|
|
876
|
+
line = _line_number(citation.get("line"))
|
|
877
|
+
if len(left) > 1 and line is not None:
|
|
878
|
+
left = [e for e in left if _line_number(e.get("source_line")) == line] or left
|
|
879
|
+
status = "found" if len(left) == 1 else "ambiguous" if left else "not-found"
|
|
880
|
+
return {"status": status, "entry": view(left[0]) if status == "found" else None,
|
|
881
|
+
"candidates": [view(e) for e in named]}
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def parse_gaps(
|
|
885
|
+
report: Path | None,
|
|
886
|
+
ledger: Path | None,
|
|
887
|
+
source_root: Path | None,
|
|
888
|
+
extensions: tuple[str, ...] | list[str] = (),
|
|
889
|
+
map_entries: list | None = None,
|
|
890
|
+
) -> dict:
|
|
891
|
+
"""Read the gaps and build the output object. Raises ParseError.
|
|
892
|
+
|
|
893
|
+
`extensions` (without the dot) add to SOURCE_EXTENSIONS. With
|
|
894
|
+
`map_entries` (a provenance map's entries[]), each gap gets its
|
|
895
|
+
`map_match`, null for a gap with no `export`.
|
|
896
|
+
"""
|
|
897
|
+
suffixes = _SOURCE_SUFFIXES | {f".{ext}" for ext in extensions}
|
|
898
|
+
warnings: list[str] = []
|
|
899
|
+
text = None
|
|
900
|
+
if report is not None:
|
|
901
|
+
if not report.is_file():
|
|
902
|
+
raise ParseError("REPORT_MISSING", f"no test report at {report}")
|
|
903
|
+
try:
|
|
904
|
+
text = report.read_text(encoding="utf-8")
|
|
905
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
906
|
+
raise ParseError("REPORT_MISSING", f"cannot read {report}: {exc}") from exc
|
|
907
|
+
if ledger is None:
|
|
908
|
+
candidate = sibling_ledger(report, text)
|
|
909
|
+
if candidate is not None and candidate.is_file():
|
|
910
|
+
ledger = candidate
|
|
911
|
+
if ledger is not None:
|
|
912
|
+
gaps = gaps_from_ledger(ledger)
|
|
913
|
+
read_from = "ledger"
|
|
914
|
+
else:
|
|
915
|
+
gaps, warnings = gaps_from_markdown(text or "")
|
|
916
|
+
read_from = "gap-report"
|
|
917
|
+
gaps.sort(key=_gap_order)
|
|
918
|
+
counts = {s: 0 for s in SEVERITIES}
|
|
919
|
+
for gap in gaps:
|
|
920
|
+
gap["remediation_paths"] = remediation_paths(gap["remediation"], suffixes)
|
|
921
|
+
if gap["severity"] in counts:
|
|
922
|
+
counts[gap["severity"]] += 1
|
|
923
|
+
if map_entries is not None:
|
|
924
|
+
gap["map_match"] = map_match(map_entries, gap["export"], gap["source_citation"]) \
|
|
925
|
+
if gap["export"] and gap["export"].strip() else None
|
|
926
|
+
out = {
|
|
927
|
+
"status": "ok",
|
|
928
|
+
"read_from": read_from,
|
|
929
|
+
"report_path": str(report) if report is not None else None,
|
|
930
|
+
"ledger_path": str(ledger) if ledger is not None else None,
|
|
931
|
+
"source_root": None,
|
|
932
|
+
"gap_count": len(gaps),
|
|
933
|
+
"counts": counts,
|
|
934
|
+
"gaps": gaps,
|
|
935
|
+
"warnings": warnings,
|
|
936
|
+
}
|
|
937
|
+
if source_root is not None:
|
|
938
|
+
root = Path(os.path.realpath(source_root))
|
|
939
|
+
union: list[str] = []
|
|
940
|
+
for gap in gaps:
|
|
941
|
+
resolved, rejected = resolve_paths(gap["remediation_paths"], root, suffixes)
|
|
942
|
+
gap["resolved_paths"] = resolved
|
|
943
|
+
gap["rejected_paths"] = rejected
|
|
944
|
+
union.extend(p for p in resolved if p not in union)
|
|
945
|
+
out["source_root"] = str(root)
|
|
946
|
+
out["resolved_files"] = union
|
|
947
|
+
return out
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
# --------------------------------------------------------------------------
|
|
951
|
+
# Translate (update-skill gap-driven.md §1)
|
|
952
|
+
# --------------------------------------------------------------------------
|
|
953
|
+
|
|
954
|
+
# gap-driven.md §1's table: each ledger category the repair routes, and the
|
|
955
|
+
# change category it takes. Any other category is not routed.
|
|
956
|
+
CHANGE_CATEGORIES = {
|
|
957
|
+
"missing-export": "NEW_EXPORT",
|
|
958
|
+
"missing-type": "NEW_EXPORT",
|
|
959
|
+
"provenance-completeness": "NEW_EXPORT",
|
|
960
|
+
"signature-mismatch": "MODIFIED_EXPORT",
|
|
961
|
+
"fabricated-signature": "MODIFIED_EXPORT",
|
|
962
|
+
"stale-documentation": "MODIFIED_EXPORT",
|
|
963
|
+
"provenance-line": "MOVED_EXPORT",
|
|
964
|
+
"split-body-mismatch": "STRUCTURAL_FIX",
|
|
965
|
+
"structural": "STRUCTURAL_FIX",
|
|
966
|
+
"broken-reference": "STRUCTURAL_FIX",
|
|
967
|
+
"inaccurate-reference": "STRUCTURAL_FIX",
|
|
968
|
+
"reference-escape": "STRUCTURAL_FIX",
|
|
969
|
+
"integration-pattern": "STRUCTURAL_FIX",
|
|
970
|
+
"migration-section": "STRUCTURAL_FIX",
|
|
971
|
+
"scripts-assets": "STRUCTURAL_FIX",
|
|
972
|
+
"metadata-drift": "metadata update",
|
|
973
|
+
"metadata": "metadata update",
|
|
974
|
+
}
|
|
975
|
+
EXPORT_CATEGORIES = ("NEW_EXPORT", "MODIFIED_EXPORT", "MOVED_EXPORT", "DELETED_EXPORT")
|
|
976
|
+
RESCOPE_CATEGORIES = ("missing-export", "missing-type") # rule R1
|
|
977
|
+
COMPLETENESS = "provenance-completeness" # rule R3
|
|
978
|
+
READ_ONLY_MODES = ("dry-run", "detect-only")
|
|
979
|
+
# The remediation gap-ledger.py generates for a missing export or type (_missing_export, and the count gap of
|
|
980
|
+
# the scalar and stack branches). No person wrote it, so it names no removal (rule R1); any other text may.
|
|
981
|
+
_LEDGER_REMEDIATION = re.compile(
|
|
982
|
+
r"Document (?:the type )?`[^`\n]+` in SKILL\.md: (?:read its definition at `[^`\n]+`"
|
|
983
|
+
r"|find its definition in the source \(`[^`\n]+` lists it\)) and "
|
|
984
|
+
r"(?:add its signature, its purpose and a usage example|list its fields or members)\."
|
|
985
|
+
r"|Find the \d+ exports the denominator counts that SKILL\.md and references/ do not name "
|
|
986
|
+
r"\(`[^`\n]+` lists the exports\) and document each with its signature\."
|
|
987
|
+
)
|
|
988
|
+
# A citation of generated markdown: the skill's SKILL.md or a references/*.md file.
|
|
989
|
+
_PACKAGE_FILE = re.compile(r"(?:^|/)(?:SKILL\.md|references/[^/]+\.md)$")
|
|
990
|
+
# The answers translate reads, with the JSON types each takes.
|
|
991
|
+
_ANSWER_TYPES = {"category": (str, type(None)), "name": (str, type(None)), "rescope": (bool,),
|
|
992
|
+
"source_file": (str, type(None))}
|
|
993
|
+
_IS_WINDOWS = os.name == "nt"
|
|
994
|
+
_REPLACE_WAIT_SECONDS = 5.0
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def in_skill_package(citation: dict | None) -> bool:
|
|
998
|
+
"""True when a citation names a line of generated markdown, not source.
|
|
999
|
+
|
|
1000
|
+
>>> [in_skill_package({"file": f, "line": 1}) for f in ("SKILL.md", "{skill_package}/references/api.md", "src/a.ts")]
|
|
1001
|
+
[True, True, False]
|
|
1002
|
+
"""
|
|
1003
|
+
path = _norm_file(citation.get("file")) if isinstance(citation, dict) else None
|
|
1004
|
+
return bool(path and _PACKAGE_FILE.search(path))
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def written_remediation(text: object) -> bool:
|
|
1008
|
+
"""True when a remediation is text a person or a model wrote, not the text gap-ledger.py generates for a
|
|
1009
|
+
missing export or type: only such text can name the removal rule R1 asks about.
|
|
1010
|
+
|
|
1011
|
+
>>> [written_remediation(t) for t in ("Document `x` in SKILL.md: read its definition at `src/a.ts` and add "
|
|
1012
|
+
... "its signature, its purpose and a usage example.", "x is internal: drop it from the surface.", None)]
|
|
1013
|
+
[False, True, False]
|
|
1014
|
+
"""
|
|
1015
|
+
return isinstance(text, str) and bool(text.strip()) and not _LEDGER_REMEDIATION.fullmatch(text.strip())
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def load_judgments(path: Path) -> dict:
|
|
1019
|
+
"""The answers of a --judgments file, keyed by gap id. Raises ParseError (INVALID_INPUT)."""
|
|
1020
|
+
try:
|
|
1021
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
1022
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
1023
|
+
raise ParseError("INVALID_INPUT", f"cannot read --judgments {path}: {exc}") from exc
|
|
1024
|
+
if not isinstance(data, dict) or not all(isinstance(v, dict) for v in data.values()):
|
|
1025
|
+
raise ParseError("INVALID_INPUT", f"--judgments {path} must map each gap id to an object of answers")
|
|
1026
|
+
for gid, answers in data.items():
|
|
1027
|
+
for key, types in _ANSWER_TYPES.items():
|
|
1028
|
+
if key in answers and not isinstance(answers[key], types):
|
|
1029
|
+
raise ParseError("INVALID_INPUT", f"--judgments {path}: {gid}'s `{key}` has the wrong type")
|
|
1030
|
+
return data
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def _answer_text(value: object) -> str | None:
|
|
1034
|
+
return value.strip() or None if isinstance(value, str) else None
|
|
1035
|
+
|
|
1036
|
+
|
|
1037
|
+
def _answer_path(value: object) -> str | None:
|
|
1038
|
+
text = _norm_file(value)
|
|
1039
|
+
return text.rstrip("/") or None if text else None
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def translate_gaps(parsed: dict, map_entries: list, judgments: dict, *, read_only: str | None = None,
|
|
1043
|
+
today: str = "") -> tuple[dict | None, dict]:
|
|
1044
|
+
"""(the change manifest, or None while a gap needs an answer; the summary). See "Translate" in the module
|
|
1045
|
+
docstring."""
|
|
1046
|
+
root_text = parsed.get("source_root")
|
|
1047
|
+
root = Path(root_text) if isinstance(root_text, str) and Path(root_text).is_dir() else None
|
|
1048
|
+
suffixes = _SOURCE_SUFFIXES | {f".{ext}" for ext in map_extensions(map_entries)}
|
|
1049
|
+
warnings = [f"test-report: {w}" for w in parsed.get("warnings") or [] if isinstance(w, str)]
|
|
1050
|
+
needs: list[dict] = []
|
|
1051
|
+
entries: list[dict] = []
|
|
1052
|
+
not_routed: list[dict] = []
|
|
1053
|
+
|
|
1054
|
+
def skip(gap: dict, category: str | None) -> None:
|
|
1055
|
+
not_routed.append({"id": gap["id"], "title": gap.get("title"), "category": category})
|
|
1056
|
+
warnings.append(f"test-report: not routed: {gap['id']} ({category or 'no category'})")
|
|
1057
|
+
|
|
1058
|
+
def ask(gap: dict, category: str | None, want: list[str]) -> None:
|
|
1059
|
+
needs.append({"gap_id": gap["id"], "title": gap.get("title"), "category": category,
|
|
1060
|
+
"issue": gap.get("issue"), "remediation": gap.get("remediation"), "needs": want})
|
|
1061
|
+
|
|
1062
|
+
for gap in parsed["gaps"]:
|
|
1063
|
+
answers = judgments.get(gap["id"]) or {}
|
|
1064
|
+
want: list[str] = []
|
|
1065
|
+
export = _answer_text(gap.get("export"))
|
|
1066
|
+
category = gap.get("category") or None
|
|
1067
|
+
if category is None and "category" in answers:
|
|
1068
|
+
category = _answer_text(answers["category"])
|
|
1069
|
+
elif category is None:
|
|
1070
|
+
want.append("category")
|
|
1071
|
+
if export is None and "name" not in answers:
|
|
1072
|
+
want.append("name")
|
|
1073
|
+
ask(gap, None, want)
|
|
1074
|
+
continue
|
|
1075
|
+
change = CHANGE_CATEGORIES.get(category or "")
|
|
1076
|
+
if change is None:
|
|
1077
|
+
skip(gap, category)
|
|
1078
|
+
continue
|
|
1079
|
+
name = export or _answer_text(answers.get("name"))
|
|
1080
|
+
if change in EXPORT_CATEGORIES and name is None:
|
|
1081
|
+
if "name" in answers: # answered null: no export to repair
|
|
1082
|
+
skip(gap, category)
|
|
1083
|
+
else:
|
|
1084
|
+
ask(gap, category, ["name"])
|
|
1085
|
+
continue
|
|
1086
|
+
citation = gap.get("source_citation")
|
|
1087
|
+
if in_skill_package(citation):
|
|
1088
|
+
citation = None
|
|
1089
|
+
match = map_match(map_entries, name, citation) if name else None
|
|
1090
|
+
rescope = False
|
|
1091
|
+
if category in RESCOPE_CATEGORIES:
|
|
1092
|
+
if "rescope" in answers:
|
|
1093
|
+
rescope = answers["rescope"]
|
|
1094
|
+
elif written_remediation(gap.get("remediation")):
|
|
1095
|
+
want.append("rescope")
|
|
1096
|
+
exclude = None
|
|
1097
|
+
if rescope:
|
|
1098
|
+
exclude = _answer_path(answers.get("source_file"))
|
|
1099
|
+
if exclude is None and match and match["status"] == "found":
|
|
1100
|
+
exclude = _answer_path(match["entry"]["source_file"])
|
|
1101
|
+
if exclude is None and citation:
|
|
1102
|
+
exclude = _answer_path(citation["file"])
|
|
1103
|
+
if exclude is None and "source_file" not in answers:
|
|
1104
|
+
want.append("source_file")
|
|
1105
|
+
elif exclude is None:
|
|
1106
|
+
warnings.append(f"test-report: {gap['id']}: no source file to exclude, so it is documented, not "
|
|
1107
|
+
f"rescoped")
|
|
1108
|
+
rescope = False
|
|
1109
|
+
resolved = list(gap.get("resolved_paths") or [])
|
|
1110
|
+
rejected = list(gap.get("rejected_paths") or [])
|
|
1111
|
+
if category == COMPLETENESS and not resolved and root is not None:
|
|
1112
|
+
cited = _answer_path(citation["file"]) if citation else None
|
|
1113
|
+
if cited is None and "source_file" in answers:
|
|
1114
|
+
cited = _answer_path(answers["source_file"])
|
|
1115
|
+
elif cited is None:
|
|
1116
|
+
want.append("source_file")
|
|
1117
|
+
if cited is not None:
|
|
1118
|
+
resolved, refused = resolve_paths([_LINE_SUFFIX.sub("", cited)], root, suffixes)
|
|
1119
|
+
rejected += refused
|
|
1120
|
+
if want:
|
|
1121
|
+
ask(gap, category, want)
|
|
1122
|
+
continue
|
|
1123
|
+
entry = {"name": name, "gap_id": gap["id"], "category": category, "severity": gap.get("severity")}
|
|
1124
|
+
if citation:
|
|
1125
|
+
entry["source_citation"] = citation
|
|
1126
|
+
entry["remediation_paths"] = list(gap.get("remediation_paths") or [])
|
|
1127
|
+
entry["resolved_paths"] = resolved
|
|
1128
|
+
entry["rejected_paths"] = rejected
|
|
1129
|
+
entry["change_category"] = "DELETED_EXPORT" if rescope else change
|
|
1130
|
+
entry["remediation"] = gap.get("remediation")
|
|
1131
|
+
if category == COMPLETENESS:
|
|
1132
|
+
entry["provenance_completeness"] = True
|
|
1133
|
+
if rescope:
|
|
1134
|
+
entry["rescope"] = {
|
|
1135
|
+
"amendment": {"path": exclude, "action": "excluded", "category": "scope-expansion",
|
|
1136
|
+
"reason": gap.get("remediation") or gap.get("title") or "", "date": today,
|
|
1137
|
+
"workflow": "skf-update-skill"},
|
|
1138
|
+
"exclude": exclude,
|
|
1139
|
+
}
|
|
1140
|
+
if read_only:
|
|
1141
|
+
warnings.append(f"proposed-amendment: excluded {exclude} (scope-expansion); not written: "
|
|
1142
|
+
f"--{read_only}")
|
|
1143
|
+
entry["map_match"] = match
|
|
1144
|
+
entries.append(entry)
|
|
1145
|
+
if needs:
|
|
1146
|
+
return None, {"status": "needs-judgment", "needs_judgment": needs}
|
|
1147
|
+
return {"mode": "gap-driven", "entries": entries}, {
|
|
1148
|
+
"status": "written", "gap_count": len(entries), "not_routed": not_routed, "warnings": warnings}
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
def write_json_atomic(path: Path, value) -> None:
|
|
1152
|
+
"""Write `value` as indented JSON through a temporary file beside `path` and one rename (retried on Windows,
|
|
1153
|
+
where a scanner may hold the target open for a moment)."""
|
|
1154
|
+
tmp = path.with_name(f".{path.name}.skf-{os.getpid()}-tmp")
|
|
1155
|
+
try:
|
|
1156
|
+
tmp.write_bytes((json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
|
|
1157
|
+
deadline = time.monotonic() + _REPLACE_WAIT_SECONDS
|
|
1158
|
+
while True:
|
|
1159
|
+
try:
|
|
1160
|
+
os.replace(tmp, path)
|
|
1161
|
+
break
|
|
1162
|
+
except PermissionError:
|
|
1163
|
+
if not _IS_WINDOWS or time.monotonic() >= deadline:
|
|
1164
|
+
raise
|
|
1165
|
+
time.sleep(0.05)
|
|
1166
|
+
except OSError:
|
|
1167
|
+
try:
|
|
1168
|
+
tmp.unlink()
|
|
1169
|
+
except OSError:
|
|
1170
|
+
pass
|
|
1171
|
+
raise
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
# --------------------------------------------------------------------------
|
|
1175
|
+
# CLI
|
|
1176
|
+
# --------------------------------------------------------------------------
|
|
1177
|
+
|
|
1178
|
+
|
|
1179
|
+
def _emit(payload: dict) -> None:
|
|
1180
|
+
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
1181
|
+
|
|
1182
|
+
|
|
1183
|
+
def map_entries_of(path: Path) -> list:
|
|
1184
|
+
"""A provenance map's entries[] ([] when it has none). Raises ParseError (INVALID_INPUT)."""
|
|
1185
|
+
try:
|
|
1186
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
1187
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
1188
|
+
raise ParseError("INVALID_INPUT", f"cannot read --provenance-map {path}: {exc}") from exc
|
|
1189
|
+
if not isinstance(data, dict):
|
|
1190
|
+
raise ParseError("INVALID_INPUT", f"--provenance-map {path} is not a JSON object")
|
|
1191
|
+
return data.get("entries") if isinstance(data.get("entries"), list) else []
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def map_extensions(entries: list) -> list[str]:
|
|
1195
|
+
"""The extension (no dot) of each file a provenance map's entries[]
|
|
1196
|
+
source_file values name."""
|
|
1197
|
+
found: list[str] = []
|
|
1198
|
+
for entry in entries:
|
|
1199
|
+
if isinstance(entry, dict) and isinstance(entry.get("source_file"), str):
|
|
1200
|
+
m = _EXTENSION.search(entry["source_file"].strip())
|
|
1201
|
+
if m and m.group(1) not in found:
|
|
1202
|
+
found.append(m.group(1))
|
|
1203
|
+
return found
|
|
1204
|
+
|
|
1205
|
+
|
|
1206
|
+
def _options(args: argparse.Namespace) -> tuple[Path | None, list[str], list | None]:
|
|
1207
|
+
"""(--source-root, the extensions --ext and --provenance-map add, the
|
|
1208
|
+
map's entries or None). Raises ParseError (INVALID_INPUT)."""
|
|
1209
|
+
root = None
|
|
1210
|
+
if args.source_root is not None:
|
|
1211
|
+
root = Path(args.source_root)
|
|
1212
|
+
if not root.is_dir():
|
|
1213
|
+
raise ParseError("INVALID_INPUT", f"--source-root is not a folder: {root}")
|
|
1214
|
+
extensions: list[str] = []
|
|
1215
|
+
for value in args.ext:
|
|
1216
|
+
m = _EXT_ARG.fullmatch(value)
|
|
1217
|
+
if not m:
|
|
1218
|
+
raise ParseError("INVALID_INPUT", f"--ext takes a file extension such as kt or .vue, got {value!r}")
|
|
1219
|
+
extensions.append(m.group(1))
|
|
1220
|
+
entries = None
|
|
1221
|
+
if args.provenance_map is not None:
|
|
1222
|
+
entries = map_entries_of(Path(args.provenance_map))
|
|
1223
|
+
extensions += map_extensions(entries)
|
|
1224
|
+
return root, extensions, entries
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
def _cmd_paths(args: argparse.Namespace) -> int:
|
|
1228
|
+
try:
|
|
1229
|
+
root, extensions, _ = _options(args)
|
|
1230
|
+
except ParseError as exc:
|
|
1231
|
+
_emit({"status": "error", "code": exc.code, "error": str(exc)})
|
|
1232
|
+
return 1
|
|
1233
|
+
real = Path(os.path.realpath(root))
|
|
1234
|
+
suffixes = _SOURCE_SUFFIXES | {f".{ext}" for ext in extensions}
|
|
1235
|
+
tokens = [_LINE_SUFFIX.sub("", path.strip()) for path in args.path if path.strip()]
|
|
1236
|
+
resolved, rejected = resolve_paths(tokens, real, suffixes)
|
|
1237
|
+
_emit({"status": "ok", "source_root": str(real), "resolved_paths": resolved, "rejected_paths": rejected})
|
|
1238
|
+
return 0
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
def _cmd_parse(args: argparse.Namespace) -> int:
|
|
1242
|
+
if args.report is None and args.ledger is None:
|
|
1243
|
+
_emit({"status": "error", "code": "INVALID_INPUT", "error": "give --report, --ledger or both"})
|
|
1244
|
+
return 1
|
|
1245
|
+
try:
|
|
1246
|
+
root, extensions, entries = _options(args)
|
|
1247
|
+
except ParseError as exc:
|
|
1248
|
+
_emit({"status": "error", "code": exc.code, "error": str(exc)})
|
|
1249
|
+
return 1
|
|
1250
|
+
try:
|
|
1251
|
+
out = parse_gaps(
|
|
1252
|
+
Path(args.report) if args.report else None,
|
|
1253
|
+
Path(args.ledger) if args.ledger else None,
|
|
1254
|
+
root,
|
|
1255
|
+
extensions,
|
|
1256
|
+
entries,
|
|
1257
|
+
)
|
|
1258
|
+
except ParseError as exc:
|
|
1259
|
+
_emit({"status": "error", "code": exc.code, "error": str(exc)})
|
|
1260
|
+
return 1
|
|
1261
|
+
_emit(out)
|
|
1262
|
+
return 0
|
|
1263
|
+
|
|
1264
|
+
|
|
1265
|
+
def _cmd_match(args: argparse.Namespace) -> int:
|
|
1266
|
+
citation = source_citation(args.citation) if args.citation is not None else None
|
|
1267
|
+
try:
|
|
1268
|
+
if args.citation is not None and citation is None:
|
|
1269
|
+
raise ParseError("INVALID_INPUT", f"--citation takes a file:line pair, got {args.citation!r}")
|
|
1270
|
+
if not args.name.strip():
|
|
1271
|
+
raise ParseError("INVALID_INPUT", "--name is empty")
|
|
1272
|
+
entries = map_entries_of(Path(args.provenance_map))
|
|
1273
|
+
except ParseError as exc:
|
|
1274
|
+
_emit({"status": "error", "code": exc.code, "error": str(exc)})
|
|
1275
|
+
return 1
|
|
1276
|
+
_emit({"status": "ok", "name": args.name, "citation": citation,
|
|
1277
|
+
"map_match": map_match(entries, args.name, citation)})
|
|
1278
|
+
return 0
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
def _parsed_gaps(path: Path) -> dict:
|
|
1282
|
+
"""parse's output in a --gaps file. Raises ParseError (INVALID_INPUT)."""
|
|
1283
|
+
try:
|
|
1284
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
1285
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
1286
|
+
raise ParseError("INVALID_INPUT", f"cannot read --gaps {path}: {exc}") from exc
|
|
1287
|
+
if not isinstance(data, dict) or data.get("status") != "ok" or not isinstance(data.get("gaps"), list) \
|
|
1288
|
+
or not all(isinstance(g, dict) and isinstance(g.get("id"), str) for g in data["gaps"]):
|
|
1289
|
+
raise ParseError("INVALID_INPUT", f"--gaps {path} is not the output of a parse that succeeded")
|
|
1290
|
+
return data
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
def _cmd_translate(args: argparse.Namespace) -> int:
|
|
1294
|
+
try:
|
|
1295
|
+
if args.date is not None and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", args.date):
|
|
1296
|
+
raise ParseError("INVALID_INPUT", f"--date takes YYYY-MM-DD, got {args.date!r}")
|
|
1297
|
+
parsed = _parsed_gaps(Path(args.gaps))
|
|
1298
|
+
entries = map_entries_of(Path(args.provenance_map))
|
|
1299
|
+
judgments = load_judgments(Path(args.judgments)) if args.judgments else {}
|
|
1300
|
+
manifest, summary = translate_gaps(
|
|
1301
|
+
parsed, entries, judgments, read_only=args.read_only,
|
|
1302
|
+
today=args.date or datetime.now(timezone.utc).date().isoformat())
|
|
1303
|
+
if manifest is not None:
|
|
1304
|
+
try:
|
|
1305
|
+
write_json_atomic(Path(args.output), manifest)
|
|
1306
|
+
except OSError as exc:
|
|
1307
|
+
raise ParseError("INVALID_INPUT", f"cannot write {args.output}: {exc}") from exc
|
|
1308
|
+
summary = {"status": summary["status"], "output": args.output,
|
|
1309
|
+
**{k: v for k, v in summary.items() if k != "status"}}
|
|
1310
|
+
except ParseError as exc:
|
|
1311
|
+
_emit({"status": "error", "code": exc.code, "error": str(exc)})
|
|
1312
|
+
return 1
|
|
1313
|
+
_emit(summary)
|
|
1314
|
+
return 0
|
|
1315
|
+
|
|
1316
|
+
|
|
1317
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
1318
|
+
parser = argparse.ArgumentParser(
|
|
1319
|
+
prog="skf-parse-gaps",
|
|
1320
|
+
description="Read a test report's gaps (ledger first, Gap Report otherwise) as JSON.",
|
|
1321
|
+
)
|
|
1322
|
+
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
1323
|
+
|
|
1324
|
+
p = sub.add_parser("parse", help="print the gaps as JSON")
|
|
1325
|
+
p.add_argument("--report", help="the test report (test-report-{skill}-{run_id}.md)")
|
|
1326
|
+
p.add_argument("--ledger", help="the gap ledger, when not the one beside the report")
|
|
1327
|
+
p.add_argument("--source-root", help="resolve remediation paths under this folder")
|
|
1328
|
+
p.set_defaults(func=_cmd_parse)
|
|
1329
|
+
|
|
1330
|
+
q = sub.add_parser("paths", help="resolve given paths under the source root")
|
|
1331
|
+
q.add_argument("--source-root", required=True, help="resolve the paths under this folder")
|
|
1332
|
+
q.add_argument("path", nargs="+", help="a path relative to the source root (a :line suffix is dropped)")
|
|
1333
|
+
q.set_defaults(func=_cmd_paths)
|
|
1334
|
+
|
|
1335
|
+
m = sub.add_parser("match", help="find an export in the provenance map, as a gap's map_match does")
|
|
1336
|
+
m.add_argument("--provenance-map", required=True, help="the provenance map to look the export up in")
|
|
1337
|
+
m.add_argument("--name", required=True, help="the export's name")
|
|
1338
|
+
m.add_argument("--citation", help="the export's source citation, file:line")
|
|
1339
|
+
m.set_defaults(func=_cmd_match)
|
|
1340
|
+
|
|
1341
|
+
t = sub.add_parser("translate", help="write update-skill's gap-driven change manifest from parse's gaps")
|
|
1342
|
+
t.add_argument("--gaps", required=True, help="the output of parse")
|
|
1343
|
+
t.add_argument("--provenance-map", required=True, help="the provenance map each entry's map_match reads")
|
|
1344
|
+
t.add_argument("--judgments", help="the answers to its needs_judgment[], keyed by gap id")
|
|
1345
|
+
t.add_argument("--read-only", choices=READ_ONLY_MODES,
|
|
1346
|
+
help="the read-only flag of the run: each rescope is a proposed amendment")
|
|
1347
|
+
t.add_argument("--date", help="the date of a rescope's amendment, YYYY-MM-DD (default: today, UTC)")
|
|
1348
|
+
t.add_argument("-o", "--output", required=True, help="where to write the change manifest")
|
|
1349
|
+
t.set_defaults(func=_cmd_translate)
|
|
1350
|
+
|
|
1351
|
+
for parser_ in (p, q):
|
|
1352
|
+
parser_.add_argument(
|
|
1353
|
+
"--ext",
|
|
1354
|
+
action="append",
|
|
1355
|
+
default=[],
|
|
1356
|
+
help="another source file extension, e.g. kt or .vue (repeatable)",
|
|
1357
|
+
)
|
|
1358
|
+
parser_.add_argument(
|
|
1359
|
+
"--provenance-map",
|
|
1360
|
+
help="add the extension of each file this provenance map's entries[] name",
|
|
1361
|
+
)
|
|
1362
|
+
|
|
1363
|
+
return parser
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
def _force_utf8(*streams) -> None:
|
|
1367
|
+
"""Reconfigure the JSON streams to UTF-8 (a Windows console uses cp1252)."""
|
|
1368
|
+
for stream in streams:
|
|
1369
|
+
if hasattr(stream, "reconfigure"):
|
|
1370
|
+
errors = getattr(stream, "errors", None)
|
|
1371
|
+
if errors is None:
|
|
1372
|
+
stream.reconfigure(encoding="utf-8")
|
|
1373
|
+
else:
|
|
1374
|
+
stream.reconfigure(encoding="utf-8", errors=errors)
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1378
|
+
_force_utf8(sys.stdout)
|
|
1379
|
+
args = _build_parser().parse_args(argv)
|
|
1380
|
+
return args.func(args)
|
|
1381
|
+
|
|
1382
|
+
|
|
1383
|
+
if __name__ == "__main__":
|
|
1384
|
+
raise SystemExit(main())
|