bmad-module-skill-forge 2.2.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +49 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +135 -18
- package/docs/verifying-a-skill.md +83 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +2 -2
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +32 -12
- package/src/shared/references/description-guard-protocol.md +12 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +30 -14
- package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +694 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
- package/src/shared/scripts/skf-description-guard.py +99 -11
- package/src/shared/scripts/skf-detect-docs.py +694 -2
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
- package/src/shared/scripts/skf-detect-tools.py +209 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +4084 -85
- package/src/shared/scripts/skf-extraction-inventory.py +746 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +661 -0
- package/src/shared/scripts/skf-hash-content.py +166 -12
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +159 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +161 -27
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +670 -119
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +5 -5
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
- package/src/shared/scripts/skf-write-skill-brief.py +367 -19
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/init.md +148 -210
- package/src/skf-audit-skill/references/re-index.md +96 -108
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +27 -18
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +75 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
- package/src/skf-brief-skill/SKILL.md +36 -25
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +17 -24
- package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +142 -125
- package/src/skf-brief-skill/references/confirm-brief.md +10 -8
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +3 -2
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
- package/src/skf-create-skill/assets/skill-sections.md +15 -5
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +107 -29
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
- package/src/skf-create-skill/references/extract.md +200 -161
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
- package/src/skf-create-skill/references/extraction-patterns.md +584 -196
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +28 -59
- package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +28 -19
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +79 -146
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +118 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -90
- package/src/skf-setup/references/detect-and-tier.md +58 -67
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +217 -254
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +163 -88
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +162 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +41 -155
- package/src/skf-test-skill/references/source-access-protocol.md +19 -92
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +287 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
- package/src/skf-test-skill/scripts/score-signatures.py +459 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +233 -222
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +250 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +180 -47
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +100 -35
- package/src/skf-update-skill/references/write.md +331 -149
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -679
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# /// script
|
|
3
|
-
# requires-python = ">=3.
|
|
4
|
-
# dependencies = []
|
|
3
|
+
# requires-python = ">=3.11"
|
|
4
|
+
# dependencies = ["pyyaml"]
|
|
5
5
|
# ///
|
|
6
6
|
"""SKF Verify Provenance Completeness — deterministic Check D for update-skill.
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
update-skill's Check D (Provenance Completeness, now in write.md §6)
|
|
9
9
|
previously asked the model to perform three deterministic set/citation
|
|
10
10
|
operations by eye, against in-context data:
|
|
11
11
|
|
|
@@ -14,7 +14,27 @@ operations by eye, against in-context data:
|
|
|
14
14
|
2. **Orphans** — flag provenance entries whose export was removed
|
|
15
15
|
(provenance `entries[].export_name` \\ metadata `exports[]`).
|
|
16
16
|
3. **Stale citations** — flag entries whose `source_file:source_line`
|
|
17
|
-
no longer resolves (file gone, or line past end of file)
|
|
17
|
+
no longer resolves (file gone, or line past end of file), or whose
|
|
18
|
+
line is not a line that defines the export (`line-not-definition`).
|
|
19
|
+
4. **Citation prefixes** (only with `--skill-dir`): flag each `[AST:]` /
|
|
20
|
+
`[SRC:]` citation in the skill's markdown whose prefix disagrees with
|
|
21
|
+
the provenance entry it cites, or an `[AST:]` citation in a skill whose
|
|
22
|
+
map holds no ast-grep entry at all.
|
|
23
|
+
5. **Node kinds** (only with `--check-node-kinds`): flag each ast-grep
|
|
24
|
+
entry whose `ast_node_type` is not a node kind ast-grep knows for the
|
|
25
|
+
entry's language.
|
|
26
|
+
|
|
27
|
+
Reference apps: when `--metadata` has `scope_type: reference-app`, 1 and 2
|
|
28
|
+
do not run. A reference app's `exports[]` is empty by design while its
|
|
29
|
+
entries are per-citation, so the two sets measure different surfaces:
|
|
30
|
+
`missing` and `orphaned` stay empty and `summary.set_diff` is
|
|
31
|
+
`not-applicable` (`checked` otherwise).
|
|
32
|
+
|
|
33
|
+
Four more subcommands act on the same rules: `fix` applies the citation
|
|
34
|
+
prefix and source-line fixes a `verify` result has one answer for,
|
|
35
|
+
`definition-lines` gives the lines that define one export,
|
|
36
|
+
`classify-stale` decides which stale names are fabricated signatures, and
|
|
37
|
+
`kind-at` gives the node kind of the recipe that matches a source line.
|
|
18
38
|
|
|
19
39
|
An LLM set-diff can silently pass a dropped or orphaned entry, and eyeballing
|
|
20
40
|
whether a cited line still exists is not something the model can do reliably.
|
|
@@ -25,9 +45,12 @@ export-name strings) and `provenance-map.json` (`entries[]` with
|
|
|
25
45
|
`write.md` §2 and §3. This script diffs the two sets and re-checks the
|
|
26
46
|
citations exactly, so Check D consumes JSON rather than computing it.
|
|
27
47
|
|
|
28
|
-
Timing:
|
|
29
|
-
(§2) and `provenance-map.json` (§3) are on disk.
|
|
30
|
-
|
|
48
|
+
Timing: update-skill runs it **post-write**, in write.md §6, after
|
|
49
|
+
`metadata.json` (§2) and `provenance-map.json` (§3) are on disk.
|
|
50
|
+
create-skill runs it too, at validate §7a against the staged package (with
|
|
51
|
+
`--skill-dir` and `--check-node-kinds`, as update-skill does), and
|
|
52
|
+
test-skill at coverage-check §4c (line check only); test-skill's
|
|
53
|
+
coverage-check §2c runs `classify-stale`.
|
|
31
54
|
|
|
32
55
|
Determinism:
|
|
33
56
|
- Set operations are pure over the two JSON inputs.
|
|
@@ -44,9 +67,157 @@ internal name of a barrel-renamed export (e.g. `_Impl` re-exported as
|
|
|
44
67
|
`skf-load-provenance.extract_reexport_map`, so an internal-named entry does
|
|
45
68
|
not read as a missing public export nor as an orphan.
|
|
46
69
|
|
|
47
|
-
|
|
70
|
+
Definition lines: after a citation resolves (the file exists and the line
|
|
71
|
+
is in range, so the older reasons win), the line must define the entry's
|
|
72
|
+
export. NAME is the raw `export_name` (not canonicalized); for a dotted
|
|
73
|
+
name (`App.update`) it is the last segment. The rules follow the
|
|
74
|
+
`source_file` extension:
|
|
75
|
+
|
|
76
|
+
- Python (`.py`, `.pyi`): the `def` / `async def` / `class NAME` line
|
|
77
|
+
itself (a decorator line above it does not count); an assignment
|
|
78
|
+
binding NAME (`NAME =`, `NAME: T =`, annotation-only `NAME: T`, a
|
|
79
|
+
target list `A, NAME = ...`, a chain `A = NAME = ...`, PEP 695
|
|
80
|
+
`type NAME =`); an import that binds NAME (`import NAME`,
|
|
81
|
+
`import X as NAME`, `from X import NAME`, `from X import Y as NAME`),
|
|
82
|
+
including NAME on its own line inside a parenthesized or
|
|
83
|
+
backslash-continued import; a `def` / `async def` / `class` line
|
|
84
|
+
ending in a backslash counts through the next line when that line
|
|
85
|
+
starts with NAME, and the name's line is the definition line (the one
|
|
86
|
+
the recipes cite). These count only on a line where a
|
|
87
|
+
statement begins (the stdlib tokenizer says so), so a docstring line,
|
|
88
|
+
a keyword argument or a parameter never matches; a file that does not
|
|
89
|
+
tokenize is matched line by line. A `from X import *` line counts only
|
|
90
|
+
when no other line of the file defines NAME (a re-export the
|
|
91
|
+
extraction could not trace is cited at its star import).
|
|
92
|
+
- TS/JS (`.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`): a
|
|
93
|
+
`function` (or `function*`), `class`, `interface`, `type`, `enum`,
|
|
94
|
+
`namespace`, `const`, `let` or `var` declaration of NAME, with any of
|
|
95
|
+
`export`, `default`, `async`, `declare`, `abstract` and decorators
|
|
96
|
+
(`@name`, `@a.b`, `@name(...)`, before `export` or between `export`
|
|
97
|
+
and `class`; a name inside a decorator's arguments is not declared);
|
|
98
|
+
a later declarator of a `const`, `let` or `var` list
|
|
99
|
+
(`export const a = 1, NAME = 2;`, or `export const a = 1,` then
|
|
100
|
+
` NAME = 2;` as Prettier writes it: see Declarator lists below); NAME
|
|
101
|
+
on the line right after a line that ends in its declaration keyword
|
|
102
|
+
(`export function`, `export interface`, `export const` and so on),
|
|
103
|
+
which is the line cited;
|
|
104
|
+
`export import NAME =`; `export * as NAME from` (NAME may be quoted);
|
|
105
|
+
`export default NAME`;
|
|
106
|
+
an `export { ... }` list whose exposed name is NAME, an identifier or
|
|
107
|
+
a quoted string on either side of `as`; CommonJS
|
|
108
|
+
`exports.NAME =`, `module.exports.NAME =`, or NAME as a key or
|
|
109
|
+
shorthand inside `module.exports = { ... }`. A list or object spanning
|
|
110
|
+
several lines counts on the line where NAME appears. For a dotted
|
|
111
|
+
name, an indented class member declaring NAME (method, property or
|
|
112
|
+
accessor, with any of `public`, `private`, `protected`, `static`,
|
|
113
|
+
`readonly`, `async`, `abstract`, `override`, `declare`, `get`, `set`)
|
|
114
|
+
also counts. An indented `NAME(...)` with no modifier, followed by `;`
|
|
115
|
+
or another expression rather than a body `{` or a return type `:`, is
|
|
116
|
+
a call, not a member; an indented `NAME: ...` or `NAME = ...` ending
|
|
117
|
+
with `,` is an object-literal entry or a destructuring default.
|
|
118
|
+
Comments never match: `//` to the end of the line and `/* ... */`
|
|
119
|
+
across lines are removed first, outside '...', "..." and `...` strings,
|
|
120
|
+
and brackets inside a string do not count toward a `module.exports`
|
|
121
|
+
object's depth. Strings are followed within one line only, so a
|
|
122
|
+
template literal spanning lines can hide or expose a comment marker.
|
|
123
|
+
|
|
124
|
+
Declarator lists: a `const`, `let` or `var` list is split at its
|
|
125
|
+
top-level commas, outside strings, regular expression literals and
|
|
126
|
+
brackets, a type-argument list included (`Map<K, V>`: a `<` right after
|
|
127
|
+
a name), up to a top-level `;`, and a destructuring pattern declares
|
|
128
|
+
nothing. The list goes on to the next line when a line ends inside a
|
|
129
|
+
bracket, after a top-level comma or after an operator waiting for its
|
|
130
|
+
operand, or when the next line starts with an operator (a member chain or
|
|
131
|
+
a ternary broken over lines), so a declarator on a line of its own
|
|
132
|
+
counts. A `<` where an expression begins (JSX, a type assertion, a generic
|
|
133
|
+
arrow function's type parameters) is not followed: no later declarator of
|
|
134
|
+
the list counts after it.
|
|
135
|
+
|
|
136
|
+
Indentation: for a dotted name every match counts. For an undotted name,
|
|
137
|
+
matches at any indentation count, but when the file has a column-0 match
|
|
138
|
+
the indented ones are dropped, so a local shadow never outranks the
|
|
139
|
+
module-level definition. A line inside a multi-line import, export list
|
|
140
|
+
or `module.exports` object takes the column of the line its statement
|
|
141
|
+
starts on, and so do the name line of a declaration split after its
|
|
142
|
+
keyword and a later declarator on a line of its own. When the recorded
|
|
143
|
+
line is itself an indented match that this rule dropped (a method
|
|
144
|
+
recorded under an undotted name beside a module-level function of the
|
|
145
|
+
same name), the finding lists the recorded line too, so it carries
|
|
146
|
+
several lines and no caller auto-moves it.
|
|
147
|
+
|
|
148
|
+
Entries in other files, and entries whose `export_type` is `module` or
|
|
149
|
+
`package` (case-insensitive; neither has a definition line), are skipped
|
|
150
|
+
and counted in `summary.line_check_skipped`. A `line-not-definition` item
|
|
151
|
+
lists every defining line in the file in `definition_lines`. An empty list
|
|
152
|
+
means the rules found no definition line, which may be a shape they do not
|
|
153
|
+
cover: the entry is unverified, not gone, and no caller auto-fixes it.
|
|
154
|
+
|
|
155
|
+
Citation check (`--skill-dir DIR`, which must hold a `SKILL.md`): scans
|
|
156
|
+
`DIR/SKILL.md` and `DIR/references/**/*.md` for `[AST:path:Ln]` /
|
|
157
|
+
`[SRC:path:Ln]`, ranges `Ln-m` / `Ln-Lm` included (the first line counts);
|
|
158
|
+
a citation with no line is ignored. A citation matches an entry when its
|
|
159
|
+
normalized path equals `source_file` and its line equals `source_line`.
|
|
160
|
+
`summary.skill_citations_matched` counts the citations that matched an
|
|
161
|
+
entry: 0 with citations scanned means the skill cites paths under another
|
|
162
|
+
root. Reasons, one per citation:
|
|
163
|
+
|
|
164
|
+
- `prefix-mismatch`: the matched entry's `extraction_method` implies the
|
|
165
|
+
other prefix (`ast-grep` / `ast_bridge` give AST, `source-read` /
|
|
166
|
+
`source_reading` give SRC). It wins over the reason below.
|
|
167
|
+
- `ast-without-ast-grep`: an `[AST:]` citation in a skill whose map has
|
|
168
|
+
no `ast-grep` or `ast_bridge` entry, raised only when every entry with
|
|
169
|
+
a source line carries a known `extraction_method` (a missing or
|
|
170
|
+
unknown method may hide an ast-grep entry).
|
|
171
|
+
|
|
172
|
+
When several entries share the cited line, the citation is a
|
|
173
|
+
`prefix-mismatch` only when none of them implies its prefix. An
|
|
174
|
+
`extraction_method` that is not a string reads as unknown (it implies no
|
|
175
|
+
prefix). Unmatched citations are not findings unless the second reason
|
|
176
|
+
applies.
|
|
177
|
+
|
|
178
|
+
Node-kind check (`--check-node-kinds`): ast-grep's JSON output never
|
|
179
|
+
reports the kind of the node a rule matched, so an entry's `ast_node_type`
|
|
180
|
+
is copied from the recipe that matched it, and nothing else proves it is a
|
|
181
|
+
real kind. For each entry whose `extraction_method` is `ast-grep` or
|
|
182
|
+
`ast_bridge` and whose `ast_node_type` is a non-blank string, the
|
|
183
|
+
`source_file` extension picks an ast-grep language (`.py` / `.pyi` python,
|
|
184
|
+
`.ts` / `.mts` / `.cts` typescript, `.tsx` tsx, `.js` / `.jsx` / `.mjs` /
|
|
185
|
+
`.cjs` javascript, `.rs` rust, `.go` go, `.java` java, `.kt` / `.kts`
|
|
186
|
+
kotlin, `.cs` csharp, `.rb` ruby, `.swift` swift, `.php` php); an entry in
|
|
187
|
+
any other file is skipped and counted in `summary.node_kind_check_skipped`.
|
|
188
|
+
Reasons, one per entry:
|
|
189
|
+
|
|
190
|
+
- `error-kind`: the kind is `ERROR`, the node tree-sitter makes of code
|
|
191
|
+
it cannot parse. ast-grep accepts it in a rule, so it is not asked.
|
|
192
|
+
- `invalid-kind`: the kind is not shaped like a kind
|
|
193
|
+
(`[A-Za-z_][A-Za-z0-9_]*`, so no whitespace or shell characters reach
|
|
194
|
+
ast-grep), judged without ast-grep; or ast-grep rejected it (its error
|
|
195
|
+
names an invalid kind).
|
|
196
|
+
|
|
197
|
+
Those two are judged whatever the ast-grep lookup finds. Every other kind is
|
|
198
|
+
asked once per distinct (language, kind): `ast-grep scan --config
|
|
199
|
+
<sgconfig.yml> --inline-rules <rule> --stdin` with a rule matching only that
|
|
200
|
+
kind, over an empty stdin, from a temporary folder holding a minimal
|
|
201
|
+
sgconfig.yml that `--config` names (so no other sgconfig.yml is read).
|
|
202
|
+
ast-grep parses the rule before reading input and rejects a kind the
|
|
203
|
+
language's grammar lacks. Before that, ast-grep is asked about a kind no
|
|
204
|
+
grammar has; unless it rejects it, its answers cannot be read and the check
|
|
205
|
+
is `skipped-unrecognized-ast-grep`. When no `ast-grep` executable resolves
|
|
206
|
+
(the CWD-shim guard applies), the check is `skipped-no-ast-grep`. With no
|
|
207
|
+
kind left to ask, ast-grep is not looked up and the check is `checked`.
|
|
208
|
+
|
|
209
|
+
An ast-grep failure that does not name the kind proves nothing about it: the
|
|
210
|
+
entry is listed in `node_kinds_unchecked[]`, not as a finding, with reason
|
|
211
|
+
`ast-grep-error`, `ast-grep-timeout` (after the first timeout the remaining
|
|
212
|
+
kinds are not asked) or `temp-folder-error` (the temporary folder could not
|
|
213
|
+
be made or written). Without the flag the output keeps its shape: there are
|
|
214
|
+
no `node_kinds` / `node_kinds_unchecked` keys, and the one key added is
|
|
215
|
+
`summary.node_kind_check: "not-requested"`.
|
|
216
|
+
|
|
217
|
+
Subcommands:
|
|
48
218
|
verify --metadata <metadata.json> --provenance <provenance-map.json>
|
|
49
|
-
[--source-root <path>] [-
|
|
219
|
+
[--source-root <path>] [--skill-dir <dir>] [--check-node-kinds]
|
|
220
|
+
[-o <out.json>]
|
|
50
221
|
|
|
51
222
|
Emit JSON:
|
|
52
223
|
{
|
|
@@ -55,9 +226,40 @@ Subcommand:
|
|
|
55
226
|
"orphaned": ["<provenance entry with no documented export>", ...],
|
|
56
227
|
"stale": [
|
|
57
228
|
{"export_name": "<raw entry name>",
|
|
229
|
+
"entry_index": <int>, # position in entries[]
|
|
58
230
|
"source_file": "<rel path>",
|
|
59
231
|
"source_line": <int|null>,
|
|
60
|
-
"reason": "file-missing" | "line-out-of-bounds" | "line-invalid"
|
|
232
|
+
"reason": "file-missing" | "line-out-of-bounds" | "line-invalid"
|
|
233
|
+
| "line-not-definition",
|
|
234
|
+
"definition_lines": [<int>, ...]}, # line-not-definition only
|
|
235
|
+
...
|
|
236
|
+
],
|
|
237
|
+
"citations": [
|
|
238
|
+
{"file": "<markdown path relative to --skill-dir>",
|
|
239
|
+
"line": <int>, # line of the markdown file
|
|
240
|
+
"citation": "[AST:<path>:L<n>]",
|
|
241
|
+
"prefix": "AST" | "SRC",
|
|
242
|
+
"cited_file": "<path as written>",
|
|
243
|
+
"cited_line": <int>,
|
|
244
|
+
"reason": "prefix-mismatch" | "ast-without-ast-grep",
|
|
245
|
+
"expected_prefix": "AST" | "SRC",
|
|
246
|
+
"export_name": "<matched entry name>" | null},
|
|
247
|
+
...
|
|
248
|
+
],
|
|
249
|
+
"node_kinds": [ # --check-node-kinds only
|
|
250
|
+
{"export_name": "<raw entry name>" | null,
|
|
251
|
+
"entry_index": <int>, # position in entries[]
|
|
252
|
+
"source_file": "<rel path>",
|
|
253
|
+
"source_line": <as recorded>,
|
|
254
|
+
"ast_node_type": "<recorded kind>",
|
|
255
|
+
"language": "<ast-grep language>",
|
|
256
|
+
"reason": "invalid-kind" | "error-kind"},
|
|
257
|
+
...
|
|
258
|
+
],
|
|
259
|
+
"node_kinds_unchecked": [ # --check-node-kinds only
|
|
260
|
+
{... the same keys ...,
|
|
261
|
+
"reason": "ast-grep-error" | "ast-grep-timeout"
|
|
262
|
+
| "temp-folder-error"},
|
|
61
263
|
...
|
|
62
264
|
],
|
|
63
265
|
"summary": {
|
|
@@ -65,39 +267,381 @@ Subcommand:
|
|
|
65
267
|
"entries_checked": <int>, # provenance entries with export_name
|
|
66
268
|
"missing_count": <int>,
|
|
67
269
|
"orphaned_count": <int>,
|
|
270
|
+
"set_diff": "checked" | "not-applicable", # reference-app
|
|
68
271
|
"stale_count": <int>,
|
|
69
272
|
"citations_checked": <int>, # entries whose citation was resolved
|
|
70
|
-
"
|
|
273
|
+
"line_check_skipped": <int>, # resolved entries with no line rule
|
|
274
|
+
"stale_check": "checked" | "skipped-no-source-root",
|
|
275
|
+
"skill_citations_scanned": <int>, # [AST:]/[SRC:] with a line
|
|
276
|
+
"skill_citations_matched": <int>, # of those, citing an entry
|
|
277
|
+
"citation_findings_count": <int>,
|
|
278
|
+
"citation_check": "checked" | "skipped-no-skill-dir",
|
|
279
|
+
"node_kind_check": "checked" | "skipped-no-ast-grep"
|
|
280
|
+
| "skipped-unrecognized-ast-grep"
|
|
281
|
+
| "not-requested",
|
|
282
|
+
# the four below appear with --check-node-kinds only
|
|
283
|
+
"node_kinds_checked": <int>, # entries judged, ERROR included
|
|
284
|
+
"node_kind_check_skipped": <int>, # entries in an unmapped file
|
|
285
|
+
"node_kind_check_errors": <int>, # len(node_kinds_unchecked)
|
|
286
|
+
"node_kind_findings_count": <int>
|
|
71
287
|
}
|
|
72
288
|
}
|
|
73
289
|
|
|
74
290
|
`missing` / `orphaned` are sorted, deduplicated public names.
|
|
75
291
|
`stale` is sorted by (export_name, source_file).
|
|
292
|
+
`citations` is sorted by (file, line, column).
|
|
293
|
+
`node_kinds` / `node_kinds_unchecked` are sorted by (export_name,
|
|
294
|
+
source_file, entry_index).
|
|
295
|
+
|
|
296
|
+
fix --verify <verify.json> --provenance <provenance-map.json>
|
|
297
|
+
--skill-dir <dir> [--manual-inventory <inventory.json>]
|
|
298
|
+
[--no-line-moves] [--dry-run] [-o <out.json>]
|
|
299
|
+
|
|
300
|
+
Applies the fixes a `verify` result (the JSON a `verify` run wrote, over
|
|
301
|
+
the same provenance map and skill folder) has one answer for:
|
|
302
|
+
|
|
303
|
+
1. Citation prefixes: each `citations[]` item's citation, found by its
|
|
304
|
+
`file`, `line` and `citation` text, takes `expected_prefix`; its
|
|
305
|
+
path and line stay.
|
|
306
|
+
2. Source lines: each `line-not-definition` item with exactly one
|
|
307
|
+
`definition_lines` value moves its entry's `source_line` there (the
|
|
308
|
+
entry at its `entry_index`, or the entries with its name, file and
|
|
309
|
+
line). Every move is planned before any is made. The citations each
|
|
310
|
+
move takes are those of either prefix, in `SKILL.md` and
|
|
311
|
+
`references/**/*.md` as they stand after step 1, whose path,
|
|
312
|
+
normalized as the citation check normalizes it, and first line are
|
|
313
|
+
the entry's. Every planned move is applied in one pass, so a
|
|
314
|
+
citation moves at most once (`LIMIT` moving from 6 to 7 and
|
|
315
|
+
`DEFAULT` from 7 to 8 move a citation of line 6 to 7, never on to
|
|
316
|
+
8), and a range moves both ends (`L10-12` to `L13-15`). A citation
|
|
317
|
+
of a line another entry also records in that file stays: it may
|
|
318
|
+
cite that other export (`shared-line`).
|
|
319
|
+
|
|
320
|
+
A citation inside a `<!-- [MANUAL:name] -->` block never changes
|
|
321
|
+
(`inside-manual-block`); the blocks are paired as skf-hash-content.py
|
|
322
|
+
pairs them. Each changed file (markdown first, the provenance map last,
|
|
323
|
+
as JSON indented by two spaces, non-ASCII kept, with a final newline;
|
|
324
|
+
bytes that are not UTF-8 survive in markdown) is written through
|
|
325
|
+
skf-atomic-write.py beside this script. `--manual-inventory` (the JSON
|
|
326
|
+
`skf-hash-content.py manual-inventory` wrote for SKILL.md) is checked
|
|
327
|
+
again after the writes. `--no-line-moves` leaves every source-line
|
|
328
|
+
finding for a person (the drift override). `--dry-run` writes nothing.
|
|
329
|
+
|
|
330
|
+
Emit JSON:
|
|
331
|
+
{
|
|
332
|
+
"applied": [
|
|
333
|
+
{"kind": "citation-prefix", "file": "<rel md>", "line": <int>,
|
|
334
|
+
"citation": "<as it was>", "fixed": "<as written>"},
|
|
335
|
+
{"kind": "source-line", "entry_index": <int>,
|
|
336
|
+
"export_name": "<name>", "source_file": "<rel path>",
|
|
337
|
+
"from": <int>, "to": <int>},
|
|
338
|
+
{"kind": "citation-line", "file": "<rel md>", "line": <int>,
|
|
339
|
+
"citation": "<as it was>", "fixed": "<as written>",
|
|
340
|
+
"export_names": ["<name>", ...]},
|
|
341
|
+
...
|
|
342
|
+
],
|
|
343
|
+
"left_as_warn": [
|
|
344
|
+
{... the verify item ..., "kind": "citation-prefix",
|
|
345
|
+
"why": "citation-not-found" | "inside-manual-block"},
|
|
346
|
+
{... the verify item ..., "kind": "source-line",
|
|
347
|
+
"why": "no-definition-line" # unverified, never moved
|
|
348
|
+
| "several-definition-lines" | "needs-decision"
|
|
349
|
+
| "line-moves-skipped" | "entry-not-found"},
|
|
350
|
+
{"kind": "citation-line", "file", "line", "citation",
|
|
351
|
+
"export_names": [...],
|
|
352
|
+
"why": "shared-line" | "inside-manual-block"},
|
|
353
|
+
...
|
|
354
|
+
],
|
|
355
|
+
"files_changed": ["<path>", ...], # as the arguments name them
|
|
356
|
+
"files_written": ["<path>", ...], # [] with --dry-run
|
|
357
|
+
"dry_run": <bool>,
|
|
358
|
+
"manual_verify": null | {"preserved", "modified", "missing",
|
|
359
|
+
"moved", "ok"},
|
|
360
|
+
"summary": {"citation_prefixes_fixed": <int>,
|
|
361
|
+
"source_lines_moved": <int>,
|
|
362
|
+
"citations_moved": <int>,
|
|
363
|
+
"left_as_warn_count": <int>}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
`fix` handles no `missing`, `orphaned` or `node_kinds` finding: read
|
|
367
|
+
those from the verify result. Run `verify` again after it.
|
|
368
|
+
|
|
369
|
+
definition-lines --file <source_file> --name <export_name>
|
|
370
|
+
[--source-root <path>] [--line <n>]
|
|
371
|
+
[--export-type <type>] [-o <out.json>]
|
|
372
|
+
|
|
373
|
+
The definition-line rules above for one export (update-skill's
|
|
374
|
+
spot-check). `--file` is relative to `--source-root` (default: the
|
|
375
|
+
current folder).
|
|
376
|
+
|
|
377
|
+
Emit JSON:
|
|
378
|
+
{
|
|
379
|
+
"source_file": "<as given>",
|
|
380
|
+
"export_name": "<as given>",
|
|
381
|
+
"line_check": "checked" | "file-missing" | "skipped-export-type"
|
|
382
|
+
| "skipped-language",
|
|
383
|
+
"definition_lines": [<int>, ...] | null, # checked only
|
|
384
|
+
"source_line": <int> | null, # --line
|
|
385
|
+
"line_is_definition": <bool> | null # --line, checked only
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
With `--line`, a recorded line that is an indented match the column-0
|
|
389
|
+
rule dropped joins `definition_lines`, as in `verify`.
|
|
390
|
+
|
|
391
|
+
classify-stale --names <coverage.json> --provenance <provenance-map.json>
|
|
392
|
+
--source-root <path> [-o <out.json>]
|
|
393
|
+
|
|
394
|
+
test-skill's fabricated-signature test, for each stale name (a
|
|
395
|
+
documented name the enumerated source surface lacks): the
|
|
396
|
+
definition-lines rules above, run for every provenance entry whose raw
|
|
397
|
+
`export_name` is the name, at its `source_file` and `source_line`, with
|
|
398
|
+
its `export_type`. `--names` holds reconcile-coverage.py's result (its
|
|
399
|
+
`stale` names) or a JSON array of names.
|
|
400
|
+
|
|
401
|
+
Emit JSON, one item per name, in the order given (a repeated name once):
|
|
402
|
+
[
|
|
403
|
+
{"name": "<as given>",
|
|
404
|
+
"fabricated": <bool>,
|
|
405
|
+
"source": "<source_file>:<source_line>" | "<source_file>" | null,
|
|
406
|
+
"reason": "not-defined" | "defined" | "unchecked" | "no-entry",
|
|
407
|
+
"entries": [{"source_file": "<as recorded>",
|
|
408
|
+
"source_line": <as recorded>,
|
|
409
|
+
"line_check": "checked" | "file-missing"
|
|
410
|
+
| "skipped-export-type"
|
|
411
|
+
| "skipped-language" | "unreadable"
|
|
412
|
+
| "no-file",
|
|
413
|
+
"definition_lines": [<int>, ...] | null}, ...]},
|
|
414
|
+
...
|
|
415
|
+
]
|
|
416
|
+
|
|
417
|
+
`fabricated` is true, with reason `not-defined`, when the name has an
|
|
418
|
+
entry and every entry's `line_check` is `file-missing`, or `checked`
|
|
419
|
+
with an empty `definition_lines`: the export is in neither the surface
|
|
420
|
+
nor the cited file. Otherwise the reason is `defined` (a cited file
|
|
421
|
+
defines the name), `unchecked` (no rule could check a cited file, and
|
|
422
|
+
none defines it) or `no-entry` (the map has no entry for the name).
|
|
423
|
+
An entry with no `source_file` (update-skill writes one for an
|
|
424
|
+
`unknown` NEW_EXPORT) cites no file to check: its `line_check` is
|
|
425
|
+
`no-file`, which, like a skipped or unreadable check, neither proves
|
|
426
|
+
the name fabricated nor shows it defined, so the name is `unchecked`
|
|
427
|
+
unless another entry defines it. `source` is the citation of the first
|
|
428
|
+
entry with a `source_file` (the file alone when its line is not a
|
|
429
|
+
number), null when no entry has one.
|
|
430
|
+
|
|
431
|
+
kind-at --file <path> --line <n> --recipes <recipes>
|
|
432
|
+
[--language <lang>] [--name <export_name>]
|
|
433
|
+
[--source-root <path>] [-o <out.json>]
|
|
434
|
+
|
|
435
|
+
The `kind` the matching recipe declares for a source line, the value an
|
|
436
|
+
ast-grep entry records as `ast_node_type`. `--recipes` is a markdown
|
|
437
|
+
file whose ```yaml fences hold the recipes (extraction-patterns.md) or
|
|
438
|
+
a YAML file: one recipe per document, a list of recipes, or a mapping
|
|
439
|
+
whose `recipes` key holds that list. A recipe is a mapping with `id`,
|
|
440
|
+
`language` and a `rule` naming its `kind`. The file's language
|
|
441
|
+
(`--language`, else its extension, mapped as for `--check-node-kinds`)
|
|
442
|
+
picks the recipes: each id's form for that language, else its
|
|
443
|
+
typescript or tsx form (for javascript, a form whose kinds the grammar
|
|
444
|
+
lacks is skipped), with `language` rewritten. Each runs once through
|
|
445
|
+
`ast-grep scan --stdin` on the file's bytes, from a temporary folder
|
|
446
|
+
with a minimal sgconfig.yml. A recipe matches when its `$NAME` is on
|
|
447
|
+
`--line` (and, with `--name`, is that name's last segment).
|
|
448
|
+
|
|
449
|
+
Emit JSON:
|
|
450
|
+
{
|
|
451
|
+
"file": "<as given>",
|
|
452
|
+
"language": "<ast-grep language>" | null,
|
|
453
|
+
"line": <int>,
|
|
454
|
+
"name": "<as given>" | null,
|
|
455
|
+
"status": "found" | "ambiguous" | "no-match" | "incomplete"
|
|
456
|
+
| "skipped-no-language" | "skipped-no-recipes"
|
|
457
|
+
| "skipped-no-ast-grep",
|
|
458
|
+
"kind": "<the one kind the matches declare>" | null, # found only
|
|
459
|
+
"matches": [{"recipe": "<id>", "kind": "<kind>",
|
|
460
|
+
"name": "<$NAME>"}, ...],
|
|
461
|
+
"errors": [{"recipe": "<id>" | null, "language": "<language>",
|
|
462
|
+
"reason": "ast-grep-error" | "ast-grep-timeout"
|
|
463
|
+
| "temp-folder-error",
|
|
464
|
+
"detail": "<text>"}, ...]
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
`incomplete`: a recipe run failed, so no kind is given even when others
|
|
468
|
+
matched. Never invent a kind: anything but `found` leaves the entry as
|
|
469
|
+
it is.
|
|
76
470
|
|
|
77
471
|
CLI examples:
|
|
78
472
|
uv run skf-verify-provenance-completeness.py verify \\
|
|
79
473
|
--metadata {skill_package}/metadata.json \\
|
|
80
474
|
--provenance {forge_version}/provenance-map.json \\
|
|
81
|
-
--source-root {source_root}
|
|
475
|
+
--source-root {source_root} \\
|
|
476
|
+
--skill-dir {skill_package} \\
|
|
477
|
+
--check-node-kinds -o {verify_json}
|
|
478
|
+
uv run skf-verify-provenance-completeness.py fix --verify {verify_json} \\
|
|
479
|
+
--provenance {forge_version}/provenance-map.json \\
|
|
480
|
+
--skill-dir {skill_package}
|
|
481
|
+
uv run skf-verify-provenance-completeness.py definition-lines \\
|
|
482
|
+
--source-root {source_root} --file src/api.py --name search --line 26
|
|
483
|
+
uv run skf-verify-provenance-completeness.py classify-stale \\
|
|
484
|
+
--names {run_dir}/coverage.json \\
|
|
485
|
+
--provenance {forge_version}/provenance-map.json \\
|
|
486
|
+
--source-root {source_root} -o {run_dir}/stale.json
|
|
487
|
+
uv run skf-verify-provenance-completeness.py kind-at \\
|
|
488
|
+
--source-root {source_root} --file src/api.ts --line 12 --name Store \\
|
|
489
|
+
--recipes {extractionPatternsData}
|
|
82
490
|
|
|
83
491
|
Exit codes:
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
492
|
+
verify
|
|
493
|
+
0 verification ran and found nothing (status "pass")
|
|
494
|
+
1 verification ran and found missing / orphaned / stale entries,
|
|
495
|
+
citation findings or node-kind findings (status "findings");
|
|
496
|
+
advisory: the caller decides how to surface it
|
|
497
|
+
fix
|
|
498
|
+
0 nothing left as a WARN (and manual_verify ok when it ran)
|
|
499
|
+
1 something left as a WARN, or manual_verify not ok
|
|
500
|
+
definition-lines
|
|
501
|
+
0 the rules answered (whatever `line_check` says)
|
|
502
|
+
classify-stale
|
|
503
|
+
0 every name was classified (whatever `fabricated` says)
|
|
504
|
+
kind-at
|
|
505
|
+
0 status "found"
|
|
506
|
+
1 any other status
|
|
507
|
+
2 error, for every subcommand (an input file or folder not found,
|
|
508
|
+
--skill-dir without a SKILL.md, malformed JSON or YAML, a verify
|
|
509
|
+
result without stale[] and citations[], --names without a list of
|
|
510
|
+
names, no recipe in --recipes, no PyYAML for kind-at, an atomic
|
|
511
|
+
write that failed, an -o file that cannot be written, or any
|
|
512
|
+
unexpected failure, such as an unreadable
|
|
513
|
+
file; once fix has started writing, stderr names the files already
|
|
514
|
+
written); one line on stderr and no JSON
|
|
88
515
|
"""
|
|
89
516
|
|
|
90
517
|
from __future__ import annotations
|
|
91
518
|
|
|
92
519
|
import argparse
|
|
520
|
+
import copy
|
|
521
|
+
import importlib.util
|
|
522
|
+
import io
|
|
93
523
|
import json
|
|
524
|
+
import os
|
|
525
|
+
import posixpath
|
|
526
|
+
import re
|
|
527
|
+
import shutil
|
|
528
|
+
import subprocess
|
|
94
529
|
import sys
|
|
530
|
+
import tempfile
|
|
531
|
+
import tokenize
|
|
95
532
|
from pathlib import Path
|
|
533
|
+
from typing import Callable
|
|
96
534
|
|
|
97
535
|
|
|
98
536
|
STALE_FILE_MISSING = "file-missing"
|
|
99
537
|
STALE_LINE_OOB = "line-out-of-bounds"
|
|
100
538
|
STALE_LINE_INVALID = "line-invalid"
|
|
539
|
+
STALE_LINE_NOT_DEFINITION = "line-not-definition"
|
|
540
|
+
|
|
541
|
+
CITATION_PREFIX_MISMATCH = "prefix-mismatch"
|
|
542
|
+
CITATION_AST_WITHOUT_AST_GREP = "ast-without-ast-grep"
|
|
543
|
+
|
|
544
|
+
PYTHON_EXTENSIONS = frozenset({".py", ".pyi"})
|
|
545
|
+
TSJS_EXTENSIONS = frozenset(
|
|
546
|
+
{".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"}
|
|
547
|
+
)
|
|
548
|
+
# export_type values that name a whole module or package: no definition line.
|
|
549
|
+
NO_DEFINITION_EXPORT_TYPES = frozenset({"module", "package"})
|
|
550
|
+
# The scope_type whose exports[] is empty by design (entries are per-citation).
|
|
551
|
+
REFERENCE_APP = "reference-app"
|
|
552
|
+
|
|
553
|
+
# extraction_method -> the citation prefix it implies
|
|
554
|
+
METHOD_PREFIX = {
|
|
555
|
+
"ast-grep": "AST",
|
|
556
|
+
"ast_bridge": "AST",
|
|
557
|
+
"source-read": "SRC",
|
|
558
|
+
"source_reading": "SRC",
|
|
559
|
+
}
|
|
560
|
+
AST_METHODS = frozenset({"ast-grep", "ast_bridge"})
|
|
561
|
+
# Every extraction_method SKF writes (the same list skf-render-metadata-stats
|
|
562
|
+
# checks labels against).
|
|
563
|
+
KNOWN_METHODS = frozenset(
|
|
564
|
+
{
|
|
565
|
+
"ast-grep",
|
|
566
|
+
"source-read",
|
|
567
|
+
"ast_bridge",
|
|
568
|
+
"source_reading",
|
|
569
|
+
"qmd_bridge",
|
|
570
|
+
"compose-from-skill",
|
|
571
|
+
}
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
NODE_KIND_INVALID = "invalid-kind"
|
|
575
|
+
NODE_KIND_ERROR = "error-kind"
|
|
576
|
+
# Why an entry's kind went unjudged (node_kinds_unchecked[].reason).
|
|
577
|
+
UNCHECKED_AST_GREP_ERROR = "ast-grep-error"
|
|
578
|
+
UNCHECKED_TIMEOUT = "ast-grep-timeout"
|
|
579
|
+
UNCHECKED_TEMP_FOLDER = "temp-folder-error"
|
|
580
|
+
# The node tree-sitter makes of code it cannot parse. ast-grep accepts it as
|
|
581
|
+
# a kind, but no recipe declares it, so an entry recording it guessed.
|
|
582
|
+
ERROR_NODE_KIND = "ERROR"
|
|
583
|
+
# Every named node kind of a grammar ast-grep ships has this shape. Anything
|
|
584
|
+
# else is judged without ast-grep: ast-grep trims surrounding whitespace, and
|
|
585
|
+
# `& | % ^` would reach cmd.exe through an npm `.cmd` shim on Windows.
|
|
586
|
+
NODE_KIND_SHAPE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
587
|
+
# A kind no grammar has. ast-grep must reject it before its answers are read.
|
|
588
|
+
BOGUS_NODE_KIND = "skf_no_such_node_kind"
|
|
589
|
+
# An explicit, valid sgconfig.yml, so no sgconfig.yml in the temporary
|
|
590
|
+
# folder's ancestors is read.
|
|
591
|
+
MINIMAL_SGCONFIG = "ruleDirs: []\n"
|
|
592
|
+
# source_file extension -> the ast-grep language whose grammar names its kinds
|
|
593
|
+
AST_GREP_LANGUAGES = {
|
|
594
|
+
".py": "python",
|
|
595
|
+
".pyi": "python",
|
|
596
|
+
".ts": "typescript",
|
|
597
|
+
".mts": "typescript",
|
|
598
|
+
".cts": "typescript",
|
|
599
|
+
".tsx": "tsx",
|
|
600
|
+
".js": "javascript",
|
|
601
|
+
".jsx": "javascript",
|
|
602
|
+
".mjs": "javascript",
|
|
603
|
+
".cjs": "javascript",
|
|
604
|
+
".rs": "rust",
|
|
605
|
+
".go": "go",
|
|
606
|
+
".java": "java",
|
|
607
|
+
".kt": "kotlin",
|
|
608
|
+
".kts": "kotlin",
|
|
609
|
+
".cs": "csharp",
|
|
610
|
+
".rb": "ruby",
|
|
611
|
+
".swift": "swift",
|
|
612
|
+
".php": "php",
|
|
613
|
+
}
|
|
614
|
+
NODE_KIND_TIMEOUT_SEC = 20 # per ast-grep call; each parses one tiny rule
|
|
615
|
+
|
|
616
|
+
# kind-at: a ```yaml fence in a markdown recipe file (extraction-patterns.md).
|
|
617
|
+
_MD_YAML_FENCE_RE = re.compile(r"^```yaml[ \t]*\n(.*?)^```", re.M | re.S)
|
|
618
|
+
# file language -> the recipe languages whose form of a recipe it runs, with
|
|
619
|
+
# `language:` rewritten, when the recipe has no form of its own for it (the
|
|
620
|
+
# language selection note in extraction-patterns-by-hand.md): typescript and tsx
|
|
621
|
+
# share their recipes, and javascript runs a TypeScript recipe with no
|
|
622
|
+
# javascript form (one whose kinds the grammar lacks is skipped).
|
|
623
|
+
RECIPE_LANGUAGE_FALLBACKS = {
|
|
624
|
+
"typescript": ("tsx",),
|
|
625
|
+
"tsx": ("typescript",),
|
|
626
|
+
"javascript": ("typescript", "tsx"),
|
|
627
|
+
}
|
|
628
|
+
KIND_AT_TIMEOUT_SEC = 30 # per ast-grep call; each scans one file
|
|
629
|
+
|
|
630
|
+
# fix: what each applied or left item fixes.
|
|
631
|
+
FIX_CITATION_PREFIX = "citation-prefix"
|
|
632
|
+
FIX_SOURCE_LINE = "source-line"
|
|
633
|
+
FIX_CITATION_LINE = "citation-line"
|
|
634
|
+
# The [MANUAL] markers skf-hash-content.py pairs (its _OPEN_RE / _CLOSE_RE,
|
|
635
|
+
# on text instead of bytes; the test pins the two copies together). ASCII,
|
|
636
|
+
# so `\s` is the whitespace a bytes pattern knows: a marker spaced with
|
|
637
|
+
# U+00A0 is no marker to either.
|
|
638
|
+
_MANUAL_OPEN_RE = re.compile(r"<!--\s*\[MANUAL:([^\]]+)\]\s*-->", re.ASCII)
|
|
639
|
+
_MANUAL_CLOSE_RE = re.compile(r"<!--\s*\[/MANUAL:([^\]]+)\]\s*-->", re.ASCII)
|
|
640
|
+
# One line with its ending, split where `check_citation` counts a line.
|
|
641
|
+
_LINE_WITH_END_RE = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+$")
|
|
642
|
+
ATOMIC_WRITE_HELPER = "skf-atomic-write.py"
|
|
643
|
+
HASH_CONTENT_HELPER = "skf-hash-content.py"
|
|
644
|
+
ATOMIC_WRITE_TIMEOUT_SEC = 60
|
|
101
645
|
|
|
102
646
|
|
|
103
647
|
# --------------------------------------------------------------------------
|
|
@@ -174,6 +718,12 @@ def extract_reexport_map(prov: dict) -> dict[str, str]:
|
|
|
174
718
|
return out
|
|
175
719
|
|
|
176
720
|
|
|
721
|
+
def is_reference_app(metadata: dict) -> bool:
|
|
722
|
+
"""True when metadata.json's `scope_type` is `reference-app`."""
|
|
723
|
+
scope_type = metadata.get("scope_type")
|
|
724
|
+
return isinstance(scope_type, str) and scope_type.strip() == REFERENCE_APP
|
|
725
|
+
|
|
726
|
+
|
|
177
727
|
def canon(name: str, reexport_map: dict[str, str]) -> str:
|
|
178
728
|
"""Canonicalize an export name to its public form via reexport_map.
|
|
179
729
|
|
|
@@ -260,6 +810,1477 @@ def check_citation(
|
|
|
260
810
|
return None
|
|
261
811
|
|
|
262
812
|
|
|
813
|
+
# --------------------------------------------------------------------------
|
|
814
|
+
# Definition lines
|
|
815
|
+
# --------------------------------------------------------------------------
|
|
816
|
+
|
|
817
|
+
_PY_FROM_IMPORT_RE = re.compile(r"^\s*from\s+[\w.]+\s+import\s+(.*)$")
|
|
818
|
+
_PY_IMPORT_RE = re.compile(r"^\s*import\s+(.*)$")
|
|
819
|
+
_PY_IMPORT_ITEM_RE = re.compile(r"^([\w.]+)(?:\s+as\s+(\w+))?$")
|
|
820
|
+
_PY_IDENT_RE = re.compile(r"[A-Za-z_]\w*")
|
|
821
|
+
# `A = value`, `A = B = value`: every name left of an `=` that is not `==`.
|
|
822
|
+
_PY_ASSIGN_CHAIN_RE = re.compile(r"^\s*((?:[A-Za-z_]\w*\s*=(?!=)\s*)+)")
|
|
823
|
+
# `A, B = value`, `(A, *B) = value`, `A, = value`.
|
|
824
|
+
_PY_TARGET_LIST_RE = re.compile(
|
|
825
|
+
r"^\s*[(\[]?\s*"
|
|
826
|
+
r"(\*?[A-Za-z_]\w*\s*,(?:\s*\*?[A-Za-z_]\w*\s*,)*(?:\s*\*?[A-Za-z_]\w*)?)"
|
|
827
|
+
r"\s*[)\]]?\s*=(?!=)"
|
|
828
|
+
)
|
|
829
|
+
# `def \` or `class \`: the name is on the next line.
|
|
830
|
+
_PY_SPLIT_DECL_RE = re.compile(r"^\s*(?:async\s+def|def|class)\s*\\$")
|
|
831
|
+
# Tokens that never begin a statement.
|
|
832
|
+
_PY_LAYOUT_TOKENS = frozenset(
|
|
833
|
+
{
|
|
834
|
+
tokenize.NL,
|
|
835
|
+
tokenize.COMMENT,
|
|
836
|
+
tokenize.INDENT,
|
|
837
|
+
tokenize.DEDENT,
|
|
838
|
+
tokenize.ENDMARKER,
|
|
839
|
+
tokenize.ENCODING,
|
|
840
|
+
}
|
|
841
|
+
)
|
|
842
|
+
|
|
843
|
+
# `@name`, `@a.b`, `@name(...)` (one level of nested brackets in the call).
|
|
844
|
+
_JS_DECORATOR = r"@[\w$]+(?:\.[\w$]+)*(?:\((?:[^()]|\([^()]*\))*\))?"
|
|
845
|
+
_JS_MODIFIERS = (
|
|
846
|
+
rf"(?:(?:export|default|async|declare|abstract)\s+|{_JS_DECORATOR}\s*)*"
|
|
847
|
+
)
|
|
848
|
+
_JS_KINDS = (
|
|
849
|
+
r"(?:function(?:\s*\*\s*|\s+)"
|
|
850
|
+
r"|(?:class|interface|type|enum|namespace|let|var)\s+"
|
|
851
|
+
r"|const\s+(?:enum\s+)?)"
|
|
852
|
+
)
|
|
853
|
+
_JS_MEMBER_MODIFIERS = (
|
|
854
|
+
r"(?:(?:public|private|protected|static|readonly|async|abstract"
|
|
855
|
+
r"|override|declare|get|set)\s+)*"
|
|
856
|
+
)
|
|
857
|
+
# An identifier, or a string name (`"a-b"`) in an export list.
|
|
858
|
+
_JS_NAME_OR_STRING = r"(?:[\w$]+|\"[^\"]*\"|'[^']*')"
|
|
859
|
+
_JS_EXPORT_LIST_RE = re.compile(r"^\s*export\s+(?:type\s+)?\{(.*)$")
|
|
860
|
+
_JS_EXPORT_ITEM_RE = re.compile(
|
|
861
|
+
rf"^(?:type\s+)?({_JS_NAME_OR_STRING})(?:\s+as\s+({_JS_NAME_OR_STRING}))?$"
|
|
862
|
+
)
|
|
863
|
+
_JS_MODULE_EXPORTS_OBJECT_RE = re.compile(r"^\s*module\.exports\s*=\s*\{(.*)$")
|
|
864
|
+
# A `const`, `let` or `var` statement: its declarator list.
|
|
865
|
+
_JS_VAR_DECL_RE = re.compile(
|
|
866
|
+
rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(?!enum(?![\w$]))(.*)$"
|
|
867
|
+
)
|
|
868
|
+
# A declaration whose name is on the next line: the line ends in its keyword.
|
|
869
|
+
_JS_SPLIT_DECL_RE = re.compile(
|
|
870
|
+
rf"^\s*{_JS_MODIFIERS}(?P<kind>function(?:\s*\*)?|class|interface|type"
|
|
871
|
+
r"|enum|namespace|let|var|const(?:\s+enum)?)$"
|
|
872
|
+
)
|
|
873
|
+
_JS_VAR_KEYWORDS = frozenset({"const", "let", "var"})
|
|
874
|
+
# How a `const` / `let` / `var` list that did not end on its line goes on
|
|
875
|
+
# (`_js_declarators`): the next line begins a declarator (this one ended
|
|
876
|
+
# with a top-level comma); it finishes the current one (this one ended
|
|
877
|
+
# inside a bracket, or after an operator waiting for its operand); or it
|
|
878
|
+
# goes on only when it starts with an operator (this one ended with a
|
|
879
|
+
# value, as a member chain or a ternary broken over lines does).
|
|
880
|
+
_DECL_NEXT = "next"
|
|
881
|
+
_DECL_OPEN = "open"
|
|
882
|
+
_DECL_MAYBE = "maybe"
|
|
883
|
+
# What `_js_declarators` gives: None, or (brackets left open, how) the list
|
|
884
|
+
# goes on with, and the pieces that begin a declarator.
|
|
885
|
+
_DeclSplit = tuple[tuple[str, str] | None, list[str]]
|
|
886
|
+
# After one of these, `/` begins a regular expression and `<` begins JSX, a
|
|
887
|
+
# type assertion or a generic arrow function's type parameters.
|
|
888
|
+
_JS_EXPRESSION_START = frozenset("([{,;=?:&|^%+-*/<>!~")
|
|
889
|
+
# A line that ends in one of these leaves an operator waiting for its operand.
|
|
890
|
+
_JS_OPERAND_PENDING = frozenset("=?:&|^%+-*/<>.")
|
|
891
|
+
# A line that begins with one of these goes on with the expression above it.
|
|
892
|
+
_JS_CONTINUATION_START = frozenset(".?:,&|^%+-*/=<>")
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def definition_name(export_name: str) -> str:
|
|
896
|
+
"""The name a definition line must carry: the last segment of a dotted
|
|
897
|
+
export name (`App.update` -> `update`), else the name itself."""
|
|
898
|
+
return export_name.rsplit(".", 1)[-1]
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
def _read_lines(path: Path) -> list[str] | None:
|
|
902
|
+
"""Read a text file as lines numbered like `check_citation` counts them
|
|
903
|
+
(universal newlines; a UTF-8 BOM is dropped so it never hides line 1).
|
|
904
|
+
None when the file cannot be read."""
|
|
905
|
+
try:
|
|
906
|
+
with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
|
|
907
|
+
return [ln.rstrip("\n") for ln in fh]
|
|
908
|
+
except OSError:
|
|
909
|
+
return None
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def _is_column_0(text: str) -> bool:
|
|
913
|
+
"""True when a line starts at column 0 (no leading whitespace)."""
|
|
914
|
+
return not text[:1].isspace()
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def _py_statement_starts(lines: list[str]) -> set[int] | None:
|
|
918
|
+
"""Line numbers on which a Python statement begins.
|
|
919
|
+
|
|
920
|
+
Uses the stdlib tokenizer, so a line inside a string or inside brackets
|
|
921
|
+
(a docstring, a keyword argument, a dict entry, a parameter on its own
|
|
922
|
+
line) never reads as a definition. None when the file does not tokenize:
|
|
923
|
+
every line is then a candidate.
|
|
924
|
+
"""
|
|
925
|
+
starts: set[int] = set()
|
|
926
|
+
at_start = True
|
|
927
|
+
readline = io.StringIO("\n".join(lines) + "\n").readline
|
|
928
|
+
try:
|
|
929
|
+
for tok in tokenize.generate_tokens(readline):
|
|
930
|
+
if tok.type == tokenize.NEWLINE:
|
|
931
|
+
at_start = True
|
|
932
|
+
elif tok.type in _PY_LAYOUT_TOKENS:
|
|
933
|
+
continue
|
|
934
|
+
elif at_start:
|
|
935
|
+
starts.add(tok.start[0])
|
|
936
|
+
at_start = False
|
|
937
|
+
except (tokenize.TokenError, SyntaxError, ValueError):
|
|
938
|
+
return None
|
|
939
|
+
return starts
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def _js_code_lines(lines: list[str]) -> list[str]:
|
|
943
|
+
"""Each TS/JS line with its comments removed: `//` to the end of the
|
|
944
|
+
line, and `/* ... */` across lines, both only outside a '...', "..." or
|
|
945
|
+
`...` string. A string is followed within its line only: a template
|
|
946
|
+
literal that spans lines is not tracked, so each line starts outside
|
|
947
|
+
any string. A backslash outside a string escapes the next character, so
|
|
948
|
+
a regex literal such as `/\\/*/` opens no comment."""
|
|
949
|
+
out: list[str] = []
|
|
950
|
+
in_block = False
|
|
951
|
+
for text in lines:
|
|
952
|
+
buf: list[str] = []
|
|
953
|
+
quote: str | None = None
|
|
954
|
+
i, n = 0, len(text)
|
|
955
|
+
while i < n:
|
|
956
|
+
if in_block:
|
|
957
|
+
end = text.find("*/", i)
|
|
958
|
+
if end < 0:
|
|
959
|
+
break
|
|
960
|
+
in_block = False
|
|
961
|
+
buf.append(" ")
|
|
962
|
+
i = end + 2
|
|
963
|
+
continue
|
|
964
|
+
ch = text[i]
|
|
965
|
+
if ch == "\\" and i + 1 < n:
|
|
966
|
+
buf.append(text[i : i + 2])
|
|
967
|
+
i += 2
|
|
968
|
+
continue
|
|
969
|
+
if quote is not None:
|
|
970
|
+
if ch == quote:
|
|
971
|
+
quote = None
|
|
972
|
+
elif ch in "'\"`":
|
|
973
|
+
quote = ch
|
|
974
|
+
elif text.startswith("//", i):
|
|
975
|
+
break
|
|
976
|
+
elif text.startswith("/*", i):
|
|
977
|
+
in_block = True
|
|
978
|
+
i += 2
|
|
979
|
+
continue
|
|
980
|
+
buf.append(ch)
|
|
981
|
+
i += 1
|
|
982
|
+
out.append("".join(buf).rstrip())
|
|
983
|
+
return out
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
class _SourceCache:
|
|
987
|
+
"""Source lines, Python statement starts and TS/JS comment-free lines,
|
|
988
|
+
read once per file, and TS/JS declarator lists, split once per line
|
|
989
|
+
(the split does not depend on the export looked up)."""
|
|
990
|
+
|
|
991
|
+
def __init__(self) -> None:
|
|
992
|
+
self._lines: dict[Path, list[str] | None] = {}
|
|
993
|
+
self._starts: dict[Path, set[int] | None] = {}
|
|
994
|
+
self._js_code: dict[Path, list[str]] = {}
|
|
995
|
+
self._js_decl: dict[tuple[str, str, bool], _DeclSplit] = {}
|
|
996
|
+
|
|
997
|
+
def lines(self, path: Path) -> list[str] | None:
|
|
998
|
+
if path not in self._lines:
|
|
999
|
+
self._lines[path] = _read_lines(path)
|
|
1000
|
+
return self._lines[path]
|
|
1001
|
+
|
|
1002
|
+
def py_statement_starts(self, path: Path, lines: list[str]) -> set[int] | None:
|
|
1003
|
+
if path not in self._starts:
|
|
1004
|
+
self._starts[path] = _py_statement_starts(lines)
|
|
1005
|
+
return self._starts[path]
|
|
1006
|
+
|
|
1007
|
+
def js_code_lines(self, path: Path, lines: list[str]) -> list[str]:
|
|
1008
|
+
if path not in self._js_code:
|
|
1009
|
+
self._js_code[path] = _js_code_lines(lines)
|
|
1010
|
+
return self._js_code[path]
|
|
1011
|
+
|
|
1012
|
+
def js_declarators(
|
|
1013
|
+
self, text: str, opened: str = "", fresh: bool = True
|
|
1014
|
+
) -> _DeclSplit:
|
|
1015
|
+
key = (text, opened, fresh)
|
|
1016
|
+
if key not in self._js_decl:
|
|
1017
|
+
self._js_decl[key] = _js_declarators(text, opened, fresh)
|
|
1018
|
+
return self._js_decl[key]
|
|
1019
|
+
|
|
1020
|
+
|
|
1021
|
+
def _py_import_binds(items: str, name: str, from_import: bool) -> bool:
|
|
1022
|
+
"""True when an import item list binds `name`."""
|
|
1023
|
+
items = items.split("#", 1)[0]
|
|
1024
|
+
for raw in items.replace("(", " ").replace(")", " ").replace("\\", " ").split(","):
|
|
1025
|
+
m = _PY_IMPORT_ITEM_RE.match(raw.strip())
|
|
1026
|
+
if not m:
|
|
1027
|
+
continue
|
|
1028
|
+
target, alias = m.group(1), m.group(2)
|
|
1029
|
+
if alias is not None:
|
|
1030
|
+
bound = alias
|
|
1031
|
+
elif from_import:
|
|
1032
|
+
bound = target
|
|
1033
|
+
else:
|
|
1034
|
+
bound = target.split(".", 1)[0] # `import a.b` binds `a`
|
|
1035
|
+
if bound == name:
|
|
1036
|
+
return True
|
|
1037
|
+
return False
|
|
1038
|
+
|
|
1039
|
+
|
|
1040
|
+
def _py_assigned_names(code: str) -> set[str]:
|
|
1041
|
+
"""Names an assignment statement binds (`A = B = v`, `A, *B = v`)."""
|
|
1042
|
+
names: set[str] = set()
|
|
1043
|
+
m = _PY_ASSIGN_CHAIN_RE.match(code)
|
|
1044
|
+
if m:
|
|
1045
|
+
names.update(_PY_IDENT_RE.findall(m.group(1)))
|
|
1046
|
+
m = _PY_TARGET_LIST_RE.match(code)
|
|
1047
|
+
if m:
|
|
1048
|
+
names.update(_PY_IDENT_RE.findall(m.group(1)))
|
|
1049
|
+
return names
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def _python_definition_lines(
|
|
1053
|
+
lines: list[str], name: str, starts: set[int] | None
|
|
1054
|
+
) -> list[tuple[int, bool]]:
|
|
1055
|
+
"""(line, at column 0) for every line that defines `name`.
|
|
1056
|
+
|
|
1057
|
+
A line inside a parenthesized or backslash-continued import takes the
|
|
1058
|
+
column of the line the import statement starts on, and so does the name
|
|
1059
|
+
line of a `def \\` or `class \\` whose name is on the next line. A
|
|
1060
|
+
`from X import *` line counts only when no other line defines `name` (a
|
|
1061
|
+
re-export the extraction could not trace further is cited at its star
|
|
1062
|
+
import).
|
|
1063
|
+
"""
|
|
1064
|
+
esc = re.escape(name)
|
|
1065
|
+
decl = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{esc}(?!\w)")
|
|
1066
|
+
leading_name = re.compile(rf"^\s*{esc}(?!\w)")
|
|
1067
|
+
annotated = re.compile(rf"^\s*{esc}\s*:(?!=)")
|
|
1068
|
+
type_alias = re.compile(rf"^\s*type\s+{esc}\s*(?:\[[^\]]*\]\s*)?=(?!=)")
|
|
1069
|
+
found: list[tuple[int, bool]] = []
|
|
1070
|
+
star_imports: list[tuple[int, bool]] = []
|
|
1071
|
+
# An import statement that continues on the next line:
|
|
1072
|
+
# (closing mark "paren" or "backslash", column 0, from-import).
|
|
1073
|
+
cont: tuple[str, bool, bool] | None = None
|
|
1074
|
+
for number, text in enumerate(lines, start=1):
|
|
1075
|
+
code = text.split("#", 1)[0]
|
|
1076
|
+
if cont is not None:
|
|
1077
|
+
mark, top, from_import = cont
|
|
1078
|
+
if mark == "paren":
|
|
1079
|
+
if ")" in code:
|
|
1080
|
+
code = code.split(")", 1)[0]
|
|
1081
|
+
cont = None
|
|
1082
|
+
else:
|
|
1083
|
+
stripped = code.rstrip()
|
|
1084
|
+
if stripped.endswith("\\"):
|
|
1085
|
+
code = stripped[:-1]
|
|
1086
|
+
else:
|
|
1087
|
+
cont = None
|
|
1088
|
+
if _py_import_binds(code, name, from_import):
|
|
1089
|
+
found.append((number, top))
|
|
1090
|
+
continue
|
|
1091
|
+
if starts is not None and number not in starts:
|
|
1092
|
+
continue
|
|
1093
|
+
top = _is_column_0(text)
|
|
1094
|
+
if (
|
|
1095
|
+
decl.match(code)
|
|
1096
|
+
or annotated.match(code)
|
|
1097
|
+
or type_alias.match(code)
|
|
1098
|
+
or name in _py_assigned_names(code)
|
|
1099
|
+
):
|
|
1100
|
+
found.append((number, top))
|
|
1101
|
+
continue
|
|
1102
|
+
if _PY_SPLIT_DECL_RE.match(code.rstrip()):
|
|
1103
|
+
# `def \` then the name: cite the name's line, as the recipes do.
|
|
1104
|
+
if number < len(lines) and leading_name.match(
|
|
1105
|
+
lines[number].split("#", 1)[0]
|
|
1106
|
+
):
|
|
1107
|
+
found.append((number + 1, top))
|
|
1108
|
+
continue
|
|
1109
|
+
m = _PY_FROM_IMPORT_RE.match(code)
|
|
1110
|
+
from_import = m is not None
|
|
1111
|
+
if m is None:
|
|
1112
|
+
m = _PY_IMPORT_RE.match(code)
|
|
1113
|
+
if m is None:
|
|
1114
|
+
continue
|
|
1115
|
+
rest = m.group(1)
|
|
1116
|
+
if from_import and rest.strip() == "*":
|
|
1117
|
+
star_imports.append((number, top))
|
|
1118
|
+
continue
|
|
1119
|
+
if from_import and rest.lstrip().startswith("(") and ")" not in rest:
|
|
1120
|
+
cont = ("paren", top, True)
|
|
1121
|
+
elif rest.rstrip().endswith("\\"):
|
|
1122
|
+
cont = ("backslash", top, from_import)
|
|
1123
|
+
rest = rest.rstrip()[:-1]
|
|
1124
|
+
if _py_import_binds(rest, name, from_import):
|
|
1125
|
+
found.append((number, top))
|
|
1126
|
+
return found or star_imports
|
|
1127
|
+
|
|
1128
|
+
|
|
1129
|
+
def _js_export_item_binds(item: str, name: str) -> bool:
|
|
1130
|
+
"""True when one `export { ... }` item exposes `name`: the name after
|
|
1131
|
+
`as`, else the item's own name, an identifier or a string."""
|
|
1132
|
+
m = _JS_EXPORT_ITEM_RE.match(item.strip())
|
|
1133
|
+
if not m:
|
|
1134
|
+
return False
|
|
1135
|
+
exposed = m.group(2) or m.group(1)
|
|
1136
|
+
if exposed[:1] in ("'", '"'):
|
|
1137
|
+
exposed = exposed[1:-1]
|
|
1138
|
+
return exposed == name
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
def _js_regex_end(text: str, start: int) -> int | None:
|
|
1142
|
+
"""The end of the regular expression literal whose opening `/` is at
|
|
1143
|
+
`start`, flags included. None when it does not close on the line, or
|
|
1144
|
+
when the `/` begins a comment."""
|
|
1145
|
+
if text.startswith(("//", "/*"), start):
|
|
1146
|
+
return None
|
|
1147
|
+
in_class = False
|
|
1148
|
+
i, n = start + 1, len(text)
|
|
1149
|
+
while i < n:
|
|
1150
|
+
ch = text[i]
|
|
1151
|
+
if ch == "\\":
|
|
1152
|
+
i += 2
|
|
1153
|
+
continue
|
|
1154
|
+
if in_class:
|
|
1155
|
+
in_class = ch != "]"
|
|
1156
|
+
elif ch == "[":
|
|
1157
|
+
in_class = True
|
|
1158
|
+
elif ch == "/":
|
|
1159
|
+
i += 1
|
|
1160
|
+
while i < n and (text[i].isalnum() or text[i] in "_$"):
|
|
1161
|
+
i += 1
|
|
1162
|
+
return i
|
|
1163
|
+
i += 1
|
|
1164
|
+
return None
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def _js_declarators(text: str, opened: str = "", fresh: bool = True) -> _DeclSplit:
|
|
1168
|
+
"""The declarators of a `const` / `let` / `var` list that begin in
|
|
1169
|
+
`text`: the pieces between its top-level commas.
|
|
1170
|
+
|
|
1171
|
+
`text` is the list's first line after the keyword, or a later line with
|
|
1172
|
+
the brackets the line above left open (`opened`, innermost last) and
|
|
1173
|
+
whether it begins a declarator (`fresh`) or finishes one, whose piece is
|
|
1174
|
+
then left out. Brackets, '...', "..." and `...` strings, a regular
|
|
1175
|
+
expression literal where an expression begins and a type-argument list
|
|
1176
|
+
(`Map<K, V>`: a `<` right after a name, closed by a `>` that is not the
|
|
1177
|
+
one of `=>`) are skipped. The list ends at a top-level `;`, at a bracket
|
|
1178
|
+
that closes one opened before it, and at a `<` where an expression
|
|
1179
|
+
begins: JSX, a type assertion or a generic arrow function's type
|
|
1180
|
+
parameters, which are not followed.
|
|
1181
|
+
|
|
1182
|
+
Returns None when the list ends on this line, else the (brackets left
|
|
1183
|
+
open, `_DECL_NEXT` | `_DECL_OPEN` | `_DECL_MAYBE`) it goes on with, and
|
|
1184
|
+
the pieces.
|
|
1185
|
+
"""
|
|
1186
|
+
pieces: list[str] = []
|
|
1187
|
+
buf: list[str] = []
|
|
1188
|
+
stack = list(opened)
|
|
1189
|
+
keep = fresh # the current piece begins a declarator
|
|
1190
|
+
last = "," if fresh else "=" # the last character outside a string
|
|
1191
|
+
quote: str | None = None
|
|
1192
|
+
escaped = False
|
|
1193
|
+
ended = False
|
|
1194
|
+
i, n = 0, len(text)
|
|
1195
|
+
while i < n:
|
|
1196
|
+
ch = text[i]
|
|
1197
|
+
i += 1
|
|
1198
|
+
if quote is not None:
|
|
1199
|
+
buf.append(ch)
|
|
1200
|
+
if escaped:
|
|
1201
|
+
escaped = False
|
|
1202
|
+
elif ch == "\\":
|
|
1203
|
+
escaped = True
|
|
1204
|
+
elif ch == quote:
|
|
1205
|
+
quote = None
|
|
1206
|
+
continue
|
|
1207
|
+
prev = text[i - 2] if i > 1 else ""
|
|
1208
|
+
if ch in "/<" and last in _JS_EXPRESSION_START and prev + ch != "<<":
|
|
1209
|
+
end = _js_regex_end(text, i - 1) if ch == "/" else None
|
|
1210
|
+
if end is None:
|
|
1211
|
+
ended = True
|
|
1212
|
+
break
|
|
1213
|
+
buf.append(text[i - 1 : end])
|
|
1214
|
+
i = end
|
|
1215
|
+
last = "a" # the literal is a value, as a name is
|
|
1216
|
+
continue
|
|
1217
|
+
if ch in "'\"`":
|
|
1218
|
+
quote = ch
|
|
1219
|
+
elif ch == "<" and (prev.isalnum() or prev in ("_", "$")) and (
|
|
1220
|
+
text[i : i + 1] not in ("<", "=")
|
|
1221
|
+
):
|
|
1222
|
+
stack.append(ch) # `Map<K, V>`, where `a < b` compares
|
|
1223
|
+
elif ch == ">" and stack[-1:] == ["<"] and prev != "=":
|
|
1224
|
+
stack.pop()
|
|
1225
|
+
buf.append(ch)
|
|
1226
|
+
last = "a" # the type is a value's, as a name is
|
|
1227
|
+
continue
|
|
1228
|
+
elif ch in "([{":
|
|
1229
|
+
stack.append(ch)
|
|
1230
|
+
elif ch in ")]}":
|
|
1231
|
+
while stack[-1:] == ["<"]:
|
|
1232
|
+
stack.pop() # a `<` that compared, written without spaces
|
|
1233
|
+
if not stack:
|
|
1234
|
+
ended = True
|
|
1235
|
+
break
|
|
1236
|
+
stack.pop()
|
|
1237
|
+
elif ch == ";" and all(b == "<" for b in stack):
|
|
1238
|
+
ended = True
|
|
1239
|
+
break
|
|
1240
|
+
elif ch == "," and not stack:
|
|
1241
|
+
if keep:
|
|
1242
|
+
pieces.append("".join(buf))
|
|
1243
|
+
buf, keep, last = [], True, ch
|
|
1244
|
+
continue
|
|
1245
|
+
if not ch.isspace():
|
|
1246
|
+
last = ch
|
|
1247
|
+
buf.append(ch)
|
|
1248
|
+
if keep:
|
|
1249
|
+
pieces.append("".join(buf))
|
|
1250
|
+
if ended or quote is not None:
|
|
1251
|
+
return None, pieces
|
|
1252
|
+
if stack:
|
|
1253
|
+
return ("".join(stack), _DECL_OPEN), pieces
|
|
1254
|
+
if last == ",":
|
|
1255
|
+
return ("", _DECL_NEXT), pieces
|
|
1256
|
+
return ("", _DECL_OPEN if last in _JS_OPERAND_PENDING else _DECL_MAYBE), pieces
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
def _js_depth1_pieces(text: str, depth: int) -> tuple[int, list[str]]:
|
|
1260
|
+
"""Split the text of a `module.exports = { ... }` object or an
|
|
1261
|
+
`export { ... }` list into its top-level pieces (the text between commas
|
|
1262
|
+
at depth 1).
|
|
1263
|
+
|
|
1264
|
+
Returns the bracket depth after `text` (0 once the object closed) and
|
|
1265
|
+
the pieces. An opening bracket at depth 1 stays in its piece, so a
|
|
1266
|
+
method shorthand `name(` and a key `name: {` keep their marker. Brackets
|
|
1267
|
+
and commas inside a string on the line are not counted.
|
|
1268
|
+
"""
|
|
1269
|
+
pieces: list[str] = []
|
|
1270
|
+
buf: list[str] = []
|
|
1271
|
+
quote: str | None = None
|
|
1272
|
+
escaped = False
|
|
1273
|
+
for ch in text:
|
|
1274
|
+
if quote is not None:
|
|
1275
|
+
# text inside a string: kept in its piece, never counted
|
|
1276
|
+
if depth == 1:
|
|
1277
|
+
buf.append(ch)
|
|
1278
|
+
if escaped:
|
|
1279
|
+
escaped = False
|
|
1280
|
+
elif ch == "\\":
|
|
1281
|
+
escaped = True
|
|
1282
|
+
elif ch == quote:
|
|
1283
|
+
quote = None
|
|
1284
|
+
continue
|
|
1285
|
+
if ch in "'\"`":
|
|
1286
|
+
quote = ch
|
|
1287
|
+
if depth == 1:
|
|
1288
|
+
buf.append(ch)
|
|
1289
|
+
elif ch in "([{":
|
|
1290
|
+
if depth == 1:
|
|
1291
|
+
buf.append(ch)
|
|
1292
|
+
depth += 1
|
|
1293
|
+
elif ch in ")]}":
|
|
1294
|
+
depth -= 1
|
|
1295
|
+
if depth == 0:
|
|
1296
|
+
break
|
|
1297
|
+
elif depth == 1:
|
|
1298
|
+
if ch == ",":
|
|
1299
|
+
pieces.append("".join(buf))
|
|
1300
|
+
buf = []
|
|
1301
|
+
else:
|
|
1302
|
+
buf.append(ch)
|
|
1303
|
+
pieces.append("".join(buf))
|
|
1304
|
+
return depth, pieces
|
|
1305
|
+
|
|
1306
|
+
|
|
1307
|
+
def _js_paren_tail_is_call(after_open: str) -> bool:
|
|
1308
|
+
"""For `name(` on an indented line: True when the text after the
|
|
1309
|
+
parameters says a call, not a method (`name(x);`). A method's
|
|
1310
|
+
parameters are followed by its body `{` or a return type `:`, or they
|
|
1311
|
+
continue on the next line."""
|
|
1312
|
+
depth = 1
|
|
1313
|
+
for i, ch in enumerate(after_open):
|
|
1314
|
+
if ch in "([{":
|
|
1315
|
+
depth += 1
|
|
1316
|
+
elif ch in ")]}":
|
|
1317
|
+
depth -= 1
|
|
1318
|
+
if depth == 0:
|
|
1319
|
+
rest = after_open[i + 1 :].strip()
|
|
1320
|
+
return not (rest == "" or rest[0] in "{:")
|
|
1321
|
+
return False
|
|
1322
|
+
|
|
1323
|
+
|
|
1324
|
+
def _tsjs_definition_lines(
|
|
1325
|
+
lines: list[str],
|
|
1326
|
+
code_lines: list[str],
|
|
1327
|
+
name: str,
|
|
1328
|
+
dotted: bool,
|
|
1329
|
+
declarators: Callable[..., _DeclSplit] = _js_declarators,
|
|
1330
|
+
) -> list[tuple[int, bool]]:
|
|
1331
|
+
"""(line, at column 0) for every line that defines `name`.
|
|
1332
|
+
|
|
1333
|
+
`code_lines` are `lines` with comments removed (`_js_code_lines`), so a
|
|
1334
|
+
JSDoc line or a commented-out declaration never matches. A line inside
|
|
1335
|
+
a multi-line `export { ... }` list or `module.exports = { ... }` object
|
|
1336
|
+
takes the column of the line the statement starts on, and so do the
|
|
1337
|
+
name line of a declaration whose keyword ends the line above it and a
|
|
1338
|
+
later declarator on a line of its own. Class members count only for a
|
|
1339
|
+
dotted export name. `declarators` splits a declarator list
|
|
1340
|
+
(`_js_declarators`, or `_SourceCache.js_declarators`, which splits each
|
|
1341
|
+
line once for every export looked up).
|
|
1342
|
+
"""
|
|
1343
|
+
esc = re.escape(name)
|
|
1344
|
+
end = r"(?![\w$])"
|
|
1345
|
+
statements = (
|
|
1346
|
+
re.compile(rf"^\s*{_JS_MODIFIERS}{_JS_KINDS}{esc}{end}"),
|
|
1347
|
+
re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
|
|
1348
|
+
re.compile(
|
|
1349
|
+
rf"^\s*export\s+(?:type\s+)?\*\s+as\s+"
|
|
1350
|
+
rf"(?:{esc}\s+|([\"']){esc}\1\s*)from{end}"
|
|
1351
|
+
),
|
|
1352
|
+
re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
|
|
1353
|
+
re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])"),
|
|
1354
|
+
)
|
|
1355
|
+
# One declarator of a `const` / `let` / `var` list: `NAME = v`,
|
|
1356
|
+
# `NAME: T = v`, `NAME!: T`, or a bare `NAME`. A destructuring pattern
|
|
1357
|
+
# starts with `{` or `[` and never matches.
|
|
1358
|
+
declarator = re.compile(rf"^\s*{esc}{end}\s*(?:!?\s*:|=|$)")
|
|
1359
|
+
leading_name = re.compile(rf"^\s*{esc}{end}")
|
|
1360
|
+
object_key = re.compile(
|
|
1361
|
+
rf"^\s*(?:(?:async|get|set)\s+|\*\s*)*(?:(['\"]){esc}\1|{esc}{end})"
|
|
1362
|
+
rf"\s*(?:[:(]|$)"
|
|
1363
|
+
)
|
|
1364
|
+
member = re.compile(
|
|
1365
|
+
rf"^\s+(?P<mods>{_JS_MEMBER_MODIFIERS})(?:\*\s*)?{esc}{end}\s*[?!]?\s*"
|
|
1366
|
+
rf"(?:<[^>]*>\s*)?(?P<tail>[(:=;]|$)"
|
|
1367
|
+
)
|
|
1368
|
+
found: list[tuple[int, bool]] = []
|
|
1369
|
+
# A statement that continues on the next line:
|
|
1370
|
+
# ("list" or "object", column 0, bracket depth).
|
|
1371
|
+
cont: tuple[str, bool, int] | None = None
|
|
1372
|
+
# A declaration whose keyword ends the line: (keyword, column 0).
|
|
1373
|
+
split: tuple[str, bool] | None = None
|
|
1374
|
+
# The `const` / `let` / `var` lists that go on on the next line, one
|
|
1375
|
+
# nested in another's initializer included: (column 0, brackets left
|
|
1376
|
+
# open, how, as `_js_declarators` gives them).
|
|
1377
|
+
decls: list[tuple[bool, str, str]] = []
|
|
1378
|
+
for number, (text, code) in enumerate(zip(lines, code_lines), start=1):
|
|
1379
|
+
# A line of a list carried from above still goes through the rules
|
|
1380
|
+
# below: it may hold a class member or a local of an arrow function.
|
|
1381
|
+
if decls and code.strip():
|
|
1382
|
+
going_on = code.lstrip()[0] in _JS_CONTINUATION_START
|
|
1383
|
+
carried = []
|
|
1384
|
+
for decl_top, opened, how in decls:
|
|
1385
|
+
if how == _DECL_MAYBE and not going_on:
|
|
1386
|
+
continue # the statement ended with the line above
|
|
1387
|
+
carry, pieces = declarators(code, opened, how == _DECL_NEXT)
|
|
1388
|
+
if carry is not None:
|
|
1389
|
+
carried.append((decl_top, *carry))
|
|
1390
|
+
if any(declarator.match(p) for p in pieces):
|
|
1391
|
+
# a later declarator on a line of its own, as Prettier
|
|
1392
|
+
# writes it: `export const a = 1,` then ` NAME = 2;`
|
|
1393
|
+
found.append((number, decl_top))
|
|
1394
|
+
decls = carried
|
|
1395
|
+
if split is not None:
|
|
1396
|
+
keyword, top = split
|
|
1397
|
+
split = None
|
|
1398
|
+
if keyword in _JS_VAR_KEYWORDS:
|
|
1399
|
+
carry, pieces = declarators(code)
|
|
1400
|
+
if carry is not None:
|
|
1401
|
+
decls.append((top, *carry))
|
|
1402
|
+
hit = any(declarator.match(p) for p in pieces)
|
|
1403
|
+
else:
|
|
1404
|
+
hit = leading_name.match(code) is not None
|
|
1405
|
+
if hit:
|
|
1406
|
+
# the name's line, as the recipes cite it
|
|
1407
|
+
found.append((number, top))
|
|
1408
|
+
continue
|
|
1409
|
+
if cont is not None:
|
|
1410
|
+
kind, top, depth = cont
|
|
1411
|
+
depth, pieces = _js_depth1_pieces(code, depth)
|
|
1412
|
+
cont = (kind, top, depth) if depth > 0 else None
|
|
1413
|
+
if kind == "list":
|
|
1414
|
+
hit = any(_js_export_item_binds(p, name) for p in pieces)
|
|
1415
|
+
else:
|
|
1416
|
+
hit = any(object_key.match(p) for p in pieces)
|
|
1417
|
+
if hit:
|
|
1418
|
+
found.append((number, top))
|
|
1419
|
+
continue
|
|
1420
|
+
top = _is_column_0(text)
|
|
1421
|
+
if any(p.match(code) for p in statements):
|
|
1422
|
+
found.append((number, top))
|
|
1423
|
+
continue
|
|
1424
|
+
m = _JS_VAR_DECL_RE.match(code)
|
|
1425
|
+
if m:
|
|
1426
|
+
carry, pieces = declarators(m.group(1))
|
|
1427
|
+
if carry is not None:
|
|
1428
|
+
decls.append((top, *carry))
|
|
1429
|
+
if any(declarator.match(p) for p in pieces):
|
|
1430
|
+
# a later declarator: `export const a = 1, NAME = 2;`
|
|
1431
|
+
found.append((number, top))
|
|
1432
|
+
continue
|
|
1433
|
+
m = _JS_SPLIT_DECL_RE.match(code)
|
|
1434
|
+
if m:
|
|
1435
|
+
split = (m.group("kind"), top)
|
|
1436
|
+
continue
|
|
1437
|
+
m = _JS_EXPORT_LIST_RE.match(code)
|
|
1438
|
+
if m:
|
|
1439
|
+
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1440
|
+
if depth > 0:
|
|
1441
|
+
cont = ("list", top, depth)
|
|
1442
|
+
if any(_js_export_item_binds(p, name) for p in pieces):
|
|
1443
|
+
found.append((number, top))
|
|
1444
|
+
continue
|
|
1445
|
+
m = _JS_MODULE_EXPORTS_OBJECT_RE.match(code)
|
|
1446
|
+
if m:
|
|
1447
|
+
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1448
|
+
if depth > 0:
|
|
1449
|
+
cont = ("object", top, depth)
|
|
1450
|
+
if any(object_key.match(p) for p in pieces):
|
|
1451
|
+
found.append((number, top))
|
|
1452
|
+
continue
|
|
1453
|
+
if not dotted:
|
|
1454
|
+
continue
|
|
1455
|
+
m = member.match(code)
|
|
1456
|
+
if not m:
|
|
1457
|
+
continue
|
|
1458
|
+
tail = m.group("tail")
|
|
1459
|
+
if tail == "(":
|
|
1460
|
+
if not m.group("mods") and _js_paren_tail_is_call(code[m.end():]):
|
|
1461
|
+
continue
|
|
1462
|
+
elif code.endswith(","):
|
|
1463
|
+
continue # an object-literal entry or a destructuring default
|
|
1464
|
+
elif tail == "=" and code[m.end() : m.end() + 1] in ("=", ">"):
|
|
1465
|
+
continue
|
|
1466
|
+
found.append((number, top))
|
|
1467
|
+
return found
|
|
1468
|
+
|
|
1469
|
+
|
|
1470
|
+
def _prefer_module_level(found: list[tuple[int, bool]], dotted: bool) -> list[int]:
|
|
1471
|
+
"""The definition lines to report. For an undotted name, a column-0
|
|
1472
|
+
match drops the indented ones, so a local shadow never outranks the
|
|
1473
|
+
module-level definition; a dotted name keeps every match."""
|
|
1474
|
+
if not dotted:
|
|
1475
|
+
top = [n for n, at_column_0 in found if at_column_0]
|
|
1476
|
+
if top:
|
|
1477
|
+
return sorted(set(top))
|
|
1478
|
+
return sorted({n for n, _ in found})
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
def line_rule_applies(source_file: str, export_type: object) -> bool:
|
|
1482
|
+
"""True when the definition-line rules cover this entry: a Python or
|
|
1483
|
+
TS/JS file, and an export that is not a whole module or package."""
|
|
1484
|
+
if isinstance(export_type, str) and (
|
|
1485
|
+
export_type.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
1486
|
+
):
|
|
1487
|
+
return False
|
|
1488
|
+
suffix = Path(source_file.strip().replace("\\", "/")).suffix.lower()
|
|
1489
|
+
return suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
|
|
1490
|
+
|
|
1491
|
+
|
|
1492
|
+
def _definition_matches(
|
|
1493
|
+
source_file: str,
|
|
1494
|
+
export_name: str,
|
|
1495
|
+
source_root: Path,
|
|
1496
|
+
cache: _SourceCache | None = None,
|
|
1497
|
+
) -> tuple[list[int], set[int]] | None:
|
|
1498
|
+
"""(definition lines to report, every matching line) for an export.
|
|
1499
|
+
|
|
1500
|
+
The first item applies the column-0 rule; the second keeps the indented
|
|
1501
|
+
matches that rule dropped. None when the file's extension has no
|
|
1502
|
+
definition rule or the file cannot be read.
|
|
1503
|
+
"""
|
|
1504
|
+
rel = source_file.strip().replace("\\", "/")
|
|
1505
|
+
suffix = Path(rel).suffix.lower()
|
|
1506
|
+
if suffix not in PYTHON_EXTENSIONS and suffix not in TSJS_EXTENSIONS:
|
|
1507
|
+
return None
|
|
1508
|
+
cache = cache if cache is not None else _SourceCache()
|
|
1509
|
+
path = (source_root / rel).resolve()
|
|
1510
|
+
lines = cache.lines(path)
|
|
1511
|
+
if lines is None:
|
|
1512
|
+
return None
|
|
1513
|
+
name = definition_name(export_name)
|
|
1514
|
+
if not name:
|
|
1515
|
+
return [], set()
|
|
1516
|
+
dotted = "." in export_name
|
|
1517
|
+
if suffix in PYTHON_EXTENSIONS:
|
|
1518
|
+
starts = cache.py_statement_starts(path, lines)
|
|
1519
|
+
found = _python_definition_lines(lines, name, starts)
|
|
1520
|
+
else:
|
|
1521
|
+
code_lines = cache.js_code_lines(path, lines)
|
|
1522
|
+
found = _tsjs_definition_lines(
|
|
1523
|
+
lines, code_lines, name, dotted, cache.js_declarators
|
|
1524
|
+
)
|
|
1525
|
+
return _prefer_module_level(found, dotted), {n for n, _ in found}
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
def find_definition_lines(
|
|
1529
|
+
source_file: str,
|
|
1530
|
+
export_name: str,
|
|
1531
|
+
source_root: Path,
|
|
1532
|
+
cache: _SourceCache | None = None,
|
|
1533
|
+
) -> list[int] | None:
|
|
1534
|
+
"""Every line of `source_file` that defines `export_name`, in order,
|
|
1535
|
+
after the column-0 rule.
|
|
1536
|
+
|
|
1537
|
+
Returns None when the file's extension has no definition rule or the
|
|
1538
|
+
file cannot be read.
|
|
1539
|
+
"""
|
|
1540
|
+
matches = _definition_matches(source_file, export_name, source_root, cache)
|
|
1541
|
+
return None if matches is None else matches[0]
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
def definition_lines_report(
|
|
1545
|
+
source_file: str,
|
|
1546
|
+
export_name: str,
|
|
1547
|
+
source_root: Path,
|
|
1548
|
+
line: int | None = None,
|
|
1549
|
+
export_type: str | None = None,
|
|
1550
|
+
) -> dict:
|
|
1551
|
+
"""What `verify` would say about one entry's line (the definition-lines
|
|
1552
|
+
subcommand). Raises ValueError when the file exists but cannot be read.
|
|
1553
|
+
|
|
1554
|
+
`line_check` is `file-missing`, `skipped-export-type` (a `module` or
|
|
1555
|
+
`package`, no definition line), `skipped-language` (no rule for the
|
|
1556
|
+
extension) or `checked`. With `line`, a recorded line that is an
|
|
1557
|
+
indented match the column-0 rule dropped joins `definition_lines`, as in
|
|
1558
|
+
`verify`, so the answer carries several lines.
|
|
1559
|
+
"""
|
|
1560
|
+
report: dict = {
|
|
1561
|
+
"source_file": source_file,
|
|
1562
|
+
"export_name": export_name,
|
|
1563
|
+
"line_check": "checked",
|
|
1564
|
+
"definition_lines": None,
|
|
1565
|
+
"source_line": line,
|
|
1566
|
+
"line_is_definition": None,
|
|
1567
|
+
}
|
|
1568
|
+
rel = source_file.strip().replace("\\", "/")
|
|
1569
|
+
if not rel or not (source_root / rel).is_file():
|
|
1570
|
+
report["line_check"] = "file-missing"
|
|
1571
|
+
return report
|
|
1572
|
+
if not line_rule_applies(source_file, export_type):
|
|
1573
|
+
suffix = Path(rel).suffix.lower()
|
|
1574
|
+
covered = suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
|
|
1575
|
+
report["line_check"] = "skipped-export-type" if covered else "skipped-language"
|
|
1576
|
+
return report
|
|
1577
|
+
matches = _definition_matches(source_file, export_name, source_root)
|
|
1578
|
+
if matches is None:
|
|
1579
|
+
raise ValueError(f"failed to read source file {source_root / rel}")
|
|
1580
|
+
defs, every = matches
|
|
1581
|
+
if line is not None:
|
|
1582
|
+
report["line_is_definition"] = line in defs
|
|
1583
|
+
if line not in defs and line in every:
|
|
1584
|
+
defs = sorted(set(defs) | {line})
|
|
1585
|
+
report["definition_lines"] = defs
|
|
1586
|
+
return report
|
|
1587
|
+
|
|
1588
|
+
|
|
1589
|
+
# --------------------------------------------------------------------------
|
|
1590
|
+
# Stale names (classify-stale)
|
|
1591
|
+
# --------------------------------------------------------------------------
|
|
1592
|
+
|
|
1593
|
+
STALE_NOT_DEFINED = "not-defined"
|
|
1594
|
+
STALE_DEFINED = "defined"
|
|
1595
|
+
STALE_UNCHECKED = "unchecked"
|
|
1596
|
+
STALE_NO_ENTRY = "no-entry"
|
|
1597
|
+
|
|
1598
|
+
|
|
1599
|
+
def stale_names(data: object) -> list[str]:
|
|
1600
|
+
"""The names `--names` holds: reconcile-coverage.py's `stale`, or a JSON
|
|
1601
|
+
array of names, each once in the order given. Raises ValueError."""
|
|
1602
|
+
names = data.get("stale") if isinstance(data, dict) else data
|
|
1603
|
+
if not isinstance(names, list) or not all(isinstance(n, str) for n in names):
|
|
1604
|
+
raise ValueError(
|
|
1605
|
+
"--names must hold a `stale` list of names or a JSON array of names"
|
|
1606
|
+
)
|
|
1607
|
+
return list(dict.fromkeys(n for n in names if n))
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
def _entry_file(entry: dict) -> str | None:
|
|
1611
|
+
"""The entry's `source_file`, None when it records none."""
|
|
1612
|
+
source_file = entry.get("source_file")
|
|
1613
|
+
if not isinstance(source_file, str) or not source_file.strip():
|
|
1614
|
+
return None
|
|
1615
|
+
return source_file
|
|
1616
|
+
|
|
1617
|
+
|
|
1618
|
+
def _entry_citation(entry: dict) -> str | None:
|
|
1619
|
+
source_file = _entry_file(entry)
|
|
1620
|
+
if source_file is None:
|
|
1621
|
+
return None
|
|
1622
|
+
line = _coerce_line(entry.get("source_line"))
|
|
1623
|
+
return f"{source_file}:{line}" if isinstance(line, int) else source_file
|
|
1624
|
+
|
|
1625
|
+
|
|
1626
|
+
def _stale_line_check(
|
|
1627
|
+
entry: dict, name: str, source_root: Path
|
|
1628
|
+
) -> tuple[str, list[int] | None]:
|
|
1629
|
+
"""(line_check, definition_lines) of one provenance entry of a stale name."""
|
|
1630
|
+
source_file = _entry_file(entry)
|
|
1631
|
+
if source_file is None:
|
|
1632
|
+
# no file to check: it neither proves the name fabricated nor clears it
|
|
1633
|
+
return "no-file", None
|
|
1634
|
+
line = _coerce_line(entry.get("source_line"))
|
|
1635
|
+
export_type = entry.get("export_type")
|
|
1636
|
+
try:
|
|
1637
|
+
report = definition_lines_report(
|
|
1638
|
+
source_file,
|
|
1639
|
+
name,
|
|
1640
|
+
source_root,
|
|
1641
|
+
line if isinstance(line, int) else None,
|
|
1642
|
+
export_type if isinstance(export_type, str) else None,
|
|
1643
|
+
)
|
|
1644
|
+
except ValueError:
|
|
1645
|
+
# the file exists but cannot be read: nothing proves it
|
|
1646
|
+
return "unreadable", None
|
|
1647
|
+
return report["line_check"], report["definition_lines"]
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def classify_stale(names: list[str], prov: dict, source_root: Path) -> list[dict]:
|
|
1651
|
+
"""The fabricated-signature test for each stale name: the definition-lines
|
|
1652
|
+
rules at every provenance entry whose raw `export_name` is the name. The
|
|
1653
|
+
classify-stale section of the module docstring gives each reason, what a
|
|
1654
|
+
`no-file` entry means and which entry gives `source`.
|
|
1655
|
+
"""
|
|
1656
|
+
entries = [e for e in prov.get("entries") or [] if isinstance(e, dict)]
|
|
1657
|
+
results: list[dict] = []
|
|
1658
|
+
for name in names:
|
|
1659
|
+
mine = [e for e in entries if e.get("export_name") == name]
|
|
1660
|
+
checks: list[dict] = []
|
|
1661
|
+
for entry in mine:
|
|
1662
|
+
line_check, defs = _stale_line_check(entry, name, source_root)
|
|
1663
|
+
checks.append(
|
|
1664
|
+
{
|
|
1665
|
+
"source_file": entry.get("source_file"),
|
|
1666
|
+
"source_line": entry.get("source_line"),
|
|
1667
|
+
"line_check": line_check,
|
|
1668
|
+
"definition_lines": defs,
|
|
1669
|
+
}
|
|
1670
|
+
)
|
|
1671
|
+
if not checks:
|
|
1672
|
+
reason = STALE_NO_ENTRY
|
|
1673
|
+
elif all(
|
|
1674
|
+
c["line_check"] == "file-missing"
|
|
1675
|
+
or (c["line_check"] == "checked" and not c["definition_lines"])
|
|
1676
|
+
for c in checks
|
|
1677
|
+
):
|
|
1678
|
+
reason = STALE_NOT_DEFINED
|
|
1679
|
+
elif any(
|
|
1680
|
+
c["line_check"] == "checked" and c["definition_lines"] for c in checks
|
|
1681
|
+
):
|
|
1682
|
+
reason = STALE_DEFINED
|
|
1683
|
+
else:
|
|
1684
|
+
reason = STALE_UNCHECKED
|
|
1685
|
+
source = next(filter(None, map(_entry_citation, mine)), None)
|
|
1686
|
+
results.append(
|
|
1687
|
+
{
|
|
1688
|
+
"name": name,
|
|
1689
|
+
"fabricated": reason == STALE_NOT_DEFINED,
|
|
1690
|
+
"source": source,
|
|
1691
|
+
"reason": reason,
|
|
1692
|
+
"entries": checks,
|
|
1693
|
+
}
|
|
1694
|
+
)
|
|
1695
|
+
return results
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
# --------------------------------------------------------------------------
|
|
1699
|
+
# Skill citations ([AST:] / [SRC:] prefixes)
|
|
1700
|
+
# --------------------------------------------------------------------------
|
|
1701
|
+
|
|
1702
|
+
# `[AST:path:L12]`, `[SRC:path:L12-34]`, `[SRC:path:L12-L34]`. A citation
|
|
1703
|
+
# with no `:L<n>` part does not match and is ignored. Groups: prefix, path,
|
|
1704
|
+
# first line, the `L` of a range's end (may be empty), the range's end.
|
|
1705
|
+
_SKILL_CITATION_RE = re.compile(
|
|
1706
|
+
r"\[(AST|SRC):([^\[\]\n]+?):L(\d+)(?:-(L?)(\d+))?\]"
|
|
1707
|
+
)
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
def normalize_path(path: str) -> str:
|
|
1711
|
+
"""Normalize a cited or recorded source path for comparison."""
|
|
1712
|
+
p = path.strip().replace("\\", "/")
|
|
1713
|
+
while p.startswith("./"):
|
|
1714
|
+
p = p[2:]
|
|
1715
|
+
return posixpath.normpath(p) if p else p
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
def skill_markdown_files(skill_dir: Path) -> list[Path]:
|
|
1719
|
+
"""`SKILL.md` then every `references/**/*.md`, in a stable order."""
|
|
1720
|
+
files: list[Path] = []
|
|
1721
|
+
skill_md = skill_dir / "SKILL.md"
|
|
1722
|
+
if skill_md.is_file():
|
|
1723
|
+
files.append(skill_md)
|
|
1724
|
+
refs = skill_dir / "references"
|
|
1725
|
+
if refs.is_dir():
|
|
1726
|
+
files.extend(sorted(p for p in refs.rglob("*.md") if p.is_file()))
|
|
1727
|
+
return files
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
def scan_skill_citations(skill_dir: Path) -> list[dict]:
|
|
1731
|
+
"""Every `[AST:]` / `[SRC:]` citation with a line, in file order."""
|
|
1732
|
+
found: list[dict] = []
|
|
1733
|
+
for md in skill_markdown_files(skill_dir):
|
|
1734
|
+
rel = md.relative_to(skill_dir).as_posix()
|
|
1735
|
+
lines = _read_lines(md) or []
|
|
1736
|
+
for number, text in enumerate(lines, start=1):
|
|
1737
|
+
for m in _SKILL_CITATION_RE.finditer(text):
|
|
1738
|
+
found.append(
|
|
1739
|
+
{
|
|
1740
|
+
"file": rel,
|
|
1741
|
+
"line": number,
|
|
1742
|
+
"column": m.start() + 1,
|
|
1743
|
+
"citation": m.group(0),
|
|
1744
|
+
"prefix": m.group(1),
|
|
1745
|
+
"cited_file": m.group(2).strip(),
|
|
1746
|
+
"cited_line": int(m.group(3)),
|
|
1747
|
+
}
|
|
1748
|
+
)
|
|
1749
|
+
return found
|
|
1750
|
+
|
|
1751
|
+
|
|
1752
|
+
def _extraction_method(entry: dict) -> str | None:
|
|
1753
|
+
"""The entry's `extraction_method`, or None when it is not a string
|
|
1754
|
+
(a list or object there reads as an unknown method, never a crash)."""
|
|
1755
|
+
method = entry.get("extraction_method")
|
|
1756
|
+
return method if isinstance(method, str) else None
|
|
1757
|
+
|
|
1758
|
+
|
|
1759
|
+
def check_skill_citations(
|
|
1760
|
+
prov: dict, skill_dir: Path
|
|
1761
|
+
) -> tuple[list[dict], int, int]:
|
|
1762
|
+
"""Compare each skill citation's prefix with the entry it cites.
|
|
1763
|
+
|
|
1764
|
+
Returns (findings, citations_scanned, citations_matched): a citation is
|
|
1765
|
+
matched when its path and line name at least one provenance entry.
|
|
1766
|
+
"""
|
|
1767
|
+
entries = prov.get("entries")
|
|
1768
|
+
entries = entries if isinstance(entries, list) else []
|
|
1769
|
+
by_location: dict[tuple[str, int], list[dict]] = {}
|
|
1770
|
+
has_ast_entry = False
|
|
1771
|
+
# `ast-without-ast-grep` needs proof that no entry came from ast-grep:
|
|
1772
|
+
# every entry with a source line must carry a known method.
|
|
1773
|
+
methods_known = True
|
|
1774
|
+
for entry in entries:
|
|
1775
|
+
if not isinstance(entry, dict):
|
|
1776
|
+
continue
|
|
1777
|
+
method = _extraction_method(entry)
|
|
1778
|
+
if method in AST_METHODS:
|
|
1779
|
+
has_ast_entry = True
|
|
1780
|
+
sf = entry.get("source_file")
|
|
1781
|
+
line = _coerce_line(entry.get("source_line"))
|
|
1782
|
+
if not isinstance(sf, str) or not sf.strip() or not isinstance(line, int):
|
|
1783
|
+
continue
|
|
1784
|
+
if method not in KNOWN_METHODS:
|
|
1785
|
+
methods_known = False
|
|
1786
|
+
by_location.setdefault((normalize_path(sf), line), []).append(entry)
|
|
1787
|
+
|
|
1788
|
+
scanned = sorted(
|
|
1789
|
+
scan_skill_citations(skill_dir),
|
|
1790
|
+
key=lambda c: (c["file"], c["line"], c["column"]),
|
|
1791
|
+
)
|
|
1792
|
+
findings: list[dict] = []
|
|
1793
|
+
matched_count = 0
|
|
1794
|
+
for cit in scanned:
|
|
1795
|
+
prefix = cit["prefix"]
|
|
1796
|
+
other = "SRC" if prefix == "AST" else "AST"
|
|
1797
|
+
matched = by_location.get(
|
|
1798
|
+
(normalize_path(cit["cited_file"]), cit["cited_line"]), []
|
|
1799
|
+
)
|
|
1800
|
+
if matched:
|
|
1801
|
+
matched_count += 1
|
|
1802
|
+
implied = [(METHOD_PREFIX.get(_extraction_method(e)), e) for e in matched]
|
|
1803
|
+
agrees = any(p == prefix for p, _ in implied)
|
|
1804
|
+
mismatch = next((e for p, e in implied if p == other), None)
|
|
1805
|
+
reason = None
|
|
1806
|
+
export_name = None
|
|
1807
|
+
if not agrees and mismatch is not None:
|
|
1808
|
+
reason = CITATION_PREFIX_MISMATCH
|
|
1809
|
+
export_name = mismatch.get("export_name")
|
|
1810
|
+
elif prefix == "AST" and not has_ast_entry and methods_known:
|
|
1811
|
+
reason = CITATION_AST_WITHOUT_AST_GREP
|
|
1812
|
+
if matched:
|
|
1813
|
+
export_name = matched[0].get("export_name")
|
|
1814
|
+
if reason is None:
|
|
1815
|
+
continue
|
|
1816
|
+
item = {k: v for k, v in cit.items() if k != "column"}
|
|
1817
|
+
item.update(
|
|
1818
|
+
{
|
|
1819
|
+
"reason": reason,
|
|
1820
|
+
"expected_prefix": other,
|
|
1821
|
+
"export_name": export_name
|
|
1822
|
+
if isinstance(export_name, str) and export_name
|
|
1823
|
+
else None,
|
|
1824
|
+
}
|
|
1825
|
+
)
|
|
1826
|
+
findings.append(item)
|
|
1827
|
+
return findings, len(scanned), matched_count
|
|
1828
|
+
|
|
1829
|
+
|
|
1830
|
+
# --------------------------------------------------------------------------
|
|
1831
|
+
# Node kinds (--check-node-kinds)
|
|
1832
|
+
# --------------------------------------------------------------------------
|
|
1833
|
+
|
|
1834
|
+
|
|
1835
|
+
def _resolve_outside_cwd(command: str) -> str | None:
|
|
1836
|
+
"""shutil.which with a CWD-shim guard. Returns the resolved path or None.
|
|
1837
|
+
|
|
1838
|
+
shutil.which on Windows searches the current directory ahead of PATH,
|
|
1839
|
+
and CWD here may be a repository SKF does not control: a bare-name
|
|
1840
|
+
lookup resolving into CWD would execute a planted shim (e.g.
|
|
1841
|
+
ast-grep.cmd). Such a resolution is treated as not-found. Explicit
|
|
1842
|
+
paths supplied by callers (containing a separator) are honored as-is.
|
|
1843
|
+
Keep the code identical to the sibling guards in
|
|
1844
|
+
skf-merge-ccc-exclusions.py, skf-detect-tools.py,
|
|
1845
|
+
skf-qmd-classify-collections.py, skf-ccc-git-hygiene.py,
|
|
1846
|
+
skf-source-tree.py and skf-tessl-review.py
|
|
1847
|
+
(test/test-skf-verify-provenance-completeness.py pins it against
|
|
1848
|
+
skf-merge-ccc-exclusions.py).
|
|
1849
|
+
"""
|
|
1850
|
+
resolved = shutil.which(command)
|
|
1851
|
+
if resolved is None:
|
|
1852
|
+
return None
|
|
1853
|
+
if os.sep in command or (os.altsep and os.altsep in command):
|
|
1854
|
+
return resolved
|
|
1855
|
+
resolved_dir = os.path.dirname(resolved)
|
|
1856
|
+
if resolved_dir:
|
|
1857
|
+
cwd = os.path.normcase(os.path.abspath(os.getcwd()))
|
|
1858
|
+
if os.path.normcase(os.path.abspath(resolved_dir)) == cwd:
|
|
1859
|
+
return None
|
|
1860
|
+
return resolved
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
def ast_grep_language(source_file: object) -> str | None:
|
|
1864
|
+
"""The ast-grep language for `source_file`'s extension, or None."""
|
|
1865
|
+
if not isinstance(source_file, str) or not source_file.strip():
|
|
1866
|
+
return None
|
|
1867
|
+
ext = posixpath.splitext(normalize_path(source_file))[1].lower()
|
|
1868
|
+
return AST_GREP_LANGUAGES.get(ext)
|
|
1869
|
+
|
|
1870
|
+
|
|
1871
|
+
def ast_grep_judges_kind(exe: str, language: str, kind: str, folder: str) -> str:
|
|
1872
|
+
"""Ask ast-grep whether `kind` is a node kind of `language`.
|
|
1873
|
+
|
|
1874
|
+
Runs one rule that matches only `kind` over an empty stdin: ast-grep
|
|
1875
|
+
parses the rule before it reads input and rejects a kind the grammar
|
|
1876
|
+
does not have. Returns "valid", "invalid" (the rule was rejected and the
|
|
1877
|
+
error names an invalid kind), "timeout", or "unknown" (a failed spawn or
|
|
1878
|
+
any other error, which proves nothing about the kind). `folder` holds
|
|
1879
|
+
the minimal sgconfig.yml passed with `--config` and is the working
|
|
1880
|
+
folder, so no other sgconfig.yml is read. The rule is JSON, which YAML
|
|
1881
|
+
reads as a flow mapping.
|
|
1882
|
+
"""
|
|
1883
|
+
rule = json.dumps(
|
|
1884
|
+
{"id": "skf-node-kind", "language": language, "rule": {"kind": kind}}
|
|
1885
|
+
)
|
|
1886
|
+
config = os.path.join(folder, "sgconfig.yml")
|
|
1887
|
+
try:
|
|
1888
|
+
result = subprocess.run(
|
|
1889
|
+
[exe, "scan", "--config", config, "--inline-rules", rule, "--stdin"],
|
|
1890
|
+
input="",
|
|
1891
|
+
capture_output=True,
|
|
1892
|
+
encoding="utf-8",
|
|
1893
|
+
errors="replace",
|
|
1894
|
+
timeout=NODE_KIND_TIMEOUT_SEC,
|
|
1895
|
+
cwd=folder,
|
|
1896
|
+
check=False,
|
|
1897
|
+
)
|
|
1898
|
+
except subprocess.TimeoutExpired:
|
|
1899
|
+
return "timeout"
|
|
1900
|
+
except (OSError, subprocess.SubprocessError):
|
|
1901
|
+
return "unknown"
|
|
1902
|
+
if result.returncode == 0:
|
|
1903
|
+
return "valid"
|
|
1904
|
+
if "invalid kind" in (result.stderr or "").lower():
|
|
1905
|
+
return "invalid"
|
|
1906
|
+
return "unknown"
|
|
1907
|
+
|
|
1908
|
+
|
|
1909
|
+
def _judge_kinds(
|
|
1910
|
+
exe: str, pairs: list[tuple[str, str]]
|
|
1911
|
+
) -> tuple[str, dict[tuple[str, str], str]]:
|
|
1912
|
+
"""Ask ast-grep about each (language, kind) pair, in order.
|
|
1913
|
+
|
|
1914
|
+
Returns (node_kind_check, verdicts): each verdict is "valid", "invalid"
|
|
1915
|
+
or the reason the pair went unjudged. ast-grep is first asked about a
|
|
1916
|
+
kind no grammar has; unless it answers "invalid", its answers cannot be
|
|
1917
|
+
read and the check is "skipped-unrecognized-ast-grep" with no verdicts.
|
|
1918
|
+
After one timeout the remaining pairs are not asked. When the temporary
|
|
1919
|
+
folder cannot be made or written, the pairs not yet asked go unjudged.
|
|
1920
|
+
"""
|
|
1921
|
+
verdicts: dict[tuple[str, str], str] = {}
|
|
1922
|
+
try:
|
|
1923
|
+
with tempfile.TemporaryDirectory(
|
|
1924
|
+
prefix="skf-node-kind-", ignore_cleanup_errors=True
|
|
1925
|
+
) as folder:
|
|
1926
|
+
with open(
|
|
1927
|
+
os.path.join(folder, "sgconfig.yml"), "w", encoding="utf-8"
|
|
1928
|
+
) as fh:
|
|
1929
|
+
fh.write(MINIMAL_SGCONFIG)
|
|
1930
|
+
calibration = ast_grep_judges_kind(
|
|
1931
|
+
exe, pairs[0][0], BOGUS_NODE_KIND, folder
|
|
1932
|
+
)
|
|
1933
|
+
if calibration != "invalid":
|
|
1934
|
+
return "skipped-unrecognized-ast-grep", {}
|
|
1935
|
+
timed_out = False
|
|
1936
|
+
for language, kind in pairs:
|
|
1937
|
+
if timed_out:
|
|
1938
|
+
verdicts[(language, kind)] = UNCHECKED_TIMEOUT
|
|
1939
|
+
continue
|
|
1940
|
+
verdict = ast_grep_judges_kind(exe, language, kind, folder)
|
|
1941
|
+
if verdict == "timeout":
|
|
1942
|
+
timed_out = True
|
|
1943
|
+
verdict = UNCHECKED_TIMEOUT
|
|
1944
|
+
elif verdict == "unknown":
|
|
1945
|
+
verdict = UNCHECKED_AST_GREP_ERROR
|
|
1946
|
+
verdicts[(language, kind)] = verdict
|
|
1947
|
+
except OSError:
|
|
1948
|
+
for pair in pairs:
|
|
1949
|
+
verdicts.setdefault(pair, UNCHECKED_TEMP_FOLDER)
|
|
1950
|
+
return "checked", verdicts
|
|
1951
|
+
|
|
1952
|
+
|
|
1953
|
+
def _node_kind_item(
|
|
1954
|
+
index: int, entry: dict, kind: str, language: str, reason: str
|
|
1955
|
+
) -> dict:
|
|
1956
|
+
name = entry.get("export_name")
|
|
1957
|
+
return {
|
|
1958
|
+
"export_name": name if isinstance(name, str) and name else None,
|
|
1959
|
+
"entry_index": index,
|
|
1960
|
+
"source_file": entry.get("source_file"),
|
|
1961
|
+
"source_line": entry.get("source_line"),
|
|
1962
|
+
"ast_node_type": kind,
|
|
1963
|
+
"language": language,
|
|
1964
|
+
"reason": reason,
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
|
|
1968
|
+
def check_node_kinds(prov: dict) -> tuple[list[dict], list[dict], dict]:
|
|
1969
|
+
"""Check each ast-grep entry's `ast_node_type` against ast-grep itself.
|
|
1970
|
+
|
|
1971
|
+
Returns (findings, unchecked, summary fields). `ERROR` and a kind that
|
|
1972
|
+
is not kind-shaped are judged without ast-grep, whatever the binary
|
|
1973
|
+
lookup finds; every other kind is asked once per (language, kind).
|
|
1974
|
+
"""
|
|
1975
|
+
entries = prov.get("entries")
|
|
1976
|
+
entries = entries if isinstance(entries, list) else []
|
|
1977
|
+
findings: list[dict] = []
|
|
1978
|
+
unchecked: list[dict] = []
|
|
1979
|
+
pending: list[tuple[int, dict, str, str]] = []
|
|
1980
|
+
judged = 0
|
|
1981
|
+
skipped = 0
|
|
1982
|
+
for index, entry in enumerate(entries):
|
|
1983
|
+
if not isinstance(entry, dict):
|
|
1984
|
+
continue
|
|
1985
|
+
if _extraction_method(entry) not in AST_METHODS:
|
|
1986
|
+
continue
|
|
1987
|
+
kind = entry.get("ast_node_type")
|
|
1988
|
+
if not isinstance(kind, str) or not kind.strip():
|
|
1989
|
+
continue
|
|
1990
|
+
language = ast_grep_language(entry.get("source_file"))
|
|
1991
|
+
if language is None:
|
|
1992
|
+
skipped += 1
|
|
1993
|
+
continue
|
|
1994
|
+
if kind == ERROR_NODE_KIND:
|
|
1995
|
+
reason = NODE_KIND_ERROR
|
|
1996
|
+
elif not NODE_KIND_SHAPE.fullmatch(kind):
|
|
1997
|
+
reason = NODE_KIND_INVALID
|
|
1998
|
+
else:
|
|
1999
|
+
pending.append((index, entry, language, kind))
|
|
2000
|
+
continue
|
|
2001
|
+
judged += 1
|
|
2002
|
+
findings.append(_node_kind_item(index, entry, kind, language, reason))
|
|
2003
|
+
|
|
2004
|
+
check = "checked"
|
|
2005
|
+
if pending:
|
|
2006
|
+
exe = _resolve_outside_cwd("ast-grep")
|
|
2007
|
+
if exe is None:
|
|
2008
|
+
check = "skipped-no-ast-grep"
|
|
2009
|
+
else:
|
|
2010
|
+
pairs = sorted({(lang, kind) for _, _, lang, kind in pending})
|
|
2011
|
+
check, verdicts = _judge_kinds(exe, pairs)
|
|
2012
|
+
if check == "checked":
|
|
2013
|
+
for index, entry, language, kind in pending:
|
|
2014
|
+
verdict = verdicts[(language, kind)]
|
|
2015
|
+
item = _node_kind_item(index, entry, kind, language, verdict)
|
|
2016
|
+
if verdict == "valid":
|
|
2017
|
+
judged += 1
|
|
2018
|
+
elif verdict == "invalid":
|
|
2019
|
+
judged += 1
|
|
2020
|
+
item["reason"] = NODE_KIND_INVALID
|
|
2021
|
+
findings.append(item)
|
|
2022
|
+
else:
|
|
2023
|
+
unchecked.append(item)
|
|
2024
|
+
|
|
2025
|
+
def order(item: dict) -> tuple:
|
|
2026
|
+
return (item["export_name"] or "", str(item["source_file"]),
|
|
2027
|
+
item["entry_index"])
|
|
2028
|
+
|
|
2029
|
+
findings.sort(key=order)
|
|
2030
|
+
unchecked.sort(key=order)
|
|
2031
|
+
summary = {
|
|
2032
|
+
"node_kind_check": check,
|
|
2033
|
+
"node_kinds_checked": judged,
|
|
2034
|
+
"node_kind_check_skipped": skipped,
|
|
2035
|
+
"node_kind_check_errors": len(unchecked),
|
|
2036
|
+
"node_kind_findings_count": len(findings),
|
|
2037
|
+
}
|
|
2038
|
+
return findings, unchecked, summary
|
|
2039
|
+
|
|
2040
|
+
|
|
2041
|
+
# --------------------------------------------------------------------------
|
|
2042
|
+
# The recipe kind at a line (kind-at)
|
|
2043
|
+
# --------------------------------------------------------------------------
|
|
2044
|
+
|
|
2045
|
+
|
|
2046
|
+
def _is_recipe(doc: object) -> bool:
|
|
2047
|
+
"""A recipe: a mapping with a string `id` and `language`, and a `rule`
|
|
2048
|
+
mapping that names its node `kind`."""
|
|
2049
|
+
if not isinstance(doc, dict):
|
|
2050
|
+
return False
|
|
2051
|
+
rule = doc.get("rule")
|
|
2052
|
+
return (
|
|
2053
|
+
isinstance(doc.get("id"), str)
|
|
2054
|
+
and isinstance(doc.get("language"), str)
|
|
2055
|
+
and isinstance(rule, dict)
|
|
2056
|
+
and isinstance(rule.get("kind"), str)
|
|
2057
|
+
)
|
|
2058
|
+
|
|
2059
|
+
|
|
2060
|
+
def load_recipes(path: Path) -> list[dict]:
|
|
2061
|
+
"""Every ast-grep recipe in `path`, first copy of each (id, language).
|
|
2062
|
+
|
|
2063
|
+
A markdown file (`.md`) gives the recipes in its ```yaml fences; any
|
|
2064
|
+
other file is read as YAML: one recipe per document, a list of recipes,
|
|
2065
|
+
or a mapping whose `recipes` key holds that list. Raises ValueError when
|
|
2066
|
+
the file cannot be read, and ImportError without PyYAML.
|
|
2067
|
+
"""
|
|
2068
|
+
import yaml # only kind-at reads recipes; `uv run` installs it
|
|
2069
|
+
|
|
2070
|
+
try:
|
|
2071
|
+
text = path.read_text(encoding="utf-8")
|
|
2072
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
2073
|
+
raise ValueError(f"failed to read recipes {path}: {exc}") from exc
|
|
2074
|
+
docs: list[object] = []
|
|
2075
|
+
if path.suffix.lower() in (".md", ".markdown"):
|
|
2076
|
+
for block in _MD_YAML_FENCE_RE.findall(text):
|
|
2077
|
+
try:
|
|
2078
|
+
docs.append(yaml.safe_load(block))
|
|
2079
|
+
except yaml.YAMLError:
|
|
2080
|
+
continue # a template, not a recipe
|
|
2081
|
+
else:
|
|
2082
|
+
try:
|
|
2083
|
+
docs = list(yaml.safe_load_all(text))
|
|
2084
|
+
except yaml.YAMLError as exc:
|
|
2085
|
+
raise ValueError(f"malformed YAML in recipes {path}: {exc}") from exc
|
|
2086
|
+
candidates: list[object] = []
|
|
2087
|
+
for doc in docs:
|
|
2088
|
+
if isinstance(doc, dict) and isinstance(doc.get("recipes"), list):
|
|
2089
|
+
candidates.extend(doc["recipes"])
|
|
2090
|
+
elif isinstance(doc, list):
|
|
2091
|
+
candidates.extend(doc)
|
|
2092
|
+
else:
|
|
2093
|
+
candidates.append(doc)
|
|
2094
|
+
recipes: list[dict] = []
|
|
2095
|
+
seen: set[tuple[str, str]] = set()
|
|
2096
|
+
for doc in candidates:
|
|
2097
|
+
if not _is_recipe(doc):
|
|
2098
|
+
continue
|
|
2099
|
+
key = (doc["id"], doc["language"].strip().lower())
|
|
2100
|
+
if key not in seen:
|
|
2101
|
+
seen.add(key)
|
|
2102
|
+
recipes.append(doc)
|
|
2103
|
+
return recipes
|
|
2104
|
+
|
|
2105
|
+
|
|
2106
|
+
def recipe_runs(recipes: list[dict], language: str) -> list[dict]:
|
|
2107
|
+
"""The recipes to run on a file of `language`, each with its `language`
|
|
2108
|
+
set to it: an id's own form for that language, else its form for the
|
|
2109
|
+
first language `RECIPE_LANGUAGE_FALLBACKS` lists (in recipe order)."""
|
|
2110
|
+
forms: dict[str, dict[str, dict]] = {}
|
|
2111
|
+
for recipe in recipes:
|
|
2112
|
+
lang = recipe["language"].strip().lower()
|
|
2113
|
+
forms.setdefault(recipe["id"], {}).setdefault(lang, recipe)
|
|
2114
|
+
runs: list[dict] = []
|
|
2115
|
+
for recipe_id, by_language in forms.items():
|
|
2116
|
+
for lang in (language, *RECIPE_LANGUAGE_FALLBACKS.get(language, ())):
|
|
2117
|
+
if lang in by_language:
|
|
2118
|
+
run = dict(by_language[lang], language=language)
|
|
2119
|
+
run["_rewritten"] = lang != language
|
|
2120
|
+
runs.append(run)
|
|
2121
|
+
break
|
|
2122
|
+
return runs
|
|
2123
|
+
|
|
2124
|
+
|
|
2125
|
+
def _scan_recipe(
|
|
2126
|
+
exe: str, run: dict, source: bytes, folder: str
|
|
2127
|
+
) -> tuple[str, list[tuple[int, str]] | str]:
|
|
2128
|
+
"""Run one recipe over `source` on stdin.
|
|
2129
|
+
|
|
2130
|
+
Returns ("ok", [($NAME's 1-based line, $NAME's text), ...]),
|
|
2131
|
+
("not-applicable", []) for a recipe whose `language` was rewritten and
|
|
2132
|
+
whose kinds that grammar lacks, ("timeout", detail) or ("error",
|
|
2133
|
+
detail). The rule goes in a file as JSON (YAML reads it as a flow
|
|
2134
|
+
mapping), never on the command line, where an npm `.cmd` shim on
|
|
2135
|
+
Windows would hand its `|` and `^` to cmd.exe.
|
|
2136
|
+
"""
|
|
2137
|
+
rule = {k: v for k, v in run.items() if not k.startswith("_")}
|
|
2138
|
+
rule_path = os.path.join(folder, "recipe.yml")
|
|
2139
|
+
with open(rule_path, "w", encoding="utf-8") as fh:
|
|
2140
|
+
fh.write(json.dumps(rule, ensure_ascii=False))
|
|
2141
|
+
config = os.path.join(folder, "sgconfig.yml")
|
|
2142
|
+
try:
|
|
2143
|
+
result = subprocess.run(
|
|
2144
|
+
[exe, "scan", "--config", config, "-r", rule_path, "--stdin",
|
|
2145
|
+
"--json=stream"],
|
|
2146
|
+
input=source,
|
|
2147
|
+
capture_output=True,
|
|
2148
|
+
timeout=KIND_AT_TIMEOUT_SEC,
|
|
2149
|
+
cwd=folder,
|
|
2150
|
+
check=False,
|
|
2151
|
+
)
|
|
2152
|
+
except subprocess.TimeoutExpired:
|
|
2153
|
+
return "timeout", f"no answer in {KIND_AT_TIMEOUT_SEC}s"
|
|
2154
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
2155
|
+
return "error", f"{type(exc).__name__}: {exc}"
|
|
2156
|
+
stdout = (result.stdout or b"").decode("utf-8", errors="replace")
|
|
2157
|
+
stderr = (result.stderr or b"").decode("utf-8", errors="replace")
|
|
2158
|
+
if result.returncode != 0 and not stdout.strip():
|
|
2159
|
+
if run.get("_rewritten") and "invalid kind" in stderr.lower():
|
|
2160
|
+
return "not-applicable", []
|
|
2161
|
+
detail = next((ln.strip() for ln in stderr.splitlines() if ln.strip()), "")
|
|
2162
|
+
return "error", f"exit {result.returncode}: {detail}"
|
|
2163
|
+
found: list[tuple[int, str]] = []
|
|
2164
|
+
for raw in stdout.splitlines():
|
|
2165
|
+
if not raw.strip():
|
|
2166
|
+
continue
|
|
2167
|
+
try:
|
|
2168
|
+
match = json.loads(raw)
|
|
2169
|
+
except ValueError:
|
|
2170
|
+
return "error", "output is not JSON lines"
|
|
2171
|
+
single = (match.get("metaVariables") or {}).get("single") or {}
|
|
2172
|
+
name = single.get("NAME")
|
|
2173
|
+
if not isinstance(name, dict):
|
|
2174
|
+
continue
|
|
2175
|
+
start = (name.get("range") or {}).get("start") or {}
|
|
2176
|
+
line = start.get("line")
|
|
2177
|
+
if isinstance(line, int) and not isinstance(line, bool):
|
|
2178
|
+
found.append((line + 1, str(name.get("text") or "")))
|
|
2179
|
+
return "ok", found
|
|
2180
|
+
|
|
2181
|
+
|
|
2182
|
+
def _unquote(name: str) -> str:
|
|
2183
|
+
"""A string name without its quotes (`"a-b"` -> `a-b`)."""
|
|
2184
|
+
if len(name) >= 2 and name[0] == name[-1] and name[0] in "'\"":
|
|
2185
|
+
return name[1:-1]
|
|
2186
|
+
return name
|
|
2187
|
+
|
|
2188
|
+
|
|
2189
|
+
def kinds_at(
|
|
2190
|
+
source: bytes,
|
|
2191
|
+
language: str | None,
|
|
2192
|
+
targets: list[tuple[int, str | None]],
|
|
2193
|
+
recipes: list[dict],
|
|
2194
|
+
) -> list[dict]:
|
|
2195
|
+
"""The node kind the recipes give each (line, export name or None)
|
|
2196
|
+
target of one file, running each recipe once.
|
|
2197
|
+
|
|
2198
|
+
Each result: {line, name, status, kind, matches[], errors[]}. A match is
|
|
2199
|
+
a recipe whose `$NAME` is on the target line (and, with a name, is that
|
|
2200
|
+
name's last segment). `status` is `found` (one kind among the matches),
|
|
2201
|
+
`ambiguous` (several), `no-match`, `incomplete` (a recipe run failed, so
|
|
2202
|
+
no kind is given), `skipped-no-language`, `skipped-no-recipes` or
|
|
2203
|
+
`skipped-no-ast-grep`.
|
|
2204
|
+
"""
|
|
2205
|
+
results = [
|
|
2206
|
+
{"line": line, "name": name, "status": "", "kind": None,
|
|
2207
|
+
"matches": [], "errors": []}
|
|
2208
|
+
for line, name in targets
|
|
2209
|
+
]
|
|
2210
|
+
|
|
2211
|
+
def skip(status: str) -> list[dict]:
|
|
2212
|
+
for result in results:
|
|
2213
|
+
result["status"] = status
|
|
2214
|
+
return results
|
|
2215
|
+
|
|
2216
|
+
if language is None:
|
|
2217
|
+
return skip("skipped-no-language")
|
|
2218
|
+
runs = recipe_runs(recipes, language)
|
|
2219
|
+
if not runs:
|
|
2220
|
+
return skip("skipped-no-recipes")
|
|
2221
|
+
exe = _resolve_outside_cwd("ast-grep")
|
|
2222
|
+
if exe is None:
|
|
2223
|
+
return skip("skipped-no-ast-grep")
|
|
2224
|
+
|
|
2225
|
+
matched: list[tuple[dict, list[tuple[int, str]]]] = []
|
|
2226
|
+
errors: list[dict] = []
|
|
2227
|
+
try:
|
|
2228
|
+
with tempfile.TemporaryDirectory(
|
|
2229
|
+
prefix="skf-kind-at-", ignore_cleanup_errors=True
|
|
2230
|
+
) as folder:
|
|
2231
|
+
with open(
|
|
2232
|
+
os.path.join(folder, "sgconfig.yml"), "w", encoding="utf-8"
|
|
2233
|
+
) as fh:
|
|
2234
|
+
fh.write(MINIMAL_SGCONFIG)
|
|
2235
|
+
timed_out = False
|
|
2236
|
+
for run in runs:
|
|
2237
|
+
if timed_out:
|
|
2238
|
+
errors.append({"recipe": run["id"], "language": language,
|
|
2239
|
+
"reason": UNCHECKED_TIMEOUT, "detail": ""})
|
|
2240
|
+
continue
|
|
2241
|
+
outcome, found = _scan_recipe(exe, run, source, folder)
|
|
2242
|
+
if outcome == "ok":
|
|
2243
|
+
matched.append((run, found))
|
|
2244
|
+
elif outcome in ("timeout", "error"):
|
|
2245
|
+
timed_out = outcome == "timeout"
|
|
2246
|
+
errors.append({
|
|
2247
|
+
"recipe": run["id"],
|
|
2248
|
+
"language": language,
|
|
2249
|
+
"reason": UNCHECKED_TIMEOUT if timed_out
|
|
2250
|
+
else UNCHECKED_AST_GREP_ERROR,
|
|
2251
|
+
"detail": found,
|
|
2252
|
+
})
|
|
2253
|
+
except OSError as exc:
|
|
2254
|
+
errors.append({"recipe": None, "language": language,
|
|
2255
|
+
"reason": UNCHECKED_TEMP_FOLDER, "detail": str(exc)})
|
|
2256
|
+
|
|
2257
|
+
for result in results:
|
|
2258
|
+
wanted = definition_name(result["name"]) if result["name"] else None
|
|
2259
|
+
for run, found in matched:
|
|
2260
|
+
for line, text in found:
|
|
2261
|
+
if line != result["line"]:
|
|
2262
|
+
continue
|
|
2263
|
+
if wanted is not None and _unquote(text) != wanted:
|
|
2264
|
+
continue
|
|
2265
|
+
item = {"recipe": run["id"], "kind": run["rule"]["kind"],
|
|
2266
|
+
"name": text}
|
|
2267
|
+
if item not in result["matches"]:
|
|
2268
|
+
result["matches"].append(item)
|
|
2269
|
+
result["matches"].sort(key=lambda m: (m["recipe"], m["name"]))
|
|
2270
|
+
result["errors"] = [dict(e) for e in errors]
|
|
2271
|
+
kinds = sorted({m["kind"] for m in result["matches"]})
|
|
2272
|
+
if errors:
|
|
2273
|
+
result["status"] = "incomplete"
|
|
2274
|
+
elif not kinds:
|
|
2275
|
+
result["status"] = "no-match"
|
|
2276
|
+
elif len(kinds) == 1:
|
|
2277
|
+
result["status"] = "found"
|
|
2278
|
+
result["kind"] = kinds[0]
|
|
2279
|
+
else:
|
|
2280
|
+
result["status"] = "ambiguous"
|
|
2281
|
+
return results
|
|
2282
|
+
|
|
2283
|
+
|
|
263
2284
|
# --------------------------------------------------------------------------
|
|
264
2285
|
# Verification
|
|
265
2286
|
# --------------------------------------------------------------------------
|
|
@@ -284,8 +2305,15 @@ def resolve_source_root(prov: dict, override: str | None) -> Path | None:
|
|
|
284
2305
|
return root if root.is_dir() else None
|
|
285
2306
|
|
|
286
2307
|
|
|
287
|
-
def verify(
|
|
288
|
-
|
|
2308
|
+
def verify(
|
|
2309
|
+
metadata: dict,
|
|
2310
|
+
prov: dict,
|
|
2311
|
+
source_root: Path | None,
|
|
2312
|
+
skill_dir: Path | None = None,
|
|
2313
|
+
check_kinds: bool = False,
|
|
2314
|
+
) -> dict:
|
|
2315
|
+
"""Compute the completeness / orphan / stale / citation / node-kind
|
|
2316
|
+
report."""
|
|
289
2317
|
reexport_map = extract_reexport_map(prov)
|
|
290
2318
|
|
|
291
2319
|
documented = extract_export_names(metadata)
|
|
@@ -294,15 +2322,24 @@ def verify(metadata: dict, prov: dict, source_root: Path | None) -> dict:
|
|
|
294
2322
|
raw_entry_names = provenance_entry_names(prov)
|
|
295
2323
|
entry_canon = {canon(n, reexport_map) for n in raw_entry_names}
|
|
296
2324
|
|
|
297
|
-
|
|
298
|
-
|
|
2325
|
+
# A reference app documents no exports (`exports[]` is empty by design)
|
|
2326
|
+
# while its entries are per-citation, so the two sets measure different
|
|
2327
|
+
# surfaces and their difference means nothing.
|
|
2328
|
+
set_diff_applies = not is_reference_app(metadata)
|
|
2329
|
+
if set_diff_applies:
|
|
2330
|
+
missing = sorted(documented_canon - entry_canon)
|
|
2331
|
+
orphaned = sorted(entry_canon - documented_canon)
|
|
2332
|
+
else:
|
|
2333
|
+
missing, orphaned = [], []
|
|
299
2334
|
|
|
300
2335
|
stale: list[dict] = []
|
|
301
2336
|
citations_checked = 0
|
|
2337
|
+
line_check_skipped = 0
|
|
302
2338
|
if source_root is not None:
|
|
2339
|
+
cache = _SourceCache()
|
|
303
2340
|
entries = prov.get("entries")
|
|
304
2341
|
if isinstance(entries, list):
|
|
305
|
-
for entry in entries:
|
|
2342
|
+
for index, entry in enumerate(entries):
|
|
306
2343
|
if not isinstance(entry, dict):
|
|
307
2344
|
continue
|
|
308
2345
|
name = entry.get("export_name")
|
|
@@ -310,40 +2347,442 @@ def verify(metadata: dict, prov: dict, source_root: Path | None) -> dict:
|
|
|
310
2347
|
continue
|
|
311
2348
|
sf = entry.get("source_file")
|
|
312
2349
|
sl = entry.get("source_line")
|
|
2350
|
+
line = _coerce_line(sl)
|
|
313
2351
|
# Only count a resolvable (non-null) citation as "checked".
|
|
314
|
-
if isinstance(sf, str) and sf.strip() and
|
|
2352
|
+
if isinstance(sf, str) and sf.strip() and line is not None:
|
|
315
2353
|
citations_checked += 1
|
|
316
2354
|
reason = check_citation(sf, sl, source_root)
|
|
2355
|
+
item = {
|
|
2356
|
+
"export_name": name,
|
|
2357
|
+
"entry_index": index,
|
|
2358
|
+
"source_file": sf if isinstance(sf, str) else None,
|
|
2359
|
+
"source_line": sl if isinstance(sl, int)
|
|
2360
|
+
and not isinstance(sl, bool) else sl,
|
|
2361
|
+
}
|
|
317
2362
|
if reason is not None:
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
)
|
|
2363
|
+
item["reason"] = reason
|
|
2364
|
+
stale.append(item)
|
|
2365
|
+
continue
|
|
2366
|
+
# The citation resolves (or is tolerated): check that the
|
|
2367
|
+
# line defines the export. A tolerated null or blank
|
|
2368
|
+
# citation has no line to check and is not counted.
|
|
2369
|
+
if (
|
|
2370
|
+
not isinstance(sf, str)
|
|
2371
|
+
or not sf.strip()
|
|
2372
|
+
or not isinstance(line, int)
|
|
2373
|
+
):
|
|
2374
|
+
continue
|
|
2375
|
+
if not line_rule_applies(sf, entry.get("export_type")):
|
|
2376
|
+
line_check_skipped += 1
|
|
2377
|
+
continue
|
|
2378
|
+
matches = _definition_matches(sf, name, source_root, cache)
|
|
2379
|
+
if matches is None:
|
|
2380
|
+
line_check_skipped += 1
|
|
2381
|
+
continue
|
|
2382
|
+
defs, every = matches
|
|
2383
|
+
if line not in defs:
|
|
2384
|
+
if line in every:
|
|
2385
|
+
# The recorded line is an indented match the
|
|
2386
|
+
# column-0 rule dropped: keep it, so the finding
|
|
2387
|
+
# lists several lines and nobody auto-moves it.
|
|
2388
|
+
defs = sorted(set(defs) | {line})
|
|
2389
|
+
item["reason"] = STALE_LINE_NOT_DEFINITION
|
|
2390
|
+
item["definition_lines"] = defs
|
|
2391
|
+
stale.append(item)
|
|
327
2392
|
stale.sort(key=lambda s: (s["export_name"], s.get("source_file") or ""))
|
|
328
2393
|
|
|
329
|
-
|
|
330
|
-
|
|
2394
|
+
citations: list[dict] = []
|
|
2395
|
+
skill_citations_scanned = 0
|
|
2396
|
+
skill_citations_matched = 0
|
|
2397
|
+
if skill_dir is not None:
|
|
2398
|
+
citations, skill_citations_scanned, skill_citations_matched = (
|
|
2399
|
+
check_skill_citations(prov, skill_dir)
|
|
2400
|
+
)
|
|
2401
|
+
|
|
2402
|
+
node_kinds: list[dict] = []
|
|
2403
|
+
node_kinds_unchecked: list[dict] = []
|
|
2404
|
+
kind_summary: dict = {"node_kind_check": "not-requested"}
|
|
2405
|
+
if check_kinds:
|
|
2406
|
+
node_kinds, node_kinds_unchecked, kind_summary = check_node_kinds(prov)
|
|
2407
|
+
|
|
2408
|
+
status = (
|
|
2409
|
+
"pass"
|
|
2410
|
+
if not (missing or orphaned or stale or citations or node_kinds)
|
|
2411
|
+
else "findings"
|
|
2412
|
+
)
|
|
2413
|
+
result: dict = {
|
|
331
2414
|
"status": status,
|
|
332
2415
|
"missing": missing,
|
|
333
2416
|
"orphaned": orphaned,
|
|
334
2417
|
"stale": stale,
|
|
335
|
-
"
|
|
336
|
-
"exports_checked": len(documented_canon),
|
|
337
|
-
"entries_checked": len(raw_entry_names),
|
|
338
|
-
"missing_count": len(missing),
|
|
339
|
-
"orphaned_count": len(orphaned),
|
|
340
|
-
"stale_count": len(stale),
|
|
341
|
-
"citations_checked": citations_checked,
|
|
342
|
-
"stale_check": "checked"
|
|
343
|
-
if source_root is not None
|
|
344
|
-
else "skipped-no-source-root",
|
|
345
|
-
},
|
|
2418
|
+
"citations": citations,
|
|
346
2419
|
}
|
|
2420
|
+
if check_kinds:
|
|
2421
|
+
result["node_kinds"] = node_kinds
|
|
2422
|
+
result["node_kinds_unchecked"] = node_kinds_unchecked
|
|
2423
|
+
result["summary"] = {
|
|
2424
|
+
"exports_checked": len(documented_canon),
|
|
2425
|
+
"entries_checked": len(raw_entry_names),
|
|
2426
|
+
"missing_count": len(missing),
|
|
2427
|
+
"orphaned_count": len(orphaned),
|
|
2428
|
+
"set_diff": "checked" if set_diff_applies else "not-applicable",
|
|
2429
|
+
"stale_count": len(stale),
|
|
2430
|
+
"citations_checked": citations_checked,
|
|
2431
|
+
"line_check_skipped": line_check_skipped,
|
|
2432
|
+
"stale_check": "checked"
|
|
2433
|
+
if source_root is not None
|
|
2434
|
+
else "skipped-no-source-root",
|
|
2435
|
+
"skill_citations_scanned": skill_citations_scanned,
|
|
2436
|
+
"skill_citations_matched": skill_citations_matched,
|
|
2437
|
+
"citation_findings_count": len(citations),
|
|
2438
|
+
"citation_check": "checked"
|
|
2439
|
+
if skill_dir is not None
|
|
2440
|
+
else "skipped-no-skill-dir",
|
|
2441
|
+
**kind_summary,
|
|
2442
|
+
}
|
|
2443
|
+
return result
|
|
2444
|
+
|
|
2445
|
+
|
|
2446
|
+
# --------------------------------------------------------------------------
|
|
2447
|
+
# Fixes with one answer (fix)
|
|
2448
|
+
# --------------------------------------------------------------------------
|
|
2449
|
+
|
|
2450
|
+
|
|
2451
|
+
def manual_interiors(text: str) -> list[tuple[int, int]]:
|
|
2452
|
+
"""(start, end) offsets of the interior of every [MANUAL] block, paired
|
|
2453
|
+
as skf-hash-content.py pairs them: an opening marker with the earliest
|
|
2454
|
+
following close marker of the same name. An unclosed marker is
|
|
2455
|
+
skipped."""
|
|
2456
|
+
closes = [
|
|
2457
|
+
(m.start(), m.group(1).strip()) for m in _MANUAL_CLOSE_RE.finditer(text)
|
|
2458
|
+
]
|
|
2459
|
+
spans: list[tuple[int, int]] = []
|
|
2460
|
+
for om in _MANUAL_OPEN_RE.finditer(text):
|
|
2461
|
+
name = om.group(1).strip()
|
|
2462
|
+
close = min(
|
|
2463
|
+
(start for start, n in closes if start >= om.end() and n == name),
|
|
2464
|
+
default=None,
|
|
2465
|
+
)
|
|
2466
|
+
if close is not None:
|
|
2467
|
+
spans.append((om.end(), close))
|
|
2468
|
+
return spans
|
|
2469
|
+
|
|
2470
|
+
|
|
2471
|
+
class _SkillFile:
|
|
2472
|
+
"""One skill markdown file: its text, its [MANUAL] interiors and every
|
|
2473
|
+
citation in it, found line by line as `scan_skill_citations` finds them.
|
|
2474
|
+
Bytes that are not UTF-8 survive the round trip (surrogateescape)."""
|
|
2475
|
+
|
|
2476
|
+
def __init__(self, skill_dir: Path, rel: str) -> None:
|
|
2477
|
+
self.rel = rel
|
|
2478
|
+
self.path = skill_dir / rel
|
|
2479
|
+
self.text = self.path.read_bytes().decode("utf-8", errors="surrogateescape")
|
|
2480
|
+
self.manual = manual_interiors(self.text)
|
|
2481
|
+
self.citations: list[dict] = []
|
|
2482
|
+
offset = 0
|
|
2483
|
+
for number, chunk in enumerate(_LINE_WITH_END_RE.findall(self.text), 1):
|
|
2484
|
+
content = chunk.rstrip("\r\n")
|
|
2485
|
+
for m in _SKILL_CITATION_RE.finditer(content):
|
|
2486
|
+
self.citations.append(
|
|
2487
|
+
{
|
|
2488
|
+
"line": number,
|
|
2489
|
+
"start": offset + m.start(),
|
|
2490
|
+
"end": offset + m.end(),
|
|
2491
|
+
"match": m,
|
|
2492
|
+
"text": m.group(0),
|
|
2493
|
+
"prefix": m.group(1),
|
|
2494
|
+
"cited_file": m.group(2).strip(),
|
|
2495
|
+
"cited_line": int(m.group(3)),
|
|
2496
|
+
}
|
|
2497
|
+
)
|
|
2498
|
+
offset += len(chunk)
|
|
2499
|
+
|
|
2500
|
+
def in_manual(self, offset: int) -> bool:
|
|
2501
|
+
return any(start <= offset < end for start, end in self.manual)
|
|
2502
|
+
|
|
2503
|
+
def fixed_text(self) -> str | None:
|
|
2504
|
+
"""The text with every changed citation rewritten, or None when no
|
|
2505
|
+
citation changed."""
|
|
2506
|
+
edits = [
|
|
2507
|
+
(c["start"], c["end"], _rewritten_citation(c))
|
|
2508
|
+
for c in self.citations
|
|
2509
|
+
if c.get("new_prefix") or c.get("shift")
|
|
2510
|
+
]
|
|
2511
|
+
if not edits:
|
|
2512
|
+
return None
|
|
2513
|
+
text = self.text
|
|
2514
|
+
for start, end, new in sorted(edits, reverse=True):
|
|
2515
|
+
text = text[:start] + new + text[end:]
|
|
2516
|
+
return text
|
|
2517
|
+
|
|
2518
|
+
|
|
2519
|
+
def _rewritten_citation(cit: dict) -> str:
|
|
2520
|
+
"""A citation with its new prefix and its lines shifted: a range moves
|
|
2521
|
+
both ends by the same amount (`L10-12` to `L13-15`)."""
|
|
2522
|
+
m = cit["match"]
|
|
2523
|
+
base = m.start()
|
|
2524
|
+
subs: list[tuple[int, int, str]] = []
|
|
2525
|
+
if cit.get("new_prefix"):
|
|
2526
|
+
subs.append((m.start(1) - base, m.end(1) - base, cit["new_prefix"]))
|
|
2527
|
+
shift = cit.get("shift", 0)
|
|
2528
|
+
if shift:
|
|
2529
|
+
subs.append((m.start(3) - base, m.end(3) - base, str(int(m.group(3)) + shift)))
|
|
2530
|
+
if m.group(5) is not None:
|
|
2531
|
+
subs.append(
|
|
2532
|
+
(m.start(5) - base, m.end(5) - base, str(int(m.group(5)) + shift))
|
|
2533
|
+
)
|
|
2534
|
+
text = m.group(0)
|
|
2535
|
+
for start, end, new in sorted(subs, reverse=True):
|
|
2536
|
+
text = text[:start] + new + text[end:]
|
|
2537
|
+
return text
|
|
2538
|
+
|
|
2539
|
+
|
|
2540
|
+
def _dict_items(value: object) -> list[dict]:
|
|
2541
|
+
return [v for v in value if isinstance(v, dict)] if isinstance(value, list) else []
|
|
2542
|
+
|
|
2543
|
+
|
|
2544
|
+
def _stale_entry_indices(entries: list, item: dict) -> list[int]:
|
|
2545
|
+
"""The provenance entries a `stale[]` item names: the one at its
|
|
2546
|
+
`entry_index` when that entry still has the item's name, file and line,
|
|
2547
|
+
else every entry that has them."""
|
|
2548
|
+
name = item.get("export_name")
|
|
2549
|
+
source_file = item.get("source_file")
|
|
2550
|
+
line = _coerce_line(item.get("source_line"))
|
|
2551
|
+
|
|
2552
|
+
def same(entry: object) -> bool:
|
|
2553
|
+
return (
|
|
2554
|
+
isinstance(entry, dict)
|
|
2555
|
+
and entry.get("export_name") == name
|
|
2556
|
+
and isinstance(source_file, str)
|
|
2557
|
+
and bool(source_file.strip())
|
|
2558
|
+
and entry.get("source_file") == source_file
|
|
2559
|
+
and isinstance(line, int)
|
|
2560
|
+
and _coerce_line(entry.get("source_line")) == line
|
|
2561
|
+
)
|
|
2562
|
+
|
|
2563
|
+
index = item.get("entry_index")
|
|
2564
|
+
if (
|
|
2565
|
+
isinstance(index, int)
|
|
2566
|
+
and not isinstance(index, bool)
|
|
2567
|
+
and 0 <= index < len(entries)
|
|
2568
|
+
and same(entries[index])
|
|
2569
|
+
):
|
|
2570
|
+
return [index]
|
|
2571
|
+
return [i for i, entry in enumerate(entries) if same(entry)]
|
|
2572
|
+
|
|
2573
|
+
|
|
2574
|
+
def plan_fixes(
|
|
2575
|
+
verify_result: dict, prov: dict, skill_dir: Path, line_moves: bool = True
|
|
2576
|
+
) -> dict:
|
|
2577
|
+
"""Plan the fixes a `verify` result has one answer for.
|
|
2578
|
+
|
|
2579
|
+
1. Citation prefixes: each `citations[]` item's citation, found by its
|
|
2580
|
+
file, line and text, takes `expected_prefix`.
|
|
2581
|
+
2. Source lines: each `line-not-definition` item with exactly one
|
|
2582
|
+
`definition_lines` value moves its entry's `source_line` there, and
|
|
2583
|
+
the citations of either prefix whose normalized path and first line
|
|
2584
|
+
are the entry's (as they stand after step 1, before any move) move
|
|
2585
|
+
with it, all in one pass, so a citation moves at most once. A
|
|
2586
|
+
citation of a line another entry also records in that file stays.
|
|
2587
|
+
|
|
2588
|
+
A citation inside a [MANUAL] block never changes. Returns {applied,
|
|
2589
|
+
left_as_warn, markdown: {rel: new text}, provenance: the new map or
|
|
2590
|
+
None}; nothing is written.
|
|
2591
|
+
"""
|
|
2592
|
+
rels = [
|
|
2593
|
+
p.relative_to(skill_dir).as_posix() for p in skill_markdown_files(skill_dir)
|
|
2594
|
+
]
|
|
2595
|
+
files = {rel: _SkillFile(skill_dir, rel) for rel in rels}
|
|
2596
|
+
applied: list[dict] = []
|
|
2597
|
+
left: list[dict] = []
|
|
2598
|
+
prefix_fixes: list[tuple[dict, dict]] = []
|
|
2599
|
+
|
|
2600
|
+
# 1. Citation prefixes.
|
|
2601
|
+
groups: dict[tuple[str, int, str, str], list[dict]] = {}
|
|
2602
|
+
for item in _dict_items(verify_result.get("citations")):
|
|
2603
|
+
key = (item.get("file"), item.get("line"), item.get("citation"),
|
|
2604
|
+
item.get("expected_prefix"))
|
|
2605
|
+
if (
|
|
2606
|
+
not isinstance(key[0], str)
|
|
2607
|
+
or not isinstance(key[1], int)
|
|
2608
|
+
or not isinstance(key[2], str)
|
|
2609
|
+
or key[3] not in ("AST", "SRC")
|
|
2610
|
+
):
|
|
2611
|
+
left.append(dict(item, kind=FIX_CITATION_PREFIX, why="citation-not-found"))
|
|
2612
|
+
continue
|
|
2613
|
+
groups.setdefault(key, []).append(item)
|
|
2614
|
+
for (rel, line, text, expected), items in groups.items():
|
|
2615
|
+
doc = files.get(rel)
|
|
2616
|
+
found = [] if doc is None else [
|
|
2617
|
+
c for c in doc.citations if c["line"] == line and c["text"] == text
|
|
2618
|
+
]
|
|
2619
|
+
if not found:
|
|
2620
|
+
for item in items:
|
|
2621
|
+
left.append(
|
|
2622
|
+
dict(item, kind=FIX_CITATION_PREFIX, why="citation-not-found")
|
|
2623
|
+
)
|
|
2624
|
+
continue
|
|
2625
|
+
for cit in found:
|
|
2626
|
+
if doc.in_manual(cit["start"]):
|
|
2627
|
+
left.append(dict(items[0], kind=FIX_CITATION_PREFIX,
|
|
2628
|
+
why="inside-manual-block"))
|
|
2629
|
+
elif cit["prefix"] != expected:
|
|
2630
|
+
cit["new_prefix"] = expected
|
|
2631
|
+
prefix_fixes.append((doc, cit))
|
|
2632
|
+
|
|
2633
|
+
# 2. Source lines: plan every move before making any.
|
|
2634
|
+
entries = prov.get("entries") if isinstance(prov.get("entries"), list) else []
|
|
2635
|
+
moves: dict[int, int] = {}
|
|
2636
|
+
for item in _dict_items(verify_result.get("stale")):
|
|
2637
|
+
defs = item.get("definition_lines")
|
|
2638
|
+
new = defs[0] if isinstance(defs, list) and len(defs) == 1 else None
|
|
2639
|
+
if item.get("reason") != STALE_LINE_NOT_DEFINITION:
|
|
2640
|
+
why = "needs-decision"
|
|
2641
|
+
elif not isinstance(defs, list) or not defs:
|
|
2642
|
+
why = "no-definition-line"
|
|
2643
|
+
elif len(defs) > 1:
|
|
2644
|
+
why = "several-definition-lines"
|
|
2645
|
+
elif not isinstance(new, int) or isinstance(new, bool) or new < 1:
|
|
2646
|
+
why = "needs-decision"
|
|
2647
|
+
elif not line_moves:
|
|
2648
|
+
why = "line-moves-skipped"
|
|
2649
|
+
else:
|
|
2650
|
+
indices = _stale_entry_indices(entries, item)
|
|
2651
|
+
if indices:
|
|
2652
|
+
for index in indices:
|
|
2653
|
+
moves[index] = new
|
|
2654
|
+
continue
|
|
2655
|
+
why = "entry-not-found"
|
|
2656
|
+
left.append(dict(item, kind=FIX_SOURCE_LINE, why=why))
|
|
2657
|
+
|
|
2658
|
+
recorded: dict[tuple[str, int], set[int]] = {}
|
|
2659
|
+
for index, entry in enumerate(entries):
|
|
2660
|
+
if not isinstance(entry, dict):
|
|
2661
|
+
continue
|
|
2662
|
+
sf = entry.get("source_file")
|
|
2663
|
+
line = _coerce_line(entry.get("source_line"))
|
|
2664
|
+
if isinstance(sf, str) and sf.strip() and isinstance(line, int):
|
|
2665
|
+
recorded.setdefault((normalize_path(sf), line), set()).add(index)
|
|
2666
|
+
movers: dict[tuple[str, int], dict[int, int]] = {}
|
|
2667
|
+
for index, new in moves.items():
|
|
2668
|
+
entry = entries[index]
|
|
2669
|
+
key = (normalize_path(entry["source_file"]), _coerce_line(entry["source_line"]))
|
|
2670
|
+
movers.setdefault(key, {})[index] = new
|
|
2671
|
+
|
|
2672
|
+
line_fixes: list[tuple[dict, dict, list[str]]] = []
|
|
2673
|
+
for rel in rels:
|
|
2674
|
+
doc = files[rel]
|
|
2675
|
+
for cit in doc.citations:
|
|
2676
|
+
key = (normalize_path(cit["cited_file"]), cit["cited_line"])
|
|
2677
|
+
moving = movers.get(key)
|
|
2678
|
+
if not moving:
|
|
2679
|
+
continue
|
|
2680
|
+
here = recorded.get(key, set()) | set(moving)
|
|
2681
|
+
names = sorted(
|
|
2682
|
+
{n for n in (entries[i].get("export_name") for i in here)
|
|
2683
|
+
if isinstance(n, str) and n}
|
|
2684
|
+
)
|
|
2685
|
+
targets = set(moving.values())
|
|
2686
|
+
if here - set(moving) or len(targets) > 1:
|
|
2687
|
+
why = "shared-line" # it may cite the other entry's export
|
|
2688
|
+
elif doc.in_manual(cit["start"]):
|
|
2689
|
+
why = "inside-manual-block"
|
|
2690
|
+
else:
|
|
2691
|
+
cit["shift"] = targets.pop() - cit["cited_line"]
|
|
2692
|
+
line_fixes.append((doc, cit, names))
|
|
2693
|
+
continue
|
|
2694
|
+
left.append({
|
|
2695
|
+
"kind": FIX_CITATION_LINE,
|
|
2696
|
+
"file": rel,
|
|
2697
|
+
"line": cit["line"],
|
|
2698
|
+
"citation": cit["text"],
|
|
2699
|
+
"export_names": names,
|
|
2700
|
+
"why": why,
|
|
2701
|
+
})
|
|
2702
|
+
|
|
2703
|
+
for doc, cit in prefix_fixes:
|
|
2704
|
+
applied.append({
|
|
2705
|
+
"kind": FIX_CITATION_PREFIX,
|
|
2706
|
+
"file": doc.rel,
|
|
2707
|
+
"line": cit["line"],
|
|
2708
|
+
"citation": cit["text"],
|
|
2709
|
+
"fixed": _rewritten_citation(cit),
|
|
2710
|
+
})
|
|
2711
|
+
new_prov = None
|
|
2712
|
+
if moves:
|
|
2713
|
+
new_prov = copy.deepcopy(prov)
|
|
2714
|
+
for index in sorted(moves):
|
|
2715
|
+
entry = new_prov["entries"][index]
|
|
2716
|
+
applied.append({
|
|
2717
|
+
"kind": FIX_SOURCE_LINE,
|
|
2718
|
+
"entry_index": index,
|
|
2719
|
+
"export_name": entry.get("export_name"),
|
|
2720
|
+
"source_file": entry.get("source_file"),
|
|
2721
|
+
"from": _coerce_line(entry.get("source_line")),
|
|
2722
|
+
"to": moves[index],
|
|
2723
|
+
})
|
|
2724
|
+
entry["source_line"] = moves[index]
|
|
2725
|
+
for doc, cit, names in line_fixes:
|
|
2726
|
+
applied.append({
|
|
2727
|
+
"kind": FIX_CITATION_LINE,
|
|
2728
|
+
"file": doc.rel,
|
|
2729
|
+
"line": cit["line"],
|
|
2730
|
+
"citation": cit["text"],
|
|
2731
|
+
"fixed": _rewritten_citation(cit),
|
|
2732
|
+
"export_names": names,
|
|
2733
|
+
})
|
|
2734
|
+
|
|
2735
|
+
markdown = {}
|
|
2736
|
+
for rel in rels:
|
|
2737
|
+
text = files[rel].fixed_text()
|
|
2738
|
+
if text is not None:
|
|
2739
|
+
markdown[rel] = text
|
|
2740
|
+
return {
|
|
2741
|
+
"applied": applied,
|
|
2742
|
+
"left_as_warn": left,
|
|
2743
|
+
"markdown": markdown,
|
|
2744
|
+
"provenance": new_prov,
|
|
2745
|
+
}
|
|
2746
|
+
|
|
2747
|
+
|
|
2748
|
+
def _sibling_helper(filename: str) -> Path | None:
|
|
2749
|
+
"""A helper installed beside this script (dev `src/shared/scripts/`
|
|
2750
|
+
and installed `_bmad/skf/shared/scripts/` alike), or None."""
|
|
2751
|
+
path = Path(__file__).resolve().parent / filename
|
|
2752
|
+
return path if path.is_file() else None
|
|
2753
|
+
|
|
2754
|
+
|
|
2755
|
+
def _load_sibling_module(filename: str, module_name: str):
|
|
2756
|
+
"""Import a sibling helper as a module. Raises ImportError."""
|
|
2757
|
+
path = _sibling_helper(filename)
|
|
2758
|
+
spec = (
|
|
2759
|
+
importlib.util.spec_from_file_location(module_name, path)
|
|
2760
|
+
if path is not None
|
|
2761
|
+
else None
|
|
2762
|
+
)
|
|
2763
|
+
if spec is None or spec.loader is None:
|
|
2764
|
+
raise ImportError(f"{filename} not found beside {Path(__file__).name}")
|
|
2765
|
+
module = importlib.util.module_from_spec(spec)
|
|
2766
|
+
spec.loader.exec_module(module)
|
|
2767
|
+
return module
|
|
2768
|
+
|
|
2769
|
+
|
|
2770
|
+
def atomic_write(helper: Path, target: Path, data: bytes) -> None:
|
|
2771
|
+
"""Write `data` to `target` through `skf-atomic-write.py write`.
|
|
2772
|
+
Raises OSError when the helper fails."""
|
|
2773
|
+
try:
|
|
2774
|
+
result = subprocess.run(
|
|
2775
|
+
[sys.executable, str(helper), "write", "--target", str(target)],
|
|
2776
|
+
input=data,
|
|
2777
|
+
capture_output=True,
|
|
2778
|
+
timeout=ATOMIC_WRITE_TIMEOUT_SEC,
|
|
2779
|
+
check=False,
|
|
2780
|
+
)
|
|
2781
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
2782
|
+
raise OSError(f"atomic write of {target} failed: {exc}") from exc
|
|
2783
|
+
if result.returncode != 0:
|
|
2784
|
+
detail = (result.stderr or b"").decode("utf-8", errors="replace").strip()
|
|
2785
|
+
raise OSError(f"atomic write of {target} failed: {detail}")
|
|
347
2786
|
|
|
348
2787
|
|
|
349
2788
|
# --------------------------------------------------------------------------
|
|
@@ -351,6 +2790,48 @@ def verify(metadata: dict, prov: dict, source_root: Path | None) -> dict:
|
|
|
351
2790
|
# --------------------------------------------------------------------------
|
|
352
2791
|
|
|
353
2792
|
|
|
2793
|
+
def _skill_dir_error(skill_dir: Path) -> str | None:
|
|
2794
|
+
"""Why `skill_dir` is not a skill package folder, or None."""
|
|
2795
|
+
if not skill_dir.exists():
|
|
2796
|
+
return f"skill dir not found: {skill_dir}"
|
|
2797
|
+
if not skill_dir.is_dir():
|
|
2798
|
+
return f"skill dir is not a directory: {skill_dir}"
|
|
2799
|
+
if not (skill_dir / "SKILL.md").is_file():
|
|
2800
|
+
return f"skill dir has no SKILL.md: {skill_dir}"
|
|
2801
|
+
return None
|
|
2802
|
+
|
|
2803
|
+
|
|
2804
|
+
def _unexpected(what: str, exc: Exception, written: list[str] | None = None) -> int:
|
|
2805
|
+
"""Report an unexpected failure on one stderr line; exit code 2. With
|
|
2806
|
+
`written` (fix), the line also names the files already written."""
|
|
2807
|
+
detail = " ".join(str(exc).split())
|
|
2808
|
+
note = ""
|
|
2809
|
+
if written is not None:
|
|
2810
|
+
note = f"; already written: {', '.join(written) if written else 'nothing'}"
|
|
2811
|
+
print(
|
|
2812
|
+
f"error: {what} failed: {type(exc).__name__}: {detail}{note}",
|
|
2813
|
+
file=sys.stderr,
|
|
2814
|
+
)
|
|
2815
|
+
return 2
|
|
2816
|
+
|
|
2817
|
+
|
|
2818
|
+
def _emit(result: dict, output: str | None, written: list[str] | None = None) -> bool:
|
|
2819
|
+
"""Write the JSON result to `output`, or to stdout when it is absent or
|
|
2820
|
+
`-`. False, after one stderr line (naming `written`, as `_unexpected`
|
|
2821
|
+
does), when it cannot be written."""
|
|
2822
|
+
out_text = json.dumps(result, indent=2) + "\n"
|
|
2823
|
+
to_file = bool(output) and output != "-"
|
|
2824
|
+
try:
|
|
2825
|
+
if to_file:
|
|
2826
|
+
Path(output).write_text(out_text, encoding="utf-8")
|
|
2827
|
+
else:
|
|
2828
|
+
sys.stdout.write(out_text)
|
|
2829
|
+
except OSError as exc:
|
|
2830
|
+
_unexpected(f"writing {output if to_file else 'stdout'}", exc, written)
|
|
2831
|
+
return False
|
|
2832
|
+
return True
|
|
2833
|
+
|
|
2834
|
+
|
|
354
2835
|
def _cmd_verify(args: argparse.Namespace) -> int:
|
|
355
2836
|
meta_path = Path(args.metadata)
|
|
356
2837
|
prov_path = Path(args.provenance)
|
|
@@ -367,6 +2848,14 @@ def _cmd_verify(args: argparse.Namespace) -> int:
|
|
|
367
2848
|
print(f"error: {exc}", file=sys.stderr)
|
|
368
2849
|
return 2
|
|
369
2850
|
|
|
2851
|
+
skill_dir: Path | None = None
|
|
2852
|
+
if args.skill_dir is not None:
|
|
2853
|
+
skill_dir = Path(args.skill_dir)
|
|
2854
|
+
problem = _skill_dir_error(skill_dir)
|
|
2855
|
+
if problem is not None:
|
|
2856
|
+
print(f"error: {problem}", file=sys.stderr)
|
|
2857
|
+
return 2
|
|
2858
|
+
|
|
370
2859
|
source_root = resolve_source_root(prov, args.source_root)
|
|
371
2860
|
if args.verbose:
|
|
372
2861
|
if source_root is None:
|
|
@@ -378,30 +2867,241 @@ def _cmd_verify(args: argparse.Namespace) -> int:
|
|
|
378
2867
|
else:
|
|
379
2868
|
print(f"verbose: resolving citations under {source_root}", file=sys.stderr)
|
|
380
2869
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
sys.stdout.write(out_text)
|
|
2870
|
+
try:
|
|
2871
|
+
result = verify(
|
|
2872
|
+
metadata, prov, source_root, skill_dir, args.check_node_kinds
|
|
2873
|
+
)
|
|
2874
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as findings
|
|
2875
|
+
return _unexpected("verification", exc)
|
|
388
2876
|
|
|
2877
|
+
if not _emit(result, args.output):
|
|
2878
|
+
return 2
|
|
389
2879
|
return 0 if result["status"] == "pass" else 1
|
|
390
2880
|
|
|
391
2881
|
|
|
2882
|
+
def _cmd_definition_lines(args: argparse.Namespace) -> int:
|
|
2883
|
+
root = Path(args.source_root) if args.source_root else Path.cwd()
|
|
2884
|
+
if not root.is_dir():
|
|
2885
|
+
print(f"error: source root not found: {root}", file=sys.stderr)
|
|
2886
|
+
return 2
|
|
2887
|
+
try:
|
|
2888
|
+
report = definition_lines_report(
|
|
2889
|
+
args.file, args.name, root, args.line, args.export_type
|
|
2890
|
+
)
|
|
2891
|
+
except ValueError as exc:
|
|
2892
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2893
|
+
return 2
|
|
2894
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
2895
|
+
return _unexpected("definition lookup", exc)
|
|
2896
|
+
return 0 if _emit(report, args.output) else 2
|
|
2897
|
+
|
|
2898
|
+
|
|
2899
|
+
def _cmd_classify_stale(args: argparse.Namespace) -> int:
|
|
2900
|
+
names_path, prov_path = Path(args.names), Path(args.provenance)
|
|
2901
|
+
root = Path(args.source_root)
|
|
2902
|
+
for label, path in (("names", names_path), ("provenance map", prov_path)):
|
|
2903
|
+
if not path.is_file():
|
|
2904
|
+
print(f"error: {label} not found: {path}", file=sys.stderr)
|
|
2905
|
+
return 2
|
|
2906
|
+
if not root.is_dir():
|
|
2907
|
+
print(f"error: source root not found: {root}", file=sys.stderr)
|
|
2908
|
+
return 2
|
|
2909
|
+
try:
|
|
2910
|
+
text = names_path.read_text(encoding="utf-8-sig")
|
|
2911
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
2912
|
+
print(f"error: failed to read names {names_path}: {exc}", file=sys.stderr)
|
|
2913
|
+
return 2
|
|
2914
|
+
try:
|
|
2915
|
+
data = json.loads(text)
|
|
2916
|
+
except json.JSONDecodeError as exc:
|
|
2917
|
+
print(f"error: malformed JSON in names {names_path}: {exc}", file=sys.stderr)
|
|
2918
|
+
return 2
|
|
2919
|
+
try:
|
|
2920
|
+
names = stale_names(data)
|
|
2921
|
+
prov = load_json_object(prov_path, "provenance map")
|
|
2922
|
+
except ValueError as exc:
|
|
2923
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2924
|
+
return 2
|
|
2925
|
+
try:
|
|
2926
|
+
result = classify_stale(names, prov, root)
|
|
2927
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
2928
|
+
return _unexpected("stale-name classification", exc)
|
|
2929
|
+
return 0 if _emit(result, args.output) else 2
|
|
2930
|
+
|
|
2931
|
+
|
|
2932
|
+
def _cmd_kind_at(args: argparse.Namespace) -> int:
|
|
2933
|
+
# a `source_file` as a provenance entry records it, as `verify` reads it
|
|
2934
|
+
path = Path(args.file.strip().replace("\\", "/"))
|
|
2935
|
+
if args.source_root and not path.is_absolute():
|
|
2936
|
+
path = Path(args.source_root) / path
|
|
2937
|
+
if not path.is_file():
|
|
2938
|
+
print(f"error: source file not found: {path}", file=sys.stderr)
|
|
2939
|
+
return 2
|
|
2940
|
+
recipes_path = Path(args.recipes)
|
|
2941
|
+
if not recipes_path.is_file():
|
|
2942
|
+
print(f"error: recipes not found: {recipes_path}", file=sys.stderr)
|
|
2943
|
+
return 2
|
|
2944
|
+
try:
|
|
2945
|
+
recipes = load_recipes(recipes_path)
|
|
2946
|
+
source = path.read_bytes()
|
|
2947
|
+
except ImportError:
|
|
2948
|
+
print(
|
|
2949
|
+
"error: kind-at reads recipes with PyYAML: run the helper with "
|
|
2950
|
+
"`uv run`, which installs it",
|
|
2951
|
+
file=sys.stderr,
|
|
2952
|
+
)
|
|
2953
|
+
return 2
|
|
2954
|
+
except (ValueError, OSError) as exc:
|
|
2955
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2956
|
+
return 2
|
|
2957
|
+
if not recipes:
|
|
2958
|
+
print(f"error: no ast-grep recipe in {recipes_path}", file=sys.stderr)
|
|
2959
|
+
return 2
|
|
2960
|
+
language = (
|
|
2961
|
+
args.language.strip().lower() if args.language
|
|
2962
|
+
else ast_grep_language(args.file)
|
|
2963
|
+
)
|
|
2964
|
+
try:
|
|
2965
|
+
(result,) = kinds_at(source, language, [(args.line, args.name)], recipes)
|
|
2966
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
2967
|
+
return _unexpected("kind lookup", exc)
|
|
2968
|
+
report = {"file": args.file, "language": language, **result}
|
|
2969
|
+
if not _emit(report, args.output):
|
|
2970
|
+
return 2
|
|
2971
|
+
return 0 if report["status"] == "found" else 1
|
|
2972
|
+
|
|
2973
|
+
|
|
2974
|
+
def _cmd_fix(args: argparse.Namespace) -> int:
|
|
2975
|
+
verify_path = Path(args.verify)
|
|
2976
|
+
prov_path = Path(args.provenance)
|
|
2977
|
+
skill_dir = Path(args.skill_dir)
|
|
2978
|
+
for path, label in ((verify_path, "verify result"), (prov_path, "provenance map")):
|
|
2979
|
+
if not path.is_file():
|
|
2980
|
+
print(f"error: {label} not found: {path}", file=sys.stderr)
|
|
2981
|
+
return 2
|
|
2982
|
+
problem = _skill_dir_error(skill_dir)
|
|
2983
|
+
if problem is not None:
|
|
2984
|
+
print(f"error: {problem}", file=sys.stderr)
|
|
2985
|
+
return 2
|
|
2986
|
+
try:
|
|
2987
|
+
verify_result = load_json_object(verify_path, "verify result")
|
|
2988
|
+
prov = load_json_object(prov_path, "provenance map")
|
|
2989
|
+
except ValueError as exc:
|
|
2990
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
2991
|
+
return 2
|
|
2992
|
+
if not all(
|
|
2993
|
+
isinstance(verify_result.get(key), list) for key in ("stale", "citations")
|
|
2994
|
+
):
|
|
2995
|
+
print(
|
|
2996
|
+
f"error: {verify_path} is not a verify result (no stale[] and "
|
|
2997
|
+
f"citations[])",
|
|
2998
|
+
file=sys.stderr,
|
|
2999
|
+
)
|
|
3000
|
+
return 2
|
|
3001
|
+
|
|
3002
|
+
inventory: list[dict] | None = None
|
|
3003
|
+
hash_content = None
|
|
3004
|
+
if args.manual_inventory is not None:
|
|
3005
|
+
try:
|
|
3006
|
+
hash_content = _load_sibling_module(
|
|
3007
|
+
HASH_CONTENT_HELPER, "skf_hash_content"
|
|
3008
|
+
)
|
|
3009
|
+
inventory = hash_content.load_inventory_blocks(Path(args.manual_inventory))
|
|
3010
|
+
except (ImportError, ValueError) as exc:
|
|
3011
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3012
|
+
return 2
|
|
3013
|
+
|
|
3014
|
+
try:
|
|
3015
|
+
plan = plan_fixes(verify_result, prov, skill_dir, not args.no_line_moves)
|
|
3016
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as a fix
|
|
3017
|
+
return _unexpected("fix", exc)
|
|
3018
|
+
|
|
3019
|
+
changes: list[tuple[Path, bytes]] = [
|
|
3020
|
+
(skill_dir / rel, text.encode("utf-8", errors="surrogateescape"))
|
|
3021
|
+
for rel, text in sorted(plan["markdown"].items())
|
|
3022
|
+
]
|
|
3023
|
+
if plan["provenance"] is not None:
|
|
3024
|
+
text = json.dumps(plan["provenance"], indent=2, ensure_ascii=False) + "\n"
|
|
3025
|
+
changes.append((prov_path, text.encode("utf-8")))
|
|
3026
|
+
written: list[str] = []
|
|
3027
|
+
if changes and not args.dry_run:
|
|
3028
|
+
helper = _sibling_helper(ATOMIC_WRITE_HELPER)
|
|
3029
|
+
if helper is None:
|
|
3030
|
+
print(
|
|
3031
|
+
f"error: {ATOMIC_WRITE_HELPER} not found beside "
|
|
3032
|
+
f"{Path(__file__).name}; nothing written",
|
|
3033
|
+
file=sys.stderr,
|
|
3034
|
+
)
|
|
3035
|
+
return 2
|
|
3036
|
+
for target, data in changes:
|
|
3037
|
+
try:
|
|
3038
|
+
atomic_write(helper, target, data)
|
|
3039
|
+
except OSError as exc:
|
|
3040
|
+
print(
|
|
3041
|
+
f"error: {exc}; already written: "
|
|
3042
|
+
f"{', '.join(written) if written else 'nothing'}",
|
|
3043
|
+
file=sys.stderr,
|
|
3044
|
+
)
|
|
3045
|
+
return 2
|
|
3046
|
+
written.append(str(target))
|
|
3047
|
+
|
|
3048
|
+
manual = None
|
|
3049
|
+
if hash_content is not None and inventory is not None and not args.dry_run:
|
|
3050
|
+
try:
|
|
3051
|
+
current = hash_content.find_manual_blocks(
|
|
3052
|
+
(skill_dir / "SKILL.md").read_bytes()
|
|
3053
|
+
)
|
|
3054
|
+
manual = hash_content.classify_manual_blocks(inventory, current)
|
|
3055
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as a WARN
|
|
3056
|
+
return _unexpected("the [MANUAL] re-check", exc, written)
|
|
3057
|
+
|
|
3058
|
+
applied = plan["applied"]
|
|
3059
|
+
left = plan["left_as_warn"]
|
|
3060
|
+
result = {
|
|
3061
|
+
"applied": applied,
|
|
3062
|
+
"left_as_warn": left,
|
|
3063
|
+
"files_changed": [str(target) for target, _ in changes],
|
|
3064
|
+
"files_written": written,
|
|
3065
|
+
"dry_run": bool(args.dry_run),
|
|
3066
|
+
"manual_verify": manual,
|
|
3067
|
+
"summary": {
|
|
3068
|
+
"citation_prefixes_fixed": sum(
|
|
3069
|
+
1 for a in applied if a["kind"] == FIX_CITATION_PREFIX
|
|
3070
|
+
),
|
|
3071
|
+
"source_lines_moved": sum(
|
|
3072
|
+
1 for a in applied if a["kind"] == FIX_SOURCE_LINE
|
|
3073
|
+
),
|
|
3074
|
+
"citations_moved": sum(
|
|
3075
|
+
1 for a in applied if a["kind"] == FIX_CITATION_LINE
|
|
3076
|
+
),
|
|
3077
|
+
"left_as_warn_count": len(left),
|
|
3078
|
+
},
|
|
3079
|
+
}
|
|
3080
|
+
if not _emit(result, args.output, written):
|
|
3081
|
+
return 2
|
|
3082
|
+
return 0 if not left and (manual is None or manual.get("ok")) else 1
|
|
3083
|
+
|
|
3084
|
+
|
|
392
3085
|
def _build_parser() -> argparse.ArgumentParser:
|
|
393
3086
|
parser = argparse.ArgumentParser(
|
|
394
3087
|
prog="skf-verify-provenance-completeness",
|
|
395
3088
|
description=(
|
|
396
3089
|
"Cross-reference documented exports (metadata.json) against "
|
|
397
|
-
"provenance-map entries
|
|
398
|
-
"file:line citations
|
|
3090
|
+
"provenance-map entries: completeness, orphans, stale or "
|
|
3091
|
+
"non-definition file:line citations, (with --skill-dir) "
|
|
3092
|
+
"citation prefixes and (with --check-node-kinds) ast-grep node "
|
|
3093
|
+
"kinds, emitting findings as JSON; fix the findings with one "
|
|
3094
|
+
"answer; look up an export's definition lines, the stale names "
|
|
3095
|
+
"no cited file defines, or the recipe kind at a line."
|
|
399
3096
|
),
|
|
400
3097
|
)
|
|
401
3098
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
402
3099
|
p = sub.add_parser(
|
|
403
3100
|
"verify",
|
|
404
|
-
help=
|
|
3101
|
+
help=(
|
|
3102
|
+
"emit provenance completeness / orphan / stale-citation / "
|
|
3103
|
+
"citation-prefix / node-kind findings"
|
|
3104
|
+
),
|
|
405
3105
|
)
|
|
406
3106
|
p.add_argument("--metadata", required=True, help="path to metadata.json")
|
|
407
3107
|
p.add_argument(
|
|
@@ -416,6 +3116,25 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
416
3116
|
"root resolves on disk."
|
|
417
3117
|
),
|
|
418
3118
|
)
|
|
3119
|
+
p.add_argument(
|
|
3120
|
+
"--skill-dir",
|
|
3121
|
+
default=None,
|
|
3122
|
+
help=(
|
|
3123
|
+
"skill package folder holding a SKILL.md; scans it and "
|
|
3124
|
+
"references/**/*.md for [AST:]/[SRC:] citations and checks each "
|
|
3125
|
+
"prefix against the provenance entry it cites. Citation check is "
|
|
3126
|
+
"skipped when absent."
|
|
3127
|
+
),
|
|
3128
|
+
)
|
|
3129
|
+
p.add_argument(
|
|
3130
|
+
"--check-node-kinds",
|
|
3131
|
+
action="store_true",
|
|
3132
|
+
help=(
|
|
3133
|
+
"ask the ast-grep CLI whether each ast-grep entry's ast_node_type "
|
|
3134
|
+
"is a node kind of its file's language; skipped when ast-grep is "
|
|
3135
|
+
"not found."
|
|
3136
|
+
),
|
|
3137
|
+
)
|
|
419
3138
|
p.add_argument(
|
|
420
3139
|
"-o",
|
|
421
3140
|
"--output",
|
|
@@ -426,6 +3145,152 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
426
3145
|
"--verbose", action="store_true", help="diagnostics to stderr"
|
|
427
3146
|
)
|
|
428
3147
|
p.set_defaults(func=_cmd_verify)
|
|
3148
|
+
|
|
3149
|
+
p = sub.add_parser(
|
|
3150
|
+
"fix",
|
|
3151
|
+
help=(
|
|
3152
|
+
"apply the citation-prefix and source-line fixes a verify result "
|
|
3153
|
+
"has one answer for"
|
|
3154
|
+
),
|
|
3155
|
+
)
|
|
3156
|
+
p.add_argument(
|
|
3157
|
+
"--verify", required=True, help="path to the JSON a verify run wrote"
|
|
3158
|
+
)
|
|
3159
|
+
p.add_argument(
|
|
3160
|
+
"--provenance",
|
|
3161
|
+
required=True,
|
|
3162
|
+
help="path to the provenance-map.json that verify run read",
|
|
3163
|
+
)
|
|
3164
|
+
p.add_argument(
|
|
3165
|
+
"--skill-dir",
|
|
3166
|
+
required=True,
|
|
3167
|
+
help="skill package folder holding a SKILL.md and references/",
|
|
3168
|
+
)
|
|
3169
|
+
p.add_argument(
|
|
3170
|
+
"--manual-inventory",
|
|
3171
|
+
default=None,
|
|
3172
|
+
help=(
|
|
3173
|
+
"a [MANUAL] inventory of SKILL.md (skf-hash-content.py "
|
|
3174
|
+
"manual-inventory); checked again after the writes"
|
|
3175
|
+
),
|
|
3176
|
+
)
|
|
3177
|
+
p.add_argument(
|
|
3178
|
+
"--no-line-moves",
|
|
3179
|
+
action="store_true",
|
|
3180
|
+
help="fix citation prefixes only; list every source-line finding",
|
|
3181
|
+
)
|
|
3182
|
+
p.add_argument(
|
|
3183
|
+
"--dry-run", action="store_true", help="plan the fixes, write nothing"
|
|
3184
|
+
)
|
|
3185
|
+
p.add_argument(
|
|
3186
|
+
"-o",
|
|
3187
|
+
"--output",
|
|
3188
|
+
default=None,
|
|
3189
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
3190
|
+
)
|
|
3191
|
+
p.set_defaults(func=_cmd_fix)
|
|
3192
|
+
|
|
3193
|
+
p = sub.add_parser(
|
|
3194
|
+
"definition-lines",
|
|
3195
|
+
help="emit the lines of a source file that define an export",
|
|
3196
|
+
)
|
|
3197
|
+
p.add_argument(
|
|
3198
|
+
"--file", required=True, help="source file, relative to --source-root"
|
|
3199
|
+
)
|
|
3200
|
+
p.add_argument("--name", required=True, help="the entry's export_name")
|
|
3201
|
+
p.add_argument(
|
|
3202
|
+
"--source-root",
|
|
3203
|
+
default=None,
|
|
3204
|
+
help="source tree root (default: the current folder)",
|
|
3205
|
+
)
|
|
3206
|
+
p.add_argument(
|
|
3207
|
+
"--line",
|
|
3208
|
+
type=int,
|
|
3209
|
+
default=None,
|
|
3210
|
+
help="the recorded source_line: report whether it defines the export",
|
|
3211
|
+
)
|
|
3212
|
+
p.add_argument(
|
|
3213
|
+
"--export-type",
|
|
3214
|
+
default=None,
|
|
3215
|
+
help="the entry's export_type (module and package have no line)",
|
|
3216
|
+
)
|
|
3217
|
+
p.add_argument(
|
|
3218
|
+
"-o",
|
|
3219
|
+
"--output",
|
|
3220
|
+
default=None,
|
|
3221
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
3222
|
+
)
|
|
3223
|
+
p.set_defaults(func=_cmd_definition_lines)
|
|
3224
|
+
|
|
3225
|
+
p = sub.add_parser(
|
|
3226
|
+
"classify-stale",
|
|
3227
|
+
help=(
|
|
3228
|
+
"decide which stale names are fabricated signatures: no cited "
|
|
3229
|
+
"file defines them"
|
|
3230
|
+
),
|
|
3231
|
+
)
|
|
3232
|
+
p.add_argument(
|
|
3233
|
+
"--names",
|
|
3234
|
+
required=True,
|
|
3235
|
+
help=(
|
|
3236
|
+
"reconcile-coverage.py's result (its `stale` names) or a JSON "
|
|
3237
|
+
"array of names"
|
|
3238
|
+
),
|
|
3239
|
+
)
|
|
3240
|
+
p.add_argument(
|
|
3241
|
+
"--provenance", required=True, help="path to provenance-map.json"
|
|
3242
|
+
)
|
|
3243
|
+
p.add_argument(
|
|
3244
|
+
"--source-root",
|
|
3245
|
+
required=True,
|
|
3246
|
+
help="source tree root the entries' source_file paths are under",
|
|
3247
|
+
)
|
|
3248
|
+
p.add_argument(
|
|
3249
|
+
"-o",
|
|
3250
|
+
"--output",
|
|
3251
|
+
default=None,
|
|
3252
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
3253
|
+
)
|
|
3254
|
+
p.set_defaults(func=_cmd_classify_stale)
|
|
3255
|
+
|
|
3256
|
+
p = sub.add_parser(
|
|
3257
|
+
"kind-at",
|
|
3258
|
+
help="emit the node kind of the recipe that matches a source line",
|
|
3259
|
+
)
|
|
3260
|
+
p.add_argument(
|
|
3261
|
+
"--file",
|
|
3262
|
+
required=True,
|
|
3263
|
+
help="source file, relative to --source-root when that is given",
|
|
3264
|
+
)
|
|
3265
|
+
p.add_argument(
|
|
3266
|
+
"--line", type=int, required=True, help="the line $NAME must be on"
|
|
3267
|
+
)
|
|
3268
|
+
p.add_argument(
|
|
3269
|
+
"--recipes",
|
|
3270
|
+
required=True,
|
|
3271
|
+
help=(
|
|
3272
|
+
"the recipes: a markdown file (its ```yaml fences, such as "
|
|
3273
|
+
"extraction-patterns.md) or a YAML file"
|
|
3274
|
+
),
|
|
3275
|
+
)
|
|
3276
|
+
p.add_argument(
|
|
3277
|
+
"--language",
|
|
3278
|
+
default=None,
|
|
3279
|
+
help="ast-grep language (default: from the file's extension)",
|
|
3280
|
+
)
|
|
3281
|
+
p.add_argument(
|
|
3282
|
+
"--name",
|
|
3283
|
+
default=None,
|
|
3284
|
+
help="export name: count only matches whose $NAME is its last segment",
|
|
3285
|
+
)
|
|
3286
|
+
p.add_argument("--source-root", default=None, help="source tree root")
|
|
3287
|
+
p.add_argument(
|
|
3288
|
+
"-o",
|
|
3289
|
+
"--output",
|
|
3290
|
+
default=None,
|
|
3291
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
3292
|
+
)
|
|
3293
|
+
p.set_defaults(func=_cmd_kind_at)
|
|
429
3294
|
return parser
|
|
430
3295
|
|
|
431
3296
|
|