bmad-module-skill-forge 2.2.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +49 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +143 -18
- package/docs/verifying-a-skill.md +87 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +3 -3
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +31 -59
- package/src/shared/references/description-guard-protocol.md +12 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +233 -64
- package/src/shared/scripts/skf-build-change-manifest.py +2753 -28
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +772 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +267 -105
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +21 -3
- package/src/shared/scripts/skf-description-guard.py +99 -11
- package/src/shared/scripts/skf-detect-docs.py +694 -2
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +197 -55
- package/src/shared/scripts/skf-detect-tools.py +209 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +5247 -96
- package/src/shared/scripts/skf-extraction-inventory.py +1280 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +664 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +469 -205
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +700 -0
- package/src/shared/scripts/skf-hash-content.py +180 -20
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +161 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +143 -20
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +176 -28
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +64 -8
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +155 -14
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +787 -121
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +120 -67
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +23 -2
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +3945 -152
- package/src/shared/scripts/skf-write-skill-brief.py +406 -25
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +28 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +149 -210
- package/src/skf-audit-skill/references/re-index.md +103 -110
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +34 -25
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +80 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +712 -0
- package/src/skf-brief-skill/SKILL.md +37 -26
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +18 -25
- package/src/skf-brief-skill/assets/skill-brief-schema.md +30 -23
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +144 -127
- package/src/skf-brief-skill/references/confirm-brief.md +11 -9
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +26 -100
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +7 -10
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +11 -8
- package/src/skf-create-skill/assets/skill-sections.md +21 -11
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +36 -58
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +119 -70
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +37 -0
- package/src/skf-create-skill/references/extract.md +204 -162
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +11 -11
- package/src/skf-create-skill/references/extraction-patterns.md +588 -201
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +37 -57
- package/src/skf-create-skill/references/source-resolution-protocols.md +58 -169
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +28 -19
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +83 -146
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +122 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -90
- package/src/skf-setup/references/detect-and-tier.md +58 -67
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +209 -264
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +164 -91
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +164 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +42 -155
- package/src/skf-test-skill/references/source-access-protocol.md +18 -94
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +380 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1666 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +1198 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +547 -79
- package/src/skf-test-skill/scripts/score-signatures.py +479 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +133 -56
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +253 -223
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +254 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +182 -49
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +102 -35
- package/src/skf-update-skill/references/write.md +357 -155
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +183 -46
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -679
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# /// script
|
|
3
|
-
# requires-python = ">=3.
|
|
4
|
-
# dependencies = []
|
|
3
|
+
# requires-python = ">=3.11"
|
|
4
|
+
# dependencies = ["pyyaml"]
|
|
5
5
|
# ///
|
|
6
6
|
"""SKF Verify Provenance Completeness — deterministic Check D for update-skill.
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
update-skill's Check D (Provenance Completeness, now in write.md §6)
|
|
9
9
|
previously asked the model to perform three deterministic set/citation
|
|
10
10
|
operations by eye, against in-context data:
|
|
11
11
|
|
|
@@ -14,7 +14,28 @@ operations by eye, against in-context data:
|
|
|
14
14
|
2. **Orphans** — flag provenance entries whose export was removed
|
|
15
15
|
(provenance `entries[].export_name` \\ metadata `exports[]`).
|
|
16
16
|
3. **Stale citations** — flag entries whose `source_file:source_line`
|
|
17
|
-
no longer resolves (file gone, or line past end of file)
|
|
17
|
+
no longer resolves (file gone, or line past end of file), or whose
|
|
18
|
+
line is not a line that defines the export (`line-not-definition`).
|
|
19
|
+
4. **Citation prefixes** (only with `--skill-dir`): flag each `[AST:]` /
|
|
20
|
+
`[SRC:]` citation in the skill's markdown whose prefix disagrees with
|
|
21
|
+
the provenance entry it cites, or an `[AST:]` citation in a skill whose
|
|
22
|
+
map holds no ast-grep entry at all.
|
|
23
|
+
5. **Node kinds** (only with `--check-node-kinds`): flag each ast-grep
|
|
24
|
+
entry whose `ast_node_type` is not a node kind ast-grep knows for the
|
|
25
|
+
entry's language.
|
|
26
|
+
|
|
27
|
+
Reference apps: when `--metadata` has `scope_type: reference-app`, 1 and 2
|
|
28
|
+
do not run. A reference app's `exports[]` is empty by design while its
|
|
29
|
+
entries are per-citation, so the two sets measure different surfaces:
|
|
30
|
+
`missing` and `orphaned` stay empty and `summary.set_diff` is
|
|
31
|
+
`not-applicable` (`checked` otherwise).
|
|
32
|
+
|
|
33
|
+
Four more subcommands act on the same rules: `fix` applies the citation
|
|
34
|
+
prefix and source-line fixes a `verify` result has one answer for,
|
|
35
|
+
`definition-lines` gives the lines that define one export,
|
|
36
|
+
`classify-stale` decides which stale names are fabricated signatures and
|
|
37
|
+
where the source declares the others, and `kind-at` gives the node kind
|
|
38
|
+
of the recipe that matches a source line.
|
|
18
39
|
|
|
19
40
|
An LLM set-diff can silently pass a dropped or orphaned entry, and eyeballing
|
|
20
41
|
whether a cited line still exists is not something the model can do reliably.
|
|
@@ -25,9 +46,12 @@ export-name strings) and `provenance-map.json` (`entries[]` with
|
|
|
25
46
|
`write.md` §2 and §3. This script diffs the two sets and re-checks the
|
|
26
47
|
citations exactly, so Check D consumes JSON rather than computing it.
|
|
27
48
|
|
|
28
|
-
Timing:
|
|
29
|
-
(§2) and `provenance-map.json` (§3) are on disk.
|
|
30
|
-
|
|
49
|
+
Timing: update-skill runs it **post-write**, in write.md §6, after
|
|
50
|
+
`metadata.json` (§2) and `provenance-map.json` (§3) are on disk.
|
|
51
|
+
create-skill runs it too, at validate §7a against the staged package (with
|
|
52
|
+
`--skill-dir` and `--check-node-kinds`, as update-skill does), and
|
|
53
|
+
test-skill at coverage-check §4c (line check only); test-skill's
|
|
54
|
+
coverage-check §2c runs `classify-stale`.
|
|
31
55
|
|
|
32
56
|
Determinism:
|
|
33
57
|
- Set operations are pure over the two JSON inputs.
|
|
@@ -44,9 +68,157 @@ internal name of a barrel-renamed export (e.g. `_Impl` re-exported as
|
|
|
44
68
|
`skf-load-provenance.extract_reexport_map`, so an internal-named entry does
|
|
45
69
|
not read as a missing public export nor as an orphan.
|
|
46
70
|
|
|
47
|
-
|
|
71
|
+
Definition lines: after a citation resolves (the file exists and the line
|
|
72
|
+
is in range, so the older reasons win), the line must define the entry's
|
|
73
|
+
export. NAME is the raw `export_name` (not canonicalized); for a dotted
|
|
74
|
+
name (`App.update`) it is the last segment. The rules follow the
|
|
75
|
+
`source_file` extension:
|
|
76
|
+
|
|
77
|
+
- Python (`.py`, `.pyi`): the `def` / `async def` / `class NAME` line
|
|
78
|
+
itself (a decorator line above it does not count); an assignment
|
|
79
|
+
binding NAME (`NAME =`, `NAME: T =`, annotation-only `NAME: T`, a
|
|
80
|
+
target list `A, NAME = ...`, a chain `A = NAME = ...`, PEP 695
|
|
81
|
+
`type NAME =`); an import that binds NAME (`import NAME`,
|
|
82
|
+
`import X as NAME`, `from X import NAME`, `from X import Y as NAME`),
|
|
83
|
+
including NAME on its own line inside a parenthesized or
|
|
84
|
+
backslash-continued import; a `def` / `async def` / `class` line
|
|
85
|
+
ending in a backslash counts through the next line when that line
|
|
86
|
+
starts with NAME, and the name's line is the definition line (the one
|
|
87
|
+
the recipes cite). These count only on a line where a
|
|
88
|
+
statement begins (the stdlib tokenizer says so), so a docstring line,
|
|
89
|
+
a keyword argument or a parameter never matches; a file that does not
|
|
90
|
+
tokenize is matched line by line. A `from X import *` line counts only
|
|
91
|
+
when no other line of the file defines NAME (a re-export the
|
|
92
|
+
extraction could not trace is cited at its star import).
|
|
93
|
+
- TS/JS (`.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`): a
|
|
94
|
+
`function` (or `function*`), `class`, `interface`, `type`, `enum`,
|
|
95
|
+
`namespace`, `const`, `let` or `var` declaration of NAME, with any of
|
|
96
|
+
`export`, `default`, `async`, `declare`, `abstract` and decorators
|
|
97
|
+
(`@name`, `@a.b`, `@name(...)`, before `export` or between `export`
|
|
98
|
+
and `class`; a name inside a decorator's arguments is not declared);
|
|
99
|
+
a later declarator of a `const`, `let` or `var` list
|
|
100
|
+
(`export const a = 1, NAME = 2;`, or `export const a = 1,` then
|
|
101
|
+
` NAME = 2;` as Prettier writes it: see Declarator lists below); NAME
|
|
102
|
+
on the line right after a line that ends in its declaration keyword
|
|
103
|
+
(`export function`, `export interface`, `export const` and so on),
|
|
104
|
+
which is the line cited;
|
|
105
|
+
`export import NAME =`; `export * as NAME from` (NAME may be quoted);
|
|
106
|
+
`export default NAME`;
|
|
107
|
+
an `export { ... }` list whose exposed name is NAME, an identifier or
|
|
108
|
+
a quoted string on either side of `as`; CommonJS
|
|
109
|
+
`exports.NAME =`, `module.exports.NAME =`, or NAME as a key or
|
|
110
|
+
shorthand inside `module.exports = { ... }`. A list or object spanning
|
|
111
|
+
several lines counts on the line where NAME appears. For a dotted
|
|
112
|
+
name, an indented class member declaring NAME (method, property or
|
|
113
|
+
accessor, with any of `public`, `private`, `protected`, `static`,
|
|
114
|
+
`readonly`, `async`, `abstract`, `override`, `declare`, `get`, `set`)
|
|
115
|
+
also counts. An indented `NAME(...)` with no modifier, followed by `;`
|
|
116
|
+
or another expression rather than a body `{` or a return type `:`, is
|
|
117
|
+
a call, not a member; an indented `NAME: ...` or `NAME = ...` ending
|
|
118
|
+
with `,` is an object-literal entry or a destructuring default.
|
|
119
|
+
Comments never match: `//` to the end of the line and `/* ... */`
|
|
120
|
+
across lines are removed first, outside '...', "..." and `...` strings,
|
|
121
|
+
and brackets inside a string do not count toward a `module.exports`
|
|
122
|
+
object's depth. Strings are followed within one line only, so a
|
|
123
|
+
template literal spanning lines can hide or expose a comment marker.
|
|
124
|
+
|
|
125
|
+
Declarator lists: a `const`, `let` or `var` list is split at its
|
|
126
|
+
top-level commas, outside strings, regular expression literals and
|
|
127
|
+
brackets, a type-argument list included (`Map<K, V>`: a `<` right after
|
|
128
|
+
a name), up to a top-level `;`, and a destructuring pattern declares
|
|
129
|
+
nothing. The list goes on to the next line when a line ends inside a
|
|
130
|
+
bracket, after a top-level comma or after an operator waiting for its
|
|
131
|
+
operand, or when the next line starts with an operator (a member chain or
|
|
132
|
+
a ternary broken over lines), so a declarator on a line of its own
|
|
133
|
+
counts. A `<` where an expression begins (JSX, a type assertion, a generic
|
|
134
|
+
arrow function's type parameters) is not followed: no later declarator of
|
|
135
|
+
the list counts after it.
|
|
136
|
+
|
|
137
|
+
Indentation: for a dotted name every match counts. For an undotted name,
|
|
138
|
+
matches at any indentation count, but when the file has a column-0 match
|
|
139
|
+
the indented ones are dropped, so a local shadow never outranks the
|
|
140
|
+
module-level definition. A line inside a multi-line import, export list
|
|
141
|
+
or `module.exports` object takes the column of the line its statement
|
|
142
|
+
starts on, and so do the name line of a declaration split after its
|
|
143
|
+
keyword and a later declarator on a line of its own. When the recorded
|
|
144
|
+
line is itself an indented match that this rule dropped (a method
|
|
145
|
+
recorded under an undotted name beside a module-level function of the
|
|
146
|
+
same name), the finding lists the recorded line too, so it carries
|
|
147
|
+
several lines and no caller auto-moves it.
|
|
148
|
+
|
|
149
|
+
Entries in other files, and entries whose `export_type` is `module` or
|
|
150
|
+
`package` (case-insensitive; neither has a definition line), are skipped
|
|
151
|
+
and counted in `summary.line_check_skipped`. A `line-not-definition` item
|
|
152
|
+
lists every defining line in the file in `definition_lines`. An empty list
|
|
153
|
+
means the rules found no definition line, which may be a shape they do not
|
|
154
|
+
cover: the entry is unverified, not gone, and no caller auto-fixes it.
|
|
155
|
+
|
|
156
|
+
Citation check (`--skill-dir DIR`, which must hold a `SKILL.md`): scans
|
|
157
|
+
`DIR/SKILL.md` and `DIR/references/**/*.md` for `[AST:path:Ln]` /
|
|
158
|
+
`[SRC:path:Ln]`, ranges `Ln-m` / `Ln-Lm` included (the first line counts);
|
|
159
|
+
a citation with no line is ignored. A citation matches an entry when its
|
|
160
|
+
normalized path equals `source_file` and its line equals `source_line`.
|
|
161
|
+
`summary.skill_citations_matched` counts the citations that matched an
|
|
162
|
+
entry: 0 with citations scanned means the skill cites paths under another
|
|
163
|
+
root. Reasons, one per citation:
|
|
164
|
+
|
|
165
|
+
- `prefix-mismatch`: the matched entry's `extraction_method` implies the
|
|
166
|
+
other prefix (`ast-grep` / `ast_bridge` give AST, `source-read` /
|
|
167
|
+
`source_reading` give SRC). It wins over the reason below.
|
|
168
|
+
- `ast-without-ast-grep`: an `[AST:]` citation in a skill whose map has
|
|
169
|
+
no `ast-grep` or `ast_bridge` entry, raised only when every entry with
|
|
170
|
+
a source line carries a known `extraction_method` (a missing or
|
|
171
|
+
unknown method may hide an ast-grep entry).
|
|
172
|
+
|
|
173
|
+
When several entries share the cited line, the citation is a
|
|
174
|
+
`prefix-mismatch` only when none of them implies its prefix. An
|
|
175
|
+
`extraction_method` that is not a string reads as unknown (it implies no
|
|
176
|
+
prefix). Unmatched citations are not findings unless the second reason
|
|
177
|
+
applies.
|
|
178
|
+
|
|
179
|
+
Node-kind check (`--check-node-kinds`): ast-grep's JSON output never
|
|
180
|
+
reports the kind of the node a rule matched, so an entry's `ast_node_type`
|
|
181
|
+
is copied from the recipe that matched it, and nothing else proves it is a
|
|
182
|
+
real kind. For each entry whose `extraction_method` is `ast-grep` or
|
|
183
|
+
`ast_bridge` and whose `ast_node_type` is a non-blank string, the
|
|
184
|
+
`source_file` extension picks an ast-grep language (`.py` / `.pyi` python,
|
|
185
|
+
`.ts` / `.mts` / `.cts` typescript, `.tsx` tsx, `.js` / `.jsx` / `.mjs` /
|
|
186
|
+
`.cjs` javascript, `.rs` rust, `.go` go, `.java` java, `.kt` / `.kts`
|
|
187
|
+
kotlin, `.cs` csharp, `.rb` ruby, `.swift` swift, `.php` php); an entry in
|
|
188
|
+
any other file is skipped and counted in `summary.node_kind_check_skipped`.
|
|
189
|
+
Reasons, one per entry:
|
|
190
|
+
|
|
191
|
+
- `error-kind`: the kind is `ERROR`, the node tree-sitter makes of code
|
|
192
|
+
it cannot parse. ast-grep accepts it in a rule, so it is not asked.
|
|
193
|
+
- `invalid-kind`: the kind is not shaped like a kind
|
|
194
|
+
(`[A-Za-z_][A-Za-z0-9_]*`, so no whitespace or shell characters reach
|
|
195
|
+
ast-grep), judged without ast-grep; or ast-grep rejected it (its error
|
|
196
|
+
names an invalid kind).
|
|
197
|
+
|
|
198
|
+
Those two are judged whatever the ast-grep lookup finds. Every other kind is
|
|
199
|
+
asked once per distinct (language, kind): `ast-grep scan --config
|
|
200
|
+
<sgconfig.yml> --inline-rules <rule> --stdin` with a rule matching only that
|
|
201
|
+
kind, over an empty stdin, from a temporary folder holding a minimal
|
|
202
|
+
sgconfig.yml that `--config` names (so no other sgconfig.yml is read).
|
|
203
|
+
ast-grep parses the rule before reading input and rejects a kind the
|
|
204
|
+
language's grammar lacks. Before that, ast-grep is asked about a kind no
|
|
205
|
+
grammar has; unless it rejects it, its answers cannot be read and the check
|
|
206
|
+
is `skipped-unrecognized-ast-grep`. When no `ast-grep` executable resolves
|
|
207
|
+
(the CWD-shim guard applies), the check is `skipped-no-ast-grep`. With no
|
|
208
|
+
kind left to ask, ast-grep is not looked up and the check is `checked`.
|
|
209
|
+
|
|
210
|
+
An ast-grep failure that does not name the kind proves nothing about it: the
|
|
211
|
+
entry is listed in `node_kinds_unchecked[]`, not as a finding, with reason
|
|
212
|
+
`ast-grep-error`, `ast-grep-timeout` (after the first timeout the remaining
|
|
213
|
+
kinds are not asked) or `temp-folder-error` (the temporary folder could not
|
|
214
|
+
be made or written). Without the flag the output keeps its shape: there are
|
|
215
|
+
no `node_kinds` / `node_kinds_unchecked` keys, and the one key added is
|
|
216
|
+
`summary.node_kind_check: "not-requested"`.
|
|
217
|
+
|
|
218
|
+
Subcommands:
|
|
48
219
|
verify --metadata <metadata.json> --provenance <provenance-map.json>
|
|
49
|
-
[--source-root <path>] [-
|
|
220
|
+
[--source-root <path>] [--skill-dir <dir>] [--check-node-kinds]
|
|
221
|
+
[-o <out.json>]
|
|
50
222
|
|
|
51
223
|
Emit JSON:
|
|
52
224
|
{
|
|
@@ -55,9 +227,40 @@ Subcommand:
|
|
|
55
227
|
"orphaned": ["<provenance entry with no documented export>", ...],
|
|
56
228
|
"stale": [
|
|
57
229
|
{"export_name": "<raw entry name>",
|
|
230
|
+
"entry_index": <int>, # position in entries[]
|
|
58
231
|
"source_file": "<rel path>",
|
|
59
232
|
"source_line": <int|null>,
|
|
60
|
-
"reason": "file-missing" | "line-out-of-bounds" | "line-invalid"
|
|
233
|
+
"reason": "file-missing" | "line-out-of-bounds" | "line-invalid"
|
|
234
|
+
| "line-not-definition",
|
|
235
|
+
"definition_lines": [<int>, ...]}, # line-not-definition only
|
|
236
|
+
...
|
|
237
|
+
],
|
|
238
|
+
"citations": [
|
|
239
|
+
{"file": "<markdown path relative to --skill-dir>",
|
|
240
|
+
"line": <int>, # line of the markdown file
|
|
241
|
+
"citation": "[AST:<path>:L<n>]",
|
|
242
|
+
"prefix": "AST" | "SRC",
|
|
243
|
+
"cited_file": "<path as written>",
|
|
244
|
+
"cited_line": <int>,
|
|
245
|
+
"reason": "prefix-mismatch" | "ast-without-ast-grep",
|
|
246
|
+
"expected_prefix": "AST" | "SRC",
|
|
247
|
+
"export_name": "<matched entry name>" | null},
|
|
248
|
+
...
|
|
249
|
+
],
|
|
250
|
+
"node_kinds": [ # --check-node-kinds only
|
|
251
|
+
{"export_name": "<raw entry name>" | null,
|
|
252
|
+
"entry_index": <int>, # position in entries[]
|
|
253
|
+
"source_file": "<rel path>",
|
|
254
|
+
"source_line": <as recorded>,
|
|
255
|
+
"ast_node_type": "<recorded kind>",
|
|
256
|
+
"language": "<ast-grep language>",
|
|
257
|
+
"reason": "invalid-kind" | "error-kind"},
|
|
258
|
+
...
|
|
259
|
+
],
|
|
260
|
+
"node_kinds_unchecked": [ # --check-node-kinds only
|
|
261
|
+
{... the same keys ...,
|
|
262
|
+
"reason": "ast-grep-error" | "ast-grep-timeout"
|
|
263
|
+
| "temp-folder-error"},
|
|
61
264
|
...
|
|
62
265
|
],
|
|
63
266
|
"summary": {
|
|
@@ -65,39 +268,462 @@ Subcommand:
|
|
|
65
268
|
"entries_checked": <int>, # provenance entries with export_name
|
|
66
269
|
"missing_count": <int>,
|
|
67
270
|
"orphaned_count": <int>,
|
|
271
|
+
"set_diff": "checked" | "not-applicable", # reference-app
|
|
68
272
|
"stale_count": <int>,
|
|
69
273
|
"citations_checked": <int>, # entries whose citation was resolved
|
|
70
|
-
"
|
|
274
|
+
"line_check_skipped": <int>, # resolved entries with no line rule
|
|
275
|
+
"stale_check": "checked" | "skipped-no-source-root",
|
|
276
|
+
"skill_citations_scanned": <int>, # [AST:]/[SRC:] with a line
|
|
277
|
+
"skill_citations_matched": <int>, # of those, citing an entry
|
|
278
|
+
"citation_findings_count": <int>,
|
|
279
|
+
"citation_check": "checked" | "skipped-no-skill-dir",
|
|
280
|
+
"node_kind_check": "checked" | "skipped-no-ast-grep"
|
|
281
|
+
| "skipped-unrecognized-ast-grep"
|
|
282
|
+
| "not-requested",
|
|
283
|
+
# the four below appear with --check-node-kinds only
|
|
284
|
+
"node_kinds_checked": <int>, # entries judged, ERROR included
|
|
285
|
+
"node_kind_check_skipped": <int>, # entries in an unmapped file
|
|
286
|
+
"node_kind_check_errors": <int>, # len(node_kinds_unchecked)
|
|
287
|
+
"node_kind_findings_count": <int>
|
|
71
288
|
}
|
|
72
289
|
}
|
|
73
290
|
|
|
74
291
|
`missing` / `orphaned` are sorted, deduplicated public names.
|
|
75
292
|
`stale` is sorted by (export_name, source_file).
|
|
293
|
+
`citations` is sorted by (file, line, column).
|
|
294
|
+
`node_kinds` / `node_kinds_unchecked` are sorted by (export_name,
|
|
295
|
+
source_file, entry_index).
|
|
296
|
+
|
|
297
|
+
fix --verify <verify.json> --provenance <provenance-map.json>
|
|
298
|
+
--skill-dir <dir> [--manual-inventory <inventory.json>]
|
|
299
|
+
[--no-line-moves] [--dry-run] [-o <out.json>]
|
|
300
|
+
|
|
301
|
+
Applies the fixes a `verify` result (the JSON a `verify` run wrote, over
|
|
302
|
+
the same provenance map and skill folder) has one answer for:
|
|
303
|
+
|
|
304
|
+
1. Citation prefixes: each `citations[]` item's citation, found by its
|
|
305
|
+
`file`, `line` and `citation` text, takes `expected_prefix`; its
|
|
306
|
+
path and line stay.
|
|
307
|
+
2. Source lines: each `line-not-definition` item with exactly one
|
|
308
|
+
`definition_lines` value moves its entry's `source_line` there (the
|
|
309
|
+
entry at its `entry_index`, or the entries with its name, file and
|
|
310
|
+
line). Every move is planned before any is made. The citations each
|
|
311
|
+
move takes are those of either prefix, in `SKILL.md` and
|
|
312
|
+
`references/**/*.md` as they stand after step 1, whose path,
|
|
313
|
+
normalized as the citation check normalizes it, and first line are
|
|
314
|
+
the entry's. Every planned move is applied in one pass, so a
|
|
315
|
+
citation moves at most once (`LIMIT` moving from 6 to 7 and
|
|
316
|
+
`DEFAULT` from 7 to 8 move a citation of line 6 to 7, never on to
|
|
317
|
+
8), and a range moves both ends (`L10-12` to `L13-15`). A citation
|
|
318
|
+
of a line another entry also records in that file stays: it may
|
|
319
|
+
cite that other export (`shared-line`).
|
|
320
|
+
|
|
321
|
+
A citation inside a `<!-- [MANUAL:name] -->` block never changes
|
|
322
|
+
(`inside-manual-block`); the blocks are paired as skf-hash-content.py
|
|
323
|
+
pairs them. Each changed file (markdown first, the provenance map last,
|
|
324
|
+
as JSON indented by two spaces, non-ASCII kept, with a final newline;
|
|
325
|
+
bytes that are not UTF-8 survive in markdown) is written through
|
|
326
|
+
skf-atomic-write.py beside this script. `--manual-inventory` (the JSON
|
|
327
|
+
`skf-hash-content.py manual-inventory` wrote for SKILL.md) is checked
|
|
328
|
+
again after the writes. `--no-line-moves` leaves every source-line
|
|
329
|
+
finding for a person (the drift override). `--dry-run` writes nothing.
|
|
330
|
+
|
|
331
|
+
Emit JSON:
|
|
332
|
+
{
|
|
333
|
+
"applied": [
|
|
334
|
+
{"kind": "citation-prefix", "file": "<rel md>", "line": <int>,
|
|
335
|
+
"citation": "<as it was>", "fixed": "<as written>"},
|
|
336
|
+
{"kind": "source-line", "entry_index": <int>,
|
|
337
|
+
"export_name": "<name>", "source_file": "<rel path>",
|
|
338
|
+
"from": <int>, "to": <int>},
|
|
339
|
+
{"kind": "citation-line", "file": "<rel md>", "line": <int>,
|
|
340
|
+
"citation": "<as it was>", "fixed": "<as written>",
|
|
341
|
+
"export_names": ["<name>", ...]},
|
|
342
|
+
...
|
|
343
|
+
],
|
|
344
|
+
"left_as_warn": [
|
|
345
|
+
{... the verify item ..., "kind": "citation-prefix",
|
|
346
|
+
"why": "citation-not-found" | "inside-manual-block"},
|
|
347
|
+
{... the verify item ..., "kind": "source-line",
|
|
348
|
+
"why": "no-definition-line" # unverified, never moved
|
|
349
|
+
| "several-definition-lines" | "needs-decision"
|
|
350
|
+
| "line-moves-skipped" | "entry-not-found"},
|
|
351
|
+
{"kind": "citation-line", "file", "line", "citation",
|
|
352
|
+
"export_names": [...],
|
|
353
|
+
"why": "shared-line" | "inside-manual-block"},
|
|
354
|
+
...
|
|
355
|
+
],
|
|
356
|
+
"files_changed": ["<path>", ...], # as the arguments name them
|
|
357
|
+
"files_written": ["<path>", ...], # [] with --dry-run
|
|
358
|
+
"dry_run": <bool>,
|
|
359
|
+
"manual_verify": null | {"preserved", "modified", "missing",
|
|
360
|
+
"moved", "ok"},
|
|
361
|
+
"summary": {"citation_prefixes_fixed": <int>,
|
|
362
|
+
"source_lines_moved": <int>,
|
|
363
|
+
"citations_moved": <int>,
|
|
364
|
+
"left_as_warn_count": <int>}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
`fix` handles no `missing`, `orphaned` or `node_kinds` finding: read
|
|
368
|
+
those from the verify result. Run `verify` again after it.
|
|
369
|
+
|
|
370
|
+
definition-lines --file <source_file> --name <export_name>
|
|
371
|
+
[--source-root <path>] [--line <n>]
|
|
372
|
+
[--export-type <type>] [-o <out.json>]
|
|
373
|
+
|
|
374
|
+
The definition-line rules above for one export (update-skill's
|
|
375
|
+
spot-check). `--file` is relative to `--source-root` (default: the
|
|
376
|
+
current folder).
|
|
377
|
+
|
|
378
|
+
Emit JSON:
|
|
379
|
+
{
|
|
380
|
+
"source_file": "<as given>",
|
|
381
|
+
"export_name": "<as given>",
|
|
382
|
+
"line_check": "checked" | "file-missing" | "skipped-export-type"
|
|
383
|
+
| "skipped-language",
|
|
384
|
+
"definition_lines": [<int>, ...] | null, # checked only
|
|
385
|
+
"source_line": <int> | null, # --line
|
|
386
|
+
"line_is_definition": <bool> | null # --line, checked only
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
With `--line`, a recorded line that is an indented match the column-0
|
|
390
|
+
rule dropped joins `definition_lines`, as in `verify`.
|
|
391
|
+
|
|
392
|
+
classify-stale --names <coverage.json> --provenance <provenance-map.json>
|
|
393
|
+
--source-root <path> [--inventory <inventory.json>]
|
|
394
|
+
[-o <out.json>]
|
|
395
|
+
|
|
396
|
+
test-skill's fabricated-signature test, for each stale name (a
|
|
397
|
+
documented name the enumerated source surface lacks): the
|
|
398
|
+
definition-lines rules above, run for every provenance entry whose raw
|
|
399
|
+
`export_name` is the name, at its `source_file` and `source_line`, with
|
|
400
|
+
its `export_type`. `--names` holds reconcile-coverage.py's result (its
|
|
401
|
+
`stale` names) or a JSON array of names. `--inventory` holds the
|
|
402
|
+
documented inventory, validate-inventory.py's `--output` file or its
|
|
403
|
+
whole result: the `kind` it gives each name decides the module rule
|
|
404
|
+
below.
|
|
405
|
+
|
|
406
|
+
Emit JSON, one item per name, in the order given (a repeated name once):
|
|
407
|
+
[
|
|
408
|
+
{"name": "<as given>",
|
|
409
|
+
"fabricated": <bool>,
|
|
410
|
+
"source": "<source_file>:<source_line>" | "<source_file>" | null,
|
|
411
|
+
"defined_at": "<path>:<line>" | null,
|
|
412
|
+
"declared_in": ["<path>:<line>", ...] | null,
|
|
413
|
+
"reason": "not-defined" | "defined" | "unchecked" | "no-entry",
|
|
414
|
+
"entries": [{"source_file": "<as recorded>",
|
|
415
|
+
"source_line": <as recorded>,
|
|
416
|
+
"line_check": "checked" | "file-missing"
|
|
417
|
+
| "skipped-export-type"
|
|
418
|
+
| "skipped-language" | "unreadable"
|
|
419
|
+
| "no-file",
|
|
420
|
+
"definition_lines": [<int>, ...] | null}, ...]},
|
|
421
|
+
...
|
|
422
|
+
]
|
|
423
|
+
|
|
424
|
+
`fabricated` is true, with reason `not-defined`, when the name has an
|
|
425
|
+
entry and every entry's `line_check` is `file-missing`, or `checked`
|
|
426
|
+
with an empty `definition_lines`: the export is in neither the surface
|
|
427
|
+
nor the cited file. Otherwise the reason is `defined` (a cited file
|
|
428
|
+
defines the name), `unchecked` (no rule could check a cited file, and
|
|
429
|
+
none defines it) or `no-entry` (the map has no entry for the name).
|
|
430
|
+
An entry with no `source_file` (update-skill writes one for an
|
|
431
|
+
`unknown` NEW_EXPORT) cites no file to check: its `line_check` is
|
|
432
|
+
`no-file`, which, like a skipped or unreadable check, neither proves
|
|
433
|
+
the name fabricated nor shows it defined, so the name is `unchecked`
|
|
434
|
+
unless another entry defines it. `source` is the citation of the first
|
|
435
|
+
entry with a `source_file` (the file alone when its line is not a
|
|
436
|
+
number), null when no entry has one.
|
|
437
|
+
|
|
438
|
+
`defined_at` is where the source declares a name that is not
|
|
439
|
+
fabricated, so test-skill can tell a documented extra (a name outside
|
|
440
|
+
the surface that the source still declares) from stale documentation;
|
|
441
|
+
a fabricated name is not looked up (null). A declaration is a line at
|
|
442
|
+
column 0 that the definition-line rules match with imports left out:
|
|
443
|
+
a `def`, `class`, assignment, annotation, type alias, TS/JS
|
|
444
|
+
declaration, `exports.NAME =`, or an alias that renames
|
|
445
|
+
(`import X as NAME`, `from X import Y as NAME`,
|
|
446
|
+
`export { Y as NAME }`, `export * as NAME from`), or a
|
|
447
|
+
`module.exports` key with a value or a method. A plain or star import,
|
|
448
|
+
an `as` that keeps the imported name (`from m import NAME as NAME`,
|
|
449
|
+
`export { NAME as NAME }`), `export import NAME =`,
|
|
450
|
+
`export default NAME`, an export-list item without `as`, a
|
|
451
|
+
`module.exports` shorthand key, a binding whose value is an import
|
|
452
|
+
(`require(...)` or `(await) import(...)`, on its line or the next:
|
|
453
|
+
`const NAME = require(...)`, `exports.NAME = require(...)`,
|
|
454
|
+
`{ NAME: require(...) }`), an indented line and any line of a Python
|
|
455
|
+
file that does not tokenize never count. A module or package named
|
|
456
|
+
NAME (`NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
|
|
457
|
+
`NAME/index.<ext>`) declares NAME too: at its line 1 when no line of it
|
|
458
|
+
declares NAME, unless `--inventory` gives NAME a `function`, `hook`,
|
|
459
|
+
`class`, `interface`, `enum` or `type` kind (any of its kinds, when it
|
|
460
|
+
gives several; a `method` kind is not read), no entry of NAME is a
|
|
461
|
+
`module` or `package` one, and the file is not a TS/JS one that is
|
|
462
|
+
itself the export (a column-0 `export default` or `module.exports =`),
|
|
463
|
+
so that a removed function named after its file is no declaration
|
|
464
|
+
(the inventory has no `module` kind). `declared_line` gives this
|
|
465
|
+
answer for one file, the module rule optional, to a script that loads
|
|
466
|
+
this one, and
|
|
467
|
+
`baseline_gaps` applies it to the map's entries an extraction left
|
|
468
|
+
out (skf-extraction-snapshot.py's baseline gaps and
|
|
469
|
+
skf-build-change-manifest.py's `baseline-gaps`, which apply the module
|
|
470
|
+
rule only to a `module` or `package` entry).
|
|
471
|
+
|
|
472
|
+
The files: when a file the name's entries cite declares it,
|
|
473
|
+
`defined_at` is that file's first declaration (the files in entry
|
|
474
|
+
order), and `declared_in` is null. Otherwise `declared_in` lists, as
|
|
475
|
+
sorted `<path>:<line>`, every file of the walk that declares the name,
|
|
476
|
+
at its first declaration (a type stub, `NAME.pyi` or `NAME.d.ts`,
|
|
477
|
+
beside its implementation is one file with it: the implementation),
|
|
478
|
+
and `defined_at` is the one item when the list has exactly one and no
|
|
479
|
+
walked Python file that does not tokenize holds the name (a homonym,
|
|
480
|
+
two files or more, gives null: test-skill decides it by the files the
|
|
481
|
+
skill cites). The walk reads, sorted by path, every file below the
|
|
482
|
+
walk root: the deepest folder that the files of the map's entries
|
|
483
|
+
share, widened to its package root (Python: the nearest folder at or
|
|
484
|
+
above it that holds `__init__.py`, then up while the parent holds
|
|
485
|
+
one, never the source root itself; TS/JS: the nearest folder at or
|
|
486
|
+
above it that holds `package.json`), leaving out the entries whose
|
|
487
|
+
file lies outside every package; the source root when none lies in
|
|
488
|
+
one. Only the rule families (Python, TS/JS) the map's entries cite are
|
|
489
|
+
read (both when no entry cites a file). The walk skips symlinks, hidden folders, test
|
|
490
|
+
files and test configuration files (`conftest.py`, `jest.config.*`),
|
|
491
|
+
`*.min.js`, and the folders generated, built, installed, vendored or
|
|
492
|
+
test output lives in (`node_modules`, `dist`, `build`, `vendor`,
|
|
493
|
+
`_vendor`, `tests`, `testing`, `fixtures`, `__mocks__`,
|
|
494
|
+
`site-packages`, `env`, `bower_components`, `esm`, `cjs`, `umd` and
|
|
495
|
+
the like), `docs`, `examples` and `benchmarks`, each judged on the
|
|
496
|
+
path below the walk root, the end-to-end test folders `e2e` and
|
|
497
|
+
`__fixtures__` wherever they lie, and, as the first folder below the
|
|
498
|
+
walk root or below a package root inside it (a folder that holds
|
|
499
|
+
`package.json`, the top of an `__init__.py` chain), `example`, `demo`,
|
|
500
|
+
`demos`, `sample`, `samples`, `scripts`, `playground`, `bench`,
|
|
501
|
+
`benchmark` and `migrations` unless it holds a file an entry cites
|
|
502
|
+
(deeper in a package such a folder can be package code). A cited file
|
|
503
|
+
is read only inside the source root (a path that is absolute or climbs
|
|
504
|
+
out of it is not read), with no symlink on its path and under the same
|
|
505
|
+
skips, those walk-only ones aside. The path is relative to the source
|
|
506
|
+
root. A dotted name, or a map whose entries cite files
|
|
507
|
+
but no Python or TS/JS one, is not looked up (`defined_at` and
|
|
508
|
+
`declared_in` null, as for a fabricated name), and a filesystem error
|
|
509
|
+
during the lookup gives every name null in both (the exit code stays
|
|
510
|
+
0).
|
|
511
|
+
|
|
512
|
+
kind-at --file <path> --line <n> --recipes <recipes>
|
|
513
|
+
[--language <lang>] [--name <export_name>]
|
|
514
|
+
[--source-root <path>] [-o <out.json>]
|
|
515
|
+
|
|
516
|
+
The `kind` the matching recipe declares for a source line, the value an
|
|
517
|
+
ast-grep entry records as `ast_node_type`. `--recipes` is a markdown
|
|
518
|
+
file whose ```yaml fences hold the recipes (extraction-patterns.md) or
|
|
519
|
+
a YAML file: one recipe per document, a list of recipes, or a mapping
|
|
520
|
+
whose `recipes` key holds that list. A recipe is a mapping with `id`,
|
|
521
|
+
`language` and a `rule` naming its `kind`. The file's language
|
|
522
|
+
(`--language`, else its extension, mapped as for `--check-node-kinds`)
|
|
523
|
+
picks the recipes: each id's form for that language, else its
|
|
524
|
+
typescript or tsx form (for javascript, a form whose kinds the grammar
|
|
525
|
+
lacks is skipped), with `language` rewritten. Each runs once through
|
|
526
|
+
`ast-grep scan --stdin` on the file's bytes, from a temporary folder
|
|
527
|
+
with a minimal sgconfig.yml. A recipe matches when its `$NAME` is on
|
|
528
|
+
`--line` (and, with `--name`, is that name's last segment).
|
|
529
|
+
|
|
530
|
+
Emit JSON:
|
|
531
|
+
{
|
|
532
|
+
"file": "<as given>",
|
|
533
|
+
"language": "<ast-grep language>" | null,
|
|
534
|
+
"line": <int>,
|
|
535
|
+
"name": "<as given>" | null,
|
|
536
|
+
"status": "found" | "ambiguous" | "no-match" | "incomplete"
|
|
537
|
+
| "skipped-no-language" | "skipped-no-recipes"
|
|
538
|
+
| "skipped-no-ast-grep",
|
|
539
|
+
"kind": "<the one kind the matches declare>" | null, # found only
|
|
540
|
+
"matches": [{"recipe": "<id>", "kind": "<kind>",
|
|
541
|
+
"name": "<$NAME>"}, ...],
|
|
542
|
+
"errors": [{"recipe": "<id>" | null, "language": "<language>",
|
|
543
|
+
"reason": "ast-grep-error" | "ast-grep-timeout"
|
|
544
|
+
| "temp-folder-error",
|
|
545
|
+
"detail": "<text>"}, ...]
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
`incomplete`: a recipe run failed, so no kind is given even when others
|
|
549
|
+
matched. Never invent a kind: anything but `found` leaves the entry as
|
|
550
|
+
it is.
|
|
76
551
|
|
|
77
552
|
CLI examples:
|
|
78
553
|
uv run skf-verify-provenance-completeness.py verify \\
|
|
79
554
|
--metadata {skill_package}/metadata.json \\
|
|
80
555
|
--provenance {forge_version}/provenance-map.json \\
|
|
81
|
-
--source-root {source_root}
|
|
556
|
+
--source-root {source_root} \\
|
|
557
|
+
--skill-dir {skill_package} \\
|
|
558
|
+
--check-node-kinds -o {verify_json}
|
|
559
|
+
uv run skf-verify-provenance-completeness.py fix --verify {verify_json} \\
|
|
560
|
+
--provenance {forge_version}/provenance-map.json \\
|
|
561
|
+
--skill-dir {skill_package}
|
|
562
|
+
uv run skf-verify-provenance-completeness.py definition-lines \\
|
|
563
|
+
--source-root {source_root} --file src/api.py --name search --line 26
|
|
564
|
+
uv run skf-verify-provenance-completeness.py classify-stale \\
|
|
565
|
+
--names {run_dir}/coverage.json \\
|
|
566
|
+
--provenance {forge_version}/provenance-map.json \\
|
|
567
|
+
--source-root {source_root} --inventory {run_dir}/inventory.json \\
|
|
568
|
+
-o {run_dir}/stale.json
|
|
569
|
+
uv run skf-verify-provenance-completeness.py kind-at \\
|
|
570
|
+
--source-root {source_root} --file src/api.ts --line 12 --name Store \\
|
|
571
|
+
--recipes {extractionPatternsData}
|
|
82
572
|
|
|
83
573
|
Exit codes:
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
574
|
+
verify
|
|
575
|
+
0 verification ran and found nothing (status "pass")
|
|
576
|
+
1 verification ran and found missing / orphaned / stale entries,
|
|
577
|
+
citation findings or node-kind findings (status "findings");
|
|
578
|
+
advisory: the caller decides how to surface it
|
|
579
|
+
fix
|
|
580
|
+
0 nothing left as a WARN (and manual_verify ok when it ran)
|
|
581
|
+
1 something left as a WARN, or manual_verify not ok
|
|
582
|
+
definition-lines
|
|
583
|
+
0 the rules answered (whatever `line_check` says)
|
|
584
|
+
classify-stale
|
|
585
|
+
0 every name was classified (whatever `fabricated` says)
|
|
586
|
+
kind-at
|
|
587
|
+
0 status "found"
|
|
588
|
+
1 any other status
|
|
589
|
+
2 error, for every subcommand (an input file or folder not found,
|
|
590
|
+
--skill-dir without a SKILL.md, malformed JSON or YAML, a verify
|
|
591
|
+
result without stale[] and citations[], --names without a list of
|
|
592
|
+
names, --inventory without an `exports` list, no recipe in
|
|
593
|
+
--recipes, no PyYAML for kind-at, an atomic write that failed, an -o
|
|
594
|
+
file that cannot be written, or any unexpected failure, such as an
|
|
595
|
+
unreadable file; once fix has started writing, stderr names the files
|
|
596
|
+
already written); one line on stderr and no JSON
|
|
88
597
|
"""
|
|
89
598
|
|
|
90
599
|
from __future__ import annotations
|
|
91
600
|
|
|
92
601
|
import argparse
|
|
602
|
+
import copy
|
|
603
|
+
import importlib.util
|
|
604
|
+
import io
|
|
93
605
|
import json
|
|
606
|
+
import os
|
|
607
|
+
import posixpath
|
|
608
|
+
import re
|
|
609
|
+
import shutil
|
|
610
|
+
import subprocess
|
|
94
611
|
import sys
|
|
612
|
+
import tempfile
|
|
613
|
+
import tokenize
|
|
95
614
|
from pathlib import Path
|
|
615
|
+
from typing import Callable
|
|
96
616
|
|
|
97
617
|
|
|
98
618
|
STALE_FILE_MISSING = "file-missing"
|
|
99
619
|
STALE_LINE_OOB = "line-out-of-bounds"
|
|
100
620
|
STALE_LINE_INVALID = "line-invalid"
|
|
621
|
+
STALE_LINE_NOT_DEFINITION = "line-not-definition"
|
|
622
|
+
|
|
623
|
+
CITATION_PREFIX_MISMATCH = "prefix-mismatch"
|
|
624
|
+
CITATION_AST_WITHOUT_AST_GREP = "ast-without-ast-grep"
|
|
625
|
+
|
|
626
|
+
PYTHON_EXTENSIONS = frozenset({".py", ".pyi"})
|
|
627
|
+
TSJS_EXTENSIONS = frozenset(
|
|
628
|
+
{".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"}
|
|
629
|
+
)
|
|
630
|
+
# export_type values that name a whole module or package: no definition line.
|
|
631
|
+
NO_DEFINITION_EXPORT_TYPES = frozenset({"module", "package"})
|
|
632
|
+
# The scope_type whose exports[] is empty by design (entries are per-citation).
|
|
633
|
+
REFERENCE_APP = "reference-app"
|
|
634
|
+
|
|
635
|
+
# extraction_method -> the citation prefix it implies
|
|
636
|
+
METHOD_PREFIX = {
|
|
637
|
+
"ast-grep": "AST",
|
|
638
|
+
"ast_bridge": "AST",
|
|
639
|
+
"source-read": "SRC",
|
|
640
|
+
"source_reading": "SRC",
|
|
641
|
+
}
|
|
642
|
+
AST_METHODS = frozenset({"ast-grep", "ast_bridge"})
|
|
643
|
+
# Every extraction_method SKF writes (the same list skf-render-metadata-stats
|
|
644
|
+
# checks labels against).
|
|
645
|
+
KNOWN_METHODS = frozenset(
|
|
646
|
+
{
|
|
647
|
+
"ast-grep",
|
|
648
|
+
"source-read",
|
|
649
|
+
"ast_bridge",
|
|
650
|
+
"source_reading",
|
|
651
|
+
"qmd_bridge",
|
|
652
|
+
"compose-from-skill",
|
|
653
|
+
}
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
NODE_KIND_INVALID = "invalid-kind"
|
|
657
|
+
NODE_KIND_ERROR = "error-kind"
|
|
658
|
+
# Why an entry's kind went unjudged (node_kinds_unchecked[].reason).
|
|
659
|
+
UNCHECKED_AST_GREP_ERROR = "ast-grep-error"
|
|
660
|
+
UNCHECKED_TIMEOUT = "ast-grep-timeout"
|
|
661
|
+
UNCHECKED_TEMP_FOLDER = "temp-folder-error"
|
|
662
|
+
# The node tree-sitter makes of code it cannot parse. ast-grep accepts it as
|
|
663
|
+
# a kind, but no recipe declares it, so an entry recording it guessed.
|
|
664
|
+
ERROR_NODE_KIND = "ERROR"
|
|
665
|
+
# Every named node kind of a grammar ast-grep ships has this shape. Anything
|
|
666
|
+
# else is judged without ast-grep: ast-grep trims surrounding whitespace, and
|
|
667
|
+
# `& | % ^` would reach cmd.exe through an npm `.cmd` shim on Windows.
|
|
668
|
+
NODE_KIND_SHAPE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
669
|
+
# A kind no grammar has. ast-grep must reject it before its answers are read.
|
|
670
|
+
BOGUS_NODE_KIND = "skf_no_such_node_kind"
|
|
671
|
+
# An explicit, valid sgconfig.yml, so no sgconfig.yml in the temporary
|
|
672
|
+
# folder's ancestors is read.
|
|
673
|
+
MINIMAL_SGCONFIG = "ruleDirs: []\n"
|
|
674
|
+
# source_file extension -> the ast-grep language whose grammar names its kinds
|
|
675
|
+
AST_GREP_LANGUAGES = {
|
|
676
|
+
".py": "python",
|
|
677
|
+
".pyi": "python",
|
|
678
|
+
".ts": "typescript",
|
|
679
|
+
".mts": "typescript",
|
|
680
|
+
".cts": "typescript",
|
|
681
|
+
".tsx": "tsx",
|
|
682
|
+
".js": "javascript",
|
|
683
|
+
".jsx": "javascript",
|
|
684
|
+
".mjs": "javascript",
|
|
685
|
+
".cjs": "javascript",
|
|
686
|
+
".rs": "rust",
|
|
687
|
+
".go": "go",
|
|
688
|
+
".java": "java",
|
|
689
|
+
".kt": "kotlin",
|
|
690
|
+
".kts": "kotlin",
|
|
691
|
+
".cs": "csharp",
|
|
692
|
+
".rb": "ruby",
|
|
693
|
+
".swift": "swift",
|
|
694
|
+
".php": "php",
|
|
695
|
+
}
|
|
696
|
+
NODE_KIND_TIMEOUT_SEC = 20 # per ast-grep call; each parses one tiny rule
|
|
697
|
+
|
|
698
|
+
# kind-at: a ```yaml fence in a markdown recipe file (extraction-patterns.md).
|
|
699
|
+
_MD_YAML_FENCE_RE = re.compile(r"^```yaml[ \t]*\n(.*?)^```", re.M | re.S)
|
|
700
|
+
# file language -> the recipe languages whose form of a recipe it runs, with
|
|
701
|
+
# `language:` rewritten, when the recipe has no form of its own for it (the
|
|
702
|
+
# language selection note in extraction-patterns-by-hand.md): typescript and tsx
|
|
703
|
+
# share their recipes, and javascript runs a TypeScript recipe with no
|
|
704
|
+
# javascript form (one whose kinds the grammar lacks is skipped).
|
|
705
|
+
RECIPE_LANGUAGE_FALLBACKS = {
|
|
706
|
+
"typescript": ("tsx",),
|
|
707
|
+
"tsx": ("typescript",),
|
|
708
|
+
"javascript": ("typescript", "tsx"),
|
|
709
|
+
}
|
|
710
|
+
KIND_AT_TIMEOUT_SEC = 30 # per ast-grep call; each scans one file
|
|
711
|
+
|
|
712
|
+
# fix: what each applied or left item fixes.
|
|
713
|
+
FIX_CITATION_PREFIX = "citation-prefix"
|
|
714
|
+
FIX_SOURCE_LINE = "source-line"
|
|
715
|
+
FIX_CITATION_LINE = "citation-line"
|
|
716
|
+
# The [MANUAL] markers skf-hash-content.py pairs (its _OPEN_RE / _CLOSE_RE,
|
|
717
|
+
# on text instead of bytes; the test pins the two copies together). ASCII,
|
|
718
|
+
# so `\s` is the whitespace a bytes pattern knows: a marker spaced with
|
|
719
|
+
# U+00A0 is no marker to either.
|
|
720
|
+
_MANUAL_OPEN_RE = re.compile(r"<!--\s*\[MANUAL:([^\]]+)\]\s*-->", re.ASCII)
|
|
721
|
+
_MANUAL_CLOSE_RE = re.compile(r"<!--\s*\[/MANUAL:([^\]]+)\]\s*-->", re.ASCII)
|
|
722
|
+
# One line with its ending, split where `check_citation` counts a line.
|
|
723
|
+
_LINE_WITH_END_RE = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+$")
|
|
724
|
+
ATOMIC_WRITE_HELPER = "skf-atomic-write.py"
|
|
725
|
+
HASH_CONTENT_HELPER = "skf-hash-content.py"
|
|
726
|
+
ATOMIC_WRITE_TIMEOUT_SEC = 60
|
|
101
727
|
|
|
102
728
|
|
|
103
729
|
# --------------------------------------------------------------------------
|
|
@@ -174,6 +800,12 @@ def extract_reexport_map(prov: dict) -> dict[str, str]:
|
|
|
174
800
|
return out
|
|
175
801
|
|
|
176
802
|
|
|
803
|
+
def is_reference_app(metadata: dict) -> bool:
|
|
804
|
+
"""True when metadata.json's `scope_type` is `reference-app`."""
|
|
805
|
+
scope_type = metadata.get("scope_type")
|
|
806
|
+
return isinstance(scope_type, str) and scope_type.strip() == REFERENCE_APP
|
|
807
|
+
|
|
808
|
+
|
|
177
809
|
def canon(name: str, reexport_map: dict[str, str]) -> str:
|
|
178
810
|
"""Canonicalize an export name to its public form via reexport_map.
|
|
179
811
|
|
|
@@ -261,171 +893,3332 @@ def check_citation(
|
|
|
261
893
|
|
|
262
894
|
|
|
263
895
|
# --------------------------------------------------------------------------
|
|
264
|
-
#
|
|
896
|
+
# Definition lines
|
|
265
897
|
# --------------------------------------------------------------------------
|
|
266
898
|
|
|
899
|
+
_PY_FROM_IMPORT_RE = re.compile(r"^\s*from\s+[\w.]+\s+import\s+(.*)$")
|
|
900
|
+
_PY_IMPORT_RE = re.compile(r"^\s*import\s+(.*)$")
|
|
901
|
+
_PY_IMPORT_ITEM_RE = re.compile(r"^([\w.]+)(?:\s+as\s+(\w+))?$")
|
|
902
|
+
_PY_IDENT_RE = re.compile(r"[A-Za-z_]\w*")
|
|
903
|
+
# `A = value`, `A = B = value`: every name left of an `=` that is not `==`.
|
|
904
|
+
_PY_ASSIGN_CHAIN_RE = re.compile(r"^\s*((?:[A-Za-z_]\w*\s*=(?!=)\s*)+)")
|
|
905
|
+
# `A, B = value`, `(A, *B) = value`, `A, = value`.
|
|
906
|
+
_PY_TARGET_LIST_RE = re.compile(
|
|
907
|
+
r"^\s*[(\[]?\s*"
|
|
908
|
+
r"(\*?[A-Za-z_]\w*\s*,(?:\s*\*?[A-Za-z_]\w*\s*,)*(?:\s*\*?[A-Za-z_]\w*)?)"
|
|
909
|
+
r"\s*[)\]]?\s*=(?!=)"
|
|
910
|
+
)
|
|
911
|
+
# `def \` or `class \`: the name is on the next line.
|
|
912
|
+
_PY_SPLIT_DECL_RE = re.compile(r"^\s*(?:async\s+def|def|class)\s*\\$")
|
|
913
|
+
# A line that can matter to a name it does not hold: an import (which may go
|
|
914
|
+
# on over the next lines) or a `def \` / `class \` whose name is on the next.
|
|
915
|
+
_PY_NAMELESS_MATCH_RE = re.compile(r"^\s*(?:from|import|async|def|class)\b")
|
|
916
|
+
# Tokens that never begin a statement.
|
|
917
|
+
_PY_LAYOUT_TOKENS = frozenset(
|
|
918
|
+
{
|
|
919
|
+
tokenize.NL,
|
|
920
|
+
tokenize.COMMENT,
|
|
921
|
+
tokenize.INDENT,
|
|
922
|
+
tokenize.DEDENT,
|
|
923
|
+
tokenize.ENDMARKER,
|
|
924
|
+
tokenize.ENCODING,
|
|
925
|
+
}
|
|
926
|
+
)
|
|
267
927
|
|
|
268
|
-
|
|
269
|
-
|
|
928
|
+
# `@name`, `@a.b`, `@name(...)` (one level of nested brackets in the call).
|
|
929
|
+
_JS_DECORATOR = r"@[\w$]+(?:\.[\w$]+)*(?:\((?:[^()]|\([^()]*\))*\))?"
|
|
930
|
+
_JS_MODIFIERS = (
|
|
931
|
+
rf"(?:(?:export|default|async|declare|abstract)\s+|{_JS_DECORATOR}\s*)*"
|
|
932
|
+
)
|
|
933
|
+
_JS_KINDS = (
|
|
934
|
+
r"(?:function(?:\s*\*\s*|\s+)"
|
|
935
|
+
r"|(?:class|interface|type|enum|namespace|let|var)\s+"
|
|
936
|
+
r"|const\s+(?:enum\s+)?)"
|
|
937
|
+
)
|
|
938
|
+
_JS_MEMBER_MODIFIERS = (
|
|
939
|
+
r"(?:(?:public|private|protected|static|readonly|async|abstract"
|
|
940
|
+
r"|override|declare|get|set)\s+)*"
|
|
941
|
+
)
|
|
942
|
+
# An identifier, or a string name (`"a-b"`) in an export list.
|
|
943
|
+
_JS_NAME_OR_STRING = r"(?:[\w$]+|\"[^\"]*\"|'[^']*')"
|
|
944
|
+
_JS_EXPORT_LIST_RE = re.compile(r"^\s*export\s+(?:type\s+)?\{(.*)$")
|
|
945
|
+
_JS_EXPORT_ITEM_RE = re.compile(
|
|
946
|
+
rf"^(?:type\s+)?({_JS_NAME_OR_STRING})(?:\s+as\s+({_JS_NAME_OR_STRING}))?$"
|
|
947
|
+
)
|
|
948
|
+
_JS_MODULE_EXPORTS_OBJECT_RE = re.compile(r"^\s*module\.exports\s*=\s*\{(.*)$")
|
|
949
|
+
# A bound value that is an import: `require(...)`, `import(...)`, awaited, in
|
|
950
|
+
# parentheses (`(await import("./m")).X`) or in one helper call
|
|
951
|
+
# (`__importDefault(require("./m"))`).
|
|
952
|
+
_JS_IMPORT_VALUE_RE = re.compile(
|
|
953
|
+
r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?(?:await\s+)?(?:require|import)\s*\("
|
|
954
|
+
)
|
|
955
|
+
# The start of such a value that goes on on the next line: nothing yet, a
|
|
956
|
+
# parenthesis, or a helper call's name and parenthesis.
|
|
957
|
+
_JS_IMPORT_VALUE_OPEN_RE = re.compile(r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?$")
|
|
958
|
+
# A `const`, `let` or `var` statement: its declarator list.
|
|
959
|
+
_JS_VAR_DECL_RE = re.compile(
|
|
960
|
+
rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(?!enum(?![\w$]))(.*)$"
|
|
961
|
+
)
|
|
962
|
+
# A declaration whose name is on the next line: the line ends in its keyword.
|
|
963
|
+
_JS_SPLIT_DECL_RE = re.compile(
|
|
964
|
+
rf"^\s*{_JS_MODIFIERS}(?P<kind>function(?:\s*\*)?|class|interface|type"
|
|
965
|
+
r"|enum|namespace|let|var|const(?:\s+enum)?)$"
|
|
966
|
+
)
|
|
967
|
+
# The keywords every line that opens a statement going on below holds (a
|
|
968
|
+
# `const` / `let` / `var` list, a split declaration, `export {`,
|
|
969
|
+
# `module.exports = {`): a line with none of them, and not the name, can
|
|
970
|
+
# match no rule while no statement is open.
|
|
971
|
+
_JS_OPENING_WORDS = ("const", "let", "var", "function", "class", "interface", "type", "enum", "namespace",
|
|
972
|
+
"export", "module")
|
|
270
973
|
|
|
271
|
-
|
|
272
|
-
|
|
974
|
+
|
|
975
|
+
def _js_quiet_lines(code_lines: list[str]) -> list[bool]:
|
|
976
|
+
"""For each comment-free line, True when it holds none of
|
|
977
|
+
`_JS_OPENING_WORDS`."""
|
|
978
|
+
return [not any(word in code for word in _JS_OPENING_WORDS) for code in code_lines]
|
|
979
|
+
_JS_VAR_KEYWORDS = frozenset({"const", "let", "var"})
|
|
980
|
+
# How a `const` / `let` / `var` list that did not end on its line goes on
|
|
981
|
+
# (`_js_declarators`): the next line begins a declarator (this one ended
|
|
982
|
+
# with a top-level comma); it finishes the current one (this one ended
|
|
983
|
+
# inside a bracket, or after an operator waiting for its operand); or it
|
|
984
|
+
# goes on only when it starts with an operator (this one ended with a
|
|
985
|
+
# value, as a member chain or a ternary broken over lines does).
|
|
986
|
+
_DECL_NEXT = "next"
|
|
987
|
+
_DECL_OPEN = "open"
|
|
988
|
+
_DECL_MAYBE = "maybe"
|
|
989
|
+
# What `_js_declarators` gives: None, or (brackets left open, how) the list
|
|
990
|
+
# goes on with, and the pieces that begin a declarator.
|
|
991
|
+
_DeclSplit = tuple[tuple[str, str] | None, list[str]]
|
|
992
|
+
# After one of these, `/` begins a regular expression and `<` begins JSX, a
|
|
993
|
+
# type assertion or a generic arrow function's type parameters.
|
|
994
|
+
_JS_EXPRESSION_START = frozenset("([{,;=?:&|^%+-*/<>!~")
|
|
995
|
+
# A line that ends in one of these leaves an operator waiting for its operand.
|
|
996
|
+
_JS_OPERAND_PENDING = frozenset("=?:&|^%+-*/<>.")
|
|
997
|
+
# A line that begins with one of these goes on with the expression above it.
|
|
998
|
+
_JS_CONTINUATION_START = frozenset(".?:,&|^%+-*/=<>")
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
def definition_name(export_name: str) -> str:
|
|
1002
|
+
"""The name a definition line must carry: the last segment of a dotted
|
|
1003
|
+
export name (`App.update` -> `update`), else the name itself."""
|
|
1004
|
+
return export_name.rsplit(".", 1)[-1]
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def _read_lines(path: Path) -> list[str] | None:
|
|
1008
|
+
"""Read a text file as lines numbered like `check_citation` counts them
|
|
1009
|
+
(universal newlines; a UTF-8 BOM is dropped so it never hides line 1).
|
|
1010
|
+
None when the file cannot be read."""
|
|
1011
|
+
try:
|
|
1012
|
+
with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
|
|
1013
|
+
return [ln.rstrip("\n") for ln in fh]
|
|
1014
|
+
except OSError:
|
|
1015
|
+
return None
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def _read_source(path: Path) -> str:
|
|
1019
|
+
"""A text file's content, decoded as `_read_lines` decodes it. Raises
|
|
1020
|
+
OSError when the file cannot be read."""
|
|
1021
|
+
with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
|
|
1022
|
+
return fh.read()
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def _text_lines(text: str) -> list[str]:
|
|
1026
|
+
"""`_read_source`'s text as the lines `_read_lines` gives for the file."""
|
|
1027
|
+
lines = text.split("\n")
|
|
1028
|
+
if lines[-1] == "":
|
|
1029
|
+
lines.pop()
|
|
1030
|
+
return lines
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def _is_column_0(text: str) -> bool:
|
|
1034
|
+
"""True when a line starts at column 0 (no leading whitespace)."""
|
|
1035
|
+
return not text[:1].isspace()
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def _py_statement_starts(lines: list[str]) -> set[int] | None:
|
|
1039
|
+
"""Line numbers on which a Python statement begins.
|
|
1040
|
+
|
|
1041
|
+
Uses the stdlib tokenizer, so a line inside a string or inside brackets
|
|
1042
|
+
(a docstring, a keyword argument, a dict entry, a parameter on its own
|
|
1043
|
+
line) never reads as a definition. None when the file does not tokenize:
|
|
1044
|
+
every line is then a candidate.
|
|
273
1045
|
"""
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
1046
|
+
starts: set[int] = set()
|
|
1047
|
+
at_start = True
|
|
1048
|
+
readline = io.StringIO("\n".join(lines) + "\n").readline
|
|
1049
|
+
try:
|
|
1050
|
+
for tok in tokenize.generate_tokens(readline):
|
|
1051
|
+
if tok.type == tokenize.NEWLINE:
|
|
1052
|
+
at_start = True
|
|
1053
|
+
elif tok.type in _PY_LAYOUT_TOKENS:
|
|
1054
|
+
continue
|
|
1055
|
+
elif at_start:
|
|
1056
|
+
starts.add(tok.start[0])
|
|
1057
|
+
at_start = False
|
|
1058
|
+
except (tokenize.TokenError, SyntaxError, ValueError):
|
|
282
1059
|
return None
|
|
283
|
-
|
|
284
|
-
return root if root.is_dir() else None
|
|
1060
|
+
return starts
|
|
285
1061
|
|
|
286
1062
|
|
|
287
|
-
def
|
|
288
|
-
"""
|
|
289
|
-
|
|
1063
|
+
def _js_code_lines(lines: list[str]) -> list[str]:
|
|
1064
|
+
"""Each TS/JS line with its comments removed: `//` to the end of the
|
|
1065
|
+
line, and `/* ... */` across lines, both only outside a '...', "..." or
|
|
1066
|
+
`...` string. A string is followed within its line only: a template
|
|
1067
|
+
literal that spans lines is not tracked, so each line starts outside
|
|
1068
|
+
any string. A backslash outside a string escapes the next character, so
|
|
1069
|
+
a regex literal such as `/\\/*/` opens no comment."""
|
|
1070
|
+
out: list[str] = []
|
|
1071
|
+
in_block = False
|
|
1072
|
+
for text in lines:
|
|
1073
|
+
buf: list[str] = []
|
|
1074
|
+
quote: str | None = None
|
|
1075
|
+
i, n = 0, len(text)
|
|
1076
|
+
while i < n:
|
|
1077
|
+
if in_block:
|
|
1078
|
+
end = text.find("*/", i)
|
|
1079
|
+
if end < 0:
|
|
1080
|
+
break
|
|
1081
|
+
in_block = False
|
|
1082
|
+
buf.append(" ")
|
|
1083
|
+
i = end + 2
|
|
1084
|
+
continue
|
|
1085
|
+
ch = text[i]
|
|
1086
|
+
if ch == "\\" and i + 1 < n:
|
|
1087
|
+
buf.append(text[i : i + 2])
|
|
1088
|
+
i += 2
|
|
1089
|
+
continue
|
|
1090
|
+
if quote is not None:
|
|
1091
|
+
if ch == quote:
|
|
1092
|
+
quote = None
|
|
1093
|
+
elif ch in "'\"`":
|
|
1094
|
+
quote = ch
|
|
1095
|
+
elif text.startswith("//", i):
|
|
1096
|
+
break
|
|
1097
|
+
elif text.startswith("/*", i):
|
|
1098
|
+
in_block = True
|
|
1099
|
+
i += 2
|
|
1100
|
+
continue
|
|
1101
|
+
buf.append(ch)
|
|
1102
|
+
i += 1
|
|
1103
|
+
out.append("".join(buf).rstrip())
|
|
1104
|
+
return out
|
|
290
1105
|
|
|
291
|
-
documented = extract_export_names(metadata)
|
|
292
|
-
documented_canon = {canon(n, reexport_map) for n in documented}
|
|
293
1106
|
|
|
294
|
-
|
|
295
|
-
|
|
1107
|
+
class _SourceCache:
|
|
1108
|
+
"""Source lines, Python statement starts and TS/JS comment-free lines,
|
|
1109
|
+
read once per file, and TS/JS declarator lists, split once per line
|
|
1110
|
+
(the split does not depend on the export looked up)."""
|
|
296
1111
|
|
|
297
|
-
|
|
298
|
-
|
|
1112
|
+
def __init__(self) -> None:
|
|
1113
|
+
self._lines: dict[Path, list[str] | None] = {}
|
|
1114
|
+
self._starts: dict[Path, set[int] | None] = {}
|
|
1115
|
+
self._js_code: dict[Path, list[str]] = {}
|
|
1116
|
+
self._js_quiet: dict[Path, list[bool]] = {}
|
|
1117
|
+
self._js_decl: dict[tuple[str, str, bool], _DeclSplit] = {}
|
|
299
1118
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
if isinstance(entries, list):
|
|
305
|
-
for entry in entries:
|
|
306
|
-
if not isinstance(entry, dict):
|
|
307
|
-
continue
|
|
308
|
-
name = entry.get("export_name")
|
|
309
|
-
if not isinstance(name, str) or not name:
|
|
310
|
-
continue
|
|
311
|
-
sf = entry.get("source_file")
|
|
312
|
-
sl = entry.get("source_line")
|
|
313
|
-
# Only count a resolvable (non-null) citation as "checked".
|
|
314
|
-
if isinstance(sf, str) and sf.strip() and _coerce_line(sl) is not None:
|
|
315
|
-
citations_checked += 1
|
|
316
|
-
reason = check_citation(sf, sl, source_root)
|
|
317
|
-
if reason is not None:
|
|
318
|
-
stale.append(
|
|
319
|
-
{
|
|
320
|
-
"export_name": name,
|
|
321
|
-
"source_file": sf if isinstance(sf, str) else None,
|
|
322
|
-
"source_line": sl if isinstance(sl, int)
|
|
323
|
-
and not isinstance(sl, bool) else sl,
|
|
324
|
-
"reason": reason,
|
|
325
|
-
}
|
|
326
|
-
)
|
|
327
|
-
stale.sort(key=lambda s: (s["export_name"], s.get("source_file") or ""))
|
|
1119
|
+
def lines(self, path: Path) -> list[str] | None:
|
|
1120
|
+
if path not in self._lines:
|
|
1121
|
+
self._lines[path] = _read_lines(path)
|
|
1122
|
+
return self._lines[path]
|
|
328
1123
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
"orphaned": orphaned,
|
|
334
|
-
"stale": stale,
|
|
335
|
-
"summary": {
|
|
336
|
-
"exports_checked": len(documented_canon),
|
|
337
|
-
"entries_checked": len(raw_entry_names),
|
|
338
|
-
"missing_count": len(missing),
|
|
339
|
-
"orphaned_count": len(orphaned),
|
|
340
|
-
"stale_count": len(stale),
|
|
341
|
-
"citations_checked": citations_checked,
|
|
342
|
-
"stale_check": "checked"
|
|
343
|
-
if source_root is not None
|
|
344
|
-
else "skipped-no-source-root",
|
|
345
|
-
},
|
|
346
|
-
}
|
|
1124
|
+
def py_statement_starts(self, path: Path, lines: list[str]) -> set[int] | None:
|
|
1125
|
+
if path not in self._starts:
|
|
1126
|
+
self._starts[path] = _py_statement_starts(lines)
|
|
1127
|
+
return self._starts[path]
|
|
347
1128
|
|
|
1129
|
+
def js_code_lines(self, path: Path, lines: list[str]) -> list[str]:
|
|
1130
|
+
if path not in self._js_code:
|
|
1131
|
+
self._js_code[path] = _js_code_lines(lines)
|
|
1132
|
+
return self._js_code[path]
|
|
348
1133
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
1134
|
+
def js_quiet_lines(self, path: Path, code_lines: list[str]) -> list[bool]:
|
|
1135
|
+
if path not in self._js_quiet:
|
|
1136
|
+
self._js_quiet[path] = _js_quiet_lines(code_lines)
|
|
1137
|
+
return self._js_quiet[path]
|
|
352
1138
|
|
|
1139
|
+
def js_declarators(
|
|
1140
|
+
self, text: str, opened: str = "", fresh: bool = True
|
|
1141
|
+
) -> _DeclSplit:
|
|
1142
|
+
key = (text, opened, fresh)
|
|
1143
|
+
if key not in self._js_decl:
|
|
1144
|
+
self._js_decl[key] = _js_declarators(text, opened, fresh)
|
|
1145
|
+
return self._js_decl[key]
|
|
353
1146
|
|
|
354
|
-
def _cmd_verify(args: argparse.Namespace) -> int:
|
|
355
|
-
meta_path = Path(args.metadata)
|
|
356
|
-
prov_path = Path(args.provenance)
|
|
357
|
-
if not meta_path.is_file():
|
|
358
|
-
print(f"error: metadata not found: {meta_path}", file=sys.stderr)
|
|
359
|
-
return 2
|
|
360
|
-
if not prov_path.is_file():
|
|
361
|
-
print(f"error: provenance map not found: {prov_path}", file=sys.stderr)
|
|
362
|
-
return 2
|
|
363
|
-
try:
|
|
364
|
-
metadata = load_json_object(meta_path, "metadata")
|
|
365
|
-
prov = load_json_object(prov_path, "provenance map")
|
|
366
|
-
except ValueError as exc:
|
|
367
|
-
print(f"error: {exc}", file=sys.stderr)
|
|
368
|
-
return 2
|
|
369
1147
|
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
1148
|
+
def _py_import_binds(
|
|
1149
|
+
items: str, name: str, from_import: bool, alias_only: bool = False
|
|
1150
|
+
) -> bool:
|
|
1151
|
+
"""True when an import item list binds `name` (with `alias_only`, only
|
|
1152
|
+
through an explicit `as name` that renames: `from m import name as
|
|
1153
|
+
name` and `import a.name as name` keep the imported name)."""
|
|
1154
|
+
items = items.split("#", 1)[0]
|
|
1155
|
+
for raw in items.replace("(", " ").replace(")", " ").replace("\\", " ").split(","):
|
|
1156
|
+
m = _PY_IMPORT_ITEM_RE.match(raw.strip())
|
|
1157
|
+
if not m:
|
|
1158
|
+
continue
|
|
1159
|
+
target, alias = m.group(1), m.group(2)
|
|
1160
|
+
if alias is not None:
|
|
1161
|
+
if alias_only and alias == target.rsplit(".", 1)[-1]:
|
|
1162
|
+
continue
|
|
1163
|
+
bound = alias
|
|
1164
|
+
elif alias_only:
|
|
1165
|
+
continue
|
|
1166
|
+
elif from_import:
|
|
1167
|
+
bound = target
|
|
378
1168
|
else:
|
|
379
|
-
|
|
1169
|
+
bound = target.split(".", 1)[0] # `import a.b` binds `a`
|
|
1170
|
+
if bound == name:
|
|
1171
|
+
return True
|
|
1172
|
+
return False
|
|
380
1173
|
|
|
381
|
-
result = verify(metadata, prov, source_root)
|
|
382
1174
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
1175
|
+
def _py_assigned_names(code: str) -> set[str]:
|
|
1176
|
+
"""Names an assignment statement binds (`A = B = v`, `A, *B = v`)."""
|
|
1177
|
+
names: set[str] = set()
|
|
1178
|
+
m = _PY_ASSIGN_CHAIN_RE.match(code)
|
|
1179
|
+
if m:
|
|
1180
|
+
names.update(_PY_IDENT_RE.findall(m.group(1)))
|
|
1181
|
+
m = _PY_TARGET_LIST_RE.match(code)
|
|
1182
|
+
if m:
|
|
1183
|
+
names.update(_PY_IDENT_RE.findall(m.group(1)))
|
|
1184
|
+
return names
|
|
388
1185
|
|
|
389
|
-
return 0 if result["status"] == "pass" else 1
|
|
390
1186
|
|
|
1187
|
+
def _python_definition_lines(
|
|
1188
|
+
lines: list[str],
|
|
1189
|
+
name: str,
|
|
1190
|
+
starts: set[int] | None,
|
|
1191
|
+
declarations_only: bool = False,
|
|
1192
|
+
) -> list[tuple[int, bool]]:
|
|
1193
|
+
"""(line, at column 0) for every line that defines `name`.
|
|
391
1194
|
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
)
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
1195
|
+
A line inside a parenthesized or backslash-continued import takes the
|
|
1196
|
+
column of the line the import statement starts on, and so does the name
|
|
1197
|
+
line of a `def \\` or `class \\` whose name is on the next line. A
|
|
1198
|
+
`from X import *` line counts only when no other line defines `name` (a
|
|
1199
|
+
re-export the extraction could not trace further is cited at its star
|
|
1200
|
+
import). With `declarations_only`, an import counts only when it binds
|
|
1201
|
+
`name` through an explicit `as name` that renames (`_py_import_binds`),
|
|
1202
|
+
and a star import never does.
|
|
1203
|
+
"""
|
|
1204
|
+
esc = re.escape(name)
|
|
1205
|
+
decl = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{esc}(?!\w)")
|
|
1206
|
+
leading_name = re.compile(rf"^\s*{esc}(?!\w)")
|
|
1207
|
+
annotated = re.compile(rf"^\s*{esc}\s*:(?!=)")
|
|
1208
|
+
type_alias = re.compile(rf"^\s*type\s+{esc}\s*(?:\[[^\]]*\]\s*)?=(?!=)")
|
|
1209
|
+
found: list[tuple[int, bool]] = []
|
|
1210
|
+
star_imports: list[tuple[int, bool]] = []
|
|
1211
|
+
# An import statement that continues on the next line:
|
|
1212
|
+
# (closing mark "paren" or "backslash", column 0, from-import).
|
|
1213
|
+
cont: tuple[str, bool, bool] | None = None
|
|
1214
|
+
for number, text in enumerate(lines, start=1):
|
|
1215
|
+
code = text.split("#", 1)[0]
|
|
1216
|
+
if cont is not None:
|
|
1217
|
+
mark, top, from_import = cont
|
|
1218
|
+
if mark == "paren":
|
|
1219
|
+
if ")" in code:
|
|
1220
|
+
code = code.split(")", 1)[0]
|
|
1221
|
+
cont = None
|
|
1222
|
+
else:
|
|
1223
|
+
stripped = code.rstrip()
|
|
1224
|
+
if stripped.endswith("\\"):
|
|
1225
|
+
code = stripped[:-1]
|
|
1226
|
+
else:
|
|
1227
|
+
cont = None
|
|
1228
|
+
if _py_import_binds(code, name, from_import, declarations_only):
|
|
1229
|
+
found.append((number, top))
|
|
1230
|
+
continue
|
|
1231
|
+
if starts is not None and number not in starts:
|
|
1232
|
+
continue
|
|
1233
|
+
if name not in code and not _PY_NAMELESS_MATCH_RE.match(code):
|
|
1234
|
+
continue # no rule below can match it: spares a scan per name
|
|
1235
|
+
top = _is_column_0(text)
|
|
1236
|
+
if (
|
|
1237
|
+
decl.match(code)
|
|
1238
|
+
or annotated.match(code)
|
|
1239
|
+
or type_alias.match(code)
|
|
1240
|
+
or name in _py_assigned_names(code)
|
|
1241
|
+
):
|
|
1242
|
+
found.append((number, top))
|
|
1243
|
+
continue
|
|
1244
|
+
if _PY_SPLIT_DECL_RE.match(code.rstrip()):
|
|
1245
|
+
# `def \` then the name: cite the name's line, as the recipes do.
|
|
1246
|
+
if number < len(lines) and leading_name.match(
|
|
1247
|
+
lines[number].split("#", 1)[0]
|
|
1248
|
+
):
|
|
1249
|
+
found.append((number + 1, top))
|
|
1250
|
+
continue
|
|
1251
|
+
m = _PY_FROM_IMPORT_RE.match(code)
|
|
1252
|
+
from_import = m is not None
|
|
1253
|
+
if m is None:
|
|
1254
|
+
m = _PY_IMPORT_RE.match(code)
|
|
1255
|
+
if m is None:
|
|
1256
|
+
continue
|
|
1257
|
+
rest = m.group(1)
|
|
1258
|
+
if from_import and rest.strip() == "*":
|
|
1259
|
+
if not declarations_only:
|
|
1260
|
+
star_imports.append((number, top))
|
|
1261
|
+
continue
|
|
1262
|
+
if from_import and rest.lstrip().startswith("(") and ")" not in rest:
|
|
1263
|
+
cont = ("paren", top, True)
|
|
1264
|
+
elif rest.rstrip().endswith("\\"):
|
|
1265
|
+
cont = ("backslash", top, from_import)
|
|
1266
|
+
rest = rest.rstrip()[:-1]
|
|
1267
|
+
if _py_import_binds(rest, name, from_import, declarations_only):
|
|
1268
|
+
found.append((number, top))
|
|
1269
|
+
return found or star_imports
|
|
1270
|
+
|
|
1271
|
+
|
|
1272
|
+
def _js_export_item_binds(item: str, name: str, alias_only: bool = False) -> bool:
|
|
1273
|
+
"""True when one `export { ... }` item exposes `name`: the name after
|
|
1274
|
+
`as`, else (unless `alias_only`) the item's own name, an identifier or
|
|
1275
|
+
a string. With `alias_only`, an `as` that keeps the item's own name
|
|
1276
|
+
(`NAME as NAME`) does not count either."""
|
|
1277
|
+
m = _JS_EXPORT_ITEM_RE.match(item.strip())
|
|
1278
|
+
if not m or (alias_only and m.group(2) is None):
|
|
1279
|
+
return False
|
|
1280
|
+
own, exposed = (
|
|
1281
|
+
s[1:-1] if s[:1] in ("'", '"') else s
|
|
1282
|
+
for s in (m.group(1), m.group(2) or m.group(1))
|
|
409
1283
|
)
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
1284
|
+
if alias_only and exposed == own:
|
|
1285
|
+
return False
|
|
1286
|
+
return exposed == name
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
def _js_regex_end(text: str, start: int) -> int | None:
|
|
1290
|
+
"""The end of the regular expression literal whose opening `/` is at
|
|
1291
|
+
`start`, flags included. None when it does not close on the line, or
|
|
1292
|
+
when the `/` begins a comment."""
|
|
1293
|
+
if text.startswith(("//", "/*"), start):
|
|
1294
|
+
return None
|
|
1295
|
+
in_class = False
|
|
1296
|
+
i, n = start + 1, len(text)
|
|
1297
|
+
while i < n:
|
|
1298
|
+
ch = text[i]
|
|
1299
|
+
if ch == "\\":
|
|
1300
|
+
i += 2
|
|
1301
|
+
continue
|
|
1302
|
+
if in_class:
|
|
1303
|
+
in_class = ch != "]"
|
|
1304
|
+
elif ch == "[":
|
|
1305
|
+
in_class = True
|
|
1306
|
+
elif ch == "/":
|
|
1307
|
+
i += 1
|
|
1308
|
+
while i < n and (text[i].isalnum() or text[i] in "_$"):
|
|
1309
|
+
i += 1
|
|
1310
|
+
return i
|
|
1311
|
+
i += 1
|
|
1312
|
+
return None
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
def _js_declarators(text: str, opened: str = "", fresh: bool = True) -> _DeclSplit:
|
|
1316
|
+
"""The declarators of a `const` / `let` / `var` list that begin in
|
|
1317
|
+
`text`: the pieces between its top-level commas.
|
|
1318
|
+
|
|
1319
|
+
`text` is the list's first line after the keyword, or a later line with
|
|
1320
|
+
the brackets the line above left open (`opened`, innermost last) and
|
|
1321
|
+
whether it begins a declarator (`fresh`) or finishes one, whose piece is
|
|
1322
|
+
then left out. Brackets, '...', "..." and `...` strings, a regular
|
|
1323
|
+
expression literal where an expression begins and a type-argument list
|
|
1324
|
+
(`Map<K, V>`: a `<` right after a name, closed by a `>` that is not the
|
|
1325
|
+
one of `=>`) are skipped. The list ends at a top-level `;`, at a bracket
|
|
1326
|
+
that closes one opened before it, and at a `<` where an expression
|
|
1327
|
+
begins: JSX, a type assertion or a generic arrow function's type
|
|
1328
|
+
parameters, which are not followed.
|
|
1329
|
+
|
|
1330
|
+
Returns None when the list ends on this line, else the (brackets left
|
|
1331
|
+
open, `_DECL_NEXT` | `_DECL_OPEN` | `_DECL_MAYBE`) it goes on with, and
|
|
1332
|
+
the pieces.
|
|
1333
|
+
"""
|
|
1334
|
+
pieces: list[str] = []
|
|
1335
|
+
buf: list[str] = []
|
|
1336
|
+
stack = list(opened)
|
|
1337
|
+
keep = fresh # the current piece begins a declarator
|
|
1338
|
+
last = "," if fresh else "=" # the last character outside a string
|
|
1339
|
+
quote: str | None = None
|
|
1340
|
+
escaped = False
|
|
1341
|
+
ended = False
|
|
1342
|
+
i, n = 0, len(text)
|
|
1343
|
+
while i < n:
|
|
1344
|
+
ch = text[i]
|
|
1345
|
+
i += 1
|
|
1346
|
+
if quote is not None:
|
|
1347
|
+
buf.append(ch)
|
|
1348
|
+
if escaped:
|
|
1349
|
+
escaped = False
|
|
1350
|
+
elif ch == "\\":
|
|
1351
|
+
escaped = True
|
|
1352
|
+
elif ch == quote:
|
|
1353
|
+
quote = None
|
|
1354
|
+
continue
|
|
1355
|
+
prev = text[i - 2] if i > 1 else ""
|
|
1356
|
+
if ch in "/<" and last in _JS_EXPRESSION_START and prev + ch != "<<":
|
|
1357
|
+
end = _js_regex_end(text, i - 1) if ch == "/" else None
|
|
1358
|
+
if end is None:
|
|
1359
|
+
ended = True
|
|
1360
|
+
break
|
|
1361
|
+
buf.append(text[i - 1 : end])
|
|
1362
|
+
i = end
|
|
1363
|
+
last = "a" # the literal is a value, as a name is
|
|
1364
|
+
continue
|
|
1365
|
+
if ch in "'\"`":
|
|
1366
|
+
quote = ch
|
|
1367
|
+
elif ch == "<" and (prev.isalnum() or prev in ("_", "$")) and (
|
|
1368
|
+
text[i : i + 1] not in ("<", "=")
|
|
1369
|
+
):
|
|
1370
|
+
stack.append(ch) # `Map<K, V>`, where `a < b` compares
|
|
1371
|
+
elif ch == ">" and stack[-1:] == ["<"] and prev != "=":
|
|
1372
|
+
stack.pop()
|
|
1373
|
+
buf.append(ch)
|
|
1374
|
+
last = "a" # the type is a value's, as a name is
|
|
1375
|
+
continue
|
|
1376
|
+
elif ch in "([{":
|
|
1377
|
+
stack.append(ch)
|
|
1378
|
+
elif ch in ")]}":
|
|
1379
|
+
while stack[-1:] == ["<"]:
|
|
1380
|
+
stack.pop() # a `<` that compared, written without spaces
|
|
1381
|
+
if not stack:
|
|
1382
|
+
ended = True
|
|
1383
|
+
break
|
|
1384
|
+
stack.pop()
|
|
1385
|
+
elif ch == ";" and all(b == "<" for b in stack):
|
|
1386
|
+
ended = True
|
|
1387
|
+
break
|
|
1388
|
+
elif ch == "," and not stack:
|
|
1389
|
+
if keep:
|
|
1390
|
+
pieces.append("".join(buf))
|
|
1391
|
+
buf, keep, last = [], True, ch
|
|
1392
|
+
continue
|
|
1393
|
+
if not ch.isspace():
|
|
1394
|
+
last = ch
|
|
1395
|
+
buf.append(ch)
|
|
1396
|
+
if keep:
|
|
1397
|
+
pieces.append("".join(buf))
|
|
1398
|
+
if ended or quote is not None:
|
|
1399
|
+
return None, pieces
|
|
1400
|
+
if stack:
|
|
1401
|
+
return ("".join(stack), _DECL_OPEN), pieces
|
|
1402
|
+
if last == ",":
|
|
1403
|
+
return ("", _DECL_NEXT), pieces
|
|
1404
|
+
return ("", _DECL_OPEN if last in _JS_OPERAND_PENDING else _DECL_MAYBE), pieces
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
def _js_depth1_pieces(text: str, depth: int) -> tuple[int, list[str]]:
|
|
1408
|
+
"""Split the text of a `module.exports = { ... }` object or an
|
|
1409
|
+
`export { ... }` list into its top-level pieces (the text between commas
|
|
1410
|
+
at depth 1).
|
|
1411
|
+
|
|
1412
|
+
Returns the bracket depth after `text` (0 once the object closed) and
|
|
1413
|
+
the pieces. An opening bracket at depth 1 stays in its piece, so a
|
|
1414
|
+
method shorthand `name(` and a key `name: {` keep their marker. Brackets
|
|
1415
|
+
and commas inside a string on the line are not counted.
|
|
1416
|
+
"""
|
|
1417
|
+
pieces: list[str] = []
|
|
1418
|
+
buf: list[str] = []
|
|
1419
|
+
quote: str | None = None
|
|
1420
|
+
escaped = False
|
|
1421
|
+
for ch in text:
|
|
1422
|
+
if quote is not None:
|
|
1423
|
+
# text inside a string: kept in its piece, never counted
|
|
1424
|
+
if depth == 1:
|
|
1425
|
+
buf.append(ch)
|
|
1426
|
+
if escaped:
|
|
1427
|
+
escaped = False
|
|
1428
|
+
elif ch == "\\":
|
|
1429
|
+
escaped = True
|
|
1430
|
+
elif ch == quote:
|
|
1431
|
+
quote = None
|
|
1432
|
+
continue
|
|
1433
|
+
if ch in "'\"`":
|
|
1434
|
+
quote = ch
|
|
1435
|
+
if depth == 1:
|
|
1436
|
+
buf.append(ch)
|
|
1437
|
+
elif ch in "([{":
|
|
1438
|
+
if depth == 1:
|
|
1439
|
+
buf.append(ch)
|
|
1440
|
+
depth += 1
|
|
1441
|
+
elif ch in ")]}":
|
|
1442
|
+
depth -= 1
|
|
1443
|
+
if depth == 0:
|
|
1444
|
+
break
|
|
1445
|
+
elif depth == 1:
|
|
1446
|
+
if ch == ",":
|
|
1447
|
+
pieces.append("".join(buf))
|
|
1448
|
+
buf = []
|
|
1449
|
+
else:
|
|
1450
|
+
buf.append(ch)
|
|
1451
|
+
pieces.append("".join(buf))
|
|
1452
|
+
return depth, pieces
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
def _js_paren_tail_is_call(after_open: str) -> bool:
|
|
1456
|
+
"""For `name(` on an indented line: True when the text after the
|
|
1457
|
+
parameters says a call, not a method (`name(x);`). A method's
|
|
1458
|
+
parameters are followed by its body `{` or a return type `:`, or they
|
|
1459
|
+
continue on the next line."""
|
|
1460
|
+
depth = 1
|
|
1461
|
+
for i, ch in enumerate(after_open):
|
|
1462
|
+
if ch in "([{":
|
|
1463
|
+
depth += 1
|
|
1464
|
+
elif ch in ")]}":
|
|
1465
|
+
depth -= 1
|
|
1466
|
+
if depth == 0:
|
|
1467
|
+
rest = after_open[i + 1 :].strip()
|
|
1468
|
+
return not (rest == "" or rest[0] in "{:")
|
|
1469
|
+
return False
|
|
1470
|
+
|
|
1471
|
+
|
|
1472
|
+
def _tsjs_definition_lines(
|
|
1473
|
+
lines: list[str],
|
|
1474
|
+
code_lines: list[str],
|
|
1475
|
+
name: str,
|
|
1476
|
+
dotted: bool,
|
|
1477
|
+
declarators: Callable[..., _DeclSplit] = _js_declarators,
|
|
1478
|
+
declarations_only: bool = False,
|
|
1479
|
+
quiet: list[bool] | None = None,
|
|
1480
|
+
) -> list[tuple[int, bool]]:
|
|
1481
|
+
"""(line, at column 0) for every line that defines `name`.
|
|
1482
|
+
|
|
1483
|
+
`quiet` is `_js_quiet_lines(code_lines)` when the caller has it (it does
|
|
1484
|
+
not depend on the name): a quiet line without the name is passed over
|
|
1485
|
+
while no statement is open. `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
|
|
1486
|
+
JSDoc line or a commented-out declaration never matches. A line inside
|
|
1487
|
+
a multi-line `export { ... }` list or `module.exports = { ... }` object
|
|
1488
|
+
takes the column of the line the statement starts on, and so do the
|
|
1489
|
+
name line of a declaration whose keyword ends the line above it and a
|
|
1490
|
+
later declarator on a line of its own. Class members count only for a
|
|
1491
|
+
dotted export name. `declarators` splits a declarator list
|
|
1492
|
+
(`_js_declarators`, or `_SourceCache.js_declarators`, which splits each
|
|
1493
|
+
line once for every export looked up). With `declarations_only`,
|
|
1494
|
+
`export import NAME =`, `export default NAME`, an export-list item
|
|
1495
|
+
without an `as` that renames, a `module.exports` shorthand key and an
|
|
1496
|
+
import-valued binding (a declarator, `exports.NAME =` or a
|
|
1497
|
+
`module.exports` key whose value starts with `require(` or an
|
|
1498
|
+
`(await) import(`, on its line or the next one) never count, nor does
|
|
1499
|
+
a class member.
|
|
1500
|
+
"""
|
|
1501
|
+
esc = re.escape(name)
|
|
1502
|
+
end = r"(?![\w$])"
|
|
1503
|
+
statements = [
|
|
1504
|
+
re.compile(rf"^\s*{_JS_MODIFIERS}{_JS_KINDS}{esc}{end}"),
|
|
1505
|
+
re.compile(
|
|
1506
|
+
rf"^\s*export\s+(?:type\s+)?\*\s+as\s+"
|
|
1507
|
+
rf"(?:{esc}\s+|([\"']){esc}\1\s*)from{end}"
|
|
417
1508
|
),
|
|
1509
|
+
re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])"),
|
|
1510
|
+
]
|
|
1511
|
+
if not declarations_only:
|
|
1512
|
+
statements += [
|
|
1513
|
+
re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
|
|
1514
|
+
re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
|
|
1515
|
+
]
|
|
1516
|
+
# One declarator of a `const` / `let` / `var` list: `NAME = v`,
|
|
1517
|
+
# `NAME: T = v`, `NAME!: T`, or a bare `NAME`. A destructuring pattern
|
|
1518
|
+
# starts with `{` or `[` and never matches.
|
|
1519
|
+
declarator = re.compile(rf"^\s*{esc}{end}\s*(?:!?\s*:|=|$)")
|
|
1520
|
+
# With declarations_only, the value bound to NAME (the rest of a
|
|
1521
|
+
# declarator, of `exports.NAME =` or of a `module.exports` key), to
|
|
1522
|
+
# test for an import.
|
|
1523
|
+
# (A type annotation holds no `=` but in `=>`: `NAME: () => T = v`.)
|
|
1524
|
+
declarator_value = re.compile(
|
|
1525
|
+
rf"^\s*{esc}{end}\s*(?:!?\s*:(?:[^=]|=>)*)?=(?![=>])(.*)$"
|
|
418
1526
|
)
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
1527
|
+
var_statement = re.compile(rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(.*)$")
|
|
1528
|
+
exports_value = re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])(.*)$")
|
|
1529
|
+
key_value = re.compile(rf"^\s*(?:(['\"]){esc}\1|{esc}{end})\s*:(.*)$")
|
|
1530
|
+
|
|
1531
|
+
def imported(value: str, number: int) -> bool:
|
|
1532
|
+
"""True when a bound value is an import: `require(...)`, or an
|
|
1533
|
+
`import(...)` awaited or not, alone or in one helper call. A value
|
|
1534
|
+
that goes on on the next line (`NAME =`, `NAME = (` or
|
|
1535
|
+
`NAME = helper(` ends line `number`) is read there too."""
|
|
1536
|
+
if _JS_IMPORT_VALUE_OPEN_RE.match(value):
|
|
1537
|
+
value += next((c for c in code_lines[number:] if c.strip()), "")
|
|
1538
|
+
return _JS_IMPORT_VALUE_RE.match(value) is not None
|
|
1539
|
+
|
|
1540
|
+
def declares(piece: str, number: int) -> bool:
|
|
1541
|
+
if declarator.match(piece) is None:
|
|
1542
|
+
return False
|
|
1543
|
+
if not declarations_only:
|
|
1544
|
+
return True
|
|
1545
|
+
m = declarator_value.match(piece)
|
|
1546
|
+
return m is None or not imported(m.group(1), number)
|
|
1547
|
+
|
|
1548
|
+
def is_statement(code: str, number: int) -> bool:
|
|
1549
|
+
if not any(p.match(code) for p in statements):
|
|
1550
|
+
return False
|
|
1551
|
+
if not declarations_only:
|
|
1552
|
+
return True
|
|
1553
|
+
m = var_statement.match(code)
|
|
1554
|
+
if m is not None:
|
|
1555
|
+
m = declarator_value.match(m.group(1))
|
|
1556
|
+
else:
|
|
1557
|
+
m = exports_value.match(code)
|
|
1558
|
+
return m is None or not imported(m.group(1), number)
|
|
1559
|
+
|
|
1560
|
+
leading_name = re.compile(rf"^\s*{esc}{end}")
|
|
1561
|
+
# A key with a value, a method, or (unless declarations_only) a shorthand.
|
|
1562
|
+
key_end = r"[:(]" if declarations_only else r"[:(]|$"
|
|
1563
|
+
object_key = re.compile(
|
|
1564
|
+
rf"^\s*(?:(?:async|get|set)\s+|\*\s*)*(?:(['\"]){esc}\1|{esc}{end})"
|
|
1565
|
+
rf"\s*(?:{key_end})"
|
|
424
1566
|
)
|
|
425
|
-
|
|
426
|
-
|
|
1567
|
+
|
|
1568
|
+
def binds_key(piece: str, number: int) -> bool:
|
|
1569
|
+
if object_key.match(piece) is None:
|
|
1570
|
+
return False
|
|
1571
|
+
if not declarations_only:
|
|
1572
|
+
return True
|
|
1573
|
+
m = key_value.match(piece)
|
|
1574
|
+
return m is None or not imported(m.group(2), number)
|
|
1575
|
+
|
|
1576
|
+
dotted = dotted and not declarations_only
|
|
1577
|
+
member = re.compile(
|
|
1578
|
+
rf"^\s+(?P<mods>{_JS_MEMBER_MODIFIERS})(?:\*\s*)?{esc}{end}\s*[?!]?\s*"
|
|
1579
|
+
rf"(?:<[^>]*>\s*)?(?P<tail>[(:=;]|$)"
|
|
427
1580
|
)
|
|
428
|
-
|
|
1581
|
+
found: list[tuple[int, bool]] = []
|
|
1582
|
+
# A statement that continues on the next line:
|
|
1583
|
+
# ("list" or "object", column 0, bracket depth).
|
|
1584
|
+
cont: tuple[str, bool, int] | None = None
|
|
1585
|
+
# A declaration whose keyword ends the line: (keyword, column 0).
|
|
1586
|
+
split: tuple[str, bool] | None = None
|
|
1587
|
+
# The `const` / `let` / `var` lists that go on on the next line, one
|
|
1588
|
+
# nested in another's initializer included: (column 0, brackets left
|
|
1589
|
+
# open, how, as `_js_declarators` gives them).
|
|
1590
|
+
decls: list[tuple[bool, str, str]] = []
|
|
1591
|
+
if quiet is None:
|
|
1592
|
+
quiet = _js_quiet_lines(code_lines)
|
|
1593
|
+
for number, (text, code) in enumerate(zip(lines, code_lines), start=1):
|
|
1594
|
+
if quiet[number - 1] and not (decls or split or cont) and name not in code:
|
|
1595
|
+
continue # no rule below can match it: spares a scan per name
|
|
1596
|
+
# A line of a list carried from above still goes through the rules
|
|
1597
|
+
# below: it may hold a class member or a local of an arrow function.
|
|
1598
|
+
if decls and code.strip():
|
|
1599
|
+
going_on = code.lstrip()[0] in _JS_CONTINUATION_START
|
|
1600
|
+
carried = []
|
|
1601
|
+
for decl_top, opened, how in decls:
|
|
1602
|
+
if how == _DECL_MAYBE and not going_on:
|
|
1603
|
+
continue # the statement ended with the line above
|
|
1604
|
+
carry, pieces = declarators(code, opened, how == _DECL_NEXT)
|
|
1605
|
+
if carry is not None:
|
|
1606
|
+
carried.append((decl_top, *carry))
|
|
1607
|
+
if any(declares(p, number) for p in pieces):
|
|
1608
|
+
# a later declarator on a line of its own, as Prettier
|
|
1609
|
+
# writes it: `export const a = 1,` then ` NAME = 2;`
|
|
1610
|
+
found.append((number, decl_top))
|
|
1611
|
+
decls = carried
|
|
1612
|
+
if split is not None:
|
|
1613
|
+
keyword, top = split
|
|
1614
|
+
split = None
|
|
1615
|
+
if keyword in _JS_VAR_KEYWORDS:
|
|
1616
|
+
carry, pieces = declarators(code)
|
|
1617
|
+
if carry is not None:
|
|
1618
|
+
decls.append((top, *carry))
|
|
1619
|
+
hit = any(declares(p, number) for p in pieces)
|
|
1620
|
+
else:
|
|
1621
|
+
hit = leading_name.match(code) is not None
|
|
1622
|
+
if hit:
|
|
1623
|
+
# the name's line, as the recipes cite it
|
|
1624
|
+
found.append((number, top))
|
|
1625
|
+
continue
|
|
1626
|
+
if cont is not None:
|
|
1627
|
+
kind, top, depth = cont
|
|
1628
|
+
depth, pieces = _js_depth1_pieces(code, depth)
|
|
1629
|
+
cont = (kind, top, depth) if depth > 0 else None
|
|
1630
|
+
if kind == "list":
|
|
1631
|
+
hit = any(
|
|
1632
|
+
_js_export_item_binds(p, name, declarations_only) for p in pieces
|
|
1633
|
+
)
|
|
1634
|
+
else:
|
|
1635
|
+
hit = any(binds_key(p, number) for p in pieces)
|
|
1636
|
+
if hit:
|
|
1637
|
+
found.append((number, top))
|
|
1638
|
+
continue
|
|
1639
|
+
top = _is_column_0(text)
|
|
1640
|
+
# every rule that matches a line needs the name in it
|
|
1641
|
+
named = name in code
|
|
1642
|
+
if named and is_statement(code, number):
|
|
1643
|
+
found.append((number, top))
|
|
1644
|
+
continue
|
|
1645
|
+
m = _JS_VAR_DECL_RE.match(code)
|
|
1646
|
+
if m:
|
|
1647
|
+
carry, pieces = declarators(m.group(1))
|
|
1648
|
+
if carry is not None:
|
|
1649
|
+
decls.append((top, *carry))
|
|
1650
|
+
if named and any(declares(p, number) for p in pieces):
|
|
1651
|
+
# a later declarator: `export const a = 1, NAME = 2;`
|
|
1652
|
+
found.append((number, top))
|
|
1653
|
+
continue
|
|
1654
|
+
m = _JS_SPLIT_DECL_RE.match(code)
|
|
1655
|
+
if m:
|
|
1656
|
+
split = (m.group("kind"), top)
|
|
1657
|
+
continue
|
|
1658
|
+
m = _JS_EXPORT_LIST_RE.match(code)
|
|
1659
|
+
if m:
|
|
1660
|
+
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1661
|
+
if depth > 0:
|
|
1662
|
+
cont = ("list", top, depth)
|
|
1663
|
+
if any(_js_export_item_binds(p, name, declarations_only) for p in pieces):
|
|
1664
|
+
found.append((number, top))
|
|
1665
|
+
continue
|
|
1666
|
+
m = _JS_MODULE_EXPORTS_OBJECT_RE.match(code)
|
|
1667
|
+
if m:
|
|
1668
|
+
depth, pieces = _js_depth1_pieces(m.group(1), 1)
|
|
1669
|
+
if depth > 0:
|
|
1670
|
+
cont = ("object", top, depth)
|
|
1671
|
+
if any(binds_key(p, number) for p in pieces):
|
|
1672
|
+
found.append((number, top))
|
|
1673
|
+
continue
|
|
1674
|
+
if not dotted:
|
|
1675
|
+
continue
|
|
1676
|
+
m = member.match(code)
|
|
1677
|
+
if not m:
|
|
1678
|
+
continue
|
|
1679
|
+
tail = m.group("tail")
|
|
1680
|
+
if tail == "(":
|
|
1681
|
+
if not m.group("mods") and _js_paren_tail_is_call(code[m.end():]):
|
|
1682
|
+
continue
|
|
1683
|
+
elif code.endswith(","):
|
|
1684
|
+
continue # an object-literal entry or a destructuring default
|
|
1685
|
+
elif tail == "=" and code[m.end() : m.end() + 1] in ("=", ">"):
|
|
1686
|
+
continue
|
|
1687
|
+
found.append((number, top))
|
|
1688
|
+
return found
|
|
1689
|
+
|
|
1690
|
+
|
|
1691
|
+
def _prefer_module_level(found: list[tuple[int, bool]], dotted: bool) -> list[int]:
|
|
1692
|
+
"""The definition lines to report. For an undotted name, a column-0
|
|
1693
|
+
match drops the indented ones, so a local shadow never outranks the
|
|
1694
|
+
module-level definition; a dotted name keeps every match."""
|
|
1695
|
+
if not dotted:
|
|
1696
|
+
top = [n for n, at_column_0 in found if at_column_0]
|
|
1697
|
+
if top:
|
|
1698
|
+
return sorted(set(top))
|
|
1699
|
+
return sorted({n for n, _ in found})
|
|
1700
|
+
|
|
1701
|
+
|
|
1702
|
+
def line_rule_applies(source_file: str, export_type: object) -> bool:
|
|
1703
|
+
"""True when the definition-line rules cover this entry: a Python or
|
|
1704
|
+
TS/JS file, and an export that is not a whole module or package."""
|
|
1705
|
+
if isinstance(export_type, str) and (
|
|
1706
|
+
export_type.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
1707
|
+
):
|
|
1708
|
+
return False
|
|
1709
|
+
suffix = Path(source_file.strip().replace("\\", "/")).suffix.lower()
|
|
1710
|
+
return suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def _definition_matches(
|
|
1714
|
+
source_file: str,
|
|
1715
|
+
export_name: str,
|
|
1716
|
+
source_root: Path,
|
|
1717
|
+
cache: _SourceCache | None = None,
|
|
1718
|
+
declarations_only: bool = False,
|
|
1719
|
+
) -> tuple[list[int], set[int]] | None:
|
|
1720
|
+
"""(definition lines to report, every matching line) for an export.
|
|
1721
|
+
|
|
1722
|
+
The first item applies the column-0 rule; the second keeps the indented
|
|
1723
|
+
matches that rule dropped. None when the file's extension has no
|
|
1724
|
+
definition rule or the file cannot be read.
|
|
1725
|
+
|
|
1726
|
+
`declarations_only` (classify-stale's `defined_at` lookup) keeps the
|
|
1727
|
+
lines that declare the name at column 0, both items alike: no plain
|
|
1728
|
+
import, star import or re-export (`_python_definition_lines` and
|
|
1729
|
+
`_tsjs_definition_lines` say which), and nothing in a Python file that
|
|
1730
|
+
does not tokenize.
|
|
1731
|
+
"""
|
|
1732
|
+
rel = source_file.strip().replace("\\", "/")
|
|
1733
|
+
suffix = Path(rel).suffix.lower()
|
|
1734
|
+
if suffix not in PYTHON_EXTENSIONS and suffix not in TSJS_EXTENSIONS:
|
|
1735
|
+
return None
|
|
1736
|
+
cache = cache if cache is not None else _SourceCache()
|
|
1737
|
+
path = (source_root / rel).resolve()
|
|
1738
|
+
lines = cache.lines(path)
|
|
1739
|
+
if lines is None:
|
|
1740
|
+
return None
|
|
1741
|
+
return _line_matches(suffix, path, lines, export_name, cache, declarations_only)
|
|
1742
|
+
|
|
1743
|
+
|
|
1744
|
+
def _line_matches(
|
|
1745
|
+
suffix: str,
|
|
1746
|
+
path: Path,
|
|
1747
|
+
lines: list[str],
|
|
1748
|
+
export_name: str,
|
|
1749
|
+
cache: _SourceCache,
|
|
1750
|
+
declarations_only: bool = False,
|
|
1751
|
+
) -> tuple[list[int], set[int]]:
|
|
1752
|
+
"""`_definition_matches` over the lines of a Python or TS/JS file already
|
|
1753
|
+
read: `suffix`, the cited file's extension, picks the rules, and `path`
|
|
1754
|
+
keys `cache`."""
|
|
1755
|
+
name = definition_name(export_name)
|
|
1756
|
+
if not name:
|
|
1757
|
+
return [], set()
|
|
1758
|
+
dotted = "." in export_name
|
|
1759
|
+
if suffix in PYTHON_EXTENSIONS:
|
|
1760
|
+
starts = cache.py_statement_starts(path, lines)
|
|
1761
|
+
if declarations_only and starts is None:
|
|
1762
|
+
# every line would be a candidate, a docstring line included
|
|
1763
|
+
return [], set()
|
|
1764
|
+
found = _python_definition_lines(lines, name, starts, declarations_only)
|
|
1765
|
+
else:
|
|
1766
|
+
code_lines = cache.js_code_lines(path, lines)
|
|
1767
|
+
found = _tsjs_definition_lines(
|
|
1768
|
+
lines, code_lines, name, dotted, cache.js_declarators, declarations_only,
|
|
1769
|
+
cache.js_quiet_lines(path, code_lines),
|
|
1770
|
+
)
|
|
1771
|
+
if declarations_only:
|
|
1772
|
+
top = {n for n, at_column_0 in found if at_column_0}
|
|
1773
|
+
return sorted(top), top
|
|
1774
|
+
return _prefer_module_level(found, dotted), {n for n, _ in found}
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
def find_definition_lines(
|
|
1778
|
+
source_file: str,
|
|
1779
|
+
export_name: str,
|
|
1780
|
+
source_root: Path,
|
|
1781
|
+
cache: _SourceCache | None = None,
|
|
1782
|
+
) -> list[int] | None:
|
|
1783
|
+
"""Every line of `source_file` that defines `export_name`, in order,
|
|
1784
|
+
after the column-0 rule.
|
|
1785
|
+
|
|
1786
|
+
Returns None when the file's extension has no definition rule or the
|
|
1787
|
+
file cannot be read.
|
|
1788
|
+
"""
|
|
1789
|
+
matches = _definition_matches(source_file, export_name, source_root, cache)
|
|
1790
|
+
return None if matches is None else matches[0]
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
def definition_lines_report(
|
|
1794
|
+
source_file: str,
|
|
1795
|
+
export_name: str,
|
|
1796
|
+
source_root: Path,
|
|
1797
|
+
line: int | None = None,
|
|
1798
|
+
export_type: str | None = None,
|
|
1799
|
+
) -> dict:
|
|
1800
|
+
"""What `verify` would say about one entry's line (the definition-lines
|
|
1801
|
+
subcommand). Raises ValueError when the file exists but cannot be read.
|
|
1802
|
+
|
|
1803
|
+
`line_check` is `file-missing`, `skipped-export-type` (a `module` or
|
|
1804
|
+
`package`, no definition line), `skipped-language` (no rule for the
|
|
1805
|
+
extension) or `checked`. With `line`, a recorded line that is an
|
|
1806
|
+
indented match the column-0 rule dropped joins `definition_lines`, as in
|
|
1807
|
+
`verify`, so the answer carries several lines.
|
|
1808
|
+
"""
|
|
1809
|
+
report: dict = {
|
|
1810
|
+
"source_file": source_file,
|
|
1811
|
+
"export_name": export_name,
|
|
1812
|
+
"line_check": "checked",
|
|
1813
|
+
"definition_lines": None,
|
|
1814
|
+
"source_line": line,
|
|
1815
|
+
"line_is_definition": None,
|
|
1816
|
+
}
|
|
1817
|
+
rel = source_file.strip().replace("\\", "/")
|
|
1818
|
+
if not rel or not (source_root / rel).is_file():
|
|
1819
|
+
report["line_check"] = "file-missing"
|
|
1820
|
+
return report
|
|
1821
|
+
if not line_rule_applies(source_file, export_type):
|
|
1822
|
+
suffix = Path(rel).suffix.lower()
|
|
1823
|
+
covered = suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
|
|
1824
|
+
report["line_check"] = "skipped-export-type" if covered else "skipped-language"
|
|
1825
|
+
return report
|
|
1826
|
+
matches = _definition_matches(source_file, export_name, source_root)
|
|
1827
|
+
if matches is None:
|
|
1828
|
+
raise ValueError(f"failed to read source file {source_root / rel}")
|
|
1829
|
+
defs, every = matches
|
|
1830
|
+
if line is not None:
|
|
1831
|
+
report["line_is_definition"] = line in defs
|
|
1832
|
+
if line not in defs and line in every:
|
|
1833
|
+
defs = sorted(set(defs) | {line})
|
|
1834
|
+
report["definition_lines"] = defs
|
|
1835
|
+
return report
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
# --------------------------------------------------------------------------
|
|
1839
|
+
# Stale names (classify-stale)
|
|
1840
|
+
# --------------------------------------------------------------------------
|
|
1841
|
+
|
|
1842
|
+
STALE_NOT_DEFINED = "not-defined"
|
|
1843
|
+
STALE_DEFINED = "defined"
|
|
1844
|
+
STALE_UNCHECKED = "unchecked"
|
|
1845
|
+
STALE_NO_ENTRY = "no-entry"
|
|
1846
|
+
|
|
1847
|
+
|
|
1848
|
+
def stale_names(data: object) -> list[str]:
|
|
1849
|
+
"""The names `--names` holds: reconcile-coverage.py's `stale`, or a JSON
|
|
1850
|
+
array of names, each once in the order given. Raises ValueError."""
|
|
1851
|
+
names = data.get("stale") if isinstance(data, dict) else data
|
|
1852
|
+
if not isinstance(names, list) or not all(isinstance(n, str) for n in names):
|
|
1853
|
+
raise ValueError(
|
|
1854
|
+
"--names must hold a `stale` list of names or a JSON array of names"
|
|
1855
|
+
)
|
|
1856
|
+
return list(dict.fromkeys(n for n in names if n))
|
|
1857
|
+
|
|
1858
|
+
|
|
1859
|
+
def _entry_file(entry: dict) -> str | None:
|
|
1860
|
+
"""The entry's `source_file`, None when it records none."""
|
|
1861
|
+
source_file = entry.get("source_file")
|
|
1862
|
+
if not isinstance(source_file, str) or not source_file.strip():
|
|
1863
|
+
return None
|
|
1864
|
+
return source_file
|
|
1865
|
+
|
|
1866
|
+
|
|
1867
|
+
def _entry_citation(entry: dict) -> str | None:
|
|
1868
|
+
source_file = _entry_file(entry)
|
|
1869
|
+
if source_file is None:
|
|
1870
|
+
return None
|
|
1871
|
+
line = _coerce_line(entry.get("source_line"))
|
|
1872
|
+
return f"{source_file}:{line}" if isinstance(line, int) else source_file
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
def _stale_line_check(
|
|
1876
|
+
entry: dict, name: str, source_root: Path
|
|
1877
|
+
) -> tuple[str, list[int] | None]:
|
|
1878
|
+
"""(line_check, definition_lines) of one provenance entry of a stale name."""
|
|
1879
|
+
source_file = _entry_file(entry)
|
|
1880
|
+
if source_file is None:
|
|
1881
|
+
# no file to check: it neither proves the name fabricated nor clears it
|
|
1882
|
+
return "no-file", None
|
|
1883
|
+
line = _coerce_line(entry.get("source_line"))
|
|
1884
|
+
export_type = entry.get("export_type")
|
|
1885
|
+
try:
|
|
1886
|
+
report = definition_lines_report(
|
|
1887
|
+
source_file,
|
|
1888
|
+
name,
|
|
1889
|
+
source_root,
|
|
1890
|
+
line if isinstance(line, int) else None,
|
|
1891
|
+
export_type if isinstance(export_type, str) else None,
|
|
1892
|
+
)
|
|
1893
|
+
except ValueError:
|
|
1894
|
+
# the file exists but cannot be read: nothing proves it
|
|
1895
|
+
return "unreadable", None
|
|
1896
|
+
return report["line_check"], report["definition_lines"]
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
# The folders and files the `defined_at` walk leaves out, judged
|
|
1900
|
+
# case-insensitively on the path below its walk root: generated, built and
|
|
1901
|
+
# installed output, a vendored copy of another package, tests and their
|
|
1902
|
+
# configuration, docs, examples and benchmarks.
|
|
1903
|
+
# Keep identical to EXCLUDED_DIR_NAMES in skf-resolve-authoritative-files.py.
|
|
1904
|
+
EXCLUDED_DIR_NAMES = {
|
|
1905
|
+
"node_modules", "__pycache__", "dist", "build", ".webpack",
|
|
1906
|
+
"target", ".next", ".nuxt", "out", "coverage", ".git",
|
|
1907
|
+
".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
|
|
1908
|
+
".ruff_cache", ".gradle", ".idea", ".vscode",
|
|
1909
|
+
}
|
|
1910
|
+
# Keep identical to _VENDORED_SEGMENTS in skf-detect-language.py.
|
|
1911
|
+
_VENDORED_SEGMENTS = frozenset(
|
|
1912
|
+
{"vendor", "vendored", "third_party", "third-party", "thirdparty"}
|
|
1913
|
+
)
|
|
1914
|
+
# Keep identical to TEST_FOLDERS in skf-disqualify-candidates.py.
|
|
1915
|
+
TEST_FOLDERS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
|
|
1916
|
+
# Keep identical to TEST_CONFIG_NAMES in skf-disqualify-candidates.py.
|
|
1917
|
+
TEST_CONFIG_NAMES = frozenset({
|
|
1918
|
+
"conftest.py", "pytest.ini", "phpunit.xml", "phpunit.xml.dist", ".rspec",
|
|
1919
|
+
})
|
|
1920
|
+
# Keep identical to TEST_CONFIG_PREFIXES in skf-disqualify-candidates.py.
|
|
1921
|
+
TEST_CONFIG_PREFIXES = (
|
|
1922
|
+
"jest.config.", "vitest.config.", "karma.conf.", "playwright.config.",
|
|
1923
|
+
"cypress.config.",
|
|
1924
|
+
)
|
|
1925
|
+
# Keep identical to TEST_FILE_RE in skf-disqualify-candidates.py.
|
|
1926
|
+
TEST_FILE_RE = re.compile(
|
|
1927
|
+
r"(\.(test|spec)\.[^./]+$)|(_test\.[^./]+$)|(^test_.+\.py$)|(_spec\.rb$)"
|
|
1928
|
+
)
|
|
1929
|
+
# Keep identical to PY_NON_CORE_TOPS in skf-extract-public-api.py.
|
|
1930
|
+
PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
|
|
1931
|
+
# The walk's own additions: pip's and setuptools' `_vendor`, test helpers
|
|
1932
|
+
# (`testing`, `fixtures`, `__mocks__`), installed packages (`site-packages`,
|
|
1933
|
+
# a virtualenv named `env`, `bower_components`) and the ESM, CommonJS and
|
|
1934
|
+
# UMD builds a JS package commits.
|
|
1935
|
+
LOOKUP_EXTRA_FOLDERS = frozenset({
|
|
1936
|
+
"_vendor", "testing", "fixtures", "__mocks__", "site-packages", "env",
|
|
1937
|
+
"bower_components", "esm", "cjs", "umd",
|
|
1938
|
+
})
|
|
1939
|
+
LOOKUP_SKIPPED_FOLDERS = frozenset(
|
|
1940
|
+
EXCLUDED_DIR_NAMES | _VENDORED_SEGMENTS | TEST_FOLDERS | PY_NON_CORE_TOPS
|
|
1941
|
+
| LOOKUP_EXTRA_FOLDERS
|
|
1942
|
+
)
|
|
1943
|
+
# The test folders the walk alone skips wherever they lie (end-to-end tests,
|
|
1944
|
+
# Jest's `__fixtures__` beside the code): a file the map cites in one is
|
|
1945
|
+
# still read.
|
|
1946
|
+
LOOKUP_WALK_FOLDERS = frozenset({"e2e", "__fixtures__"})
|
|
1947
|
+
# The folders the walk alone skips as the first folder below its walk root
|
|
1948
|
+
# or below a package root it passes through (`_package_root`'s roots):
|
|
1949
|
+
# examples and demos, samples, scripts, playgrounds, benchmarks and
|
|
1950
|
+
# migrations, which hold code that uses or changes the package rather than
|
|
1951
|
+
# its API. Deeper in a package such a folder can be live package code
|
|
1952
|
+
# (cognee's `cognee/modules/migrations/`), so it is walked there; one that
|
|
1953
|
+
# holds a file the map cites is walked too, and that file is still read.
|
|
1954
|
+
LOOKUP_TOP_FOLDERS = frozenset({
|
|
1955
|
+
"example", "demo", "demos", "sample", "samples", "scripts", "playground",
|
|
1956
|
+
"bench", "benchmark", "migrations",
|
|
1957
|
+
})
|
|
1958
|
+
# The kinds of a documented inventory (validate-inventory.py's `kind`) that a
|
|
1959
|
+
# module cannot be: such a name takes no line 1 from a module so named. A
|
|
1960
|
+
# `method` is left out: reconcile-coverage.py never lists a method as stale,
|
|
1961
|
+
# and a module may share a method's name.
|
|
1962
|
+
NO_MODULE_KINDS = frozenset({"function", "hook", "class", "interface", "enum", "type"})
|
|
1963
|
+
# A TS/JS file that is itself the export: its module so named declares the
|
|
1964
|
+
# name whatever its kind.
|
|
1965
|
+
_SELF_EXPORT_RE = re.compile(r"^(?:export\s+default\b|module\.exports\s*=(?!=))", re.M)
|
|
1966
|
+
# A minified bundle: built output, whatever folder holds it.
|
|
1967
|
+
LOOKUP_SKIPPED_SUFFIXES = (".min.js",)
|
|
1968
|
+
# The errors a lookup gives up on: every `defined_at` is then null.
|
|
1969
|
+
_LOOKUP_ERRORS = (OSError, ValueError, RuntimeError)
|
|
1970
|
+
|
|
1971
|
+
|
|
1972
|
+
def _rule_family(path: str) -> str | None:
|
|
1973
|
+
"""The definition rules a file's extension picks: `python`, `tsjs` or
|
|
1974
|
+
None."""
|
|
1975
|
+
suffix = posixpath.splitext(path)[1].lower()
|
|
1976
|
+
if suffix in PYTHON_EXTENSIONS:
|
|
1977
|
+
return "python"
|
|
1978
|
+
return "tsjs" if suffix in TSJS_EXTENSIONS else None
|
|
1979
|
+
|
|
1980
|
+
|
|
1981
|
+
def _lookup_skips_folder(segment: str) -> bool:
|
|
1982
|
+
"""True when the `defined_at` walk never goes into this folder."""
|
|
1983
|
+
segment = segment.lower()
|
|
1984
|
+
return segment.startswith(".") or segment in LOOKUP_SKIPPED_FOLDERS
|
|
1985
|
+
|
|
1986
|
+
|
|
1987
|
+
def _lookup_skips_file(filename: str) -> bool:
|
|
1988
|
+
"""True when the `defined_at` lookup never reads this file: a test file,
|
|
1989
|
+
as skf-disqualify-candidates.py's `_is_test_file` judges its name (a
|
|
1990
|
+
test configuration file included), or a minified bundle."""
|
|
1991
|
+
name = filename.lower()
|
|
1992
|
+
return (
|
|
1993
|
+
name in TEST_CONFIG_NAMES
|
|
1994
|
+
or name.startswith(TEST_CONFIG_PREFIXES)
|
|
1995
|
+
or TEST_FILE_RE.search(name) is not None
|
|
1996
|
+
or name.endswith(LOOKUP_SKIPPED_SUFFIXES)
|
|
1997
|
+
)
|
|
1998
|
+
|
|
1999
|
+
|
|
2000
|
+
def _symlinked(root: Path, parts: tuple[str, ...]) -> bool:
|
|
2001
|
+
"""True when a path below `root` goes through a symlink, itself
|
|
2002
|
+
included."""
|
|
2003
|
+
return any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1))
|
|
2004
|
+
|
|
2005
|
+
|
|
2006
|
+
def _readable_below(root: Path, parts: tuple[str, ...], real_root: Path | None = None) -> bool:
|
|
2007
|
+
"""True when the `defined_at` lookup may read the file at `parts` below
|
|
2008
|
+
`root`: not a file it skips (`_lookup_skips_file`), no symlink on its
|
|
2009
|
+
path, and its real path a file inside the root (`real_root`, the root
|
|
2010
|
+
resolved)."""
|
|
2011
|
+
if _lookup_skips_file(parts[-1]) or _symlinked(root, parts):
|
|
2012
|
+
return False
|
|
2013
|
+
real = root.joinpath(*parts).resolve()
|
|
2014
|
+
return real.is_relative_to(real_root or root.resolve()) and real.is_file()
|
|
2015
|
+
|
|
2016
|
+
|
|
2017
|
+
def _cited_parts(source_file: str) -> tuple[str, ...] | None:
|
|
2018
|
+
"""A recorded `source_file` as path segments below the source root, None
|
|
2019
|
+
when it is absolute or climbs out of the root."""
|
|
2020
|
+
path = normalize_path(source_file)
|
|
2021
|
+
if not path or path == "." or posixpath.isabs(path):
|
|
2022
|
+
return None
|
|
2023
|
+
parts = tuple(path.split("/"))
|
|
2024
|
+
return None if parts[0] == ".." or ":" in parts[0] else parts
|
|
2025
|
+
|
|
2026
|
+
|
|
2027
|
+
# The file whose folder is a package, by rule family.
|
|
2028
|
+
_PACKAGE_MARKERS = {"python": "__init__.py", "tsjs": "package.json"}
|
|
2029
|
+
# Type stubs: a declaration file beside the module it types.
|
|
2030
|
+
_STUB_SUFFIXES = {".pyi": "python", ".d.ts": "tsjs", ".d.mts": "tsjs", ".d.cts": "tsjs"}
|
|
2031
|
+
|
|
2032
|
+
|
|
2033
|
+
def _stub_of(path: str) -> tuple[str, str] | None:
|
|
2034
|
+
"""(the path without its suffix, family) of a type stub (`NAME.pyi`,
|
|
2035
|
+
`NAME.d.ts`), None for any other file."""
|
|
2036
|
+
lower = path.lower()
|
|
2037
|
+
for suffix, family in _STUB_SUFFIXES.items():
|
|
2038
|
+
if lower.endswith(suffix):
|
|
2039
|
+
return path[: -len(suffix)], family
|
|
2040
|
+
return None
|
|
2041
|
+
|
|
2042
|
+
|
|
2043
|
+
def _without_stubs(hits: list[str]) -> list[str]:
|
|
2044
|
+
"""`file:line` items less each stub whose implementation (`NAME.py`
|
|
2045
|
+
beside `NAME.pyi`, a TS/JS `NAME.<ext>` beside `NAME.d.ts`) is an item
|
|
2046
|
+
too: the two are one declaring file, the implementation."""
|
|
2047
|
+
files = [h.rsplit(":", 1)[0] for h in hits]
|
|
2048
|
+
implementations = {
|
|
2049
|
+
(posixpath.splitext(f)[0], _rule_family(f))
|
|
2050
|
+
for f in files
|
|
2051
|
+
if _stub_of(f) is None and _rule_family(f) is not None
|
|
2052
|
+
}
|
|
2053
|
+
return [h for h, f in zip(hits, files) if _stub_of(f) not in implementations]
|
|
2054
|
+
|
|
2055
|
+
|
|
2056
|
+
def _module_name(parts: tuple[str, ...]) -> str | None:
|
|
2057
|
+
"""The name a file is the module or package of, which it declares at
|
|
2058
|
+
its line 1: `NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
|
|
2059
|
+
`NAME/index.<ext>`. None for any other file, and for a package file
|
|
2060
|
+
with no folder above it below the source root."""
|
|
2061
|
+
stem, ext = posixpath.splitext(parts[-1])
|
|
2062
|
+
ext = ext.lower()
|
|
2063
|
+
if ext == ".py":
|
|
2064
|
+
package = "__init__"
|
|
2065
|
+
elif ext in TSJS_EXTENSIONS:
|
|
2066
|
+
package = "index"
|
|
2067
|
+
else:
|
|
2068
|
+
return None
|
|
2069
|
+
if stem != package:
|
|
2070
|
+
return stem
|
|
2071
|
+
return parts[-2] if len(parts) > 1 else None
|
|
2072
|
+
|
|
2073
|
+
|
|
2074
|
+
def _exports_itself(parts: tuple[str, ...], text: str) -> bool:
|
|
2075
|
+
"""True for a TS/JS file that is itself the export, with a column-0
|
|
2076
|
+
`export default` or `module.exports =`: its module so named declares
|
|
2077
|
+
the name whatever kind the inventory gives it."""
|
|
2078
|
+
return _rule_family(parts[-1]) == "tsjs" and _SELF_EXPORT_RE.search(text) is not None
|
|
2079
|
+
|
|
2080
|
+
|
|
2081
|
+
def declared_line(
|
|
2082
|
+
source_file: str | tuple[str, ...],
|
|
2083
|
+
name: str,
|
|
2084
|
+
source_root: Path | None = None,
|
|
2085
|
+
*,
|
|
2086
|
+
lines: list[str] | None = None,
|
|
2087
|
+
cache: _SourceCache | None = None,
|
|
2088
|
+
module_fallback: bool = True,
|
|
2089
|
+
) -> int | None:
|
|
2090
|
+
"""The line at which one file declares `name`, by classify-stale's
|
|
2091
|
+
`defined_at` rule (see the module docstring): its first line at column
|
|
2092
|
+
0 that the definition-line rules match with imports left out (a `def`,
|
|
2093
|
+
`class`, assignment, annotation, type alias, TS/JS declaration, or an
|
|
2094
|
+
alias that renames, such as `from X import Y as NAME`; a plain or star
|
|
2095
|
+
import, an `as` that keeps the imported name, a re-export and an
|
|
2096
|
+
import-valued binding never count), else, with `module_fallback`, its
|
|
2097
|
+
line 1 when it is the module or package so named (`_module_name`).
|
|
2098
|
+
|
|
2099
|
+
`source_file` is the file's path below the source root, as recorded
|
|
2100
|
+
(one that is absolute or climbs out of the root gives None) or as its
|
|
2101
|
+
segments. `lines` are its lines when the caller read them already;
|
|
2102
|
+
otherwise the file is read below `source_root` under the lookup's
|
|
2103
|
+
guards (`_readable_below`: no symlink on its path, its real path
|
|
2104
|
+
inside the root, not a file the lookup skips). Pass one `cache` for
|
|
2105
|
+
the names of one source root, and each file is read and parsed once.
|
|
2106
|
+
None when no line declares the name, the name is empty or dotted, the
|
|
2107
|
+
file is not a Python or TS/JS one, or it cannot be read.
|
|
2108
|
+
"""
|
|
2109
|
+
parts = _cited_parts(source_file) if isinstance(source_file, str) else tuple(source_file)
|
|
2110
|
+
if not name or "." in name or not parts or _rule_family(parts[-1]) is None:
|
|
2111
|
+
return None
|
|
2112
|
+
cache = cache if cache is not None else _SourceCache()
|
|
2113
|
+
if lines is None:
|
|
2114
|
+
if source_root is None:
|
|
2115
|
+
raise ValueError("declared_line reads the file below source_root when no lines are given")
|
|
2116
|
+
if not _readable_below(source_root, parts):
|
|
2117
|
+
return None
|
|
2118
|
+
lines = cache.lines(source_root.joinpath(*parts))
|
|
2119
|
+
if lines is None:
|
|
2120
|
+
return None
|
|
2121
|
+
path = Path(*parts)
|
|
2122
|
+
top, _ = _line_matches(path.suffix.lower(), path, lines, name, cache, True)
|
|
2123
|
+
if top:
|
|
2124
|
+
return top[0]
|
|
2125
|
+
return 1 if module_fallback and _module_name(parts) == name else None
|
|
2126
|
+
|
|
2127
|
+
|
|
2128
|
+
# declared_line's `cache`, public for a script that loads this one.
|
|
2129
|
+
SourceCache = _SourceCache
|
|
2130
|
+
|
|
2131
|
+
|
|
2132
|
+
def _baseline_path(path: str) -> str:
|
|
2133
|
+
"""A cited path as an extraction writes it: forward slashes, no
|
|
2134
|
+
leading `./`."""
|
|
2135
|
+
rel = path.strip().replace("\\", "/")
|
|
2136
|
+
while rel.startswith("./"):
|
|
2137
|
+
rel = rel[2:]
|
|
2138
|
+
return rel
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def _declaration_refused(source_root: Path, file: str) -> bool:
|
|
2142
|
+
"""True when `declared_line` would not read a file that is there: a
|
|
2143
|
+
path outside the source root, a file the lookup skips (a test file, a
|
|
2144
|
+
minified bundle) or one with a symlink on its path. A file that is gone
|
|
2145
|
+
is not refused: its entries are deleted."""
|
|
2146
|
+
parts = _cited_parts(file)
|
|
2147
|
+
if parts is None:
|
|
2148
|
+
return True
|
|
2149
|
+
path = source_root.joinpath(*parts)
|
|
2150
|
+
return os.path.lexists(path) and not _readable_below(source_root, parts)
|
|
2151
|
+
|
|
2152
|
+
|
|
2153
|
+
def baseline_gaps(
|
|
2154
|
+
provenance: dict,
|
|
2155
|
+
source_root: Path,
|
|
2156
|
+
held: set[tuple[str, str]],
|
|
2157
|
+
files: set[str],
|
|
2158
|
+
listed: set[tuple] | frozenset = frozenset(),
|
|
2159
|
+
*,
|
|
2160
|
+
cache: _SourceCache | None = None,
|
|
2161
|
+
) -> tuple[list[dict], list[dict]]:
|
|
2162
|
+
"""The provenance entries an extraction left out whose cited file still
|
|
2163
|
+
declares them: the recipe runner never reports a module, an alias that
|
|
2164
|
+
renames, a dunder such as `__version__` or another underscore name, so
|
|
2165
|
+
a diff would read each as removed.
|
|
2166
|
+
|
|
2167
|
+
`held` holds the (name, file) of every export the extraction found,
|
|
2168
|
+
`files` the files whose entries are checked (the ones read), and
|
|
2169
|
+
`listed` the (name, file) of the gaps already listed elsewhere (the
|
|
2170
|
+
runner's own extraction gaps), each path with forward slashes. An
|
|
2171
|
+
entry is left out when its name is dotted (a member) and it is no
|
|
2172
|
+
`module` or `package` entry, when its file is not in `files`, when the
|
|
2173
|
+
extraction holds the name or its re-export target (`reexport_map`,
|
|
2174
|
+
else the entry's `reexported_as`) at that file, or when it is listed
|
|
2175
|
+
already. Each name and file is taken once.
|
|
2176
|
+
|
|
2177
|
+
Returns (gaps, unchecked). A gap is {"name", "file", "line", "entry":
|
|
2178
|
+
null}, plus the entry's `export_type` when it has one, at the line
|
|
2179
|
+
`declared_line` gives (only a `module` or `package` entry takes the
|
|
2180
|
+
module rule); an entry whose file declares it no more is no gap.
|
|
2181
|
+
`unchecked` holds {"name", "file", "export_type"} for each entry no
|
|
2182
|
+
rule can check, to read by eye: one whose file is neither Python nor
|
|
2183
|
+
TS/JS, one whose file the lookup will not read (outside the source
|
|
2184
|
+
root, a test file or a minified bundle, a symlink on its path), and a
|
|
2185
|
+
dotted `module` or `package` entry (`api.v1`). The entries are read
|
|
2186
|
+
file by file over one cache, so each file is read and parsed once."""
|
|
2187
|
+
checkable = PYTHON_EXTENSIONS | TSJS_EXTENSIONS
|
|
2188
|
+
renamed = extract_reexport_map(provenance)
|
|
2189
|
+
seen = set(listed)
|
|
2190
|
+
by_file: dict[str, list[tuple[str, object]]] = {}
|
|
2191
|
+
unchecked: list[dict] = []
|
|
2192
|
+
entries = provenance.get("entries")
|
|
2193
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
2194
|
+
if not isinstance(entry, dict):
|
|
2195
|
+
continue
|
|
2196
|
+
name, cited = entry.get("export_name"), entry.get("source_file")
|
|
2197
|
+
if not isinstance(name, str) or not isinstance(cited, str):
|
|
2198
|
+
continue
|
|
2199
|
+
name, file, kind = name.strip(), _baseline_path(cited), entry.get("export_type")
|
|
2200
|
+
module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2201
|
+
if not name or ("." in name and not module) or (name, file) in seen:
|
|
2202
|
+
continue
|
|
2203
|
+
if cited.replace("\\", "/") not in files and file not in files:
|
|
2204
|
+
continue
|
|
2205
|
+
if (name, file) in held or (renamed.get(name), file) in held:
|
|
2206
|
+
continue
|
|
2207
|
+
seen.add((name, file))
|
|
2208
|
+
if "." in name or Path(file).suffix.lower() not in checkable or _declaration_refused(source_root, file):
|
|
2209
|
+
unchecked.append({"name": name, "file": file, "export_type": kind})
|
|
2210
|
+
continue
|
|
2211
|
+
by_file.setdefault(file, []).append((name, kind))
|
|
2212
|
+
cache = cache if cache is not None else _SourceCache()
|
|
2213
|
+
gaps = []
|
|
2214
|
+
for file, named in by_file.items():
|
|
2215
|
+
for name, kind in named:
|
|
2216
|
+
module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2217
|
+
line = declared_line(file, name, source_root, cache=cache, module_fallback=module)
|
|
2218
|
+
if line is None:
|
|
2219
|
+
continue
|
|
2220
|
+
gap = {"name": name, "file": file, "line": line, "entry": None}
|
|
2221
|
+
if kind is not None:
|
|
2222
|
+
gap["export_type"] = kind
|
|
2223
|
+
gaps.append(gap)
|
|
2224
|
+
return gaps, unchecked
|
|
2225
|
+
|
|
2226
|
+
|
|
2227
|
+
class _DeclarationLookup:
|
|
2228
|
+
"""Where the source declares the stale names (classify-stale's
|
|
2229
|
+
`defined_at`): a column-0 declaration, an alias that renames included, a
|
|
2230
|
+
plain or star import and a re-export never counting.
|
|
2231
|
+
|
|
2232
|
+
A module or package named after the name (`_module_name`) declares it,
|
|
2233
|
+
at its line 1 when no line of it does, unless `find` takes the name as
|
|
2234
|
+
`strict` and the file is not a TS/JS one that is itself the export
|
|
2235
|
+
(`_exports_itself`). A file the name's entries cite that declares it
|
|
2236
|
+
gives its first such line. Otherwise the walk lists every file that
|
|
2237
|
+
declares it (`declared_in`), and exactly one must: the walk reads,
|
|
2238
|
+
sorted, the files below the walk root (`_walk_root`), leaving out the
|
|
2239
|
+
folders `_lookup_skips_folder` names, the `LOOKUP_WALK_FOLDERS`, the
|
|
2240
|
+
`LOOKUP_TOP_FOLDERS` right below the walk root and a package root
|
|
2241
|
+
(`_is_package_root`) that hold no cited file, the files
|
|
2242
|
+
`_lookup_skips_file` names and every symlink. Only the rule families
|
|
2243
|
+
(Python, TS/JS) the entries cite are read (both when no entry cites a
|
|
2244
|
+
file). A cited file is read inside the source root only, with no symlink
|
|
2245
|
+
on its path and the same skips but the walk-only ones, judged below the
|
|
2246
|
+
walk root when it lies there. The walk runs once for all the names no
|
|
2247
|
+
cited file settles: each file is read once, and only a file whose text
|
|
2248
|
+
holds a name goes through the rules. Any error of the filesystem raises
|
|
2249
|
+
(`_LOOKUP_ERRORS`)."""
|
|
2250
|
+
|
|
2251
|
+
def __init__(self, entries: list[dict], source_root: Path) -> None:
|
|
2252
|
+
self.root = source_root
|
|
2253
|
+
self.real_root = source_root.resolve()
|
|
2254
|
+
recorded = [f for f in map(_entry_file, entries) if f is not None]
|
|
2255
|
+
# a map that cites no file names no language: both families
|
|
2256
|
+
families = {_rule_family(f) for f in recorded}
|
|
2257
|
+
self.families = families - {None} if recorded else {"python", "tsjs"}
|
|
2258
|
+
self.folder = self._walk_root(recorded)
|
|
2259
|
+
# the folders that hold a cited file: a LOOKUP_TOP_FOLDERS one is walked
|
|
2260
|
+
self._cited_folders = {
|
|
2261
|
+
parts[:i]
|
|
2262
|
+
for parts in filter(None, map(_cited_parts, recorded))
|
|
2263
|
+
for i in range(1, len(parts))
|
|
2264
|
+
}
|
|
2265
|
+
# the cited files read, kept for the walk: a few files at most
|
|
2266
|
+
self._cited: dict[tuple[str, ...], list[str] | None] = {}
|
|
2267
|
+
self._cited_cache = _SourceCache()
|
|
2268
|
+
|
|
2269
|
+
# -- the walk root ------------------------------------------------------
|
|
2270
|
+
|
|
2271
|
+
def _symlinked(self, parts: tuple[str, ...]) -> bool:
|
|
2272
|
+
"""True when a path below the source root goes through a symlink,
|
|
2273
|
+
itself included."""
|
|
2274
|
+
return _symlinked(self.root, parts)
|
|
2275
|
+
|
|
2276
|
+
def _real_folder(self, parts: tuple[str, ...]) -> tuple[str, ...]:
|
|
2277
|
+
"""The deepest folder of a path below the source root that is one,
|
|
2278
|
+
with no symlink on its path."""
|
|
2279
|
+
while parts and (
|
|
2280
|
+
self._symlinked(parts) or not self.root.joinpath(*parts).is_dir()
|
|
2281
|
+
):
|
|
2282
|
+
parts = parts[:-1]
|
|
2283
|
+
return parts
|
|
2284
|
+
|
|
2285
|
+
def _holds(self, folder: tuple[str, ...], filename: str) -> bool:
|
|
2286
|
+
return (self.root.joinpath(*folder) / filename).is_file()
|
|
2287
|
+
|
|
2288
|
+
def _package_root(
|
|
2289
|
+
self, folder: tuple[str, ...], family: str
|
|
2290
|
+
) -> tuple[str, ...] | None:
|
|
2291
|
+
"""The root of the package a folder below the source root lies in,
|
|
2292
|
+
None when it lies in none. Python: the nearest folder at or above it
|
|
2293
|
+
that holds `__init__.py`, then up while the parent holds one, never
|
|
2294
|
+
the source root itself (a stray `__init__.py` there would make the
|
|
2295
|
+
whole tree one package). TS/JS: the nearest folder at or above it
|
|
2296
|
+
that holds `package.json`."""
|
|
2297
|
+
marker = _PACKAGE_MARKERS[family]
|
|
2298
|
+
python = family == "python"
|
|
2299
|
+
here = folder
|
|
2300
|
+
while not (self._holds(here, marker) and (here or not python)):
|
|
2301
|
+
if not here:
|
|
2302
|
+
return None
|
|
2303
|
+
here = here[:-1]
|
|
2304
|
+
if python:
|
|
2305
|
+
while len(here) > 1 and self._holds(here[:-1], marker):
|
|
2306
|
+
here = here[:-1]
|
|
2307
|
+
return here
|
|
2308
|
+
|
|
2309
|
+
def _walk_root(self, recorded: list[str]) -> tuple[str, ...]:
|
|
2310
|
+
"""The folder the walk reads: the deepest folder the entries' files
|
|
2311
|
+
share, widened to its package root, leaving out an entry outside
|
|
2312
|
+
every package (the source root when no entry lies in one)."""
|
|
2313
|
+
folders: list[tuple[str, ...]] = []
|
|
2314
|
+
widen: set[str] = set()
|
|
2315
|
+
for source_file in recorded:
|
|
2316
|
+
parts = _cited_parts(source_file)
|
|
2317
|
+
family = _rule_family(source_file) if parts else None
|
|
2318
|
+
if family is None:
|
|
2319
|
+
continue
|
|
2320
|
+
if self._package_root(self._real_folder(parts[:-1]), family) is None:
|
|
2321
|
+
continue # outside every package: a script, a docs page
|
|
2322
|
+
folders.append(parts[:-1])
|
|
2323
|
+
widen.add(family)
|
|
2324
|
+
if not folders:
|
|
2325
|
+
return ()
|
|
2326
|
+
shared = self._real_folder(tuple(os.path.commonprefix(folders)))
|
|
2327
|
+
roots = [self._package_root(shared, family) for family in sorted(widen)]
|
|
2328
|
+
return min((r for r in roots if r is not None), key=len, default=shared)
|
|
2329
|
+
|
|
2330
|
+
# -- reading ------------------------------------------------------------
|
|
2331
|
+
|
|
2332
|
+
def _below_walk_root(self, parts: tuple[str, ...]) -> tuple[str, ...]:
|
|
2333
|
+
depth = len(self.folder)
|
|
2334
|
+
return parts[depth:] if parts[:depth] == self.folder else parts
|
|
2335
|
+
|
|
2336
|
+
def _reads_cited(self, parts: tuple[str, ...]) -> bool:
|
|
2337
|
+
"""True when the lookup reads a cited file: one of a family the
|
|
2338
|
+
entries cite, in no skipped place, with no symlink on its path and
|
|
2339
|
+
inside the source root."""
|
|
2340
|
+
if _rule_family(parts[-1]) not in self.families:
|
|
2341
|
+
return False
|
|
2342
|
+
below = self._below_walk_root(parts)
|
|
2343
|
+
if any(_lookup_skips_folder(s) for s in below[:-1]):
|
|
2344
|
+
return False
|
|
2345
|
+
return _readable_below(self.root, parts, self.real_root)
|
|
2346
|
+
|
|
2347
|
+
def _cited_lines(self, parts: tuple[str, ...]) -> list[str] | None:
|
|
2348
|
+
if parts not in self._cited:
|
|
2349
|
+
readable = self._reads_cited(parts)
|
|
2350
|
+
path = self.root.joinpath(*parts)
|
|
2351
|
+
self._cited[parts] = _text_lines(_read_source(path)) if readable else None
|
|
2352
|
+
return self._cited[parts]
|
|
2353
|
+
|
|
2354
|
+
def _is_package_root(self, folder: tuple[str, ...], filenames: list[str]) -> bool:
|
|
2355
|
+
"""True when a walked folder (its `filenames`) is a package root of a
|
|
2356
|
+
family the walk reads, as `_package_root` finds one: TS/JS, a folder
|
|
2357
|
+
that holds `package.json`; Python, the top of an `__init__.py`
|
|
2358
|
+
chain, never the source root."""
|
|
2359
|
+
if "tsjs" in self.families and "package.json" in filenames:
|
|
2360
|
+
return True
|
|
2361
|
+
return (
|
|
2362
|
+
"python" in self.families and bool(folder) and "__init__.py" in filenames
|
|
2363
|
+
and (len(folder) == 1 or not self._holds(folder[:-1], "__init__.py"))
|
|
2364
|
+
)
|
|
2365
|
+
|
|
2366
|
+
def _tree(self) -> list[tuple[str, ...]]:
|
|
2367
|
+
"""The files the walk reads, as segments below the source root,
|
|
2368
|
+
sorted."""
|
|
2369
|
+
top = self.root.joinpath(*self.folder)
|
|
2370
|
+
found: list[tuple[str, ...]] = []
|
|
2371
|
+
|
|
2372
|
+
def fail(exc: OSError) -> None:
|
|
2373
|
+
raise exc
|
|
2374
|
+
|
|
2375
|
+
for dirpath, dirnames, filenames in os.walk(top, onerror=fail):
|
|
2376
|
+
here = Path(dirpath)
|
|
2377
|
+
inner = here.relative_to(top).parts
|
|
2378
|
+
below = self.folder + inner
|
|
2379
|
+
# a LOOKUP_TOP_FOLDERS name is skipped right below the walk root
|
|
2380
|
+
# and a package root, unless it holds a cited file
|
|
2381
|
+
root = not inner or self._is_package_root(below, filenames)
|
|
2382
|
+
dirnames[:] = [
|
|
2383
|
+
d for d in dirnames
|
|
2384
|
+
if not _lookup_skips_folder(d)
|
|
2385
|
+
and d.lower() not in LOOKUP_WALK_FOLDERS
|
|
2386
|
+
and not (
|
|
2387
|
+
root and d.lower() in LOOKUP_TOP_FOLDERS
|
|
2388
|
+
and (*below, d) not in self._cited_folders
|
|
2389
|
+
)
|
|
2390
|
+
and not (here / d).is_symlink()
|
|
2391
|
+
]
|
|
2392
|
+
for filename in filenames:
|
|
2393
|
+
path = here / filename
|
|
2394
|
+
if (
|
|
2395
|
+
_rule_family(filename) in self.families
|
|
2396
|
+
and not _lookup_skips_file(filename)
|
|
2397
|
+
and not path.is_symlink()
|
|
2398
|
+
and path.is_file()
|
|
2399
|
+
):
|
|
2400
|
+
found.append((*below, filename))
|
|
2401
|
+
return sorted(found)
|
|
2402
|
+
|
|
2403
|
+
@staticmethod
|
|
2404
|
+
def _first(
|
|
2405
|
+
parts: tuple[str, ...],
|
|
2406
|
+
lines: list[str],
|
|
2407
|
+
name: str,
|
|
2408
|
+
cache: _SourceCache,
|
|
2409
|
+
module_fallback: bool = True,
|
|
2410
|
+
) -> str | None:
|
|
2411
|
+
"""`file:line` of the first line of a file that declares `name`, else,
|
|
2412
|
+
with `module_fallback`, its line 1 when it is the module or package
|
|
2413
|
+
so named (`declared_line`)."""
|
|
2414
|
+
line = declared_line(
|
|
2415
|
+
parts, name, lines=lines, cache=cache, module_fallback=module_fallback
|
|
2416
|
+
)
|
|
2417
|
+
return f"{'/'.join(parts)}:{line}" if line is not None else None
|
|
2418
|
+
|
|
2419
|
+
# -- the answer ---------------------------------------------------------
|
|
2420
|
+
|
|
2421
|
+
def find(
|
|
2422
|
+
self, cited: dict[str, list[str]], strict: frozenset[str] = frozenset()
|
|
2423
|
+
) -> tuple[dict[str, str], dict[str, list[str]]]:
|
|
2424
|
+
"""For the names of `cited` (name -> the source_file values its
|
|
2425
|
+
entries record): name -> `file:line` for each one the source
|
|
2426
|
+
declares, and name -> the sorted `file:line` of every walked file
|
|
2427
|
+
that declares it, for each name the walk looked for (the names no
|
|
2428
|
+
cited file settles). A dotted name is never looked up, and a name of
|
|
2429
|
+
`strict` takes no line 1 from a module or package so named that is
|
|
2430
|
+
not itself the export (`_exports_itself`), in a cited file or in
|
|
2431
|
+
the walk."""
|
|
2432
|
+
found: dict[str, str] = {}
|
|
2433
|
+
if not self.families:
|
|
2434
|
+
return found, {}
|
|
2435
|
+
pending: list[str] = []
|
|
2436
|
+
for name, files in cited.items():
|
|
2437
|
+
if not name or "." in name:
|
|
2438
|
+
continue
|
|
2439
|
+
for parts in dict.fromkeys(filter(None, map(_cited_parts, files))):
|
|
2440
|
+
lines = self._cited_lines(parts)
|
|
2441
|
+
hit = None if lines is None else self._first(
|
|
2442
|
+
parts, lines, name, self._cited_cache,
|
|
2443
|
+
name not in strict or _exports_itself(parts, "\n".join(lines)),
|
|
2444
|
+
)
|
|
2445
|
+
if hit is not None:
|
|
2446
|
+
found[name] = hit
|
|
2447
|
+
break
|
|
2448
|
+
else:
|
|
2449
|
+
pending.append(name)
|
|
2450
|
+
declared, unsure = self._walk(pending, strict) if pending else ({}, set())
|
|
2451
|
+
# exactly one walked file must declare a name, and none it holds
|
|
2452
|
+
# in an untokenizable Python file may
|
|
2453
|
+
found.update({
|
|
2454
|
+
name: h[0] for name, h in declared.items()
|
|
2455
|
+
if len(h) == 1 and name not in unsure
|
|
2456
|
+
})
|
|
2457
|
+
return found, declared
|
|
2458
|
+
|
|
2459
|
+
def _walk(
|
|
2460
|
+
self, names: list[str], strict: frozenset[str] = frozenset()
|
|
2461
|
+
) -> tuple[dict[str, list[str]], set[str]]:
|
|
2462
|
+
"""name -> the sorted `file:line` of each walked file that declares
|
|
2463
|
+
it, at the file's first declaration (a stub beside its
|
|
2464
|
+
implementation left out: `_without_stubs`; a module so named
|
|
2465
|
+
declaring no name of `strict` unless it is the export), and the
|
|
2466
|
+
names a Python file that
|
|
2467
|
+
does not tokenize holds, which it may declare."""
|
|
2468
|
+
hits: dict[str, list[str]] = {name: [] for name in names}
|
|
2469
|
+
unsure: set[str] = set()
|
|
2470
|
+
for parts in self._tree():
|
|
2471
|
+
module = _module_name(parts)
|
|
2472
|
+
lines = self._cited.get(parts)
|
|
2473
|
+
text = (
|
|
2474
|
+
_read_source(self.root.joinpath(*parts))
|
|
2475
|
+
if lines is None
|
|
2476
|
+
else "\n".join(lines) # a cited file, read already
|
|
2477
|
+
)
|
|
2478
|
+
# a strict name's module so named counts when it is the export
|
|
2479
|
+
fallback = module not in strict or _exports_itself(parts, text)
|
|
2480
|
+
# the substring test spares the rules most files and names
|
|
2481
|
+
here = sorted(
|
|
2482
|
+
n for n in names if n in text or (n == module and fallback)
|
|
2483
|
+
)
|
|
2484
|
+
if not here:
|
|
2485
|
+
continue
|
|
2486
|
+
if lines is None:
|
|
2487
|
+
lines = _text_lines(text)
|
|
2488
|
+
cache = _SourceCache() # this file's rules only: nothing kept
|
|
2489
|
+
path = Path(*parts)
|
|
2490
|
+
if path.suffix.lower() in PYTHON_EXTENSIONS and (
|
|
2491
|
+
cache.py_statement_starts(path, lines) is None
|
|
2492
|
+
):
|
|
2493
|
+
unsure.update(n for n in here if n in text)
|
|
2494
|
+
for name in here:
|
|
2495
|
+
hit = self._first(parts, lines, name, cache, fallback)
|
|
2496
|
+
if hit is not None:
|
|
2497
|
+
hits[name].append(hit)
|
|
2498
|
+
return {name: sorted(_without_stubs(h)) for name, h in hits.items()}, unsure
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
def inventory_kinds(data: object, path: str) -> dict[str, frozenset[str]]:
|
|
2502
|
+
"""name -> the kinds a documented inventory gives it (`--inventory`):
|
|
2503
|
+
validate-inventory.py's `--output` file, `{"exports": [...]}`, or its
|
|
2504
|
+
whole result, the inventory under `inventory`. An item without a string
|
|
2505
|
+
`name` and `kind` is left out, and a kind is kept in lower case. Raises
|
|
2506
|
+
ValueError when no `exports` list is there."""
|
|
2507
|
+
if isinstance(data, dict) and isinstance(data.get("inventory"), dict):
|
|
2508
|
+
data = data["inventory"] # validate-inventory.py's whole result
|
|
2509
|
+
if not isinstance(data, dict) or not isinstance(data.get("exports"), list):
|
|
2510
|
+
raise ValueError(f"--inventory {path} holds no `exports` list")
|
|
2511
|
+
kinds: dict[str, set[str]] = {}
|
|
2512
|
+
for item in data["exports"]:
|
|
2513
|
+
if not isinstance(item, dict):
|
|
2514
|
+
continue
|
|
2515
|
+
name, kind = item.get("name"), item.get("kind")
|
|
2516
|
+
if isinstance(name, str) and isinstance(kind, str):
|
|
2517
|
+
kinds.setdefault(name, set()).add(kind.strip().lower())
|
|
2518
|
+
return {name: frozenset(k) for name, k in kinds.items()}
|
|
2519
|
+
|
|
2520
|
+
|
|
2521
|
+
def _module_entry(entry: dict) -> bool:
|
|
2522
|
+
"""True for a `module` or `package` entry."""
|
|
2523
|
+
kind = entry.get("export_type")
|
|
2524
|
+
return isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
|
|
2525
|
+
|
|
2526
|
+
|
|
2527
|
+
def classify_stale(
|
|
2528
|
+
names: list[str],
|
|
2529
|
+
prov: dict,
|
|
2530
|
+
source_root: Path,
|
|
2531
|
+
kinds: dict[str, frozenset[str]] | None = None,
|
|
2532
|
+
) -> list[dict]:
|
|
2533
|
+
"""The fabricated-signature test for each stale name: the definition-lines
|
|
2534
|
+
rules at every provenance entry whose raw `export_name` is the name, and
|
|
2535
|
+
for a name that is not fabricated, where the source declares it
|
|
2536
|
+
(`defined_at`, and `declared_in` when the walk looked for it). The
|
|
2537
|
+
classify-stale section of the module docstring gives each reason, what
|
|
2538
|
+
a `no-file` entry means, which entry gives `source` and how
|
|
2539
|
+
`defined_at` and `declared_in` are found. `kinds` (`inventory_kinds`,
|
|
2540
|
+
None without `--inventory`) keeps a name of a `NO_MODULE_KINDS` kind
|
|
2541
|
+
from a module or package so named, unless an entry of the name is a
|
|
2542
|
+
`module` or `package` one.
|
|
2543
|
+
"""
|
|
2544
|
+
entries = [e for e in prov.get("entries") or [] if isinstance(e, dict)]
|
|
2545
|
+
results: list[dict] = []
|
|
2546
|
+
# the names to look up (never a fabricated one): name -> its cited files
|
|
2547
|
+
cited: dict[str, list[str]] = {}
|
|
2548
|
+
# the names a module so named does not declare
|
|
2549
|
+
strict: set[str] = set()
|
|
2550
|
+
for name in names:
|
|
2551
|
+
mine = [e for e in entries if e.get("export_name") == name]
|
|
2552
|
+
checks: list[dict] = []
|
|
2553
|
+
for entry in mine:
|
|
2554
|
+
line_check, defs = _stale_line_check(entry, name, source_root)
|
|
2555
|
+
checks.append(
|
|
2556
|
+
{
|
|
2557
|
+
"source_file": entry.get("source_file"),
|
|
2558
|
+
"source_line": entry.get("source_line"),
|
|
2559
|
+
"line_check": line_check,
|
|
2560
|
+
"definition_lines": defs,
|
|
2561
|
+
}
|
|
2562
|
+
)
|
|
2563
|
+
if not checks:
|
|
2564
|
+
reason = STALE_NO_ENTRY
|
|
2565
|
+
elif all(
|
|
2566
|
+
c["line_check"] == "file-missing"
|
|
2567
|
+
or (c["line_check"] == "checked" and not c["definition_lines"])
|
|
2568
|
+
for c in checks
|
|
2569
|
+
):
|
|
2570
|
+
reason = STALE_NOT_DEFINED
|
|
2571
|
+
elif any(
|
|
2572
|
+
c["line_check"] == "checked" and c["definition_lines"] for c in checks
|
|
2573
|
+
):
|
|
2574
|
+
reason = STALE_DEFINED
|
|
2575
|
+
else:
|
|
2576
|
+
reason = STALE_UNCHECKED
|
|
2577
|
+
source = next(filter(None, map(_entry_citation, mine)), None)
|
|
2578
|
+
fabricated = reason == STALE_NOT_DEFINED
|
|
2579
|
+
if not fabricated:
|
|
2580
|
+
cited[name] = [f for f in map(_entry_file, mine) if f is not None]
|
|
2581
|
+
documented = (kinds or {}).get(name, frozenset())
|
|
2582
|
+
if documented & NO_MODULE_KINDS and not any(map(_module_entry, mine)):
|
|
2583
|
+
strict.add(name)
|
|
2584
|
+
results.append(
|
|
2585
|
+
{
|
|
2586
|
+
"name": name,
|
|
2587
|
+
"fabricated": fabricated,
|
|
2588
|
+
"source": source,
|
|
2589
|
+
"defined_at": None,
|
|
2590
|
+
"declared_in": None,
|
|
2591
|
+
"reason": reason,
|
|
2592
|
+
"entries": checks,
|
|
2593
|
+
}
|
|
2594
|
+
)
|
|
2595
|
+
try:
|
|
2596
|
+
found, declared = _DeclarationLookup(entries, source_root).find(
|
|
2597
|
+
cited, frozenset(strict)
|
|
2598
|
+
)
|
|
2599
|
+
except _LOOKUP_ERRORS:
|
|
2600
|
+
# a lookup that fails proves nothing: every name stays null
|
|
2601
|
+
found, declared = {}, {}
|
|
2602
|
+
for item in results:
|
|
2603
|
+
item["defined_at"] = found.get(item["name"])
|
|
2604
|
+
item["declared_in"] = declared.get(item["name"])
|
|
2605
|
+
return results
|
|
2606
|
+
|
|
2607
|
+
|
|
2608
|
+
# --------------------------------------------------------------------------
|
|
2609
|
+
# Skill citations ([AST:] / [SRC:] prefixes)
|
|
2610
|
+
# --------------------------------------------------------------------------
|
|
2611
|
+
|
|
2612
|
+
# `[AST:path:L12]`, `[SRC:path:L12-34]`, `[SRC:path:L12-L34]`. A citation
|
|
2613
|
+
# with no `:L<n>` part does not match and is ignored. Groups: prefix, path,
|
|
2614
|
+
# first line, the `L` of a range's end (may be empty), the range's end.
|
|
2615
|
+
_SKILL_CITATION_RE = re.compile(
|
|
2616
|
+
r"\[(AST|SRC):([^\[\]\n]+?):L(\d+)(?:-(L?)(\d+))?\]"
|
|
2617
|
+
)
|
|
2618
|
+
|
|
2619
|
+
|
|
2620
|
+
def normalize_path(path: str) -> str:
|
|
2621
|
+
"""Normalize a cited or recorded source path for comparison."""
|
|
2622
|
+
p = path.strip().replace("\\", "/")
|
|
2623
|
+
while p.startswith("./"):
|
|
2624
|
+
p = p[2:]
|
|
2625
|
+
return posixpath.normpath(p) if p else p
|
|
2626
|
+
|
|
2627
|
+
|
|
2628
|
+
def skill_markdown_files(skill_dir: Path) -> list[Path]:
|
|
2629
|
+
"""`SKILL.md` then every `references/**/*.md`, in a stable order."""
|
|
2630
|
+
files: list[Path] = []
|
|
2631
|
+
skill_md = skill_dir / "SKILL.md"
|
|
2632
|
+
if skill_md.is_file():
|
|
2633
|
+
files.append(skill_md)
|
|
2634
|
+
refs = skill_dir / "references"
|
|
2635
|
+
if refs.is_dir():
|
|
2636
|
+
files.extend(sorted(p for p in refs.rglob("*.md") if p.is_file()))
|
|
2637
|
+
return files
|
|
2638
|
+
|
|
2639
|
+
|
|
2640
|
+
def scan_skill_citations(skill_dir: Path) -> list[dict]:
|
|
2641
|
+
"""Every `[AST:]` / `[SRC:]` citation with a line, in file order."""
|
|
2642
|
+
found: list[dict] = []
|
|
2643
|
+
for md in skill_markdown_files(skill_dir):
|
|
2644
|
+
rel = md.relative_to(skill_dir).as_posix()
|
|
2645
|
+
lines = _read_lines(md) or []
|
|
2646
|
+
for number, text in enumerate(lines, start=1):
|
|
2647
|
+
for m in _SKILL_CITATION_RE.finditer(text):
|
|
2648
|
+
found.append(
|
|
2649
|
+
{
|
|
2650
|
+
"file": rel,
|
|
2651
|
+
"line": number,
|
|
2652
|
+
"column": m.start() + 1,
|
|
2653
|
+
"citation": m.group(0),
|
|
2654
|
+
"prefix": m.group(1),
|
|
2655
|
+
"cited_file": m.group(2).strip(),
|
|
2656
|
+
"cited_line": int(m.group(3)),
|
|
2657
|
+
}
|
|
2658
|
+
)
|
|
2659
|
+
return found
|
|
2660
|
+
|
|
2661
|
+
|
|
2662
|
+
def _extraction_method(entry: dict) -> str | None:
|
|
2663
|
+
"""The entry's `extraction_method`, or None when it is not a string
|
|
2664
|
+
(a list or object there reads as an unknown method, never a crash)."""
|
|
2665
|
+
method = entry.get("extraction_method")
|
|
2666
|
+
return method if isinstance(method, str) else None
|
|
2667
|
+
|
|
2668
|
+
|
|
2669
|
+
def check_skill_citations(
|
|
2670
|
+
prov: dict, skill_dir: Path
|
|
2671
|
+
) -> tuple[list[dict], int, int]:
|
|
2672
|
+
"""Compare each skill citation's prefix with the entry it cites.
|
|
2673
|
+
|
|
2674
|
+
Returns (findings, citations_scanned, citations_matched): a citation is
|
|
2675
|
+
matched when its path and line name at least one provenance entry.
|
|
2676
|
+
"""
|
|
2677
|
+
entries = prov.get("entries")
|
|
2678
|
+
entries = entries if isinstance(entries, list) else []
|
|
2679
|
+
by_location: dict[tuple[str, int], list[dict]] = {}
|
|
2680
|
+
has_ast_entry = False
|
|
2681
|
+
# `ast-without-ast-grep` needs proof that no entry came from ast-grep:
|
|
2682
|
+
# every entry with a source line must carry a known method.
|
|
2683
|
+
methods_known = True
|
|
2684
|
+
for entry in entries:
|
|
2685
|
+
if not isinstance(entry, dict):
|
|
2686
|
+
continue
|
|
2687
|
+
method = _extraction_method(entry)
|
|
2688
|
+
if method in AST_METHODS:
|
|
2689
|
+
has_ast_entry = True
|
|
2690
|
+
sf = entry.get("source_file")
|
|
2691
|
+
line = _coerce_line(entry.get("source_line"))
|
|
2692
|
+
if not isinstance(sf, str) or not sf.strip() or not isinstance(line, int):
|
|
2693
|
+
continue
|
|
2694
|
+
if method not in KNOWN_METHODS:
|
|
2695
|
+
methods_known = False
|
|
2696
|
+
by_location.setdefault((normalize_path(sf), line), []).append(entry)
|
|
2697
|
+
|
|
2698
|
+
scanned = sorted(
|
|
2699
|
+
scan_skill_citations(skill_dir),
|
|
2700
|
+
key=lambda c: (c["file"], c["line"], c["column"]),
|
|
2701
|
+
)
|
|
2702
|
+
findings: list[dict] = []
|
|
2703
|
+
matched_count = 0
|
|
2704
|
+
for cit in scanned:
|
|
2705
|
+
prefix = cit["prefix"]
|
|
2706
|
+
other = "SRC" if prefix == "AST" else "AST"
|
|
2707
|
+
matched = by_location.get(
|
|
2708
|
+
(normalize_path(cit["cited_file"]), cit["cited_line"]), []
|
|
2709
|
+
)
|
|
2710
|
+
if matched:
|
|
2711
|
+
matched_count += 1
|
|
2712
|
+
implied = [(METHOD_PREFIX.get(_extraction_method(e)), e) for e in matched]
|
|
2713
|
+
agrees = any(p == prefix for p, _ in implied)
|
|
2714
|
+
mismatch = next((e for p, e in implied if p == other), None)
|
|
2715
|
+
reason = None
|
|
2716
|
+
export_name = None
|
|
2717
|
+
if not agrees and mismatch is not None:
|
|
2718
|
+
reason = CITATION_PREFIX_MISMATCH
|
|
2719
|
+
export_name = mismatch.get("export_name")
|
|
2720
|
+
elif prefix == "AST" and not has_ast_entry and methods_known:
|
|
2721
|
+
reason = CITATION_AST_WITHOUT_AST_GREP
|
|
2722
|
+
if matched:
|
|
2723
|
+
export_name = matched[0].get("export_name")
|
|
2724
|
+
if reason is None:
|
|
2725
|
+
continue
|
|
2726
|
+
item = {k: v for k, v in cit.items() if k != "column"}
|
|
2727
|
+
item.update(
|
|
2728
|
+
{
|
|
2729
|
+
"reason": reason,
|
|
2730
|
+
"expected_prefix": other,
|
|
2731
|
+
"export_name": export_name
|
|
2732
|
+
if isinstance(export_name, str) and export_name
|
|
2733
|
+
else None,
|
|
2734
|
+
}
|
|
2735
|
+
)
|
|
2736
|
+
findings.append(item)
|
|
2737
|
+
return findings, len(scanned), matched_count
|
|
2738
|
+
|
|
2739
|
+
|
|
2740
|
+
# --------------------------------------------------------------------------
|
|
2741
|
+
# Node kinds (--check-node-kinds)
|
|
2742
|
+
# --------------------------------------------------------------------------
|
|
2743
|
+
|
|
2744
|
+
|
|
2745
|
+
def _resolve_outside_cwd(command: str) -> str | None:
|
|
2746
|
+
"""shutil.which with a CWD-shim guard. Returns the resolved path or None.
|
|
2747
|
+
|
|
2748
|
+
shutil.which on Windows searches the current directory ahead of PATH,
|
|
2749
|
+
and CWD here may be a repository SKF does not control: a bare-name
|
|
2750
|
+
lookup resolving into CWD would execute a planted shim (e.g.
|
|
2751
|
+
ast-grep.cmd). Such a resolution is treated as not-found. Explicit
|
|
2752
|
+
paths supplied by callers (containing a separator) are honored as-is.
|
|
2753
|
+
Keep the code identical to the sibling guards in
|
|
2754
|
+
skf-merge-ccc-exclusions.py, skf-detect-tools.py,
|
|
2755
|
+
skf-qmd-classify-collections.py, skf-ccc-git-hygiene.py,
|
|
2756
|
+
skf-source-tree.py and skf-tessl-review.py
|
|
2757
|
+
(test/test-skf-verify-provenance-completeness.py pins it against
|
|
2758
|
+
skf-merge-ccc-exclusions.py).
|
|
2759
|
+
"""
|
|
2760
|
+
resolved = shutil.which(command)
|
|
2761
|
+
if resolved is None:
|
|
2762
|
+
return None
|
|
2763
|
+
if os.sep in command or (os.altsep and os.altsep in command):
|
|
2764
|
+
return resolved
|
|
2765
|
+
resolved_dir = os.path.dirname(resolved)
|
|
2766
|
+
if resolved_dir:
|
|
2767
|
+
cwd = os.path.normcase(os.path.abspath(os.getcwd()))
|
|
2768
|
+
if os.path.normcase(os.path.abspath(resolved_dir)) == cwd:
|
|
2769
|
+
return None
|
|
2770
|
+
return resolved
|
|
2771
|
+
|
|
2772
|
+
|
|
2773
|
+
def ast_grep_language(source_file: object) -> str | None:
|
|
2774
|
+
"""The ast-grep language for `source_file`'s extension, or None."""
|
|
2775
|
+
if not isinstance(source_file, str) or not source_file.strip():
|
|
2776
|
+
return None
|
|
2777
|
+
ext = posixpath.splitext(normalize_path(source_file))[1].lower()
|
|
2778
|
+
return AST_GREP_LANGUAGES.get(ext)
|
|
2779
|
+
|
|
2780
|
+
|
|
2781
|
+
def ast_grep_judges_kind(exe: str, language: str, kind: str, folder: str) -> str:
|
|
2782
|
+
"""Ask ast-grep whether `kind` is a node kind of `language`.
|
|
2783
|
+
|
|
2784
|
+
Runs one rule that matches only `kind` over an empty stdin: ast-grep
|
|
2785
|
+
parses the rule before it reads input and rejects a kind the grammar
|
|
2786
|
+
does not have. Returns "valid", "invalid" (the rule was rejected and the
|
|
2787
|
+
error names an invalid kind), "timeout", or "unknown" (a failed spawn or
|
|
2788
|
+
any other error, which proves nothing about the kind). `folder` holds
|
|
2789
|
+
the minimal sgconfig.yml passed with `--config` and is the working
|
|
2790
|
+
folder, so no other sgconfig.yml is read. The rule is JSON, which YAML
|
|
2791
|
+
reads as a flow mapping.
|
|
2792
|
+
"""
|
|
2793
|
+
rule = json.dumps(
|
|
2794
|
+
{"id": "skf-node-kind", "language": language, "rule": {"kind": kind}}
|
|
2795
|
+
)
|
|
2796
|
+
config = os.path.join(folder, "sgconfig.yml")
|
|
2797
|
+
try:
|
|
2798
|
+
result = subprocess.run(
|
|
2799
|
+
[exe, "scan", "--config", config, "--inline-rules", rule, "--stdin"],
|
|
2800
|
+
input="",
|
|
2801
|
+
capture_output=True,
|
|
2802
|
+
encoding="utf-8",
|
|
2803
|
+
errors="replace",
|
|
2804
|
+
timeout=NODE_KIND_TIMEOUT_SEC,
|
|
2805
|
+
cwd=folder,
|
|
2806
|
+
check=False,
|
|
2807
|
+
)
|
|
2808
|
+
except subprocess.TimeoutExpired:
|
|
2809
|
+
return "timeout"
|
|
2810
|
+
except (OSError, subprocess.SubprocessError):
|
|
2811
|
+
return "unknown"
|
|
2812
|
+
if result.returncode == 0:
|
|
2813
|
+
return "valid"
|
|
2814
|
+
if "invalid kind" in (result.stderr or "").lower():
|
|
2815
|
+
return "invalid"
|
|
2816
|
+
return "unknown"
|
|
2817
|
+
|
|
2818
|
+
|
|
2819
|
+
def _judge_kinds(
|
|
2820
|
+
exe: str, pairs: list[tuple[str, str]]
|
|
2821
|
+
) -> tuple[str, dict[tuple[str, str], str]]:
|
|
2822
|
+
"""Ask ast-grep about each (language, kind) pair, in order.
|
|
2823
|
+
|
|
2824
|
+
Returns (node_kind_check, verdicts): each verdict is "valid", "invalid"
|
|
2825
|
+
or the reason the pair went unjudged. ast-grep is first asked about a
|
|
2826
|
+
kind no grammar has; unless it answers "invalid", its answers cannot be
|
|
2827
|
+
read and the check is "skipped-unrecognized-ast-grep" with no verdicts.
|
|
2828
|
+
After one timeout the remaining pairs are not asked. When the temporary
|
|
2829
|
+
folder cannot be made or written, the pairs not yet asked go unjudged.
|
|
2830
|
+
"""
|
|
2831
|
+
verdicts: dict[tuple[str, str], str] = {}
|
|
2832
|
+
try:
|
|
2833
|
+
with tempfile.TemporaryDirectory(
|
|
2834
|
+
prefix="skf-node-kind-", ignore_cleanup_errors=True
|
|
2835
|
+
) as folder:
|
|
2836
|
+
with open(
|
|
2837
|
+
os.path.join(folder, "sgconfig.yml"), "w", encoding="utf-8"
|
|
2838
|
+
) as fh:
|
|
2839
|
+
fh.write(MINIMAL_SGCONFIG)
|
|
2840
|
+
calibration = ast_grep_judges_kind(
|
|
2841
|
+
exe, pairs[0][0], BOGUS_NODE_KIND, folder
|
|
2842
|
+
)
|
|
2843
|
+
if calibration != "invalid":
|
|
2844
|
+
return "skipped-unrecognized-ast-grep", {}
|
|
2845
|
+
timed_out = False
|
|
2846
|
+
for language, kind in pairs:
|
|
2847
|
+
if timed_out:
|
|
2848
|
+
verdicts[(language, kind)] = UNCHECKED_TIMEOUT
|
|
2849
|
+
continue
|
|
2850
|
+
verdict = ast_grep_judges_kind(exe, language, kind, folder)
|
|
2851
|
+
if verdict == "timeout":
|
|
2852
|
+
timed_out = True
|
|
2853
|
+
verdict = UNCHECKED_TIMEOUT
|
|
2854
|
+
elif verdict == "unknown":
|
|
2855
|
+
verdict = UNCHECKED_AST_GREP_ERROR
|
|
2856
|
+
verdicts[(language, kind)] = verdict
|
|
2857
|
+
except OSError:
|
|
2858
|
+
for pair in pairs:
|
|
2859
|
+
verdicts.setdefault(pair, UNCHECKED_TEMP_FOLDER)
|
|
2860
|
+
return "checked", verdicts
|
|
2861
|
+
|
|
2862
|
+
|
|
2863
|
+
def _node_kind_item(
|
|
2864
|
+
index: int, entry: dict, kind: str, language: str, reason: str
|
|
2865
|
+
) -> dict:
|
|
2866
|
+
name = entry.get("export_name")
|
|
2867
|
+
return {
|
|
2868
|
+
"export_name": name if isinstance(name, str) and name else None,
|
|
2869
|
+
"entry_index": index,
|
|
2870
|
+
"source_file": entry.get("source_file"),
|
|
2871
|
+
"source_line": entry.get("source_line"),
|
|
2872
|
+
"ast_node_type": kind,
|
|
2873
|
+
"language": language,
|
|
2874
|
+
"reason": reason,
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2877
|
+
|
|
2878
|
+
def check_node_kinds(prov: dict) -> tuple[list[dict], list[dict], dict]:
|
|
2879
|
+
"""Check each ast-grep entry's `ast_node_type` against ast-grep itself.
|
|
2880
|
+
|
|
2881
|
+
Returns (findings, unchecked, summary fields). `ERROR` and a kind that
|
|
2882
|
+
is not kind-shaped are judged without ast-grep, whatever the binary
|
|
2883
|
+
lookup finds; every other kind is asked once per (language, kind).
|
|
2884
|
+
"""
|
|
2885
|
+
entries = prov.get("entries")
|
|
2886
|
+
entries = entries if isinstance(entries, list) else []
|
|
2887
|
+
findings: list[dict] = []
|
|
2888
|
+
unchecked: list[dict] = []
|
|
2889
|
+
pending: list[tuple[int, dict, str, str]] = []
|
|
2890
|
+
judged = 0
|
|
2891
|
+
skipped = 0
|
|
2892
|
+
for index, entry in enumerate(entries):
|
|
2893
|
+
if not isinstance(entry, dict):
|
|
2894
|
+
continue
|
|
2895
|
+
if _extraction_method(entry) not in AST_METHODS:
|
|
2896
|
+
continue
|
|
2897
|
+
kind = entry.get("ast_node_type")
|
|
2898
|
+
if not isinstance(kind, str) or not kind.strip():
|
|
2899
|
+
continue
|
|
2900
|
+
language = ast_grep_language(entry.get("source_file"))
|
|
2901
|
+
if language is None:
|
|
2902
|
+
skipped += 1
|
|
2903
|
+
continue
|
|
2904
|
+
if kind == ERROR_NODE_KIND:
|
|
2905
|
+
reason = NODE_KIND_ERROR
|
|
2906
|
+
elif not NODE_KIND_SHAPE.fullmatch(kind):
|
|
2907
|
+
reason = NODE_KIND_INVALID
|
|
2908
|
+
else:
|
|
2909
|
+
pending.append((index, entry, language, kind))
|
|
2910
|
+
continue
|
|
2911
|
+
judged += 1
|
|
2912
|
+
findings.append(_node_kind_item(index, entry, kind, language, reason))
|
|
2913
|
+
|
|
2914
|
+
check = "checked"
|
|
2915
|
+
if pending:
|
|
2916
|
+
exe = _resolve_outside_cwd("ast-grep")
|
|
2917
|
+
if exe is None:
|
|
2918
|
+
check = "skipped-no-ast-grep"
|
|
2919
|
+
else:
|
|
2920
|
+
pairs = sorted({(lang, kind) for _, _, lang, kind in pending})
|
|
2921
|
+
check, verdicts = _judge_kinds(exe, pairs)
|
|
2922
|
+
if check == "checked":
|
|
2923
|
+
for index, entry, language, kind in pending:
|
|
2924
|
+
verdict = verdicts[(language, kind)]
|
|
2925
|
+
item = _node_kind_item(index, entry, kind, language, verdict)
|
|
2926
|
+
if verdict == "valid":
|
|
2927
|
+
judged += 1
|
|
2928
|
+
elif verdict == "invalid":
|
|
2929
|
+
judged += 1
|
|
2930
|
+
item["reason"] = NODE_KIND_INVALID
|
|
2931
|
+
findings.append(item)
|
|
2932
|
+
else:
|
|
2933
|
+
unchecked.append(item)
|
|
2934
|
+
|
|
2935
|
+
def order(item: dict) -> tuple:
|
|
2936
|
+
return (item["export_name"] or "", str(item["source_file"]),
|
|
2937
|
+
item["entry_index"])
|
|
2938
|
+
|
|
2939
|
+
findings.sort(key=order)
|
|
2940
|
+
unchecked.sort(key=order)
|
|
2941
|
+
summary = {
|
|
2942
|
+
"node_kind_check": check,
|
|
2943
|
+
"node_kinds_checked": judged,
|
|
2944
|
+
"node_kind_check_skipped": skipped,
|
|
2945
|
+
"node_kind_check_errors": len(unchecked),
|
|
2946
|
+
"node_kind_findings_count": len(findings),
|
|
2947
|
+
}
|
|
2948
|
+
return findings, unchecked, summary
|
|
2949
|
+
|
|
2950
|
+
|
|
2951
|
+
# --------------------------------------------------------------------------
|
|
2952
|
+
# The recipe kind at a line (kind-at)
|
|
2953
|
+
# --------------------------------------------------------------------------
|
|
2954
|
+
|
|
2955
|
+
|
|
2956
|
+
def _is_recipe(doc: object) -> bool:
|
|
2957
|
+
"""A recipe: a mapping with a string `id` and `language`, and a `rule`
|
|
2958
|
+
mapping that names its node `kind`."""
|
|
2959
|
+
if not isinstance(doc, dict):
|
|
2960
|
+
return False
|
|
2961
|
+
rule = doc.get("rule")
|
|
2962
|
+
return (
|
|
2963
|
+
isinstance(doc.get("id"), str)
|
|
2964
|
+
and isinstance(doc.get("language"), str)
|
|
2965
|
+
and isinstance(rule, dict)
|
|
2966
|
+
and isinstance(rule.get("kind"), str)
|
|
2967
|
+
)
|
|
2968
|
+
|
|
2969
|
+
|
|
2970
|
+
def load_recipes(path: Path) -> list[dict]:
|
|
2971
|
+
"""Every ast-grep recipe in `path`, first copy of each (id, language).
|
|
2972
|
+
|
|
2973
|
+
A markdown file (`.md`) gives the recipes in its ```yaml fences; any
|
|
2974
|
+
other file is read as YAML: one recipe per document, a list of recipes,
|
|
2975
|
+
or a mapping whose `recipes` key holds that list. Raises ValueError when
|
|
2976
|
+
the file cannot be read, and ImportError without PyYAML.
|
|
2977
|
+
"""
|
|
2978
|
+
import yaml # only kind-at reads recipes; `uv run` installs it
|
|
2979
|
+
|
|
2980
|
+
try:
|
|
2981
|
+
text = path.read_text(encoding="utf-8")
|
|
2982
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
2983
|
+
raise ValueError(f"failed to read recipes {path}: {exc}") from exc
|
|
2984
|
+
docs: list[object] = []
|
|
2985
|
+
if path.suffix.lower() in (".md", ".markdown"):
|
|
2986
|
+
for block in _MD_YAML_FENCE_RE.findall(text):
|
|
2987
|
+
try:
|
|
2988
|
+
docs.append(yaml.safe_load(block))
|
|
2989
|
+
except yaml.YAMLError:
|
|
2990
|
+
continue # a template, not a recipe
|
|
2991
|
+
else:
|
|
2992
|
+
try:
|
|
2993
|
+
docs = list(yaml.safe_load_all(text))
|
|
2994
|
+
except yaml.YAMLError as exc:
|
|
2995
|
+
raise ValueError(f"malformed YAML in recipes {path}: {exc}") from exc
|
|
2996
|
+
candidates: list[object] = []
|
|
2997
|
+
for doc in docs:
|
|
2998
|
+
if isinstance(doc, dict) and isinstance(doc.get("recipes"), list):
|
|
2999
|
+
candidates.extend(doc["recipes"])
|
|
3000
|
+
elif isinstance(doc, list):
|
|
3001
|
+
candidates.extend(doc)
|
|
3002
|
+
else:
|
|
3003
|
+
candidates.append(doc)
|
|
3004
|
+
recipes: list[dict] = []
|
|
3005
|
+
seen: set[tuple[str, str]] = set()
|
|
3006
|
+
for doc in candidates:
|
|
3007
|
+
if not _is_recipe(doc):
|
|
3008
|
+
continue
|
|
3009
|
+
key = (doc["id"], doc["language"].strip().lower())
|
|
3010
|
+
if key not in seen:
|
|
3011
|
+
seen.add(key)
|
|
3012
|
+
recipes.append(doc)
|
|
3013
|
+
return recipes
|
|
3014
|
+
|
|
3015
|
+
|
|
3016
|
+
def recipe_runs(recipes: list[dict], language: str) -> list[dict]:
|
|
3017
|
+
"""The recipes to run on a file of `language`, each with its `language`
|
|
3018
|
+
set to it: an id's own form for that language, else its form for the
|
|
3019
|
+
first language `RECIPE_LANGUAGE_FALLBACKS` lists (in recipe order)."""
|
|
3020
|
+
forms: dict[str, dict[str, dict]] = {}
|
|
3021
|
+
for recipe in recipes:
|
|
3022
|
+
lang = recipe["language"].strip().lower()
|
|
3023
|
+
forms.setdefault(recipe["id"], {}).setdefault(lang, recipe)
|
|
3024
|
+
runs: list[dict] = []
|
|
3025
|
+
for recipe_id, by_language in forms.items():
|
|
3026
|
+
for lang in (language, *RECIPE_LANGUAGE_FALLBACKS.get(language, ())):
|
|
3027
|
+
if lang in by_language:
|
|
3028
|
+
run = dict(by_language[lang], language=language)
|
|
3029
|
+
run["_rewritten"] = lang != language
|
|
3030
|
+
runs.append(run)
|
|
3031
|
+
break
|
|
3032
|
+
return runs
|
|
3033
|
+
|
|
3034
|
+
|
|
3035
|
+
def _scan_recipe(
|
|
3036
|
+
exe: str, run: dict, source: bytes, folder: str
|
|
3037
|
+
) -> tuple[str, list[tuple[int, str]] | str]:
|
|
3038
|
+
"""Run one recipe over `source` on stdin.
|
|
3039
|
+
|
|
3040
|
+
Returns ("ok", [($NAME's 1-based line, $NAME's text), ...]),
|
|
3041
|
+
("not-applicable", []) for a recipe whose `language` was rewritten and
|
|
3042
|
+
whose kinds that grammar lacks, ("timeout", detail) or ("error",
|
|
3043
|
+
detail). The rule goes in a file as JSON (YAML reads it as a flow
|
|
3044
|
+
mapping), never on the command line, where an npm `.cmd` shim on
|
|
3045
|
+
Windows would hand its `|` and `^` to cmd.exe.
|
|
3046
|
+
"""
|
|
3047
|
+
rule = {k: v for k, v in run.items() if not k.startswith("_")}
|
|
3048
|
+
rule_path = os.path.join(folder, "recipe.yml")
|
|
3049
|
+
with open(rule_path, "w", encoding="utf-8") as fh:
|
|
3050
|
+
fh.write(json.dumps(rule, ensure_ascii=False))
|
|
3051
|
+
config = os.path.join(folder, "sgconfig.yml")
|
|
3052
|
+
try:
|
|
3053
|
+
result = subprocess.run(
|
|
3054
|
+
[exe, "scan", "--config", config, "-r", rule_path, "--stdin",
|
|
3055
|
+
"--json=stream"],
|
|
3056
|
+
input=source,
|
|
3057
|
+
capture_output=True,
|
|
3058
|
+
timeout=KIND_AT_TIMEOUT_SEC,
|
|
3059
|
+
cwd=folder,
|
|
3060
|
+
check=False,
|
|
3061
|
+
)
|
|
3062
|
+
except subprocess.TimeoutExpired:
|
|
3063
|
+
return "timeout", f"no answer in {KIND_AT_TIMEOUT_SEC}s"
|
|
3064
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
3065
|
+
return "error", f"{type(exc).__name__}: {exc}"
|
|
3066
|
+
stdout = (result.stdout or b"").decode("utf-8", errors="replace")
|
|
3067
|
+
stderr = (result.stderr or b"").decode("utf-8", errors="replace")
|
|
3068
|
+
if result.returncode != 0 and not stdout.strip():
|
|
3069
|
+
if run.get("_rewritten") and "invalid kind" in stderr.lower():
|
|
3070
|
+
return "not-applicable", []
|
|
3071
|
+
detail = next((ln.strip() for ln in stderr.splitlines() if ln.strip()), "")
|
|
3072
|
+
return "error", f"exit {result.returncode}: {detail}"
|
|
3073
|
+
found: list[tuple[int, str]] = []
|
|
3074
|
+
for raw in stdout.splitlines():
|
|
3075
|
+
if not raw.strip():
|
|
3076
|
+
continue
|
|
3077
|
+
try:
|
|
3078
|
+
match = json.loads(raw)
|
|
3079
|
+
except ValueError:
|
|
3080
|
+
return "error", "output is not JSON lines"
|
|
3081
|
+
single = (match.get("metaVariables") or {}).get("single") or {}
|
|
3082
|
+
name = single.get("NAME")
|
|
3083
|
+
if not isinstance(name, dict):
|
|
3084
|
+
continue
|
|
3085
|
+
start = (name.get("range") or {}).get("start") or {}
|
|
3086
|
+
line = start.get("line")
|
|
3087
|
+
if isinstance(line, int) and not isinstance(line, bool):
|
|
3088
|
+
found.append((line + 1, str(name.get("text") or "")))
|
|
3089
|
+
return "ok", found
|
|
3090
|
+
|
|
3091
|
+
|
|
3092
|
+
def _unquote(name: str) -> str:
|
|
3093
|
+
"""A string name without its quotes (`"a-b"` -> `a-b`)."""
|
|
3094
|
+
if len(name) >= 2 and name[0] == name[-1] and name[0] in "'\"":
|
|
3095
|
+
return name[1:-1]
|
|
3096
|
+
return name
|
|
3097
|
+
|
|
3098
|
+
|
|
3099
|
+
def kinds_at(
|
|
3100
|
+
source: bytes,
|
|
3101
|
+
language: str | None,
|
|
3102
|
+
targets: list[tuple[int, str | None]],
|
|
3103
|
+
recipes: list[dict],
|
|
3104
|
+
) -> list[dict]:
|
|
3105
|
+
"""The node kind the recipes give each (line, export name or None)
|
|
3106
|
+
target of one file, running each recipe once.
|
|
3107
|
+
|
|
3108
|
+
Each result: {line, name, status, kind, matches[], errors[]}. A match is
|
|
3109
|
+
a recipe whose `$NAME` is on the target line (and, with a name, is that
|
|
3110
|
+
name's last segment). `status` is `found` (one kind among the matches),
|
|
3111
|
+
`ambiguous` (several), `no-match`, `incomplete` (a recipe run failed, so
|
|
3112
|
+
no kind is given), `skipped-no-language`, `skipped-no-recipes` or
|
|
3113
|
+
`skipped-no-ast-grep`.
|
|
3114
|
+
"""
|
|
3115
|
+
results = [
|
|
3116
|
+
{"line": line, "name": name, "status": "", "kind": None,
|
|
3117
|
+
"matches": [], "errors": []}
|
|
3118
|
+
for line, name in targets
|
|
3119
|
+
]
|
|
3120
|
+
|
|
3121
|
+
def skip(status: str) -> list[dict]:
|
|
3122
|
+
for result in results:
|
|
3123
|
+
result["status"] = status
|
|
3124
|
+
return results
|
|
3125
|
+
|
|
3126
|
+
if language is None:
|
|
3127
|
+
return skip("skipped-no-language")
|
|
3128
|
+
runs = recipe_runs(recipes, language)
|
|
3129
|
+
if not runs:
|
|
3130
|
+
return skip("skipped-no-recipes")
|
|
3131
|
+
exe = _resolve_outside_cwd("ast-grep")
|
|
3132
|
+
if exe is None:
|
|
3133
|
+
return skip("skipped-no-ast-grep")
|
|
3134
|
+
|
|
3135
|
+
matched: list[tuple[dict, list[tuple[int, str]]]] = []
|
|
3136
|
+
errors: list[dict] = []
|
|
3137
|
+
try:
|
|
3138
|
+
with tempfile.TemporaryDirectory(
|
|
3139
|
+
prefix="skf-kind-at-", ignore_cleanup_errors=True
|
|
3140
|
+
) as folder:
|
|
3141
|
+
with open(
|
|
3142
|
+
os.path.join(folder, "sgconfig.yml"), "w", encoding="utf-8"
|
|
3143
|
+
) as fh:
|
|
3144
|
+
fh.write(MINIMAL_SGCONFIG)
|
|
3145
|
+
timed_out = False
|
|
3146
|
+
for run in runs:
|
|
3147
|
+
if timed_out:
|
|
3148
|
+
errors.append({"recipe": run["id"], "language": language,
|
|
3149
|
+
"reason": UNCHECKED_TIMEOUT, "detail": ""})
|
|
3150
|
+
continue
|
|
3151
|
+
outcome, found = _scan_recipe(exe, run, source, folder)
|
|
3152
|
+
if outcome == "ok":
|
|
3153
|
+
matched.append((run, found))
|
|
3154
|
+
elif outcome in ("timeout", "error"):
|
|
3155
|
+
timed_out = outcome == "timeout"
|
|
3156
|
+
errors.append({
|
|
3157
|
+
"recipe": run["id"],
|
|
3158
|
+
"language": language,
|
|
3159
|
+
"reason": UNCHECKED_TIMEOUT if timed_out
|
|
3160
|
+
else UNCHECKED_AST_GREP_ERROR,
|
|
3161
|
+
"detail": found,
|
|
3162
|
+
})
|
|
3163
|
+
except OSError as exc:
|
|
3164
|
+
errors.append({"recipe": None, "language": language,
|
|
3165
|
+
"reason": UNCHECKED_TEMP_FOLDER, "detail": str(exc)})
|
|
3166
|
+
|
|
3167
|
+
for result in results:
|
|
3168
|
+
wanted = definition_name(result["name"]) if result["name"] else None
|
|
3169
|
+
for run, found in matched:
|
|
3170
|
+
for line, text in found:
|
|
3171
|
+
if line != result["line"]:
|
|
3172
|
+
continue
|
|
3173
|
+
if wanted is not None and _unquote(text) != wanted:
|
|
3174
|
+
continue
|
|
3175
|
+
item = {"recipe": run["id"], "kind": run["rule"]["kind"],
|
|
3176
|
+
"name": text}
|
|
3177
|
+
if item not in result["matches"]:
|
|
3178
|
+
result["matches"].append(item)
|
|
3179
|
+
result["matches"].sort(key=lambda m: (m["recipe"], m["name"]))
|
|
3180
|
+
result["errors"] = [dict(e) for e in errors]
|
|
3181
|
+
kinds = sorted({m["kind"] for m in result["matches"]})
|
|
3182
|
+
if errors:
|
|
3183
|
+
result["status"] = "incomplete"
|
|
3184
|
+
elif not kinds:
|
|
3185
|
+
result["status"] = "no-match"
|
|
3186
|
+
elif len(kinds) == 1:
|
|
3187
|
+
result["status"] = "found"
|
|
3188
|
+
result["kind"] = kinds[0]
|
|
3189
|
+
else:
|
|
3190
|
+
result["status"] = "ambiguous"
|
|
3191
|
+
return results
|
|
3192
|
+
|
|
3193
|
+
|
|
3194
|
+
# --------------------------------------------------------------------------
|
|
3195
|
+
# Verification
|
|
3196
|
+
# --------------------------------------------------------------------------
|
|
3197
|
+
|
|
3198
|
+
|
|
3199
|
+
def resolve_source_root(prov: dict, override: str | None) -> Path | None:
|
|
3200
|
+
"""Pick the source root for citation resolution.
|
|
3201
|
+
|
|
3202
|
+
`--source-root` override wins; otherwise the provenance map's top-level
|
|
3203
|
+
`source_root`. Returns None when neither resolves to an on-disk dir.
|
|
3204
|
+
"""
|
|
3205
|
+
candidate: str | None = None
|
|
3206
|
+
if override:
|
|
3207
|
+
candidate = override
|
|
3208
|
+
else:
|
|
3209
|
+
sr = prov.get("source_root")
|
|
3210
|
+
if isinstance(sr, str) and sr:
|
|
3211
|
+
candidate = sr
|
|
3212
|
+
if not candidate:
|
|
3213
|
+
return None
|
|
3214
|
+
root = Path(candidate)
|
|
3215
|
+
return root if root.is_dir() else None
|
|
3216
|
+
|
|
3217
|
+
|
|
3218
|
+
def verify(
|
|
3219
|
+
metadata: dict,
|
|
3220
|
+
prov: dict,
|
|
3221
|
+
source_root: Path | None,
|
|
3222
|
+
skill_dir: Path | None = None,
|
|
3223
|
+
check_kinds: bool = False,
|
|
3224
|
+
) -> dict:
|
|
3225
|
+
"""Compute the completeness / orphan / stale / citation / node-kind
|
|
3226
|
+
report."""
|
|
3227
|
+
reexport_map = extract_reexport_map(prov)
|
|
3228
|
+
|
|
3229
|
+
documented = extract_export_names(metadata)
|
|
3230
|
+
documented_canon = {canon(n, reexport_map) for n in documented}
|
|
3231
|
+
|
|
3232
|
+
raw_entry_names = provenance_entry_names(prov)
|
|
3233
|
+
entry_canon = {canon(n, reexport_map) for n in raw_entry_names}
|
|
3234
|
+
|
|
3235
|
+
# A reference app documents no exports (`exports[]` is empty by design)
|
|
3236
|
+
# while its entries are per-citation, so the two sets measure different
|
|
3237
|
+
# surfaces and their difference means nothing.
|
|
3238
|
+
set_diff_applies = not is_reference_app(metadata)
|
|
3239
|
+
if set_diff_applies:
|
|
3240
|
+
missing = sorted(documented_canon - entry_canon)
|
|
3241
|
+
orphaned = sorted(entry_canon - documented_canon)
|
|
3242
|
+
else:
|
|
3243
|
+
missing, orphaned = [], []
|
|
3244
|
+
|
|
3245
|
+
stale: list[dict] = []
|
|
3246
|
+
citations_checked = 0
|
|
3247
|
+
line_check_skipped = 0
|
|
3248
|
+
if source_root is not None:
|
|
3249
|
+
cache = _SourceCache()
|
|
3250
|
+
entries = prov.get("entries")
|
|
3251
|
+
if isinstance(entries, list):
|
|
3252
|
+
for index, entry in enumerate(entries):
|
|
3253
|
+
if not isinstance(entry, dict):
|
|
3254
|
+
continue
|
|
3255
|
+
name = entry.get("export_name")
|
|
3256
|
+
if not isinstance(name, str) or not name:
|
|
3257
|
+
continue
|
|
3258
|
+
sf = entry.get("source_file")
|
|
3259
|
+
sl = entry.get("source_line")
|
|
3260
|
+
line = _coerce_line(sl)
|
|
3261
|
+
# Only count a resolvable (non-null) citation as "checked".
|
|
3262
|
+
if isinstance(sf, str) and sf.strip() and line is not None:
|
|
3263
|
+
citations_checked += 1
|
|
3264
|
+
reason = check_citation(sf, sl, source_root)
|
|
3265
|
+
item = {
|
|
3266
|
+
"export_name": name,
|
|
3267
|
+
"entry_index": index,
|
|
3268
|
+
"source_file": sf if isinstance(sf, str) else None,
|
|
3269
|
+
"source_line": sl if isinstance(sl, int)
|
|
3270
|
+
and not isinstance(sl, bool) else sl,
|
|
3271
|
+
}
|
|
3272
|
+
if reason is not None:
|
|
3273
|
+
item["reason"] = reason
|
|
3274
|
+
stale.append(item)
|
|
3275
|
+
continue
|
|
3276
|
+
# The citation resolves (or is tolerated): check that the
|
|
3277
|
+
# line defines the export. A tolerated null or blank
|
|
3278
|
+
# citation has no line to check and is not counted.
|
|
3279
|
+
if (
|
|
3280
|
+
not isinstance(sf, str)
|
|
3281
|
+
or not sf.strip()
|
|
3282
|
+
or not isinstance(line, int)
|
|
3283
|
+
):
|
|
3284
|
+
continue
|
|
3285
|
+
if not line_rule_applies(sf, entry.get("export_type")):
|
|
3286
|
+
line_check_skipped += 1
|
|
3287
|
+
continue
|
|
3288
|
+
matches = _definition_matches(sf, name, source_root, cache)
|
|
3289
|
+
if matches is None:
|
|
3290
|
+
line_check_skipped += 1
|
|
3291
|
+
continue
|
|
3292
|
+
defs, every = matches
|
|
3293
|
+
if line not in defs:
|
|
3294
|
+
if line in every:
|
|
3295
|
+
# The recorded line is an indented match the
|
|
3296
|
+
# column-0 rule dropped: keep it, so the finding
|
|
3297
|
+
# lists several lines and nobody auto-moves it.
|
|
3298
|
+
defs = sorted(set(defs) | {line})
|
|
3299
|
+
item["reason"] = STALE_LINE_NOT_DEFINITION
|
|
3300
|
+
item["definition_lines"] = defs
|
|
3301
|
+
stale.append(item)
|
|
3302
|
+
stale.sort(key=lambda s: (s["export_name"], s.get("source_file") or ""))
|
|
3303
|
+
|
|
3304
|
+
citations: list[dict] = []
|
|
3305
|
+
skill_citations_scanned = 0
|
|
3306
|
+
skill_citations_matched = 0
|
|
3307
|
+
if skill_dir is not None:
|
|
3308
|
+
citations, skill_citations_scanned, skill_citations_matched = (
|
|
3309
|
+
check_skill_citations(prov, skill_dir)
|
|
3310
|
+
)
|
|
3311
|
+
|
|
3312
|
+
node_kinds: list[dict] = []
|
|
3313
|
+
node_kinds_unchecked: list[dict] = []
|
|
3314
|
+
kind_summary: dict = {"node_kind_check": "not-requested"}
|
|
3315
|
+
if check_kinds:
|
|
3316
|
+
node_kinds, node_kinds_unchecked, kind_summary = check_node_kinds(prov)
|
|
3317
|
+
|
|
3318
|
+
status = (
|
|
3319
|
+
"pass"
|
|
3320
|
+
if not (missing or orphaned or stale or citations or node_kinds)
|
|
3321
|
+
else "findings"
|
|
3322
|
+
)
|
|
3323
|
+
result: dict = {
|
|
3324
|
+
"status": status,
|
|
3325
|
+
"missing": missing,
|
|
3326
|
+
"orphaned": orphaned,
|
|
3327
|
+
"stale": stale,
|
|
3328
|
+
"citations": citations,
|
|
3329
|
+
}
|
|
3330
|
+
if check_kinds:
|
|
3331
|
+
result["node_kinds"] = node_kinds
|
|
3332
|
+
result["node_kinds_unchecked"] = node_kinds_unchecked
|
|
3333
|
+
result["summary"] = {
|
|
3334
|
+
"exports_checked": len(documented_canon),
|
|
3335
|
+
"entries_checked": len(raw_entry_names),
|
|
3336
|
+
"missing_count": len(missing),
|
|
3337
|
+
"orphaned_count": len(orphaned),
|
|
3338
|
+
"set_diff": "checked" if set_diff_applies else "not-applicable",
|
|
3339
|
+
"stale_count": len(stale),
|
|
3340
|
+
"citations_checked": citations_checked,
|
|
3341
|
+
"line_check_skipped": line_check_skipped,
|
|
3342
|
+
"stale_check": "checked"
|
|
3343
|
+
if source_root is not None
|
|
3344
|
+
else "skipped-no-source-root",
|
|
3345
|
+
"skill_citations_scanned": skill_citations_scanned,
|
|
3346
|
+
"skill_citations_matched": skill_citations_matched,
|
|
3347
|
+
"citation_findings_count": len(citations),
|
|
3348
|
+
"citation_check": "checked"
|
|
3349
|
+
if skill_dir is not None
|
|
3350
|
+
else "skipped-no-skill-dir",
|
|
3351
|
+
**kind_summary,
|
|
3352
|
+
}
|
|
3353
|
+
return result
|
|
3354
|
+
|
|
3355
|
+
|
|
3356
|
+
# --------------------------------------------------------------------------
|
|
3357
|
+
# Fixes with one answer (fix)
|
|
3358
|
+
# --------------------------------------------------------------------------
|
|
3359
|
+
|
|
3360
|
+
|
|
3361
|
+
def manual_interiors(text: str) -> list[tuple[int, int]]:
|
|
3362
|
+
"""(start, end) offsets of the interior of every [MANUAL] block, paired
|
|
3363
|
+
as skf-hash-content.py pairs them: an opening marker with the earliest
|
|
3364
|
+
following close marker of the same name. An unclosed marker is
|
|
3365
|
+
skipped."""
|
|
3366
|
+
closes = [
|
|
3367
|
+
(m.start(), m.group(1).strip()) for m in _MANUAL_CLOSE_RE.finditer(text)
|
|
3368
|
+
]
|
|
3369
|
+
spans: list[tuple[int, int]] = []
|
|
3370
|
+
for om in _MANUAL_OPEN_RE.finditer(text):
|
|
3371
|
+
name = om.group(1).strip()
|
|
3372
|
+
close = min(
|
|
3373
|
+
(start for start, n in closes if start >= om.end() and n == name),
|
|
3374
|
+
default=None,
|
|
3375
|
+
)
|
|
3376
|
+
if close is not None:
|
|
3377
|
+
spans.append((om.end(), close))
|
|
3378
|
+
return spans
|
|
3379
|
+
|
|
3380
|
+
|
|
3381
|
+
class _SkillFile:
|
|
3382
|
+
"""One skill markdown file: its text, its [MANUAL] interiors and every
|
|
3383
|
+
citation in it, found line by line as `scan_skill_citations` finds them.
|
|
3384
|
+
Bytes that are not UTF-8 survive the round trip (surrogateescape)."""
|
|
3385
|
+
|
|
3386
|
+
def __init__(self, skill_dir: Path, rel: str) -> None:
|
|
3387
|
+
self.rel = rel
|
|
3388
|
+
self.path = skill_dir / rel
|
|
3389
|
+
self.text = self.path.read_bytes().decode("utf-8", errors="surrogateescape")
|
|
3390
|
+
self.manual = manual_interiors(self.text)
|
|
3391
|
+
self.citations: list[dict] = []
|
|
3392
|
+
offset = 0
|
|
3393
|
+
for number, chunk in enumerate(_LINE_WITH_END_RE.findall(self.text), 1):
|
|
3394
|
+
content = chunk.rstrip("\r\n")
|
|
3395
|
+
for m in _SKILL_CITATION_RE.finditer(content):
|
|
3396
|
+
self.citations.append(
|
|
3397
|
+
{
|
|
3398
|
+
"line": number,
|
|
3399
|
+
"start": offset + m.start(),
|
|
3400
|
+
"end": offset + m.end(),
|
|
3401
|
+
"match": m,
|
|
3402
|
+
"text": m.group(0),
|
|
3403
|
+
"prefix": m.group(1),
|
|
3404
|
+
"cited_file": m.group(2).strip(),
|
|
3405
|
+
"cited_line": int(m.group(3)),
|
|
3406
|
+
}
|
|
3407
|
+
)
|
|
3408
|
+
offset += len(chunk)
|
|
3409
|
+
|
|
3410
|
+
def in_manual(self, offset: int) -> bool:
|
|
3411
|
+
return any(start <= offset < end for start, end in self.manual)
|
|
3412
|
+
|
|
3413
|
+
def fixed_text(self) -> str | None:
|
|
3414
|
+
"""The text with every changed citation rewritten, or None when no
|
|
3415
|
+
citation changed."""
|
|
3416
|
+
edits = [
|
|
3417
|
+
(c["start"], c["end"], _rewritten_citation(c))
|
|
3418
|
+
for c in self.citations
|
|
3419
|
+
if c.get("new_prefix") or c.get("shift")
|
|
3420
|
+
]
|
|
3421
|
+
if not edits:
|
|
3422
|
+
return None
|
|
3423
|
+
text = self.text
|
|
3424
|
+
for start, end, new in sorted(edits, reverse=True):
|
|
3425
|
+
text = text[:start] + new + text[end:]
|
|
3426
|
+
return text
|
|
3427
|
+
|
|
3428
|
+
|
|
3429
|
+
def _rewritten_citation(cit: dict) -> str:
|
|
3430
|
+
"""A citation with its new prefix and its lines shifted: a range moves
|
|
3431
|
+
both ends by the same amount (`L10-12` to `L13-15`)."""
|
|
3432
|
+
m = cit["match"]
|
|
3433
|
+
base = m.start()
|
|
3434
|
+
subs: list[tuple[int, int, str]] = []
|
|
3435
|
+
if cit.get("new_prefix"):
|
|
3436
|
+
subs.append((m.start(1) - base, m.end(1) - base, cit["new_prefix"]))
|
|
3437
|
+
shift = cit.get("shift", 0)
|
|
3438
|
+
if shift:
|
|
3439
|
+
subs.append((m.start(3) - base, m.end(3) - base, str(int(m.group(3)) + shift)))
|
|
3440
|
+
if m.group(5) is not None:
|
|
3441
|
+
subs.append(
|
|
3442
|
+
(m.start(5) - base, m.end(5) - base, str(int(m.group(5)) + shift))
|
|
3443
|
+
)
|
|
3444
|
+
text = m.group(0)
|
|
3445
|
+
for start, end, new in sorted(subs, reverse=True):
|
|
3446
|
+
text = text[:start] + new + text[end:]
|
|
3447
|
+
return text
|
|
3448
|
+
|
|
3449
|
+
|
|
3450
|
+
def _dict_items(value: object) -> list[dict]:
|
|
3451
|
+
return [v for v in value if isinstance(v, dict)] if isinstance(value, list) else []
|
|
3452
|
+
|
|
3453
|
+
|
|
3454
|
+
def _stale_entry_indices(entries: list, item: dict) -> list[int]:
|
|
3455
|
+
"""The provenance entries a `stale[]` item names: the one at its
|
|
3456
|
+
`entry_index` when that entry still has the item's name, file and line,
|
|
3457
|
+
else every entry that has them."""
|
|
3458
|
+
name = item.get("export_name")
|
|
3459
|
+
source_file = item.get("source_file")
|
|
3460
|
+
line = _coerce_line(item.get("source_line"))
|
|
3461
|
+
|
|
3462
|
+
def same(entry: object) -> bool:
|
|
3463
|
+
return (
|
|
3464
|
+
isinstance(entry, dict)
|
|
3465
|
+
and entry.get("export_name") == name
|
|
3466
|
+
and isinstance(source_file, str)
|
|
3467
|
+
and bool(source_file.strip())
|
|
3468
|
+
and entry.get("source_file") == source_file
|
|
3469
|
+
and isinstance(line, int)
|
|
3470
|
+
and _coerce_line(entry.get("source_line")) == line
|
|
3471
|
+
)
|
|
3472
|
+
|
|
3473
|
+
index = item.get("entry_index")
|
|
3474
|
+
if (
|
|
3475
|
+
isinstance(index, int)
|
|
3476
|
+
and not isinstance(index, bool)
|
|
3477
|
+
and 0 <= index < len(entries)
|
|
3478
|
+
and same(entries[index])
|
|
3479
|
+
):
|
|
3480
|
+
return [index]
|
|
3481
|
+
return [i for i, entry in enumerate(entries) if same(entry)]
|
|
3482
|
+
|
|
3483
|
+
|
|
3484
|
+
def plan_fixes(
|
|
3485
|
+
verify_result: dict, prov: dict, skill_dir: Path, line_moves: bool = True
|
|
3486
|
+
) -> dict:
|
|
3487
|
+
"""Plan the fixes a `verify` result has one answer for.
|
|
3488
|
+
|
|
3489
|
+
1. Citation prefixes: each `citations[]` item's citation, found by its
|
|
3490
|
+
file, line and text, takes `expected_prefix`.
|
|
3491
|
+
2. Source lines: each `line-not-definition` item with exactly one
|
|
3492
|
+
`definition_lines` value moves its entry's `source_line` there, and
|
|
3493
|
+
the citations of either prefix whose normalized path and first line
|
|
3494
|
+
are the entry's (as they stand after step 1, before any move) move
|
|
3495
|
+
with it, all in one pass, so a citation moves at most once. A
|
|
3496
|
+
citation of a line another entry also records in that file stays.
|
|
3497
|
+
|
|
3498
|
+
A citation inside a [MANUAL] block never changes. Returns {applied,
|
|
3499
|
+
left_as_warn, markdown: {rel: new text}, provenance: the new map or
|
|
3500
|
+
None}; nothing is written.
|
|
3501
|
+
"""
|
|
3502
|
+
rels = [
|
|
3503
|
+
p.relative_to(skill_dir).as_posix() for p in skill_markdown_files(skill_dir)
|
|
3504
|
+
]
|
|
3505
|
+
files = {rel: _SkillFile(skill_dir, rel) for rel in rels}
|
|
3506
|
+
applied: list[dict] = []
|
|
3507
|
+
left: list[dict] = []
|
|
3508
|
+
prefix_fixes: list[tuple[dict, dict]] = []
|
|
3509
|
+
|
|
3510
|
+
# 1. Citation prefixes.
|
|
3511
|
+
groups: dict[tuple[str, int, str, str], list[dict]] = {}
|
|
3512
|
+
for item in _dict_items(verify_result.get("citations")):
|
|
3513
|
+
key = (item.get("file"), item.get("line"), item.get("citation"),
|
|
3514
|
+
item.get("expected_prefix"))
|
|
3515
|
+
if (
|
|
3516
|
+
not isinstance(key[0], str)
|
|
3517
|
+
or not isinstance(key[1], int)
|
|
3518
|
+
or not isinstance(key[2], str)
|
|
3519
|
+
or key[3] not in ("AST", "SRC")
|
|
3520
|
+
):
|
|
3521
|
+
left.append(dict(item, kind=FIX_CITATION_PREFIX, why="citation-not-found"))
|
|
3522
|
+
continue
|
|
3523
|
+
groups.setdefault(key, []).append(item)
|
|
3524
|
+
for (rel, line, text, expected), items in groups.items():
|
|
3525
|
+
doc = files.get(rel)
|
|
3526
|
+
found = [] if doc is None else [
|
|
3527
|
+
c for c in doc.citations if c["line"] == line and c["text"] == text
|
|
3528
|
+
]
|
|
3529
|
+
if not found:
|
|
3530
|
+
for item in items:
|
|
3531
|
+
left.append(
|
|
3532
|
+
dict(item, kind=FIX_CITATION_PREFIX, why="citation-not-found")
|
|
3533
|
+
)
|
|
3534
|
+
continue
|
|
3535
|
+
for cit in found:
|
|
3536
|
+
if doc.in_manual(cit["start"]):
|
|
3537
|
+
left.append(dict(items[0], kind=FIX_CITATION_PREFIX,
|
|
3538
|
+
why="inside-manual-block"))
|
|
3539
|
+
elif cit["prefix"] != expected:
|
|
3540
|
+
cit["new_prefix"] = expected
|
|
3541
|
+
prefix_fixes.append((doc, cit))
|
|
3542
|
+
|
|
3543
|
+
# 2. Source lines: plan every move before making any.
|
|
3544
|
+
entries = prov.get("entries") if isinstance(prov.get("entries"), list) else []
|
|
3545
|
+
moves: dict[int, int] = {}
|
|
3546
|
+
for item in _dict_items(verify_result.get("stale")):
|
|
3547
|
+
defs = item.get("definition_lines")
|
|
3548
|
+
new = defs[0] if isinstance(defs, list) and len(defs) == 1 else None
|
|
3549
|
+
if item.get("reason") != STALE_LINE_NOT_DEFINITION:
|
|
3550
|
+
why = "needs-decision"
|
|
3551
|
+
elif not isinstance(defs, list) or not defs:
|
|
3552
|
+
why = "no-definition-line"
|
|
3553
|
+
elif len(defs) > 1:
|
|
3554
|
+
why = "several-definition-lines"
|
|
3555
|
+
elif not isinstance(new, int) or isinstance(new, bool) or new < 1:
|
|
3556
|
+
why = "needs-decision"
|
|
3557
|
+
elif not line_moves:
|
|
3558
|
+
why = "line-moves-skipped"
|
|
3559
|
+
else:
|
|
3560
|
+
indices = _stale_entry_indices(entries, item)
|
|
3561
|
+
if indices:
|
|
3562
|
+
for index in indices:
|
|
3563
|
+
moves[index] = new
|
|
3564
|
+
continue
|
|
3565
|
+
why = "entry-not-found"
|
|
3566
|
+
left.append(dict(item, kind=FIX_SOURCE_LINE, why=why))
|
|
3567
|
+
|
|
3568
|
+
recorded: dict[tuple[str, int], set[int]] = {}
|
|
3569
|
+
for index, entry in enumerate(entries):
|
|
3570
|
+
if not isinstance(entry, dict):
|
|
3571
|
+
continue
|
|
3572
|
+
sf = entry.get("source_file")
|
|
3573
|
+
line = _coerce_line(entry.get("source_line"))
|
|
3574
|
+
if isinstance(sf, str) and sf.strip() and isinstance(line, int):
|
|
3575
|
+
recorded.setdefault((normalize_path(sf), line), set()).add(index)
|
|
3576
|
+
movers: dict[tuple[str, int], dict[int, int]] = {}
|
|
3577
|
+
for index, new in moves.items():
|
|
3578
|
+
entry = entries[index]
|
|
3579
|
+
key = (normalize_path(entry["source_file"]), _coerce_line(entry["source_line"]))
|
|
3580
|
+
movers.setdefault(key, {})[index] = new
|
|
3581
|
+
|
|
3582
|
+
line_fixes: list[tuple[dict, dict, list[str]]] = []
|
|
3583
|
+
for rel in rels:
|
|
3584
|
+
doc = files[rel]
|
|
3585
|
+
for cit in doc.citations:
|
|
3586
|
+
key = (normalize_path(cit["cited_file"]), cit["cited_line"])
|
|
3587
|
+
moving = movers.get(key)
|
|
3588
|
+
if not moving:
|
|
3589
|
+
continue
|
|
3590
|
+
here = recorded.get(key, set()) | set(moving)
|
|
3591
|
+
names = sorted(
|
|
3592
|
+
{n for n in (entries[i].get("export_name") for i in here)
|
|
3593
|
+
if isinstance(n, str) and n}
|
|
3594
|
+
)
|
|
3595
|
+
targets = set(moving.values())
|
|
3596
|
+
if here - set(moving) or len(targets) > 1:
|
|
3597
|
+
why = "shared-line" # it may cite the other entry's export
|
|
3598
|
+
elif doc.in_manual(cit["start"]):
|
|
3599
|
+
why = "inside-manual-block"
|
|
3600
|
+
else:
|
|
3601
|
+
cit["shift"] = targets.pop() - cit["cited_line"]
|
|
3602
|
+
line_fixes.append((doc, cit, names))
|
|
3603
|
+
continue
|
|
3604
|
+
left.append({
|
|
3605
|
+
"kind": FIX_CITATION_LINE,
|
|
3606
|
+
"file": rel,
|
|
3607
|
+
"line": cit["line"],
|
|
3608
|
+
"citation": cit["text"],
|
|
3609
|
+
"export_names": names,
|
|
3610
|
+
"why": why,
|
|
3611
|
+
})
|
|
3612
|
+
|
|
3613
|
+
for doc, cit in prefix_fixes:
|
|
3614
|
+
applied.append({
|
|
3615
|
+
"kind": FIX_CITATION_PREFIX,
|
|
3616
|
+
"file": doc.rel,
|
|
3617
|
+
"line": cit["line"],
|
|
3618
|
+
"citation": cit["text"],
|
|
3619
|
+
"fixed": _rewritten_citation(cit),
|
|
3620
|
+
})
|
|
3621
|
+
new_prov = None
|
|
3622
|
+
if moves:
|
|
3623
|
+
new_prov = copy.deepcopy(prov)
|
|
3624
|
+
for index in sorted(moves):
|
|
3625
|
+
entry = new_prov["entries"][index]
|
|
3626
|
+
applied.append({
|
|
3627
|
+
"kind": FIX_SOURCE_LINE,
|
|
3628
|
+
"entry_index": index,
|
|
3629
|
+
"export_name": entry.get("export_name"),
|
|
3630
|
+
"source_file": entry.get("source_file"),
|
|
3631
|
+
"from": _coerce_line(entry.get("source_line")),
|
|
3632
|
+
"to": moves[index],
|
|
3633
|
+
})
|
|
3634
|
+
entry["source_line"] = moves[index]
|
|
3635
|
+
for doc, cit, names in line_fixes:
|
|
3636
|
+
applied.append({
|
|
3637
|
+
"kind": FIX_CITATION_LINE,
|
|
3638
|
+
"file": doc.rel,
|
|
3639
|
+
"line": cit["line"],
|
|
3640
|
+
"citation": cit["text"],
|
|
3641
|
+
"fixed": _rewritten_citation(cit),
|
|
3642
|
+
"export_names": names,
|
|
3643
|
+
})
|
|
3644
|
+
|
|
3645
|
+
markdown = {}
|
|
3646
|
+
for rel in rels:
|
|
3647
|
+
text = files[rel].fixed_text()
|
|
3648
|
+
if text is not None:
|
|
3649
|
+
markdown[rel] = text
|
|
3650
|
+
return {
|
|
3651
|
+
"applied": applied,
|
|
3652
|
+
"left_as_warn": left,
|
|
3653
|
+
"markdown": markdown,
|
|
3654
|
+
"provenance": new_prov,
|
|
3655
|
+
}
|
|
3656
|
+
|
|
3657
|
+
|
|
3658
|
+
def _sibling_helper(filename: str) -> Path | None:
|
|
3659
|
+
"""A helper installed beside this script (dev `src/shared/scripts/`
|
|
3660
|
+
and installed `_bmad/skf/shared/scripts/` alike), or None."""
|
|
3661
|
+
path = Path(__file__).resolve().parent / filename
|
|
3662
|
+
return path if path.is_file() else None
|
|
3663
|
+
|
|
3664
|
+
|
|
3665
|
+
def _load_sibling_module(filename: str, module_name: str):
|
|
3666
|
+
"""Import a sibling helper as a module. Raises ImportError."""
|
|
3667
|
+
path = _sibling_helper(filename)
|
|
3668
|
+
spec = (
|
|
3669
|
+
importlib.util.spec_from_file_location(module_name, path)
|
|
3670
|
+
if path is not None
|
|
3671
|
+
else None
|
|
3672
|
+
)
|
|
3673
|
+
if spec is None or spec.loader is None:
|
|
3674
|
+
raise ImportError(f"{filename} not found beside {Path(__file__).name}")
|
|
3675
|
+
module = importlib.util.module_from_spec(spec)
|
|
3676
|
+
spec.loader.exec_module(module)
|
|
3677
|
+
return module
|
|
3678
|
+
|
|
3679
|
+
|
|
3680
|
+
def atomic_write(helper: Path, target: Path, data: bytes) -> None:
|
|
3681
|
+
"""Write `data` to `target` through `skf-atomic-write.py write`.
|
|
3682
|
+
Raises OSError when the helper fails."""
|
|
3683
|
+
try:
|
|
3684
|
+
result = subprocess.run(
|
|
3685
|
+
[sys.executable, str(helper), "write", "--target", str(target)],
|
|
3686
|
+
input=data,
|
|
3687
|
+
capture_output=True,
|
|
3688
|
+
timeout=ATOMIC_WRITE_TIMEOUT_SEC,
|
|
3689
|
+
check=False,
|
|
3690
|
+
)
|
|
3691
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
3692
|
+
raise OSError(f"atomic write of {target} failed: {exc}") from exc
|
|
3693
|
+
if result.returncode != 0:
|
|
3694
|
+
detail = (result.stderr or b"").decode("utf-8", errors="replace").strip()
|
|
3695
|
+
raise OSError(f"atomic write of {target} failed: {detail}")
|
|
3696
|
+
|
|
3697
|
+
|
|
3698
|
+
# --------------------------------------------------------------------------
|
|
3699
|
+
# CLI
|
|
3700
|
+
# --------------------------------------------------------------------------
|
|
3701
|
+
|
|
3702
|
+
|
|
3703
|
+
def _skill_dir_error(skill_dir: Path) -> str | None:
|
|
3704
|
+
"""Why `skill_dir` is not a skill package folder, or None."""
|
|
3705
|
+
if not skill_dir.exists():
|
|
3706
|
+
return f"skill dir not found: {skill_dir}"
|
|
3707
|
+
if not skill_dir.is_dir():
|
|
3708
|
+
return f"skill dir is not a directory: {skill_dir}"
|
|
3709
|
+
if not (skill_dir / "SKILL.md").is_file():
|
|
3710
|
+
return f"skill dir has no SKILL.md: {skill_dir}"
|
|
3711
|
+
return None
|
|
3712
|
+
|
|
3713
|
+
|
|
3714
|
+
def _unexpected(what: str, exc: Exception, written: list[str] | None = None) -> int:
|
|
3715
|
+
"""Report an unexpected failure on one stderr line; exit code 2. With
|
|
3716
|
+
`written` (fix), the line also names the files already written."""
|
|
3717
|
+
detail = " ".join(str(exc).split())
|
|
3718
|
+
note = ""
|
|
3719
|
+
if written is not None:
|
|
3720
|
+
note = f"; already written: {', '.join(written) if written else 'nothing'}"
|
|
3721
|
+
print(
|
|
3722
|
+
f"error: {what} failed: {type(exc).__name__}: {detail}{note}",
|
|
3723
|
+
file=sys.stderr,
|
|
3724
|
+
)
|
|
3725
|
+
return 2
|
|
3726
|
+
|
|
3727
|
+
|
|
3728
|
+
def _emit(result: dict, output: str | None, written: list[str] | None = None) -> bool:
|
|
3729
|
+
"""Write the JSON result to `output`, or to stdout when it is absent or
|
|
3730
|
+
`-`. False, after one stderr line (naming `written`, as `_unexpected`
|
|
3731
|
+
does), when it cannot be written."""
|
|
3732
|
+
out_text = json.dumps(result, indent=2) + "\n"
|
|
3733
|
+
to_file = bool(output) and output != "-"
|
|
3734
|
+
try:
|
|
3735
|
+
if to_file:
|
|
3736
|
+
Path(output).write_text(out_text, encoding="utf-8")
|
|
3737
|
+
else:
|
|
3738
|
+
sys.stdout.write(out_text)
|
|
3739
|
+
except OSError as exc:
|
|
3740
|
+
_unexpected(f"writing {output if to_file else 'stdout'}", exc, written)
|
|
3741
|
+
return False
|
|
3742
|
+
return True
|
|
3743
|
+
|
|
3744
|
+
|
|
3745
|
+
def _cmd_verify(args: argparse.Namespace) -> int:
|
|
3746
|
+
meta_path = Path(args.metadata)
|
|
3747
|
+
prov_path = Path(args.provenance)
|
|
3748
|
+
if not meta_path.is_file():
|
|
3749
|
+
print(f"error: metadata not found: {meta_path}", file=sys.stderr)
|
|
3750
|
+
return 2
|
|
3751
|
+
if not prov_path.is_file():
|
|
3752
|
+
print(f"error: provenance map not found: {prov_path}", file=sys.stderr)
|
|
3753
|
+
return 2
|
|
3754
|
+
try:
|
|
3755
|
+
metadata = load_json_object(meta_path, "metadata")
|
|
3756
|
+
prov = load_json_object(prov_path, "provenance map")
|
|
3757
|
+
except ValueError as exc:
|
|
3758
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3759
|
+
return 2
|
|
3760
|
+
|
|
3761
|
+
skill_dir: Path | None = None
|
|
3762
|
+
if args.skill_dir is not None:
|
|
3763
|
+
skill_dir = Path(args.skill_dir)
|
|
3764
|
+
problem = _skill_dir_error(skill_dir)
|
|
3765
|
+
if problem is not None:
|
|
3766
|
+
print(f"error: {problem}", file=sys.stderr)
|
|
3767
|
+
return 2
|
|
3768
|
+
|
|
3769
|
+
source_root = resolve_source_root(prov, args.source_root)
|
|
3770
|
+
if args.verbose:
|
|
3771
|
+
if source_root is None:
|
|
3772
|
+
print(
|
|
3773
|
+
"verbose: no source root resolved — skipping stale-citation "
|
|
3774
|
+
"check (completeness + orphan diffs only)",
|
|
3775
|
+
file=sys.stderr,
|
|
3776
|
+
)
|
|
3777
|
+
else:
|
|
3778
|
+
print(f"verbose: resolving citations under {source_root}", file=sys.stderr)
|
|
3779
|
+
|
|
3780
|
+
try:
|
|
3781
|
+
result = verify(
|
|
3782
|
+
metadata, prov, source_root, skill_dir, args.check_node_kinds
|
|
3783
|
+
)
|
|
3784
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as findings
|
|
3785
|
+
return _unexpected("verification", exc)
|
|
3786
|
+
|
|
3787
|
+
if not _emit(result, args.output):
|
|
3788
|
+
return 2
|
|
3789
|
+
return 0 if result["status"] == "pass" else 1
|
|
3790
|
+
|
|
3791
|
+
|
|
3792
|
+
def _cmd_definition_lines(args: argparse.Namespace) -> int:
|
|
3793
|
+
root = Path(args.source_root) if args.source_root else Path.cwd()
|
|
3794
|
+
if not root.is_dir():
|
|
3795
|
+
print(f"error: source root not found: {root}", file=sys.stderr)
|
|
3796
|
+
return 2
|
|
3797
|
+
try:
|
|
3798
|
+
report = definition_lines_report(
|
|
3799
|
+
args.file, args.name, root, args.line, args.export_type
|
|
3800
|
+
)
|
|
3801
|
+
except ValueError as exc:
|
|
3802
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3803
|
+
return 2
|
|
3804
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
3805
|
+
return _unexpected("definition lookup", exc)
|
|
3806
|
+
return 0 if _emit(report, args.output) else 2
|
|
3807
|
+
|
|
3808
|
+
|
|
3809
|
+
def _cmd_classify_stale(args: argparse.Namespace) -> int:
|
|
3810
|
+
names_path, prov_path = Path(args.names), Path(args.provenance)
|
|
3811
|
+
inv_path = Path(args.inventory) if args.inventory is not None else None
|
|
3812
|
+
root = Path(args.source_root)
|
|
3813
|
+
inputs = [("names", names_path), ("provenance map", prov_path)]
|
|
3814
|
+
if inv_path is not None:
|
|
3815
|
+
inputs.append(("inventory", inv_path))
|
|
3816
|
+
for label, path in inputs:
|
|
3817
|
+
if not path.is_file():
|
|
3818
|
+
print(f"error: {label} not found: {path}", file=sys.stderr)
|
|
3819
|
+
return 2
|
|
3820
|
+
if not root.is_dir():
|
|
3821
|
+
print(f"error: source root not found: {root}", file=sys.stderr)
|
|
3822
|
+
return 2
|
|
3823
|
+
try:
|
|
3824
|
+
text = names_path.read_text(encoding="utf-8-sig")
|
|
3825
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
3826
|
+
print(f"error: failed to read names {names_path}: {exc}", file=sys.stderr)
|
|
3827
|
+
return 2
|
|
3828
|
+
try:
|
|
3829
|
+
data = json.loads(text)
|
|
3830
|
+
except json.JSONDecodeError as exc:
|
|
3831
|
+
print(f"error: malformed JSON in names {names_path}: {exc}", file=sys.stderr)
|
|
3832
|
+
return 2
|
|
3833
|
+
try:
|
|
3834
|
+
names = stale_names(data)
|
|
3835
|
+
prov = load_json_object(prov_path, "provenance map")
|
|
3836
|
+
kinds = None if inv_path is None else inventory_kinds(
|
|
3837
|
+
load_json_object(inv_path, "inventory"), args.inventory
|
|
3838
|
+
)
|
|
3839
|
+
except ValueError as exc:
|
|
3840
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3841
|
+
return 2
|
|
3842
|
+
try:
|
|
3843
|
+
result = classify_stale(names, prov, root, kinds)
|
|
3844
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
3845
|
+
return _unexpected("stale-name classification", exc)
|
|
3846
|
+
return 0 if _emit(result, args.output) else 2
|
|
3847
|
+
|
|
3848
|
+
|
|
3849
|
+
def _cmd_kind_at(args: argparse.Namespace) -> int:
|
|
3850
|
+
# a `source_file` as a provenance entry records it, as `verify` reads it
|
|
3851
|
+
path = Path(args.file.strip().replace("\\", "/"))
|
|
3852
|
+
if args.source_root and not path.is_absolute():
|
|
3853
|
+
path = Path(args.source_root) / path
|
|
3854
|
+
if not path.is_file():
|
|
3855
|
+
print(f"error: source file not found: {path}", file=sys.stderr)
|
|
3856
|
+
return 2
|
|
3857
|
+
recipes_path = Path(args.recipes)
|
|
3858
|
+
if not recipes_path.is_file():
|
|
3859
|
+
print(f"error: recipes not found: {recipes_path}", file=sys.stderr)
|
|
3860
|
+
return 2
|
|
3861
|
+
try:
|
|
3862
|
+
recipes = load_recipes(recipes_path)
|
|
3863
|
+
source = path.read_bytes()
|
|
3864
|
+
except ImportError:
|
|
3865
|
+
print(
|
|
3866
|
+
"error: kind-at reads recipes with PyYAML: run the helper with "
|
|
3867
|
+
"`uv run`, which installs it",
|
|
3868
|
+
file=sys.stderr,
|
|
3869
|
+
)
|
|
3870
|
+
return 2
|
|
3871
|
+
except (ValueError, OSError) as exc:
|
|
3872
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3873
|
+
return 2
|
|
3874
|
+
if not recipes:
|
|
3875
|
+
print(f"error: no ast-grep recipe in {recipes_path}", file=sys.stderr)
|
|
3876
|
+
return 2
|
|
3877
|
+
language = (
|
|
3878
|
+
args.language.strip().lower() if args.language
|
|
3879
|
+
else ast_grep_language(args.file)
|
|
3880
|
+
)
|
|
3881
|
+
try:
|
|
3882
|
+
(result,) = kinds_at(source, language, [(args.line, args.name)], recipes)
|
|
3883
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
|
|
3884
|
+
return _unexpected("kind lookup", exc)
|
|
3885
|
+
report = {"file": args.file, "language": language, **result}
|
|
3886
|
+
if not _emit(report, args.output):
|
|
3887
|
+
return 2
|
|
3888
|
+
return 0 if report["status"] == "found" else 1
|
|
3889
|
+
|
|
3890
|
+
|
|
3891
|
+
def _cmd_fix(args: argparse.Namespace) -> int:
|
|
3892
|
+
verify_path = Path(args.verify)
|
|
3893
|
+
prov_path = Path(args.provenance)
|
|
3894
|
+
skill_dir = Path(args.skill_dir)
|
|
3895
|
+
for path, label in ((verify_path, "verify result"), (prov_path, "provenance map")):
|
|
3896
|
+
if not path.is_file():
|
|
3897
|
+
print(f"error: {label} not found: {path}", file=sys.stderr)
|
|
3898
|
+
return 2
|
|
3899
|
+
problem = _skill_dir_error(skill_dir)
|
|
3900
|
+
if problem is not None:
|
|
3901
|
+
print(f"error: {problem}", file=sys.stderr)
|
|
3902
|
+
return 2
|
|
3903
|
+
try:
|
|
3904
|
+
verify_result = load_json_object(verify_path, "verify result")
|
|
3905
|
+
prov = load_json_object(prov_path, "provenance map")
|
|
3906
|
+
except ValueError as exc:
|
|
3907
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3908
|
+
return 2
|
|
3909
|
+
if not all(
|
|
3910
|
+
isinstance(verify_result.get(key), list) for key in ("stale", "citations")
|
|
3911
|
+
):
|
|
3912
|
+
print(
|
|
3913
|
+
f"error: {verify_path} is not a verify result (no stale[] and "
|
|
3914
|
+
f"citations[])",
|
|
3915
|
+
file=sys.stderr,
|
|
3916
|
+
)
|
|
3917
|
+
return 2
|
|
3918
|
+
|
|
3919
|
+
inventory: list[dict] | None = None
|
|
3920
|
+
hash_content = None
|
|
3921
|
+
if args.manual_inventory is not None:
|
|
3922
|
+
try:
|
|
3923
|
+
hash_content = _load_sibling_module(
|
|
3924
|
+
HASH_CONTENT_HELPER, "skf_hash_content"
|
|
3925
|
+
)
|
|
3926
|
+
inventory = hash_content.load_inventory_blocks(Path(args.manual_inventory))
|
|
3927
|
+
except (ImportError, ValueError) as exc:
|
|
3928
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
3929
|
+
return 2
|
|
3930
|
+
|
|
3931
|
+
try:
|
|
3932
|
+
plan = plan_fixes(verify_result, prov, skill_dir, not args.no_line_moves)
|
|
3933
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as a fix
|
|
3934
|
+
return _unexpected("fix", exc)
|
|
3935
|
+
|
|
3936
|
+
changes: list[tuple[Path, bytes]] = [
|
|
3937
|
+
(skill_dir / rel, text.encode("utf-8", errors="surrogateescape"))
|
|
3938
|
+
for rel, text in sorted(plan["markdown"].items())
|
|
3939
|
+
]
|
|
3940
|
+
if plan["provenance"] is not None:
|
|
3941
|
+
text = json.dumps(plan["provenance"], indent=2, ensure_ascii=False) + "\n"
|
|
3942
|
+
changes.append((prov_path, text.encode("utf-8")))
|
|
3943
|
+
written: list[str] = []
|
|
3944
|
+
if changes and not args.dry_run:
|
|
3945
|
+
helper = _sibling_helper(ATOMIC_WRITE_HELPER)
|
|
3946
|
+
if helper is None:
|
|
3947
|
+
print(
|
|
3948
|
+
f"error: {ATOMIC_WRITE_HELPER} not found beside "
|
|
3949
|
+
f"{Path(__file__).name}; nothing written",
|
|
3950
|
+
file=sys.stderr,
|
|
3951
|
+
)
|
|
3952
|
+
return 2
|
|
3953
|
+
for target, data in changes:
|
|
3954
|
+
try:
|
|
3955
|
+
atomic_write(helper, target, data)
|
|
3956
|
+
except OSError as exc:
|
|
3957
|
+
print(
|
|
3958
|
+
f"error: {exc}; already written: "
|
|
3959
|
+
f"{', '.join(written) if written else 'nothing'}",
|
|
3960
|
+
file=sys.stderr,
|
|
3961
|
+
)
|
|
3962
|
+
return 2
|
|
3963
|
+
written.append(str(target))
|
|
3964
|
+
|
|
3965
|
+
manual = None
|
|
3966
|
+
if hash_content is not None and inventory is not None and not args.dry_run:
|
|
3967
|
+
try:
|
|
3968
|
+
current = hash_content.find_manual_blocks(
|
|
3969
|
+
(skill_dir / "SKILL.md").read_bytes()
|
|
3970
|
+
)
|
|
3971
|
+
manual = hash_content.classify_manual_blocks(inventory, current)
|
|
3972
|
+
except Exception as exc: # noqa: BLE001 - exit 2, never read as a WARN
|
|
3973
|
+
return _unexpected("the [MANUAL] re-check", exc, written)
|
|
3974
|
+
|
|
3975
|
+
applied = plan["applied"]
|
|
3976
|
+
left = plan["left_as_warn"]
|
|
3977
|
+
result = {
|
|
3978
|
+
"applied": applied,
|
|
3979
|
+
"left_as_warn": left,
|
|
3980
|
+
"files_changed": [str(target) for target, _ in changes],
|
|
3981
|
+
"files_written": written,
|
|
3982
|
+
"dry_run": bool(args.dry_run),
|
|
3983
|
+
"manual_verify": manual,
|
|
3984
|
+
"summary": {
|
|
3985
|
+
"citation_prefixes_fixed": sum(
|
|
3986
|
+
1 for a in applied if a["kind"] == FIX_CITATION_PREFIX
|
|
3987
|
+
),
|
|
3988
|
+
"source_lines_moved": sum(
|
|
3989
|
+
1 for a in applied if a["kind"] == FIX_SOURCE_LINE
|
|
3990
|
+
),
|
|
3991
|
+
"citations_moved": sum(
|
|
3992
|
+
1 for a in applied if a["kind"] == FIX_CITATION_LINE
|
|
3993
|
+
),
|
|
3994
|
+
"left_as_warn_count": len(left),
|
|
3995
|
+
},
|
|
3996
|
+
}
|
|
3997
|
+
if not _emit(result, args.output, written):
|
|
3998
|
+
return 2
|
|
3999
|
+
return 0 if not left and (manual is None or manual.get("ok")) else 1
|
|
4000
|
+
|
|
4001
|
+
|
|
4002
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
4003
|
+
parser = argparse.ArgumentParser(
|
|
4004
|
+
prog="skf-verify-provenance-completeness",
|
|
4005
|
+
description=(
|
|
4006
|
+
"Cross-reference documented exports (metadata.json) against "
|
|
4007
|
+
"provenance-map entries: completeness, orphans, stale or "
|
|
4008
|
+
"non-definition file:line citations, (with --skill-dir) "
|
|
4009
|
+
"citation prefixes and (with --check-node-kinds) ast-grep node "
|
|
4010
|
+
"kinds, emitting findings as JSON; fix the findings with one "
|
|
4011
|
+
"answer; look up an export's definition lines, the stale names "
|
|
4012
|
+
"no cited file defines, or the recipe kind at a line."
|
|
4013
|
+
),
|
|
4014
|
+
)
|
|
4015
|
+
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
4016
|
+
p = sub.add_parser(
|
|
4017
|
+
"verify",
|
|
4018
|
+
help=(
|
|
4019
|
+
"emit provenance completeness / orphan / stale-citation / "
|
|
4020
|
+
"citation-prefix / node-kind findings"
|
|
4021
|
+
),
|
|
4022
|
+
)
|
|
4023
|
+
p.add_argument("--metadata", required=True, help="path to metadata.json")
|
|
4024
|
+
p.add_argument(
|
|
4025
|
+
"--provenance", required=True, help="path to provenance-map.json"
|
|
4026
|
+
)
|
|
4027
|
+
p.add_argument(
|
|
4028
|
+
"--source-root",
|
|
4029
|
+
default=None,
|
|
4030
|
+
help=(
|
|
4031
|
+
"source tree root for citation resolution; overrides the "
|
|
4032
|
+
"provenance map's source_root. Stale check is skipped when no "
|
|
4033
|
+
"root resolves on disk."
|
|
4034
|
+
),
|
|
4035
|
+
)
|
|
4036
|
+
p.add_argument(
|
|
4037
|
+
"--skill-dir",
|
|
4038
|
+
default=None,
|
|
4039
|
+
help=(
|
|
4040
|
+
"skill package folder holding a SKILL.md; scans it and "
|
|
4041
|
+
"references/**/*.md for [AST:]/[SRC:] citations and checks each "
|
|
4042
|
+
"prefix against the provenance entry it cites. Citation check is "
|
|
4043
|
+
"skipped when absent."
|
|
4044
|
+
),
|
|
4045
|
+
)
|
|
4046
|
+
p.add_argument(
|
|
4047
|
+
"--check-node-kinds",
|
|
4048
|
+
action="store_true",
|
|
4049
|
+
help=(
|
|
4050
|
+
"ask the ast-grep CLI whether each ast-grep entry's ast_node_type "
|
|
4051
|
+
"is a node kind of its file's language; skipped when ast-grep is "
|
|
4052
|
+
"not found."
|
|
4053
|
+
),
|
|
4054
|
+
)
|
|
4055
|
+
p.add_argument(
|
|
4056
|
+
"-o",
|
|
4057
|
+
"--output",
|
|
4058
|
+
default=None,
|
|
4059
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
4060
|
+
)
|
|
4061
|
+
p.add_argument(
|
|
4062
|
+
"--verbose", action="store_true", help="diagnostics to stderr"
|
|
4063
|
+
)
|
|
4064
|
+
p.set_defaults(func=_cmd_verify)
|
|
4065
|
+
|
|
4066
|
+
p = sub.add_parser(
|
|
4067
|
+
"fix",
|
|
4068
|
+
help=(
|
|
4069
|
+
"apply the citation-prefix and source-line fixes a verify result "
|
|
4070
|
+
"has one answer for"
|
|
4071
|
+
),
|
|
4072
|
+
)
|
|
4073
|
+
p.add_argument(
|
|
4074
|
+
"--verify", required=True, help="path to the JSON a verify run wrote"
|
|
4075
|
+
)
|
|
4076
|
+
p.add_argument(
|
|
4077
|
+
"--provenance",
|
|
4078
|
+
required=True,
|
|
4079
|
+
help="path to the provenance-map.json that verify run read",
|
|
4080
|
+
)
|
|
4081
|
+
p.add_argument(
|
|
4082
|
+
"--skill-dir",
|
|
4083
|
+
required=True,
|
|
4084
|
+
help="skill package folder holding a SKILL.md and references/",
|
|
4085
|
+
)
|
|
4086
|
+
p.add_argument(
|
|
4087
|
+
"--manual-inventory",
|
|
4088
|
+
default=None,
|
|
4089
|
+
help=(
|
|
4090
|
+
"a [MANUAL] inventory of SKILL.md (skf-hash-content.py "
|
|
4091
|
+
"manual-inventory); checked again after the writes"
|
|
4092
|
+
),
|
|
4093
|
+
)
|
|
4094
|
+
p.add_argument(
|
|
4095
|
+
"--no-line-moves",
|
|
4096
|
+
action="store_true",
|
|
4097
|
+
help="fix citation prefixes only; list every source-line finding",
|
|
4098
|
+
)
|
|
4099
|
+
p.add_argument(
|
|
4100
|
+
"--dry-run", action="store_true", help="plan the fixes, write nothing"
|
|
4101
|
+
)
|
|
4102
|
+
p.add_argument(
|
|
4103
|
+
"-o",
|
|
4104
|
+
"--output",
|
|
4105
|
+
default=None,
|
|
4106
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
4107
|
+
)
|
|
4108
|
+
p.set_defaults(func=_cmd_fix)
|
|
4109
|
+
|
|
4110
|
+
p = sub.add_parser(
|
|
4111
|
+
"definition-lines",
|
|
4112
|
+
help="emit the lines of a source file that define an export",
|
|
4113
|
+
)
|
|
4114
|
+
p.add_argument(
|
|
4115
|
+
"--file", required=True, help="source file, relative to --source-root"
|
|
4116
|
+
)
|
|
4117
|
+
p.add_argument("--name", required=True, help="the entry's export_name")
|
|
4118
|
+
p.add_argument(
|
|
4119
|
+
"--source-root",
|
|
4120
|
+
default=None,
|
|
4121
|
+
help="source tree root (default: the current folder)",
|
|
4122
|
+
)
|
|
4123
|
+
p.add_argument(
|
|
4124
|
+
"--line",
|
|
4125
|
+
type=int,
|
|
4126
|
+
default=None,
|
|
4127
|
+
help="the recorded source_line: report whether it defines the export",
|
|
4128
|
+
)
|
|
4129
|
+
p.add_argument(
|
|
4130
|
+
"--export-type",
|
|
4131
|
+
default=None,
|
|
4132
|
+
help="the entry's export_type (module and package have no line)",
|
|
4133
|
+
)
|
|
4134
|
+
p.add_argument(
|
|
4135
|
+
"-o",
|
|
4136
|
+
"--output",
|
|
4137
|
+
default=None,
|
|
4138
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
4139
|
+
)
|
|
4140
|
+
p.set_defaults(func=_cmd_definition_lines)
|
|
4141
|
+
|
|
4142
|
+
p = sub.add_parser(
|
|
4143
|
+
"classify-stale",
|
|
4144
|
+
help=(
|
|
4145
|
+
"decide which stale names are fabricated signatures (no cited "
|
|
4146
|
+
"file defines them) and where the source declares the others"
|
|
4147
|
+
),
|
|
4148
|
+
)
|
|
4149
|
+
p.add_argument(
|
|
4150
|
+
"--names",
|
|
4151
|
+
required=True,
|
|
4152
|
+
help=(
|
|
4153
|
+
"reconcile-coverage.py's result (its `stale` names) or a JSON "
|
|
4154
|
+
"array of names"
|
|
4155
|
+
),
|
|
4156
|
+
)
|
|
4157
|
+
p.add_argument(
|
|
4158
|
+
"--provenance", required=True, help="path to provenance-map.json"
|
|
4159
|
+
)
|
|
4160
|
+
p.add_argument(
|
|
4161
|
+
"--source-root",
|
|
4162
|
+
required=True,
|
|
4163
|
+
help="source tree root the entries' source_file paths are under",
|
|
4164
|
+
)
|
|
4165
|
+
p.add_argument(
|
|
4166
|
+
"--inventory",
|
|
4167
|
+
default=None,
|
|
4168
|
+
help=(
|
|
4169
|
+
"validate-inventory.py's documented inventory: a name it gives a "
|
|
4170
|
+
"function, hook, class, interface, enum or type kind takes no "
|
|
4171
|
+
"line 1 from a module so named, unless a provenance map entry of "
|
|
4172
|
+
"the name is a module or package one or the module is a TS/JS "
|
|
4173
|
+
"file that is itself the export"
|
|
4174
|
+
),
|
|
4175
|
+
)
|
|
4176
|
+
p.add_argument(
|
|
4177
|
+
"-o",
|
|
4178
|
+
"--output",
|
|
4179
|
+
default=None,
|
|
4180
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
4181
|
+
)
|
|
4182
|
+
p.set_defaults(func=_cmd_classify_stale)
|
|
4183
|
+
|
|
4184
|
+
p = sub.add_parser(
|
|
4185
|
+
"kind-at",
|
|
4186
|
+
help="emit the node kind of the recipe that matches a source line",
|
|
4187
|
+
)
|
|
4188
|
+
p.add_argument(
|
|
4189
|
+
"--file",
|
|
4190
|
+
required=True,
|
|
4191
|
+
help="source file, relative to --source-root when that is given",
|
|
4192
|
+
)
|
|
4193
|
+
p.add_argument(
|
|
4194
|
+
"--line", type=int, required=True, help="the line $NAME must be on"
|
|
4195
|
+
)
|
|
4196
|
+
p.add_argument(
|
|
4197
|
+
"--recipes",
|
|
4198
|
+
required=True,
|
|
4199
|
+
help=(
|
|
4200
|
+
"the recipes: a markdown file (its ```yaml fences, such as "
|
|
4201
|
+
"extraction-patterns.md) or a YAML file"
|
|
4202
|
+
),
|
|
4203
|
+
)
|
|
4204
|
+
p.add_argument(
|
|
4205
|
+
"--language",
|
|
4206
|
+
default=None,
|
|
4207
|
+
help="ast-grep language (default: from the file's extension)",
|
|
4208
|
+
)
|
|
4209
|
+
p.add_argument(
|
|
4210
|
+
"--name",
|
|
4211
|
+
default=None,
|
|
4212
|
+
help="export name: count only matches whose $NAME is its last segment",
|
|
4213
|
+
)
|
|
4214
|
+
p.add_argument("--source-root", default=None, help="source tree root")
|
|
4215
|
+
p.add_argument(
|
|
4216
|
+
"-o",
|
|
4217
|
+
"--output",
|
|
4218
|
+
default=None,
|
|
4219
|
+
help="write JSON to this file instead of stdout ('-' for stdout)",
|
|
4220
|
+
)
|
|
4221
|
+
p.set_defaults(func=_cmd_kind_at)
|
|
429
4222
|
return parser
|
|
430
4223
|
|
|
431
4224
|
|