bmad-module-skill-forge 2.2.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +49 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +135 -18
- package/docs/verifying-a-skill.md +83 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +2 -2
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +32 -12
- package/src/shared/references/description-guard-protocol.md +12 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +30 -14
- package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +694 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
- package/src/shared/scripts/skf-description-guard.py +99 -11
- package/src/shared/scripts/skf-detect-docs.py +694 -2
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
- package/src/shared/scripts/skf-detect-tools.py +209 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +4084 -85
- package/src/shared/scripts/skf-extraction-inventory.py +746 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +661 -0
- package/src/shared/scripts/skf-hash-content.py +166 -12
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +159 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +161 -27
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +670 -119
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +5 -5
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
- package/src/shared/scripts/skf-write-skill-brief.py +367 -19
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/init.md +148 -210
- package/src/skf-audit-skill/references/re-index.md +96 -108
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +27 -18
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +75 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
- package/src/skf-brief-skill/SKILL.md +36 -25
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +17 -24
- package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +142 -125
- package/src/skf-brief-skill/references/confirm-brief.md +10 -8
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +3 -2
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
- package/src/skf-create-skill/assets/skill-sections.md +15 -5
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +107 -29
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
- package/src/skf-create-skill/references/extract.md +200 -161
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
- package/src/skf-create-skill/references/extraction-patterns.md +584 -196
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +28 -59
- package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +28 -19
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +79 -146
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +118 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -90
- package/src/skf-setup/references/detect-and-tier.md +58 -67
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +217 -254
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +163 -88
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +162 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +41 -155
- package/src/skf-test-skill/references/source-access-protocol.md +19 -92
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +287 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
- package/src/skf-test-skill/scripts/score-signatures.py +459 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +233 -222
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +250 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +180 -47
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +100 -35
- package/src/skf-update-skill/references/write.md +331 -149
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -679
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -1,40 +1,155 @@
|
|
|
1
1
|
# /// script
|
|
2
2
|
# requires-python = ">=3.11"
|
|
3
|
-
# dependencies = []
|
|
3
|
+
# dependencies = ["pyyaml"]
|
|
4
4
|
# ///
|
|
5
|
-
"""SKF Extract Public API
|
|
5
|
+
"""SKF Extract Public API: one package's public surface, by parser or by recipe.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
manifest, one or more entry-point files) and emits a structured
|
|
9
|
-
extraction inventory: package name, version, description, public
|
|
10
|
-
exports, declared dependencies, and (for Maven/Gradle) discovered
|
|
11
|
-
sub-modules. The helper does NO file I/O — the caller fetches files
|
|
12
|
-
(via gh api / web browsing / local filesystem / wherever) and pipes
|
|
13
|
-
their contents in. Multi-module monorepos are caller-orchestrated:
|
|
14
|
-
invoke once per module, aggregate the results.
|
|
7
|
+
Three modes:
|
|
15
8
|
|
|
16
|
-
|
|
9
|
+
--mode quick (the default) A pure parser: takes one logical package (one
|
|
10
|
+
manifest, one or more entry-point files) and emits its
|
|
11
|
+
package name, version, description, public exports,
|
|
12
|
+
declared dependencies and (for Maven/Gradle) sub-modules.
|
|
13
|
+
The files come as a JSON payload on stdin, or by path with
|
|
14
|
+
--manifest-file and --entry-file, which the script reads
|
|
15
|
+
from disk (the caller stages them first: skf-github-fetch.py
|
|
16
|
+
raw fetches into a folder, or a local checkout), so a file's
|
|
17
|
+
text never passes through the model or a shell string.
|
|
18
|
+
--follow also reads the modules an `export *` chain passes
|
|
19
|
+
names on from, fetching them first with --repo. Multi-module
|
|
20
|
+
monorepos are caller-orchestrated: invoke once per module,
|
|
21
|
+
aggregate.
|
|
22
|
+
--mode full The ast-grep recipe runner (skf-create-skill and the
|
|
23
|
+
workflows that extract as it does): runs the recipes in
|
|
24
|
+
src/shared/data/ast-grep-recipes.yaml over a source tree
|
|
25
|
+
and emits every export with its exact file, line, name and
|
|
26
|
+
node kind, the export surface the package's entry points
|
|
27
|
+
declare, and the counts compile needs. See "Full mode" below.
|
|
28
|
+
--mode entries The files quick mode reads for one package, found in the
|
|
29
|
+
repository's file listing with full mode's entry-point
|
|
30
|
+
rules, so no caller derives an entry file from a
|
|
31
|
+
distribution name or reads a manifest by eye. See "Entries
|
|
32
|
+
mode" below.
|
|
33
|
+
|
|
34
|
+
Quick mode
|
|
35
|
+
----------
|
|
36
|
+
|
|
37
|
+
Supported languages (the payload's `language`, or --language; first listed
|
|
38
|
+
is preferred):
|
|
17
39
|
|
|
18
40
|
js, ts, javascript, typescript package.json index.{js,ts}, src/index.{js,ts}
|
|
19
|
-
python pyproject.toml __init__.py / setup.py
|
|
41
|
+
python pyproject.toml __init__.py / setup.py / setup.cfg
|
|
20
42
|
rust Cargo.toml src/lib.rs
|
|
21
43
|
go go.mod top-level *.go
|
|
22
44
|
java pom.xml src/main/java/**/*.java
|
|
23
45
|
kotlin build.gradle* src/main/kotlin/**/*.kt
|
|
46
|
+
swift Package.swift Sources/<target>/*.swift
|
|
24
47
|
|
|
25
|
-
|
|
48
|
+
Top-level exports only, read from the entry files. Quick mode reads no other
|
|
49
|
+
file (--follow aside, below), so a name an entry file passes on from another
|
|
50
|
+
module without naming it is not an export here: a warning names each such
|
|
51
|
+
statement, and an `unlisted[]` record gives its file, line, statement,
|
|
52
|
+
specifier and `module_file`, so the caller can pass that module as a
|
|
53
|
+
--follow-file, or have --follow read it (a module_file), or read it by eye
|
|
54
|
+
(none). A warning also names
|
|
55
|
+
each statement below whose names the file holds in a form quick mode does
|
|
56
|
+
not read (an anonymous default export, a destructuring declaration, an
|
|
57
|
+
assignment inside a block).
|
|
26
58
|
|
|
27
|
-
|
|
59
|
+
JavaScript / TypeScript read with comments, strings and regular
|
|
60
|
+
expressions blanked: column-0 `export` declarations (`const`, `let`
|
|
61
|
+
and `var`, every declarator of the statement, `function`,
|
|
62
|
+
`function*`, `async function`, `class`, `abstract class`,
|
|
63
|
+
`interface`, `type`, `enum`, `const enum`, `namespace`, each may be
|
|
64
|
+
`default` or `declare`), typed by keyword; `export { ... }` and
|
|
65
|
+
`export type { ... }` lists (`re-export`); `export * as ns from`
|
|
66
|
+
(`namespace`); `export default <name>` (the type of that local
|
|
67
|
+
declaration, else `re-export`); TypeScript's `export = <value>`, read
|
|
68
|
+
as `module.exports = <value>` is; and CommonJS: each `exports.<name> =`
|
|
69
|
+
and `module.exports.<name> =` (each name of a chain such as tsc's
|
|
70
|
+
`exports.a = exports.b = void 0`), the keys of a `module.exports =
|
|
71
|
+
{ ... }` object, and the one name of `module.exports = <name>`
|
|
72
|
+
(`exports = module.exports = <name>` included), a named function or a
|
|
73
|
+
named class (`function`, `class`, or the local declaration's type,
|
|
74
|
+
else `variable`). Warned: `export * from`, `module.exports =
|
|
75
|
+
require(...)`, a spread in the `module.exports` object, a
|
|
76
|
+
`module.exports` or `export =` that is anonymous or an expression, an
|
|
77
|
+
`export default` that is anonymous or an expression, an `export const`
|
|
78
|
+
that destructures, and a `module.exports`, `exports` or
|
|
79
|
+
`exports.<name>` assignment that does not open its line (in an `if`, a
|
|
80
|
+
block or a function, so which one runs the file alone cannot tell).
|
|
81
|
+
Python with `__all__` (literal lists and tuples, however the module
|
|
82
|
+
builds it, read as full mode reads it), every public name it lists,
|
|
83
|
+
typed by what binds it in the file: `def` (async def included),
|
|
84
|
+
`class`, `variable` (an assignment), `module` (`import x`),
|
|
85
|
+
`re-export` (an import, or no binding in the file); a part that is no
|
|
86
|
+
literal (another module's `__all__`) is warned. Without `__all__`, the
|
|
87
|
+
public def, async def and class names, and the names imported from
|
|
88
|
+
the package itself (`re-export`: `from .x import Y`, or, in an
|
|
89
|
+
`__init__.py`, an absolute import that names the package by any
|
|
90
|
+
trailing run of its folders: `storage`, `cloud.storage` or
|
|
91
|
+
`acme.cloud.storage` for acme/cloud/storage/__init__.py); a star
|
|
92
|
+
import from the package is warned. Imports under `if TYPE_CHECKING:`
|
|
93
|
+
do not count. A module this Python cannot parse is read from its
|
|
94
|
+
text, with a warning.
|
|
95
|
+
Rust column-0 unrestricted `pub` items, qualifiers included (`const`,
|
|
96
|
+
`async`, `unsafe`, `extern "C"`: `pub const fn f` is the fn `f`),
|
|
97
|
+
typed by item keyword; each `#[macro_export]` `macro_rules!` macro
|
|
98
|
+
(`macro`); and each name a column-0 `pub use` brings in (`re-export`,
|
|
99
|
+
under its alias when it has one); a glob `pub use x::*` is warned. An
|
|
100
|
+
indented `pub` item is a method or sits in an inline module, not a
|
|
101
|
+
top-level export.
|
|
102
|
+
Go capitalized top-level func, type, var and const names.
|
|
103
|
+
Java public classes, interfaces, enums and records, and the classes a
|
|
104
|
+
Spring or Jakarta annotation marks.
|
|
105
|
+
Kotlin declarations not marked internal or private.
|
|
106
|
+
Swift public and open declarations.
|
|
28
107
|
|
|
29
|
-
Input JSON
|
|
108
|
+
Input: a JSON payload on stdin,
|
|
30
109
|
|
|
31
110
|
{
|
|
32
111
|
"language": "python",
|
|
33
112
|
"manifest": {"path": "pyproject.toml", "content": "..."},
|
|
34
|
-
"entries": [{"path": "src/foo/__init__.py", "content": "..."
|
|
113
|
+
"entries": [{"path": "src/foo/__init__.py", "content": "...",
|
|
114
|
+
"followed": false}, ...],
|
|
35
115
|
"mode": "quick"
|
|
36
116
|
}
|
|
37
117
|
|
|
118
|
+
or the files, read from disk (stdin is then never read):
|
|
119
|
+
|
|
120
|
+
--mode quick --language <language> [--source-root <dir>]
|
|
121
|
+
[--manifest-file <path>] [--entry-file <path>]...
|
|
122
|
+
[--follow-file <path>]... [--tree-file <listing>] [--fetch-list <file>]
|
|
123
|
+
[--follow [--repo <owner/repo> [--ref <ref>]]]
|
|
124
|
+
|
|
125
|
+
An entry with `followed` true (a --follow-file) is a module an `unlisted[]`
|
|
126
|
+
record named, read as an entry is, less a JS/TS `export default`, which
|
|
127
|
+
`export *` does not pass on.
|
|
128
|
+
|
|
129
|
+
--follow (with --tree-file and --source-root) reads those modules itself,
|
|
130
|
+
so no caller loops over the files: while `unlisted[]` names a module_file
|
|
131
|
+
not tried yet, at most 5 rounds, each one is read as one more --follow-file
|
|
132
|
+
and the files are parsed again, so an `export *` chain is read to its end.
|
|
133
|
+
With --repo, each is first fetched into --source-root at --ref (HEAD by
|
|
134
|
+
default) by skf-github-fetch.py, which reads it from
|
|
135
|
+
raw.githubusercontent.com and through gh only for a private repository;
|
|
136
|
+
without it, each is read where --source-root holds it (a local checkout).
|
|
137
|
+
A module that cannot be fetched or read stays in `unlisted[]`, and a
|
|
138
|
+
warning names it and why. The output then also lists `followed`, the
|
|
139
|
+
modules read this way, in the order they were read.
|
|
140
|
+
|
|
141
|
+
Each path is relative to --source-root, the folder the files were staged in
|
|
142
|
+
laid out like the repository (or a local checkout), else to the current
|
|
143
|
+
folder, and is the `path` the output names (`source_file`), written with
|
|
144
|
+
`/`. A path under --source-root may not be absolute or lead outside it. A
|
|
145
|
+
file's text is read as UTF-8 (a byte order mark dropped, an invalid byte
|
|
146
|
+
replaced), so a quote or an apostrophe in it reaches the parser as it is.
|
|
147
|
+
An entry named twice is read once (a file both an --entry-file and a
|
|
148
|
+
--follow-file is an entry). Without --manifest-file the package metadata is
|
|
149
|
+
empty, as for a payload whose manifest `content` is empty. A python
|
|
150
|
+
manifest is read by its name: setup.py and setup.cfg by their own parsers,
|
|
151
|
+
any other as pyproject.toml.
|
|
152
|
+
|
|
38
153
|
Output JSON shape (stdout):
|
|
39
154
|
|
|
40
155
|
{
|
|
@@ -45,26 +160,343 @@ Output JSON shape (stdout):
|
|
|
45
160
|
"exports": [{"name": "Bar", "type": "class", "source_file": "..."}, ...],
|
|
46
161
|
"dependencies": ["requests", "pydantic", ...],
|
|
47
162
|
"modules": ["server", "client"], (Maven/Gradle only)
|
|
163
|
+
"module_folders": ["server", "client"], (each module's folder in
|
|
164
|
+
the repository)
|
|
48
165
|
"extra": {"group_id": "com.example"}, (Maven only)
|
|
49
|
-
"warnings": ["..."]
|
|
166
|
+
"warnings": ["..."], (parse failures, fallbacks)
|
|
167
|
+
"unlisted": [{"file", "line", "statement", "specifier",
|
|
168
|
+
"module_file"}, ...],
|
|
169
|
+
"followed": ["src/core.ts", ...] (--follow only)
|
|
50
170
|
}
|
|
51
171
|
|
|
52
|
-
|
|
172
|
+
`unlisted` holds one record per statement whose names come from a module
|
|
173
|
+
quick mode did not read: `export * from`, `module.exports = require(...)`
|
|
174
|
+
(and such an assignment in a block), a spread in the `module.exports`
|
|
175
|
+
object, a Python star import from the package and a Rust `pub use x::*`.
|
|
176
|
+
`module_file` is the file of the listing (--tree-file, or a payload's
|
|
177
|
+
`tree`) that module is, found as full mode finds it (resolve_js_module, a
|
|
178
|
+
NodeNext `./x.js` naming x.ts, and _py_module_file); null without a
|
|
179
|
+
listing, and for a package import, a spread, a Rust glob or a module the
|
|
180
|
+
listing does not hold. A statement whose module_file is already an entry
|
|
181
|
+
file is left out, with its warning: its names are read. --fetch-list
|
|
182
|
+
writes each module_file still to read, one per line, so a caller fetches
|
|
183
|
+
them, passes them as more --follow-file and runs again until the list is
|
|
184
|
+
empty, which --follow does itself. `module_folders` gives each of
|
|
185
|
+
`modules` as a folder of the repository: the manifest's folder joined with
|
|
186
|
+
the module (a Gradle `core:api` read as `core/api`).
|
|
187
|
+
|
|
188
|
+
Exit codes (quick):
|
|
53
189
|
|
|
54
190
|
0 success
|
|
55
|
-
1 payload-level error (
|
|
56
|
-
2 stdin
|
|
191
|
+
1 payload-level error (a language quick mode does not parse)
|
|
192
|
+
2 input error: stdin, argparse or JSON-decode error, --language missing
|
|
193
|
+
or given twice with the file inputs, a --manifest-file, --entry-file
|
|
194
|
+
or --follow-file that cannot be read or leads outside --source-root,
|
|
195
|
+
a --tree-file that cannot be read or a --fetch-list that cannot be
|
|
196
|
+
written, --follow without --tree-file and --source-root, --repo or
|
|
197
|
+
--ref without --follow (--ref without --repo), or a --repo or --ref
|
|
198
|
+
skf-github-fetch.py does not accept. A module --follow could not
|
|
199
|
+
read changes no exit code.
|
|
200
|
+
|
|
201
|
+
Entries mode
|
|
202
|
+
------------
|
|
203
|
+
|
|
204
|
+
--mode entries --language <language> --tree-file <listing>
|
|
205
|
+
--source-root <dir> [--scope <folder>] [--fetch-list <file>]
|
|
206
|
+
|
|
207
|
+
The entry files of the one package at --scope (the repository root
|
|
208
|
+
without it), read from the listing (skf-github-probe.py tree output, or any
|
|
209
|
+
listing skf-detect-language.py's --tree-file reads). The package's manifest
|
|
210
|
+
is read from --source-root, the folder it was fetched into (a folder not
|
|
211
|
+
made yet holds none):
|
|
212
|
+
|
|
213
|
+
js, ts the files its package.json exports map, else its
|
|
214
|
+
types/module/main/browser fields, name (a build output not
|
|
215
|
+
committed mapped back to its source: dist/index.js ->
|
|
216
|
+
src/index.ts), else index.* or src/index.* (_js_entries)
|
|
217
|
+
python the top __init__.py of each package among its .py files
|
|
218
|
+
(_py_entries), less those under a top folder named test*,
|
|
219
|
+
docs, examples or benchmarks, the one whose folder is the
|
|
220
|
+
manifest name as Python imports it preferred; with no
|
|
221
|
+
package, <name>.py or src/<name>.py (a single-module
|
|
222
|
+
package such as six). The name is the first one the
|
|
223
|
+
manifests give, read in the order pyproject.toml ([project]
|
|
224
|
+
or [tool.poetry]), setup.py, setup.cfg ([metadata]), and
|
|
225
|
+
`manifest` is the one that gave it (else the first listed)
|
|
226
|
+
rust the crate root, `[lib] path` else src/lib.rs (_rust_entries)
|
|
227
|
+
go the .go files directly in the folder, less *_test.go
|
|
228
|
+
java src/main/java/<groupId as a path>/*.java (the pom's groupId,
|
|
229
|
+
else its parent's)
|
|
230
|
+
kotlin src/main/kotlin/**/*.kt
|
|
231
|
+
|
|
232
|
+
Go, Java and Kotlin read the first 5 files in path order, and no language
|
|
233
|
+
more than 25. Output (stdout):
|
|
234
|
+
|
|
235
|
+
{
|
|
236
|
+
"mode": "entries",
|
|
237
|
+
"language": "python",
|
|
238
|
+
"scope": "." | "<folder>",
|
|
239
|
+
"manifest": "<path in the listing>" | null,
|
|
240
|
+
"entry_files": ["src/PIL/__init__.py", ...],
|
|
241
|
+
"unresolved": [{"package", "subpath", "targets"}, ...],
|
|
242
|
+
"warnings": ["..."]
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
`manifest` is the manifest the language reads at the folder (null when the
|
|
246
|
+
listing has none), `unresolved` names what no listed file answered (a
|
|
247
|
+
package.json target, or the paths tried), and --fetch-list writes
|
|
248
|
+
`entry_files` one per line. A language quick mode parses but has no rule
|
|
249
|
+
here (swift) gives no entry file and a warning.
|
|
250
|
+
|
|
251
|
+
Exit codes (entries): 0 (an empty `entry_files` included); 1 for a language
|
|
252
|
+
quick mode does not parse; 2 for an input error (a --tree-file that cannot
|
|
253
|
+
be read, a --source-root that is a file, a --fetch-list that cannot be
|
|
254
|
+
written, a flag of another mode).
|
|
255
|
+
|
|
256
|
+
Full mode
|
|
257
|
+
---------
|
|
258
|
+
|
|
259
|
+
--mode full --source-root <dir>
|
|
260
|
+
[--brief <skill-brief.yaml>] [--include GLOB]... [--exclude GLOB]...
|
|
261
|
+
[--tier-a-include GLOB]... [--scope-type TYPE] [--language LANG]...
|
|
262
|
+
[--files-from <file>] [--recipe-set standard|component-library]
|
|
263
|
+
[--tier Quick|Forge|Forge+|Deep] [--head-cap N] [--recipes <file>]
|
|
264
|
+
[--timeout <seconds>] [-o <out.json>]
|
|
265
|
+
|
|
266
|
+
Which files. The files under --source-root are those git lists there
|
|
267
|
+
(tracked, and untracked but not ignored), or every file when it is not in
|
|
268
|
+
a git work tree, leaving out hidden entries and symbolic links: the files
|
|
269
|
+
ast-grep itself would walk. With --files-from (paths relative to
|
|
270
|
+
--source-root: a JSON list when the whole file is one, else one path per
|
|
271
|
+
line, so a first path such as `[slug]/page.tsx` is read as a path) only
|
|
272
|
+
the files it names are read, and a named file that is missing, or whose
|
|
273
|
+
language has no recipe, is listed in `file_issues`. A file is in scope when an include glob matches it (every
|
|
274
|
+
file when there is none) and no exclude glob does, and when its language
|
|
275
|
+
is one the recipes read (python, typescript, tsx, javascript, rust, go and
|
|
276
|
+
.vue files) and --language allows it. Globs follow
|
|
277
|
+
skf-resolve-authoritative-files.py's rules: `**` spans any number of path
|
|
278
|
+
segments, none included, and `*` and `?` stay inside one, so `src/**/*.ts`
|
|
279
|
+
matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`.
|
|
280
|
+
|
|
281
|
+
--brief supplies scope.include, scope.exclude, scope.tier_a_include,
|
|
282
|
+
scope.type and language; --include, --exclude, --tier-a-include,
|
|
283
|
+
--scope-type and --language replace the brief's value when given.
|
|
284
|
+
--language takes a language family (`typescript`, `javascript`, `ts`,
|
|
285
|
+
`js`, `tsx` and `vue` all select .ts, .tsx, .js and .vue files; `python`,
|
|
286
|
+
`rust`, `go`).
|
|
287
|
+
|
|
288
|
+
Which recipes. --recipe-set picks the recipes whose metadata.sets lists it:
|
|
289
|
+
`component-library` when the scope type is component-library, else
|
|
290
|
+
`standard`. Each recipe runs in the languages its metadata.languages
|
|
291
|
+
lists. A match whose name starts with `_` is never an export. --head-cap
|
|
292
|
+
keeps at most N matches of each recipe, in file and line order (0: no
|
|
293
|
+
cap); without it the cap is 200, or at Forge+ and Deep tiers 500 for a
|
|
294
|
+
full-library scope and 300 for a component-library one. `truncated` is
|
|
295
|
+
true when a recipe had more.
|
|
296
|
+
|
|
297
|
+
Merging. The matches of one name in one file become one export, at the
|
|
298
|
+
first line that names it (overload signatures, and a definition in both
|
|
299
|
+
branches of an `if`, repeat a name). When several recipes match that line,
|
|
300
|
+
the export records the one no other of them lists under
|
|
301
|
+
metadata.prefer_over, else the first in the recipe file. Its `source_line`
|
|
302
|
+
is the line of `$NAME` (a decorator or an `export` line can come first),
|
|
303
|
+
`signature_line` the whole source line holding it (the full list line for
|
|
304
|
+
a list item), `signature` the declaration that line opens on one line (the
|
|
305
|
+
lines after it joined while a ( or [ it opens is still open, each without
|
|
306
|
+
its line comment, and a trailing comma before the closing bracket dropped,
|
|
307
|
+
so a parameter list a formatter split one per line reads as its one-line
|
|
308
|
+
form; at most 40 lines; otherwise `signature_line`), `ast_node_type` the
|
|
309
|
+
kind the recipe declares, and
|
|
310
|
+
`export_type` its metadata.export_type (for a list, the declaration
|
|
311
|
+
keyword before the name picks one). A function (export_type `function`)
|
|
312
|
+
also gets `params`, each parameter read from `signature` as {name, type,
|
|
313
|
+
default, optional} (Python's through its own parser; a rest or variadic
|
|
314
|
+
parameter, `...rest`, `*args`, `**kwargs` or Go's `...T`, is optional, and
|
|
315
|
+
a Go name shares the type written after it), and `return_type`, the type
|
|
316
|
+
the declaration states, null when it states none. Both are null for any
|
|
317
|
+
other export, and for a signature the reader does not parse (one cut at
|
|
318
|
+
the line cap, say), which the caller then reads by eye. An item of a local `export { ... }`
|
|
319
|
+
list takes instead the type of the top-level binding it names in its file:
|
|
320
|
+
what the recipes record for that declaration exported in place (a const
|
|
321
|
+
bound to an arrow function or a function expression is a function), or
|
|
322
|
+
`re-export` for an import.
|
|
323
|
+
|
|
324
|
+
Entry points. The export surface each package declares: every subpath of a
|
|
325
|
+
package.json `exports` map (committed .d.ts/.d.mts targets outside src/
|
|
326
|
+
included, a build output mapped back to its source when the target is not
|
|
327
|
+
committed, a pattern subpath's `*` matched against the tree, one entry per
|
|
328
|
+
file, less the subpaths a `null` target takes out), else its
|
|
329
|
+
types/module/main fields, else index.* or src/index.*; a Python package's
|
|
330
|
+
top __init__.py (its __all__ when it has one, else the public names it
|
|
331
|
+
defines or imports from the package, and those its star imports give); a
|
|
332
|
+
crate's lib.rs (its `pub use` items, globs included, and column-0
|
|
333
|
+
unrestricted `pub` items); a Go package's capitalized top-level names
|
|
334
|
+
outside _test.go, package main and internal/ folders. Each entry point is
|
|
335
|
+
`in_scope` when it is one of the files in scope (a warning names each
|
|
336
|
+
JS/TS one that is not). Chains are followed to the end, a cycle cut: an
|
|
337
|
+
`export *`, a star import or a glob passes on what its module exports,
|
|
338
|
+
that module's own chains included; a re-exported name is traced through
|
|
339
|
+
each re-export, import and glob between to the file that defines it (a
|
|
340
|
+
default export to the name it declares). A namespace (`export * as ns`, a
|
|
341
|
+
Python submodule, a Rust `pub mod`) counts once, and what it makes
|
|
342
|
+
reachable is not internal: the names its module passes on, the modules
|
|
343
|
+
among them in turn, and for Rust every recipe match in the crate root and
|
|
344
|
+
in each module file a `pub mod` or `pub use` of a reachable module names.
|
|
345
|
+
A chain the trace cannot follow (a module no file here holds, a `#[path]`
|
|
346
|
+
module) is listed in `unresolved`. The diff against the recipe inventory,
|
|
347
|
+
one language family at a time (JavaScript/TypeScript, Python, Rust, Go):
|
|
348
|
+
`public` (the entry points' names), `internal` (names the recipes found
|
|
349
|
+
that no entry point exports or makes reachable), `extraction_gaps` (names
|
|
350
|
+
an entry point exports that no recipe found, defined in a file in scope or
|
|
351
|
+
in none the trace reached: read them by eye) and `outside_scope` (such
|
|
352
|
+
names defined in a file out of scope: widen scope.include to document
|
|
353
|
+
them). In a family with no entry point, or only empty ones, every name the
|
|
354
|
+
recipes found is public.
|
|
355
|
+
|
|
356
|
+
Counts. `exports_public_api` counts the public names (the package's whole
|
|
357
|
+
public API, `outside_scope` names included) and `exports_internal` the
|
|
358
|
+
internal ones. `effective_denominator` counts the public names whose
|
|
359
|
+
definition file matches scope.tier_a_include (else scope.include) and no
|
|
360
|
+
scope.exclude glob: each name once, so a type counts once whatever its
|
|
361
|
+
methods, and a barrel that only re-exports adds nothing of its own. `arms` gives compile's effective_denominator arms:
|
|
362
|
+
(a) `monorepo` from skf-detect-workspaces.py, (b) `specific_modules`
|
|
363
|
+
(scope type specific-modules on a repository that is not a monorepo) and
|
|
364
|
+
(c) `multi_subpath_exports` (an in-scope package.json whose `exports` map
|
|
365
|
+
has more than one non-root subpath without a `*`, on a repository that is
|
|
366
|
+
not a monorepo).
|
|
367
|
+
|
|
368
|
+
Output JSON (stdout, or -o):
|
|
369
|
+
|
|
370
|
+
{
|
|
371
|
+
"mode": "full",
|
|
372
|
+
"status": "ok" | "incomplete" | "no-ast-grep",
|
|
373
|
+
"source_root": "<as given>",
|
|
374
|
+
"recipes_file": "<path>",
|
|
375
|
+
"recipe_set": "standard" | "component-library",
|
|
376
|
+
"ast_grep": {"path": "<exe>" | null, "version": "<x.y.z>" | null},
|
|
377
|
+
"scope": {"include": [...], "exclude": [...],
|
|
378
|
+
"tier_a_include": [...] | null, "type": "<type>" | null,
|
|
379
|
+
"languages": [...], "files_from": "<file>" | null},
|
|
380
|
+
"files_in_scope": N,
|
|
381
|
+
"files_by_language": {"<language>": N, ...},
|
|
382
|
+
"files_without_recipes": {".java": N, ...}, # in scope, no recipe reads them
|
|
383
|
+
"file_issues": [{"file", "issue": "missing" | "no-recipes" | "not-utf8"
|
|
384
|
+
| "unreadable" | "syntax-errors",
|
|
385
|
+
"count": N, "line": N}, ...], # count, line: syntax-errors
|
|
386
|
+
"head_cap": N | null,
|
|
387
|
+
"truncated": bool,
|
|
388
|
+
"recipes": [{"id", "languages": [...], "matches": N, "truncated": bool}, ...],
|
|
389
|
+
"exports": [{"export_name", "source_file", "source_line", "signature_line",
|
|
390
|
+
"signature", "params": [{"name", "type", "default",
|
|
391
|
+
"optional"}, ...] | null, "return_type": "<type>" | null,
|
|
392
|
+
"citation": "[AST:<file>:L<line>]", "ast_recipe",
|
|
393
|
+
"ast_node_type", "export_type", "language",
|
|
394
|
+
"from": "<module>" | null, "from_file": "<file>" | null,
|
|
395
|
+
"confidence": "T1", "extraction_method": "ast-grep"}, ...],
|
|
396
|
+
"aggregates": {"exports": N, "by_type": {"function": N, ...},
|
|
397
|
+
"t1": N, "t1_low": 0},
|
|
398
|
+
"entry_points": {"status": "barrel" | "empty-barrel" | "no-entry-point"
|
|
399
|
+
| null, # null: no-ast-grep
|
|
400
|
+
"by_language": {"<family>": "<status>", ...},
|
|
401
|
+
"files": [{"language", "file", "package", "subpath",
|
|
402
|
+
"resolution": "file" | "source-guess"
|
|
403
|
+
| "conventional" | "package",
|
|
404
|
+
"in_scope": bool}, ...],
|
|
405
|
+
"exports_maps": [{"package", "subpaths": [...],
|
|
406
|
+
"non_root_subpaths": N,
|
|
407
|
+
"wildcard_subpaths": N}, ...],
|
|
408
|
+
"unresolved": [{"package", "subpath", "targets"}
|
|
409
|
+
| {"entry", "name", "from", "file"}, ...]},
|
|
410
|
+
"entry_point_diff": {
|
|
411
|
+
"public": [{"name", "language", "entry", "via", "from", "local",
|
|
412
|
+
"file", "line"}, ...],
|
|
413
|
+
"internal": [{"name", "language", "source_file", "source_line"}, ...],
|
|
414
|
+
"extraction_gaps": [{"name", "language", "entry", "file", "line"}, ...],
|
|
415
|
+
"outside_scope": [{"name", "language", "entry", "file", "line"}, ...]},
|
|
416
|
+
"reexport_targets": [{"name", "language", "local", "from", "entry",
|
|
417
|
+
"file", "line", "via"}, ...],
|
|
418
|
+
"counts": {"exports_public_api": N, "exports_internal": N,
|
|
419
|
+
"effective_denominator": N,
|
|
420
|
+
"effective_denominator_basis": "tier_a_include" | "scope.include"
|
|
421
|
+
| "all-files",
|
|
422
|
+
"denominator_files": N},
|
|
423
|
+
"arms": {"monorepo": bool, "monorepo_kind": "<kind>" | null,
|
|
424
|
+
"specific_modules": bool, "multi_subpath_exports": bool},
|
|
425
|
+
"errors": [{"reason": "ast-grep-error" | "ast-grep-timeout" | "time-limit",
|
|
426
|
+
"files": N, "first_file", "detail",
|
|
427
|
+
"unread": [...]}, ...], # unread: time-limit only
|
|
428
|
+
"warnings": ["..."]
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
`via` says how an entry point exports a name: `declaration` (defined in the
|
|
432
|
+
entry point), `re-export` (from another file, which `file` names, `local`
|
|
433
|
+
giving the name it has there when it differs), `star` (through `export *`,
|
|
434
|
+
`from .x import *` or `pub use x::*`) or `namespace` (a module: `export *
|
|
435
|
+
as ns`, a Python submodule, a Rust module). `line` is null where no line
|
|
436
|
+
defines the name (a module, a name from another package). An `unresolved`
|
|
437
|
+
item names a subpath whose targets name no file, or a chain met tracing
|
|
438
|
+
`entry` that stops at `file`, the module `from` naming no file here.
|
|
439
|
+
Without ast-grep the JSON still has the entry points' `files`, the
|
|
440
|
+
`exports_maps`, the unresolved subpaths and the `arms`.
|
|
441
|
+
`file_issues` lists the files to read by eye: a file ast-grep skips (not
|
|
442
|
+
UTF-8), and a file where the parser met code it could not read (an ERROR
|
|
443
|
+
node), where a recipe can miss an export. `t1_low` is 0: every export here
|
|
444
|
+
comes from a recipe, and an export the caller then reads by eye (an
|
|
445
|
+
extraction gap, a Known Limitation #11 form) is T1-low.
|
|
446
|
+
|
|
447
|
+
Time limit. The ast-grep runs stop once --timeout seconds (540 by default,
|
|
448
|
+
under the 10-minute limit an agent's shell call often has) have passed
|
|
449
|
+
since the run started: the run going on is stopped, no other starts, and
|
|
450
|
+
an `errors[]` item with the reason `time-limit` lists in `unread` the
|
|
451
|
+
files left unread (each ast-grep call also stops after 600 seconds,
|
|
452
|
+
`ast-grep-timeout`). The JSON is still written, status "incomplete".
|
|
453
|
+
|
|
454
|
+
Exit codes (full):
|
|
455
|
+
|
|
456
|
+
0 the recipes ran on every file in scope ("ok")
|
|
457
|
+
1 an ast-grep run failed (exited non-zero) or timed out, or the
|
|
458
|
+
--timeout ran out: its files may be unread and the result is
|
|
459
|
+
incomplete ("incomplete"); `errors` lists them
|
|
460
|
+
2 input error (no source root, an unreadable brief, file list or
|
|
461
|
+
recipe file, a recipe without its metadata, no PyYAML, an -o file
|
|
462
|
+
that cannot be written, a --timeout not above 0), or a failure the
|
|
463
|
+
runner did not foresee: one line on stderr and no JSON
|
|
464
|
+
3 no ast-grep the runner can run ("no-ast-grep"): none on PATH, or
|
|
465
|
+
only an npm .cmd/.bat shim (Windows) with no native ast-grep.exe
|
|
466
|
+
beside it (cmd.exe would read a repository path's `&` as syntax);
|
|
467
|
+
the JSON has the scope and no exports; extract by source reading
|
|
468
|
+
|
|
469
|
+
CLI examples:
|
|
470
|
+
uv run skf-extract-public-api.py --mode quick < {payload_json}
|
|
471
|
+
uv run skf-extract-public-api.py --mode entries --language python \\
|
|
472
|
+
--tree-file {tree_json} --source-root {staged_dir} --fetch-list {entries_txt}
|
|
473
|
+
uv run skf-extract-public-api.py --mode quick --language rust \\
|
|
474
|
+
--source-root {staged_dir} --manifest-file Cargo.toml --entry-file src/lib.rs
|
|
475
|
+
uv run skf-extract-public-api.py --mode full --source-root {source_root} \\
|
|
476
|
+
--brief {brief_path} --tier Forge -o {extraction_json}
|
|
477
|
+
uv run skf-extract-public-api.py --mode full --source-root {source_root} \\
|
|
478
|
+
--files-from {scan_list} --head-cap 0
|
|
57
479
|
"""
|
|
58
480
|
|
|
59
481
|
from __future__ import annotations
|
|
60
482
|
|
|
61
483
|
import argparse
|
|
484
|
+
import ast
|
|
485
|
+
import configparser
|
|
486
|
+
import importlib.util
|
|
62
487
|
import json
|
|
488
|
+
import os
|
|
489
|
+
import posixpath
|
|
63
490
|
import re
|
|
491
|
+
import subprocess
|
|
64
492
|
import sys
|
|
493
|
+
import tempfile
|
|
494
|
+
import time
|
|
65
495
|
import tomllib
|
|
66
496
|
import xml.etree.ElementTree as ET
|
|
67
|
-
from
|
|
497
|
+
from collections import Counter
|
|
498
|
+
from pathlib import Path
|
|
499
|
+
from typing import Callable, Iterator
|
|
68
500
|
|
|
69
501
|
# --------------------------------------------------------------------------
|
|
70
502
|
# Manifest parsers
|
|
@@ -127,6 +559,31 @@ def parse_setup_py(content: str) -> dict:
|
|
|
127
559
|
}
|
|
128
560
|
|
|
129
561
|
|
|
562
|
+
def parse_setup_cfg(content: str) -> dict:
|
|
563
|
+
"""setup.cfg's [metadata] (a literal version only) and [options]
|
|
564
|
+
install_requires, read with configparser."""
|
|
565
|
+
parser = configparser.ConfigParser(interpolation=None, strict=False)
|
|
566
|
+
try:
|
|
567
|
+
parser.read_string(content)
|
|
568
|
+
except configparser.Error as e:
|
|
569
|
+
return {"_parse_error": f"setup.cfg parse error: {e}"}
|
|
570
|
+
meta = parser["metadata"] if parser.has_section("metadata") else {}
|
|
571
|
+
version = (meta.get("version") or "").strip()
|
|
572
|
+
deps: list[str] = []
|
|
573
|
+
if parser.has_option("options", "install_requires"):
|
|
574
|
+
for line in parser.get("options", "install_requires").splitlines():
|
|
575
|
+
m = re.match(r"^([A-Za-z0-9_.\-]+)", line.strip())
|
|
576
|
+
if m:
|
|
577
|
+
deps.append(m.group(1))
|
|
578
|
+
return {
|
|
579
|
+
"name": (meta.get("name") or "").strip() or None,
|
|
580
|
+
"version": None if not version or version.startswith(("attr:", "file:")) else version,
|
|
581
|
+
"description": (meta.get("description") or "").strip() or None,
|
|
582
|
+
"dependencies": deps,
|
|
583
|
+
"modules": [],
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
|
|
130
587
|
def parse_cargo_toml(content: str) -> dict:
|
|
131
588
|
try:
|
|
132
589
|
data = tomllib.loads(content)
|
|
@@ -288,69 +745,449 @@ def parse_package_swift(content: str) -> dict:
|
|
|
288
745
|
# Export scanners
|
|
289
746
|
# --------------------------------------------------------------------------
|
|
290
747
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
)
|
|
295
|
-
_JS_REEXPORT_RE = re.compile(r"^export\s*\{([^}]+)\}", re.MULTILINE)
|
|
748
|
+
def _note(warnings: list[str] | None, message: str) -> None:
|
|
749
|
+
if warnings is not None:
|
|
750
|
+
warnings.append(message)
|
|
296
751
|
|
|
297
752
|
|
|
298
|
-
def
|
|
753
|
+
def _unlisted(source_file: str, line: int, statement: str) -> str:
|
|
754
|
+
"""The warning for a statement that exports names quick mode cannot list."""
|
|
755
|
+
return (f"{source_file} line {line}: {statement} passes on names from a module quick mode does not read: "
|
|
756
|
+
"pass that module as an entry too, or read it by eye")
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def _note_unlisted(warnings: list[str] | None, unlisted: list[dict] | None, source_file: str, line: int,
|
|
760
|
+
statement: str, specifier: str | None = None, module: tuple | None = None,
|
|
761
|
+
shown: str | None = None) -> None:
|
|
762
|
+
"""Note a statement whose names come from a module quick mode does not
|
|
763
|
+
read: its warning (_unlisted, naming `shown`, else the statement in
|
|
764
|
+
backticks) and, when `unlisted` collects them, its record. `module` says
|
|
765
|
+
how extract() finds the module's file: ("js", specifier) or ("python",
|
|
766
|
+
level, module); None when it cannot (a package import, a Rust glob)."""
|
|
767
|
+
message = _unlisted(source_file, line, shown or f"`{statement}`")
|
|
768
|
+
_note(warnings, message)
|
|
769
|
+
if unlisted is not None:
|
|
770
|
+
unlisted.append({"file": source_file, "line": line, "statement": statement, "specifier": specifier,
|
|
771
|
+
"_module": module, "_warning": message})
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def _exports(found: list[tuple[int, str, str]], source_file: str) -> list[dict]:
|
|
775
|
+
"""The export records of (offset, name, type) found, in file order, each
|
|
776
|
+
name once (its first form wins)."""
|
|
299
777
|
out: list[dict] = []
|
|
300
778
|
seen: set[str] = set()
|
|
301
|
-
for
|
|
302
|
-
kind, name = m.group(1), m.group(2)
|
|
779
|
+
for _, name, kind in sorted(found, key=lambda item: item[0]):
|
|
303
780
|
if name not in seen:
|
|
304
781
|
seen.add(name)
|
|
305
782
|
out.append({"name": name, "type": kind, "source_file": source_file})
|
|
306
|
-
for m in _JS_REEXPORT_RE.finditer(content):
|
|
307
|
-
for piece in m.group(1).split(","):
|
|
308
|
-
piece = piece.strip()
|
|
309
|
-
if not piece:
|
|
310
|
-
continue
|
|
311
|
-
# Handle `foo as bar` — keep the local name (foo) and the alias (bar);
|
|
312
|
-
# we only emit one entry, the exposed name.
|
|
313
|
-
if " as " in piece:
|
|
314
|
-
_, _, exposed = piece.partition(" as ")
|
|
315
|
-
exposed = exposed.strip()
|
|
316
|
-
else:
|
|
317
|
-
exposed = piece
|
|
318
|
-
m2 = re.match(r"\w+$", exposed)
|
|
319
|
-
if m2 and exposed not in seen:
|
|
320
|
-
seen.add(exposed)
|
|
321
|
-
out.append({"name": exposed, "type": "re-export", "source_file": source_file})
|
|
322
783
|
return out
|
|
323
784
|
|
|
324
785
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
786
|
+
# A column-0 declaration: `export` (with `declare` or `default`) when it is
|
|
787
|
+
# exported, its keyword and its name. A generator's `*` may touch the name.
|
|
788
|
+
_JS_DECL_RE = re.compile(
|
|
789
|
+
r"^(export\s+(?:declare\s+)?(?:default\s+)?)?(?:declare\s+)?(?:abstract\s+)?(?:async\s+)?"
|
|
790
|
+
r"(const\s+enum|const|let|var|function\s*\*|function|class|interface|type|enum|namespace|module)"
|
|
791
|
+
r"(?:(?<=\*)\s*|\s+)([A-Za-z_$][\w$]*)",
|
|
792
|
+
re.MULTILINE,
|
|
793
|
+
)
|
|
794
|
+
_JS_LIST_RE = re.compile(r"^export\s+(?:type\s+)?\{([^}]*)\}", re.MULTILINE)
|
|
795
|
+
_JS_STAR_RE = re.compile(
|
|
796
|
+
r"^export\s+(?:type\s+)?\*\s*(?:as\s+([A-Za-z_$][\w$]*)\s*)?from\s*(['\"])", re.MULTILINE)
|
|
797
|
+
_JS_DEFAULT_RE = re.compile(r"^export\s+default\s+([A-Za-z_$][\w$]*)\s*(?:;|$)", re.MULTILINE)
|
|
798
|
+
_JS_ANY_DEFAULT_RE = re.compile(r"^export\s+default\b\s*", re.MULTILINE)
|
|
799
|
+
# `export const { a, b } = obj` or `export let [x, y] = pair`
|
|
800
|
+
_JS_DESTRUCTURE_RE = re.compile(r"^export\s+(?:declare\s+)?(const|let|var)\s*[{[]", re.MULTILINE)
|
|
801
|
+
# A name after a comma of a `const`/`let`/`var` statement: its next declarator.
|
|
802
|
+
_JS_DECLARATOR_RE = re.compile(r"\s*([A-Za-z_$][\w$]*)\s*(?:[=,;:!]|$)", re.MULTILINE)
|
|
803
|
+
_JS_LEADING_COMMA_RE = re.compile(r"\s*,")
|
|
804
|
+
# TypeScript's `export = <value>`, read as `module.exports = <value>` is.
|
|
805
|
+
_TS_EXPORT_ASSIGN_RE = re.compile(r"^export\s*=(?![=>])\s*", re.MULTILINE)
|
|
806
|
+
_CJS_NAMED_RE = re.compile(r"^(?:module\.)?exports\.([A-Za-z_$][\w$]*)\s*=(?![=>])", re.MULTILINE)
|
|
807
|
+
# A column-0 `module.exports =`, also as Express writes it: `exports =
|
|
808
|
+
# module.exports = <value>` (or `module.exports = exports = <value>`).
|
|
809
|
+
_CJS_MODULE_RE = re.compile(
|
|
810
|
+
r"^(?:exports\s*=\s*)?module\.exports\s*=(?![=>])\s*(?:exports\s*=(?![=>])\s*)?", re.MULTILINE)
|
|
811
|
+
# Any `module.exports`, `exports` or `exports.<name>` assignment; the ones
|
|
812
|
+
# that do not open their line or follow a chain that does (inside an `if`, a
|
|
813
|
+
# block, a function) are warned, since which of them runs the entry file
|
|
814
|
+
# alone cannot tell. TypeScript's CommonJS output opens with a chain:
|
|
815
|
+
# `exports.a = exports.b = void 0;`.
|
|
816
|
+
_CJS_ANY_RE = re.compile(r"(?<![\w$.])(?:module\.exports|exports)(?:\.([A-Za-z_$][\w$]*))?\s*=(?![=>])")
|
|
817
|
+
_CJS_CHAIN_RE = re.compile(r"(?:(?:module\.)?exports(?:\.[A-Za-z_$][\w$]*)?\s*=(?![=>])\s*)*")
|
|
818
|
+
_JS_NAME_RE = re.compile(r"[A-Za-z_$][\w$]*")
|
|
819
|
+
_JS_SPACE_RE = re.compile(r"\s*")
|
|
820
|
+
_JS_REQUIRE_RE = re.compile(r"require\s*\(\s*(['\"])")
|
|
821
|
+
_JS_NAMED_VALUE_RE = re.compile(r"(?:async\s+)?function\s*\*?\s*([A-Za-z_$][\w$]*)|class\s+([A-Za-z_$][\w$]*)")
|
|
822
|
+
_JS_ANONYMOUS_VALUE_RE = re.compile(r"(?:async\s+)?(?:function\b|class\b|\([^()]*\)\s*=>|[A-Za-z_$][\w$]*\s*=>)")
|
|
823
|
+
_JS_FUNCTION_VALUE_RE = re.compile(r"\s*(?:async\s+)?(?:function\b|(?:\([^()]*\)|[A-Za-z_$][\w$]*)\s*=>)")
|
|
824
|
+
_JS_CLASS_VALUE_RE = re.compile(r"\s*class\b")
|
|
825
|
+
_JS_NAME_VALUE_RE = re.compile(r"\s*([A-Za-z_$][\w$]*)\s*(?:[;,})\]]|$)", re.MULTILINE)
|
|
826
|
+
_JS_MEMBER_PREFIX_RE = re.compile(r"(?:(?:get|set|static|async)\s+(?=[\w$'\"*\[]))*\*?\s*")
|
|
827
|
+
# Words the name patterns can meet that name no export: `class extends
|
|
828
|
+
# Base`, `export default true`, and the interop flag compilers set.
|
|
829
|
+
_JS_NOT_NAMES = frozenset({
|
|
830
|
+
"extends", "implements", "true", "false", "null", "undefined", "this", "new", "void", "typeof",
|
|
831
|
+
"function", "class", "async", "await", "__esModule",
|
|
832
|
+
})
|
|
330
833
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
834
|
+
|
|
835
|
+
def _js_type(keyword: str) -> str:
|
|
836
|
+
"""The export type of a declaration keyword: `function*` and `async
|
|
837
|
+
function` are functions, a `const enum` an enum, a `module` a
|
|
838
|
+
namespace."""
|
|
839
|
+
keyword = " ".join(keyword.split())
|
|
840
|
+
if keyword.startswith("function"):
|
|
841
|
+
return "function"
|
|
842
|
+
return {"const enum": "enum", "module": "namespace"}.get(keyword, keyword)
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def _js_string(content: str, code: str, quote: int) -> str:
|
|
846
|
+
"""The text of the string literal whose opening quote is at `quote`."""
|
|
847
|
+
close = code.find(code[quote], quote + 1)
|
|
848
|
+
return content[quote + 1: close if close >= 0 else len(content)]
|
|
849
|
+
|
|
850
|
+
|
|
851
|
+
def _js_value_type(code: str, pos: int, declared: dict[str, str]) -> str:
|
|
852
|
+
"""The export type of the value assigned at `pos`: a function or class
|
|
853
|
+
expression, the local declaration a name refers to, else a variable."""
|
|
854
|
+
if _JS_FUNCTION_VALUE_RE.match(code, pos):
|
|
855
|
+
return "function"
|
|
856
|
+
if _JS_CLASS_VALUE_RE.match(code, pos):
|
|
857
|
+
return "class"
|
|
858
|
+
name = _JS_NAME_VALUE_RE.match(code, pos)
|
|
859
|
+
return declared.get(name.group(1), "variable") if name else "variable"
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def _js_object_members(content: str, code: str, start: int,
|
|
863
|
+
declared: dict[str, str]) -> tuple[list[tuple[int, str, str]], list[str]]:
|
|
864
|
+
"""((offset, key, type) of each member of the object literal whose `{`
|
|
865
|
+
is at `start` in `code`, the spread expressions it holds). A method is
|
|
866
|
+
a function, `key: value` takes the value's type, a shorthand key the
|
|
867
|
+
type of its local declaration; a computed key names nothing."""
|
|
868
|
+
spans: list[tuple[int, int]] = []
|
|
869
|
+
depth, begin = 0, start + 1
|
|
870
|
+
for i in range(start, len(code)):
|
|
871
|
+
ch = code[i]
|
|
872
|
+
if ch in "{[(":
|
|
873
|
+
depth += 1
|
|
874
|
+
elif ch in "}])":
|
|
875
|
+
depth -= 1
|
|
876
|
+
if depth == 0:
|
|
877
|
+
spans.append((begin, i))
|
|
878
|
+
break
|
|
879
|
+
elif ch == "," and depth == 1:
|
|
880
|
+
spans.append((begin, i))
|
|
881
|
+
begin = i + 1
|
|
882
|
+
members: list[tuple[int, str, str]] = []
|
|
883
|
+
spreads: list[str] = []
|
|
884
|
+
for b, e in spans:
|
|
885
|
+
pos = b + len(code[b:e]) - len(code[b:e].lstrip())
|
|
886
|
+
if pos >= e:
|
|
335
887
|
continue
|
|
336
|
-
if
|
|
888
|
+
if code.startswith("...", pos):
|
|
889
|
+
spreads.append(" ".join(content[pos:e].split()))
|
|
337
890
|
continue
|
|
338
|
-
|
|
339
|
-
|
|
891
|
+
pos = _JS_MEMBER_PREFIX_RE.match(code, pos, e).end()
|
|
892
|
+
if pos >= e:
|
|
893
|
+
continue
|
|
894
|
+
if code[pos] in "'\"":
|
|
895
|
+
close = code.find(code[pos], pos + 1, e)
|
|
896
|
+
if close < 0:
|
|
897
|
+
continue
|
|
898
|
+
key, after = content[pos + 1:close], close + 1
|
|
899
|
+
else:
|
|
900
|
+
ident = _JS_NAME_RE.match(code, pos, e)
|
|
901
|
+
if not ident:
|
|
902
|
+
continue
|
|
903
|
+
key, after = ident.group(0), ident.end()
|
|
904
|
+
rest = code[after:e].lstrip()
|
|
905
|
+
if rest.startswith("("):
|
|
906
|
+
kind = "function"
|
|
907
|
+
elif rest.startswith(":"):
|
|
908
|
+
kind = _js_value_type(code, code.index(":", after, e) + 1, declared)
|
|
909
|
+
else:
|
|
910
|
+
kind = declared.get(key, "variable")
|
|
911
|
+
if key:
|
|
912
|
+
members.append((pos, key, kind))
|
|
913
|
+
return members, spreads
|
|
340
914
|
|
|
341
915
|
|
|
342
|
-
def
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
)
|
|
349
|
-
|
|
350
|
-
|
|
916
|
+
def _js_module_exports(content: str, code: str, match: re.Match, source_file: str, declared: dict[str, str],
|
|
917
|
+
found: list[tuple[int, str, str]], warnings: list[str] | None,
|
|
918
|
+
target: str = "module.exports", unlisted: list[dict] | None = None) -> None:
|
|
919
|
+
"""Add the names one `module.exports = <value>` (or TypeScript's
|
|
920
|
+
`export = <value>`, the `target`) exports."""
|
|
921
|
+
pos, line = match.end(), _line_of(code, match.start())
|
|
922
|
+
assign = target if target.endswith("=") else f"{target} ="
|
|
923
|
+
if code.startswith("{", pos):
|
|
924
|
+
members, spreads = _js_object_members(content, code, pos, declared)
|
|
925
|
+
found.extend(members)
|
|
926
|
+
for spread in spreads:
|
|
927
|
+
_note_unlisted(warnings, unlisted, source_file, line, spread,
|
|
928
|
+
shown=f"the `{target}` object's `{spread}`")
|
|
929
|
+
return
|
|
930
|
+
require = _JS_REQUIRE_RE.match(code, pos)
|
|
931
|
+
if require:
|
|
932
|
+
spec = _js_string(content, code, require.end(1) - 1)
|
|
933
|
+
_note_unlisted(warnings, unlisted, source_file, line, f"{assign} require('{spec}')", spec, ("js", spec))
|
|
934
|
+
return
|
|
935
|
+
named = _JS_NAMED_VALUE_RE.match(code, pos)
|
|
936
|
+
if named:
|
|
937
|
+
found.append((pos, named.group(1) or named.group(2), "function" if named.group(1) else "class"))
|
|
938
|
+
return
|
|
939
|
+
if _JS_ANONYMOUS_VALUE_RE.match(code, pos):
|
|
940
|
+
_note(warnings, f"{source_file} line {line}: `{target}` is an anonymous function or class: "
|
|
941
|
+
"the package's one export has no name here")
|
|
942
|
+
return
|
|
943
|
+
name = _JS_NAME_VALUE_RE.match(code, pos)
|
|
944
|
+
if name and name.group(1) not in _JS_NOT_NAMES:
|
|
945
|
+
found.append((pos, name.group(1), declared.get(name.group(1), "variable")))
|
|
946
|
+
return
|
|
947
|
+
_note(warnings, f"{source_file} line {line}: `{target}` is set to an expression quick mode does not "
|
|
948
|
+
"read: read the names it exports by eye")
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def _js_more_declarators(code: str, pos: int) -> list[tuple[int, str]]:
|
|
952
|
+
"""(offset, name) of each declarator after the first of the `const`,
|
|
953
|
+
`let` or `var` statement whose first name ends at `pos`: `export const
|
|
954
|
+
a = 1, b = 2` declares `b` too. The statement ends at a `;`, a bracket
|
|
955
|
+
it did not open, or a line end that no comma joins to the next line."""
|
|
956
|
+
names: list[tuple[int, str]] = []
|
|
957
|
+
depth, last, i = 0, "", pos
|
|
958
|
+
while i < len(code):
|
|
959
|
+
ch = code[i]
|
|
960
|
+
if ch in "([{":
|
|
961
|
+
depth += 1
|
|
962
|
+
elif ch in ")]}":
|
|
963
|
+
if depth == 0:
|
|
964
|
+
break
|
|
965
|
+
depth -= 1
|
|
966
|
+
elif depth == 0 and ch == ";":
|
|
967
|
+
break
|
|
968
|
+
elif depth == 0 and ch == "\n":
|
|
969
|
+
if last != "," and not _JS_LEADING_COMMA_RE.match(code, i):
|
|
970
|
+
break
|
|
971
|
+
elif depth == 0 and ch == ",":
|
|
972
|
+
declarator = _JS_DECLARATOR_RE.match(code, i + 1)
|
|
973
|
+
if declarator and declarator.group(1) not in _JS_NOT_NAMES:
|
|
974
|
+
names.append((declarator.start(1), declarator.group(1)))
|
|
975
|
+
if not ch.isspace():
|
|
976
|
+
last = ch
|
|
977
|
+
i += 1
|
|
978
|
+
return names
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def scan_exports_js(content: str, source_file: str, warnings: list[str] | None = None,
|
|
982
|
+
unlisted: list[dict] | None = None, defaults: bool = True) -> list[dict]:
|
|
983
|
+
"""The names a JS/TS entry file exports (see "Quick mode" above), read
|
|
984
|
+
from its text with comments, string contents and regular expressions
|
|
985
|
+
blanked. Without `defaults` (a module reached through `export *`) its
|
|
986
|
+
`export default` is left out, unread and unwarned."""
|
|
987
|
+
code = _blank(content, "js")
|
|
988
|
+
declared: dict[str, str] = {}
|
|
989
|
+
found: list[tuple[int, str, str]] = []
|
|
990
|
+
read_defaults: set[int] = set() # the `export default` statements a name was read from
|
|
991
|
+
for m in _JS_DECL_RE.finditer(code):
|
|
992
|
+
name, kind = m.group(3), _js_type(m.group(2))
|
|
993
|
+
if name in _JS_NOT_NAMES:
|
|
994
|
+
continue
|
|
995
|
+
declared.setdefault(name, kind)
|
|
996
|
+
if m.group(1):
|
|
997
|
+
if "default" in m.group(1).split():
|
|
998
|
+
read_defaults.add(m.start())
|
|
999
|
+
if not defaults:
|
|
1000
|
+
continue
|
|
1001
|
+
found.append((m.start(), name, kind))
|
|
1002
|
+
if kind in ("const", "let", "var"):
|
|
1003
|
+
for offset, more in _js_more_declarators(code, m.end(3)):
|
|
1004
|
+
declared.setdefault(more, kind)
|
|
1005
|
+
if m.group(1):
|
|
1006
|
+
found.append((offset, more, kind))
|
|
1007
|
+
for m in _JS_DESTRUCTURE_RE.finditer(code):
|
|
1008
|
+
_note(warnings, f"{source_file} line {_line_of(code, m.start())}: `export {m.group(1)}` destructures, "
|
|
1009
|
+
"and quick mode does not read the names it binds: read them by eye")
|
|
1010
|
+
for m in _JS_LIST_RE.finditer(code):
|
|
1011
|
+
for piece in m.group(1).split(","):
|
|
1012
|
+
# `a`, `a as b`, `type T`, `"a-b" as c`: the exposed name comes last
|
|
1013
|
+
words = piece.split()
|
|
1014
|
+
if words and _JS_NAME_RE.fullmatch(words[-1]):
|
|
1015
|
+
found.append((m.start(), words[-1], "re-export"))
|
|
1016
|
+
for m in _JS_STAR_RE.finditer(code):
|
|
1017
|
+
if m.group(1):
|
|
1018
|
+
found.append((m.start(), m.group(1), "namespace"))
|
|
1019
|
+
else:
|
|
1020
|
+
spec = _js_string(content, code, m.end(2) - 1)
|
|
1021
|
+
_note_unlisted(warnings, unlisted, source_file, _line_of(code, m.start()), f"export * from '{spec}'",
|
|
1022
|
+
spec, ("js", spec))
|
|
1023
|
+
for m in _JS_DEFAULT_RE.finditer(code):
|
|
1024
|
+
if m.group(1) not in _JS_NOT_NAMES and defaults:
|
|
1025
|
+
found.append((m.start(), m.group(1), declared.get(m.group(1), "re-export")))
|
|
1026
|
+
read_defaults.add(m.start())
|
|
1027
|
+
for m in _JS_ANY_DEFAULT_RE.finditer(code):
|
|
1028
|
+
if m.start() in read_defaults or not defaults:
|
|
1029
|
+
continue
|
|
1030
|
+
line = _line_of(code, m.start())
|
|
1031
|
+
if _JS_ANONYMOUS_VALUE_RE.match(code, m.end()):
|
|
1032
|
+
_note(warnings, f"{source_file} line {line}: `export default` is an anonymous function or class: "
|
|
1033
|
+
"the module's default export has no name here")
|
|
1034
|
+
else:
|
|
1035
|
+
_note(warnings, f"{source_file} line {line}: `export default` is set to an expression quick mode "
|
|
1036
|
+
"does not read: read the names it exports by eye")
|
|
1037
|
+
for m in _TS_EXPORT_ASSIGN_RE.finditer(code):
|
|
1038
|
+
_js_module_exports(content, code, m, source_file, declared, found, warnings, "export =", unlisted)
|
|
1039
|
+
for m in _CJS_NAMED_RE.finditer(code):
|
|
1040
|
+
if m.group(1) not in _JS_NOT_NAMES:
|
|
1041
|
+
found.append((m.start(), m.group(1), _js_value_type(code, m.end(), declared)))
|
|
1042
|
+
for m in _CJS_MODULE_RE.finditer(code):
|
|
1043
|
+
_js_module_exports(content, code, m, source_file, declared, found, warnings, unlisted=unlisted)
|
|
1044
|
+
nested_lines: set[int] = set()
|
|
1045
|
+
for m in _CJS_ANY_RE.finditer(code):
|
|
1046
|
+
line_start = code.rfind("\n", 0, m.start()) + 1
|
|
1047
|
+
if _CJS_CHAIN_RE.fullmatch(code, line_start, m.start()):
|
|
1048
|
+
# A statement that opens its line, its first target read above;
|
|
1049
|
+
# a later `exports.<name>` of its chain is one more export.
|
|
1050
|
+
if m.start() > line_start and m.group(1) and m.group(1) not in _JS_NOT_NAMES:
|
|
1051
|
+
found.append((m.start(), m.group(1), _js_value_type(code, m.end(), declared)))
|
|
1052
|
+
continue
|
|
1053
|
+
line = _line_of(code, m.start())
|
|
1054
|
+
if line in nested_lines:
|
|
1055
|
+
continue
|
|
1056
|
+
nested_lines.add(line)
|
|
1057
|
+
target = " ".join(m.group(0).rstrip("=").split())
|
|
1058
|
+
require = _JS_REQUIRE_RE.match(code, _JS_SPACE_RE.match(code, m.end()).end())
|
|
1059
|
+
if require:
|
|
1060
|
+
spec = _js_string(content, code, require.end(1) - 1)
|
|
1061
|
+
_note_unlisted(warnings, unlisted, source_file, line, f"{target} = require('{spec}')", spec, ("js", spec))
|
|
1062
|
+
else:
|
|
1063
|
+
_note(warnings, f"{source_file} line {line}: `{target}` is assigned inside a block or a condition: "
|
|
1064
|
+
"read the names it exports by eye")
|
|
1065
|
+
return _exports(found, source_file)
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
def _py_export_type(binding: dict | None) -> str:
|
|
1069
|
+
"""The quick-mode type of a name by what binds it in the module."""
|
|
1070
|
+
if binding is None or binding["kind"] == "import":
|
|
1071
|
+
return "re-export"
|
|
1072
|
+
if binding["kind"] == "module-import":
|
|
1073
|
+
return "module"
|
|
1074
|
+
return binding.get("form", "def")
|
|
1075
|
+
|
|
1076
|
+
|
|
1077
|
+
def _py_all_unread(node: ast.AST | None) -> bool:
|
|
1078
|
+
"""Whether an `__all__` value holds a part that is no string literal
|
|
1079
|
+
(another module's `__all__`, a list built at run time)."""
|
|
1080
|
+
if isinstance(node, ast.Constant):
|
|
1081
|
+
return not isinstance(node.value, str)
|
|
1082
|
+
if isinstance(node, (ast.List, ast.Tuple)):
|
|
1083
|
+
return any(_py_all_unread(e) for e in node.elts)
|
|
1084
|
+
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
|
|
1085
|
+
return _py_all_unread(node.left) or _py_all_unread(node.right)
|
|
1086
|
+
return True
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def _py_all_unread_line(text: str) -> int | None:
|
|
1090
|
+
"""The line of the first statement that builds `__all__` from a part
|
|
1091
|
+
that is no string literal, whose names no reader of this file alone can
|
|
1092
|
+
list; None when there is none or the module does not parse."""
|
|
1093
|
+
try:
|
|
1094
|
+
tree = ast.parse(text)
|
|
1095
|
+
except SyntaxError:
|
|
1096
|
+
return None
|
|
1097
|
+
for node in _module_statements(tree.body):
|
|
1098
|
+
value = None
|
|
1099
|
+
if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
|
|
1100
|
+
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
|
1101
|
+
if "__all__" in [n for t in targets for n in _target_names(t)]:
|
|
1102
|
+
value = node.value
|
|
1103
|
+
elif (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
|
|
1104
|
+
and isinstance(node.value.func, ast.Attribute)
|
|
1105
|
+
and isinstance(node.value.func.value, ast.Name) and node.value.func.value.id == "__all__"
|
|
1106
|
+
and node.value.func.attr in ("extend", "append") and node.value.args):
|
|
1107
|
+
value = node.value.args[0]
|
|
1108
|
+
if value is not None and _py_all_unread(value):
|
|
1109
|
+
return node.lineno
|
|
1110
|
+
return None
|
|
1111
|
+
|
|
1112
|
+
|
|
1113
|
+
def scan_exports_python(content: str, source_file: str, warnings: list[str] | None = None,
|
|
1114
|
+
unlisted: list[dict] | None = None) -> list[dict]:
|
|
1115
|
+
"""The names a Python entry file exports (see "Quick mode" above), read
|
|
1116
|
+
with full mode's module reader: __all__ when the module has one, else
|
|
1117
|
+
its public definitions and the names it imports from its own
|
|
1118
|
+
package."""
|
|
1119
|
+
try:
|
|
1120
|
+
all_names, bound, stars = _py_bindings(content)
|
|
1121
|
+
except SyntaxError as exc:
|
|
1122
|
+
_note(warnings, f"{source_file}: this Python cannot parse it ({exc.msg}, line {exc.lineno}); "
|
|
1123
|
+
"its names are read from its text")
|
|
1124
|
+
all_names, bound, stars = _py_text_bindings(content)
|
|
1125
|
+
# The package an __init__.py opens, whose own modules its absolute imports
|
|
1126
|
+
# can name: by any trailing run of its folders, as a nested or namespace
|
|
1127
|
+
# package is imported (storage, cloud.storage, acme.cloud.storage).
|
|
1128
|
+
packages: list[str] = []
|
|
1129
|
+
if posixpath.basename(source_file) == "__init__.py":
|
|
1130
|
+
folders = [p for p in posixpath.dirname(source_file).split("/") if p and p != "."]
|
|
1131
|
+
packages = [".".join(folders[i:]) for i in range(len(folders))]
|
|
351
1132
|
|
|
1133
|
+
def own(module: str | None, level: int) -> bool:
|
|
1134
|
+
return level > 0 or bool(module and any(module == p or module.startswith(p + ".") for p in packages))
|
|
352
1135
|
|
|
353
|
-
|
|
1136
|
+
unread = _py_all_unread_line(content)
|
|
1137
|
+
if unread is not None:
|
|
1138
|
+
_note(warnings, f"{source_file} line {unread}: `__all__` takes names quick mode cannot read (another "
|
|
1139
|
+
"module's `__all__`, or a list built at run time): read the names it exports by eye")
|
|
1140
|
+
if all_names is not None:
|
|
1141
|
+
return [{"name": name, "type": _py_export_type(bound.get(name)), "source_file": source_file}
|
|
1142
|
+
for name in dict.fromkeys(all_names) if name.isidentifier() and not name.startswith("_")]
|
|
1143
|
+
found: list[tuple[int, str, str]] = []
|
|
1144
|
+
for name, binding in bound.items():
|
|
1145
|
+
if name.startswith("_"):
|
|
1146
|
+
continue
|
|
1147
|
+
if binding["kind"] == "definition" and binding.get("form", "def") in ("def", "class"):
|
|
1148
|
+
found.append((binding["line"], name, binding.get("form", "def")))
|
|
1149
|
+
elif binding["kind"] == "import" and own(binding["module"], binding["level"]):
|
|
1150
|
+
found.append((binding["line"], name, "re-export"))
|
|
1151
|
+
for star in stars:
|
|
1152
|
+
if own(star["module"], star["level"]):
|
|
1153
|
+
source = "." * star["level"] + (star["module"] or "")
|
|
1154
|
+
_note_unlisted(warnings, unlisted, source_file, star["line"], f"from {source} import *", source,
|
|
1155
|
+
("python", star["level"], star["module"]))
|
|
1156
|
+
return _exports(found, source_file)
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
# A `#[macro_export]` macro, exported at the crate root wherever it sits;
|
|
1160
|
+
# other attributes may stand between the two.
|
|
1161
|
+
_RUST_MACRO_EXPORT_RE = re.compile(
|
|
1162
|
+
r"^[ \t]*#\[macro_export\b[^\]]*\](?:\s*#\[[^\]]*\])*\s*macro_rules!\s*((?:r#)?[A-Za-z_]\w*)", re.M)
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def scan_exports_rust(content: str, source_file: str, warnings: list[str] | None = None,
|
|
1166
|
+
unlisted: list[dict] | None = None) -> list[dict]:
|
|
1167
|
+
"""The names a Rust entry file exports (see "Quick mode" above): its
|
|
1168
|
+
column-0 unrestricted `pub` items, `#[macro_export]` macros and `pub
|
|
1169
|
+
use` names, read with full mode's readers from its text with comments
|
|
1170
|
+
and strings blanked."""
|
|
1171
|
+
code = _blank(content, "rust")
|
|
1172
|
+
found = [(line, name, kind) for name, kind, line in _rust_items(code)]
|
|
1173
|
+
found += [(_line_of(code, m.start(1)), m.group(1), "macro") for m in _RUST_MACRO_EXPORT_RE.finditer(code)]
|
|
1174
|
+
for use in _RUST_USE_RE.finditer(code):
|
|
1175
|
+
line = _line_of(code, use.start())
|
|
1176
|
+
for path, alias, glob in _rust_use_leaves(use.group(1)):
|
|
1177
|
+
if path and path[-1] == "self" and not glob:
|
|
1178
|
+
path = path[:-1]
|
|
1179
|
+
if not path or alias == "_":
|
|
1180
|
+
continue
|
|
1181
|
+
if glob:
|
|
1182
|
+
_note_unlisted(warnings, unlisted, source_file, line, f"pub use {'::'.join(path)}::*",
|
|
1183
|
+
"::".join(path))
|
|
1184
|
+
else:
|
|
1185
|
+
found.append((line, alias or path[-1], "re-export"))
|
|
1186
|
+
return _exports(found, source_file)
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
def scan_exports_go(content: str, source_file: str, warnings: list[str] | None = None,
|
|
1190
|
+
unlisted: list[dict] | None = None) -> list[dict]:
|
|
354
1191
|
out: list[dict] = []
|
|
355
1192
|
for m in re.finditer(r"^(func|type|var|const)\s+([A-Z]\w*)", content, re.MULTILINE):
|
|
356
1193
|
out.append({"name": m.group(2), "type": m.group(1), "source_file": source_file})
|
|
@@ -367,7 +1204,8 @@ _JAVA_ANNOTATION_RE = re.compile(
|
|
|
367
1204
|
)
|
|
368
1205
|
|
|
369
1206
|
|
|
370
|
-
def scan_exports_java(content: str, source_file: str
|
|
1207
|
+
def scan_exports_java(content: str, source_file: str, warnings: list[str] | None = None,
|
|
1208
|
+
unlisted: list[dict] | None = None) -> list[dict]:
|
|
371
1209
|
out: list[dict] = []
|
|
372
1210
|
seen: set[str] = set()
|
|
373
1211
|
for m in _JAVA_PUBLIC_RE.finditer(content):
|
|
@@ -398,7 +1236,8 @@ _KOTLIN_DECL_RE = re.compile(
|
|
|
398
1236
|
)
|
|
399
1237
|
|
|
400
1238
|
|
|
401
|
-
def scan_exports_kotlin(content: str, source_file: str
|
|
1239
|
+
def scan_exports_kotlin(content: str, source_file: str, warnings: list[str] | None = None,
|
|
1240
|
+
unlisted: list[dict] | None = None) -> list[dict]:
|
|
402
1241
|
"""Kotlin defaults to public — omit internal/private declarations."""
|
|
403
1242
|
out: list[dict] = []
|
|
404
1243
|
seen: set[str] = set()
|
|
@@ -425,7 +1264,8 @@ _SWIFT_DECL_KEYWORDS = {
|
|
|
425
1264
|
}
|
|
426
1265
|
|
|
427
1266
|
|
|
428
|
-
def scan_exports_swift(content: str, source_file: str
|
|
1267
|
+
def scan_exports_swift(content: str, source_file: str, warnings: list[str] | None = None,
|
|
1268
|
+
unlisted: list[dict] | None = None) -> list[dict]:
|
|
429
1269
|
"""Swift defaults to internal — emit only public/open declarations."""
|
|
430
1270
|
out: list[dict] = []
|
|
431
1271
|
seen: set[str] = set()
|
|
@@ -445,7 +1285,9 @@ def scan_exports_swift(content: str, source_file: str) -> list[dict]:
|
|
|
445
1285
|
# --------------------------------------------------------------------------
|
|
446
1286
|
|
|
447
1287
|
ManifestParser = Callable[[str], dict]
|
|
448
|
-
|
|
1288
|
+
# (content, source file, warnings, unlisted): a scanner may note a form it
|
|
1289
|
+
# cannot list, and record each statement whose names another module gives.
|
|
1290
|
+
ExportScanner = Callable[[str, str, list[str], list[dict]], list[dict]]
|
|
449
1291
|
|
|
450
1292
|
LANGUAGE_DISPATCH: dict[str, tuple[ManifestParser, ExportScanner]] = {
|
|
451
1293
|
"js": (parse_package_json, scan_exports_js),
|
|
@@ -462,9 +1304,11 @@ LANGUAGE_DISPATCH: dict[str, tuple[ManifestParser, ExportScanner]] = {
|
|
|
462
1304
|
|
|
463
1305
|
|
|
464
1306
|
def _select_manifest_parser(language: str, manifest_path: str) -> ManifestParser:
|
|
465
|
-
"""Pick the right manifest parser, special-casing Python's setup.py."""
|
|
1307
|
+
"""Pick the right manifest parser, special-casing Python's setup.py and setup.cfg."""
|
|
466
1308
|
if language == "python" and manifest_path.endswith("setup.py"):
|
|
467
1309
|
return parse_setup_py
|
|
1310
|
+
if language == "python" and manifest_path.endswith("setup.cfg"):
|
|
1311
|
+
return parse_setup_cfg
|
|
468
1312
|
return LANGUAGE_DISPATCH[language][0]
|
|
469
1313
|
|
|
470
1314
|
|
|
@@ -520,15 +1364,22 @@ def extract(payload: dict) -> dict:
|
|
|
520
1364
|
|
|
521
1365
|
_, scanner = LANGUAGE_DISPATCH[language]
|
|
522
1366
|
exports: list[dict] = []
|
|
1367
|
+
unlisted: list[dict] = []
|
|
1368
|
+
entry_paths: set[str] = set()
|
|
523
1369
|
for entry in payload.get("entries") or []:
|
|
524
1370
|
if not isinstance(entry, dict):
|
|
525
1371
|
continue
|
|
526
1372
|
content = entry.get("content") or ""
|
|
527
1373
|
path = entry.get("path") or ""
|
|
1374
|
+
entry_paths.add(path)
|
|
528
1375
|
if not content:
|
|
529
1376
|
continue
|
|
530
1377
|
try:
|
|
531
|
-
|
|
1378
|
+
if entry.get("followed") and scanner is scan_exports_js:
|
|
1379
|
+
# `export *` passes on no default export
|
|
1380
|
+
exports.extend(scan_exports_js(content, path, warnings, unlisted, defaults=False))
|
|
1381
|
+
else:
|
|
1382
|
+
exports.extend(scanner(content, path, warnings, unlisted))
|
|
532
1383
|
except Exception as e: # noqa: BLE001 — best-effort: scanner errors are warnings, not fatal
|
|
533
1384
|
warnings.append(f"export scan failed for {path}: {e}")
|
|
534
1385
|
|
|
@@ -540,29 +1391,3177 @@ def extract(payload: dict) -> dict:
|
|
|
540
1391
|
"exports": exports,
|
|
541
1392
|
"dependencies": parsed.get("dependencies", []),
|
|
542
1393
|
"modules": parsed.get("modules", []),
|
|
1394
|
+
"module_folders": [_module_folder(manifest_path, module) for module in parsed.get("modules", [])],
|
|
543
1395
|
"warnings": warnings,
|
|
1396
|
+
"unlisted": _resolve_unlisted(unlisted, payload.get("tree"), entry_paths, warnings),
|
|
544
1397
|
}
|
|
545
1398
|
if "_extra" in parsed:
|
|
546
1399
|
result["extra"] = parsed["_extra"]
|
|
547
1400
|
return result
|
|
548
1401
|
|
|
549
1402
|
|
|
1403
|
+
def _module_folder(manifest_path: str, module: str) -> str:
|
|
1404
|
+
"""The folder of a Maven or Gradle module in the repository: the
|
|
1405
|
+
manifest's folder joined with the module, a Gradle `core:api` read as
|
|
1406
|
+
`core/api`."""
|
|
1407
|
+
joined = posixpath.join(posixpath.dirname(manifest_path), module.strip(":").replace(":", "/"))
|
|
1408
|
+
return posixpath.normpath(joined)
|
|
1409
|
+
|
|
1410
|
+
|
|
1411
|
+
def _resolve_unlisted(unlisted: list[dict], tree: object, entry_paths: set[str],
|
|
1412
|
+
warnings: list[str]) -> list[dict]:
|
|
1413
|
+
"""The `unlisted` records, each with its `module_file`: the file in the
|
|
1414
|
+
listing (`tree`) that the statement's module names, by full mode's
|
|
1415
|
+
resolvers (resolve_js_module for `export * from` and `module.exports =
|
|
1416
|
+
require(...)`, _py_module_file for a Python star import), else None. A
|
|
1417
|
+
record whose module is already an entry file is dropped with its
|
|
1418
|
+
warning: its names are read. Without a listing every module_file is
|
|
1419
|
+
None, since only the entry files are known."""
|
|
1420
|
+
listed = set(tree) if isinstance(tree, list) else None
|
|
1421
|
+
known = (listed or set()) | entry_paths
|
|
1422
|
+
records = []
|
|
1423
|
+
for item in unlisted:
|
|
1424
|
+
module, message = item.pop("_module"), item.pop("_warning")
|
|
1425
|
+
module_file = None
|
|
1426
|
+
if module and module[0] == "js":
|
|
1427
|
+
module_file = resolve_js_module(module[1], item["file"], known)
|
|
1428
|
+
elif module and module[0] == "python":
|
|
1429
|
+
module_file = _py_module_file(posixpath.dirname(item["file"]), module[1], module[2], known)
|
|
1430
|
+
if module_file is not None and module_file in entry_paths:
|
|
1431
|
+
if message in warnings:
|
|
1432
|
+
warnings.remove(message)
|
|
1433
|
+
continue
|
|
1434
|
+
item["module_file"] = module_file if listed is not None else None
|
|
1435
|
+
records.append(item)
|
|
1436
|
+
return records
|
|
1437
|
+
|
|
1438
|
+
|
|
550
1439
|
# --------------------------------------------------------------------------
|
|
551
|
-
#
|
|
1440
|
+
# Full mode: the ast-grep recipe runner
|
|
552
1441
|
# --------------------------------------------------------------------------
|
|
553
1442
|
|
|
1443
|
+
RECIPES_FILE = Path(__file__).resolve().parent.parent / "data" / "ast-grep-recipes.yaml"
|
|
1444
|
+
RECIPE_SETS = ("standard", "component-library")
|
|
1445
|
+
RECIPE_LANGUAGES = ("python", "typescript", "tsx", "javascript", "rust", "go")
|
|
1446
|
+
EXPORT_TYPES = ("function", "class", "const", "interface", "type", "enum", "re-export")
|
|
1447
|
+
TIERS = ("Quick", "Forge", "Forge+", "Deep")
|
|
554
1448
|
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
1449
|
+
# A file's extension -> its language here: the ast-grep language its
|
|
1450
|
+
# recipes run in, or `vue`, a file ast-grep reads as HTML through the
|
|
1451
|
+
# scratch sgconfig.yml, where the typescript and tsx recipes run in its
|
|
1452
|
+
# <script> blocks (the Vue note in extraction-patterns-by-hand.md).
|
|
1453
|
+
EXTENSION_LANGUAGES = {
|
|
1454
|
+
".py": "python",
|
|
1455
|
+
".pyi": "python",
|
|
1456
|
+
".ts": "typescript",
|
|
1457
|
+
".mts": "typescript",
|
|
1458
|
+
".cts": "typescript",
|
|
1459
|
+
".tsx": "tsx",
|
|
1460
|
+
".js": "javascript",
|
|
1461
|
+
".jsx": "javascript",
|
|
1462
|
+
".mjs": "javascript",
|
|
1463
|
+
".cjs": "javascript",
|
|
1464
|
+
".rs": "rust",
|
|
1465
|
+
".go": "go",
|
|
1466
|
+
".vue": "vue",
|
|
1467
|
+
}
|
|
1468
|
+
JS_FAMILY = ("typescript", "tsx", "javascript", "vue")
|
|
1469
|
+
# A file language -> the language family whose entry points declare its
|
|
1470
|
+
# exports: TypeScript, JavaScript and .vue files share package.json barrels.
|
|
1471
|
+
FAMILY_OF = {
|
|
1472
|
+
"python": "python",
|
|
1473
|
+
"typescript": "javascript",
|
|
1474
|
+
"tsx": "javascript",
|
|
1475
|
+
"javascript": "javascript",
|
|
1476
|
+
"vue": "javascript",
|
|
1477
|
+
"rust": "rust",
|
|
1478
|
+
"go": "go",
|
|
1479
|
+
}
|
|
1480
|
+
FAMILIES = ("javascript", "python", "rust", "go")
|
|
1481
|
+
# --language (or a brief's language) -> the file languages it selects.
|
|
1482
|
+
LANGUAGE_FAMILIES = {
|
|
1483
|
+
"python": ("python",),
|
|
1484
|
+
"rust": ("rust",),
|
|
1485
|
+
"go": ("go",),
|
|
1486
|
+
"golang": ("go",),
|
|
1487
|
+
**{name: JS_FAMILY for name in ("typescript", "ts", "tsx", "javascript", "js", "jsx", "vue")},
|
|
1488
|
+
}
|
|
1489
|
+
# Source files no recipe reads, counted in `files_without_recipes`.
|
|
1490
|
+
NO_RECIPE_EXTENSIONS = frozenset({
|
|
1491
|
+
".java", ".kt", ".kts", ".swift", ".rb", ".php", ".cs", ".scala", ".c", ".h",
|
|
1492
|
+
".cc", ".cpp", ".hpp", ".m", ".mm", ".dart", ".ex", ".exs", ".lua", ".zig",
|
|
1493
|
+
})
|
|
1494
|
+
DEFAULT_HEAD_CAP = 200
|
|
1495
|
+
# (tier, scope type) -> head cap, as extraction-patterns.md's head cap
|
|
1496
|
+
# selection sets it; DEFAULT_HEAD_CAP otherwise.
|
|
1497
|
+
HEAD_CAPS = {
|
|
1498
|
+
("Forge+", "full-library"): 500,
|
|
1499
|
+
("Deep", "full-library"): 500,
|
|
1500
|
+
("Forge+", "component-library"): 300,
|
|
1501
|
+
("Deep", "component-library"): 300,
|
|
1502
|
+
}
|
|
1503
|
+
SCAN_TIMEOUT_SEC = 600 # per ast-grep call
|
|
1504
|
+
# The whole run's limit (--timeout), below the 10-minute cap many agents put
|
|
1505
|
+
# on one shell call, so the JSON is written before the shell stops the run.
|
|
1506
|
+
DEFAULT_TIMEOUT_SEC = 540
|
|
1507
|
+
# The most source lines one export's `signature` spans (a formatter splits a
|
|
1508
|
+
# long parameter list one parameter per line).
|
|
1509
|
+
SIGNATURE_MAX_LINES = 40
|
|
1510
|
+
# Characters of file paths per ast-grep call: Windows caps a whole command
|
|
1511
|
+
# line at 32,767 characters.
|
|
1512
|
+
ARGV_BUDGET = 24000
|
|
1513
|
+
SGCONFIG = 'ruleDirs: []\nlanguageGlobs:\n html: ["*.vue"]\n'
|
|
1514
|
+
# Rules a scan runs beside the recipes; a match of one is never an export.
|
|
1515
|
+
SYNTAX_ERROR_RULE = "skf-syntax-error"
|
|
1516
|
+
STAR_RULE = "skf-star-reexport"
|
|
1517
|
+
IMPORT_NAMED_RULE = "skf-import-named"
|
|
1518
|
+
IMPORT_DEFAULT_RULE = "skf-import-default"
|
|
1519
|
+
IMPORT_NAMESPACE_RULE = "skf-import-namespace"
|
|
1520
|
+
IMPORT_RULES = (IMPORT_NAMED_RULE, IMPORT_DEFAULT_RULE, IMPORT_NAMESPACE_RULE)
|
|
1521
|
+
BINDING_RULE = "skf-binding" # skf-binding-<form>: a top-level binding a local list names
|
|
1522
|
+
# The build folders a package.json target can name when the build output is
|
|
1523
|
+
# not committed (`dist/index.js`): the target maps back to its source.
|
|
1524
|
+
BUILD_DIRS = frozenset({"dist", "build", "lib", "out", "esm", "cjs", "es", "module", "umd", "types", "typings"})
|
|
1525
|
+
JS_SOURCE_EXTENSIONS = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs")
|
|
1526
|
+
JS_RESOLVE_EXTENSIONS = (
|
|
1527
|
+
".ts", ".tsx", ".d.ts", ".mts", ".d.mts", ".cts", ".d.cts", ".js", ".jsx", ".mjs", ".cjs",
|
|
1528
|
+
)
|
|
1529
|
+
# A `.js` import specifier names its TypeScript source (`./x.js` -> x.ts).
|
|
1530
|
+
JS_SOURCE_OF = {
|
|
1531
|
+
".js": (".ts", ".tsx", ".d.ts"),
|
|
1532
|
+
".jsx": (".tsx",),
|
|
1533
|
+
".mjs": (".mts", ".d.mts"),
|
|
1534
|
+
".cjs": (".cts", ".d.cts"),
|
|
1535
|
+
}
|
|
1536
|
+
JS_TARGET_SUFFIXES = (".d.mts", ".d.cts", ".d.ts", ".mjs", ".cjs", ".jsx", ".js", ".mts", ".cts", ".tsx", ".ts")
|
|
1537
|
+
PACKAGE_ENTRY_FIELDS = ("types", "typings", "module", "main", "browser")
|
|
1538
|
+
WORKSPACE_MANIFESTS = ("package.json", "pnpm-workspace.yaml", "lerna.json", "Cargo.toml", "rush.json", "nx.json")
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
class RunnerError(Exception):
|
|
1542
|
+
"""An input full mode cannot use (exit 2)."""
|
|
1543
|
+
|
|
1544
|
+
|
|
1545
|
+
class _Clock:
|
|
1546
|
+
"""The --timeout of one full run: the seconds left before it runs out."""
|
|
1547
|
+
|
|
1548
|
+
def __init__(self, seconds: float):
|
|
1549
|
+
self.seconds = seconds
|
|
1550
|
+
self.end = time.monotonic() + seconds
|
|
1551
|
+
|
|
1552
|
+
def left(self) -> float:
|
|
1553
|
+
return self.end - time.monotonic()
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
_SIBLINGS: dict[str, object] = {}
|
|
1557
|
+
|
|
1558
|
+
|
|
1559
|
+
def _sibling(filename: str):
|
|
1560
|
+
"""A helper installed beside this script, loaded once as a module:
|
|
1561
|
+
skf-source-tree.py (git without the caller's hooks and git variables,
|
|
1562
|
+
and a tool lookup that never takes a shim planted in the current
|
|
1563
|
+
folder), skf-resolve-authoritative-files.py (the brief's glob rules) and
|
|
1564
|
+
skf-detect-workspaces.py (monorepo detection). Raises RunnerError."""
|
|
1565
|
+
module = _SIBLINGS.get(filename)
|
|
1566
|
+
if module is None:
|
|
1567
|
+
path = Path(__file__).resolve().parent / filename
|
|
1568
|
+
name = "skf_" + filename.removeprefix("skf-").removesuffix(".py").replace("-", "_")
|
|
1569
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
1570
|
+
if spec is None or spec.loader is None:
|
|
1571
|
+
raise RunnerError(f"cannot load {filename} beside {Path(__file__).name}")
|
|
1572
|
+
module = importlib.util.module_from_spec(spec)
|
|
1573
|
+
try:
|
|
1574
|
+
spec.loader.exec_module(module)
|
|
1575
|
+
except (OSError, ImportError, SyntaxError) as exc:
|
|
1576
|
+
raise RunnerError(f"cannot load {filename} beside {Path(__file__).name}: {exc}") from exc
|
|
1577
|
+
_SIBLINGS[filename] = module
|
|
1578
|
+
return module
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def _yaml():
|
|
1582
|
+
try:
|
|
1583
|
+
import yaml # full mode only; `uv run` installs it from the header
|
|
1584
|
+
except ImportError as exc:
|
|
1585
|
+
raise RunnerError("full mode reads YAML with PyYAML: run the helper with `uv run`") from exc
|
|
1586
|
+
return yaml
|
|
1587
|
+
|
|
1588
|
+
|
|
1589
|
+
def _norm(path: object) -> str:
|
|
1590
|
+
"""A path or glob as the runner compares them: forward slashes, no
|
|
1591
|
+
leading `./` or `/`."""
|
|
1592
|
+
text = str(path or "").strip().replace("\\", "/")
|
|
1593
|
+
while text.startswith("./"):
|
|
1594
|
+
text = text[2:]
|
|
1595
|
+
return text.lstrip("/")
|
|
1596
|
+
|
|
1597
|
+
|
|
1598
|
+
def _line_of(text: str, offset: int) -> int:
|
|
1599
|
+
return text.count("\n", 0, offset) + 1
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
# --------------------------------------------------------------------------
|
|
1603
|
+
# Recipes
|
|
1604
|
+
# --------------------------------------------------------------------------
|
|
1605
|
+
|
|
1606
|
+
|
|
1607
|
+
def _recipe_problem(doc: object) -> str | None:
|
|
1608
|
+
"""Why `doc` is not a recipe the runner can run, or None."""
|
|
1609
|
+
if not isinstance(doc, dict) or not isinstance(doc.get("id"), str) or not doc["id"]:
|
|
1610
|
+
return "a document that is not a recipe with an id"
|
|
1611
|
+
rid, rule, meta = doc["id"], doc.get("rule"), doc.get("metadata")
|
|
1612
|
+
if doc.get("language") not in RECIPE_LANGUAGES or not isinstance(rule, dict) or not isinstance(rule.get("kind"), str):
|
|
1613
|
+
return f"{rid}: a language the runner does not know, or a rule that names no kind"
|
|
1614
|
+
if not isinstance(meta, dict):
|
|
1615
|
+
return f"{rid}: no metadata"
|
|
1616
|
+
languages, sets = meta.get("languages"), meta.get("sets")
|
|
1617
|
+
if (not isinstance(languages, list) or not languages or languages[0] != doc["language"]
|
|
1618
|
+
or not all(language in RECIPE_LANGUAGES for language in languages)):
|
|
1619
|
+
return f"{rid}: metadata.languages must list its own language first, then other recipe languages"
|
|
1620
|
+
if not isinstance(sets, list) or not sets or not all(s in RECIPE_SETS for s in sets):
|
|
1621
|
+
return f"{rid}: metadata.sets must list {' or '.join(RECIPE_SETS)}"
|
|
1622
|
+
types = meta.get("export_type")
|
|
1623
|
+
types = types if isinstance(types, list) else [types]
|
|
1624
|
+
if not types or not all(t in EXPORT_TYPES for t in types):
|
|
1625
|
+
return f"{rid}: metadata.export_type must be one of {', '.join(EXPORT_TYPES)}, or a list of them"
|
|
1626
|
+
prefer = meta.get("prefer_over", [])
|
|
1627
|
+
if not isinstance(prefer, list) or not all(isinstance(p, str) for p in prefer):
|
|
1628
|
+
return f"{rid}: metadata.prefer_over must be a list of recipe ids"
|
|
1629
|
+
return None
|
|
1630
|
+
|
|
1631
|
+
|
|
1632
|
+
def load_recipe_file(path: Path) -> list[dict]:
|
|
1633
|
+
"""The recipes in `path`, in file order, each checked for the metadata
|
|
1634
|
+
the runner reads: the `recipes` list of a YAML mapping (the module's
|
|
1635
|
+
data file), a list, or one recipe per YAML document, as kind-at reads
|
|
1636
|
+
them. Raises RunnerError."""
|
|
1637
|
+
yaml = _yaml()
|
|
1638
|
+
try:
|
|
1639
|
+
loaded = list(yaml.safe_load_all(path.read_text(encoding="utf-8")))
|
|
1640
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
1641
|
+
raise RunnerError(f"cannot read recipes {path}: {exc}") from exc
|
|
1642
|
+
except yaml.YAMLError as exc:
|
|
1643
|
+
raise RunnerError(f"malformed YAML in recipes {path}: {exc}") from exc
|
|
1644
|
+
docs: list[object] = []
|
|
1645
|
+
for doc in loaded:
|
|
1646
|
+
if isinstance(doc, dict) and isinstance(doc.get("recipes"), list):
|
|
1647
|
+
docs.extend(doc["recipes"])
|
|
1648
|
+
elif isinstance(doc, list):
|
|
1649
|
+
docs.extend(doc)
|
|
1650
|
+
elif doc is not None:
|
|
1651
|
+
docs.append(doc)
|
|
1652
|
+
if not docs:
|
|
1653
|
+
raise RunnerError(f"no recipe in {path}")
|
|
1654
|
+
seen: set[tuple[str, str]] = set()
|
|
1655
|
+
for doc in docs:
|
|
1656
|
+
problem = _recipe_problem(doc)
|
|
1657
|
+
for language in [] if problem else doc["metadata"]["languages"]:
|
|
1658
|
+
if (doc["id"], language) in seen:
|
|
1659
|
+
problem = f"{doc['id']} runs in {language} twice"
|
|
1660
|
+
seen.add((doc["id"], language))
|
|
1661
|
+
if problem is not None:
|
|
1662
|
+
raise RunnerError(f"recipes {path}: {problem}")
|
|
1663
|
+
return docs
|
|
1664
|
+
|
|
1665
|
+
|
|
1666
|
+
def _forms(recipes: list[dict]) -> dict[tuple[str, str], dict]:
|
|
1667
|
+
"""(recipe id, language) -> the recipe form that runs in that language."""
|
|
1668
|
+
return {(r["id"], language): r for r in recipes for language in r["metadata"]["languages"]}
|
|
1669
|
+
|
|
1670
|
+
|
|
1671
|
+
def _recipe_rules(recipes: list[dict], languages: set[str]) -> list[dict]:
|
|
1672
|
+
"""One ast-grep rule per recipe form and language it runs in, among
|
|
1673
|
+
`languages`, with `language` rewritten and the metadata left out."""
|
|
1674
|
+
rules = []
|
|
1675
|
+
for recipe in recipes:
|
|
1676
|
+
for language in recipe["metadata"]["languages"]:
|
|
1677
|
+
if language in languages:
|
|
1678
|
+
rule = {k: v for k, v in recipe.items() if k != "metadata"}
|
|
1679
|
+
rule["language"] = language
|
|
1680
|
+
rules.append(rule)
|
|
1681
|
+
return rules
|
|
1682
|
+
|
|
1683
|
+
|
|
1684
|
+
def _syntax_error_rule(language: str) -> dict:
|
|
1685
|
+
"""An outermost ERROR node: code the parser could not read."""
|
|
1686
|
+
return {"id": SYNTAX_ERROR_RULE, "language": language,
|
|
1687
|
+
"rule": {"kind": "ERROR", "not": {"inside": {"kind": "ERROR", "stopBy": "end"}}}}
|
|
1688
|
+
|
|
1689
|
+
|
|
1690
|
+
def _trace_rules(language: str) -> list[dict]:
|
|
1691
|
+
"""A JS/TS file's `export * from` statements and its imported names,
|
|
1692
|
+
which one-level re-export tracing follows."""
|
|
1693
|
+
source = {"field": "source", "has": {"kind": "string_fragment", "pattern": "$SOURCE"}}
|
|
1694
|
+
clause = {"kind": "import_clause",
|
|
1695
|
+
"inside": {"kind": "import_statement", "inside": {"kind": "program"}, "has": source}}
|
|
1696
|
+
named = {"kind": "import_specifier", "inside": {"kind": "named_imports", "inside": clause},
|
|
1697
|
+
"any": [{"has": {"field": "alias", "pattern": "$NAME"}},
|
|
1698
|
+
{"all": [{"not": {"has": {"field": "alias", "regex": "."}}},
|
|
1699
|
+
{"has": {"field": "name", "pattern": "$NAME"}}]}]}
|
|
1700
|
+
star = {"kind": "export_statement", "inside": {"kind": "program"}, "has": source,
|
|
1701
|
+
"not": {"any": [{"has": {"kind": "export_clause"}}, {"has": {"kind": "namespace_export"}}]}}
|
|
1702
|
+
return [
|
|
1703
|
+
{"id": STAR_RULE, "language": language, "rule": star},
|
|
1704
|
+
{"id": IMPORT_NAMED_RULE, "language": language, "rule": named},
|
|
1705
|
+
{"id": IMPORT_DEFAULT_RULE, "language": language,
|
|
1706
|
+
"rule": {"kind": "identifier", "pattern": "$NAME", "inside": clause}},
|
|
1707
|
+
{"id": IMPORT_NAMESPACE_RULE, "language": language,
|
|
1708
|
+
"rule": {"kind": "identifier", "pattern": "$NAME",
|
|
1709
|
+
"inside": {"kind": "namespace_import", "inside": clause}}},
|
|
1710
|
+
]
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
# A binding's form -> the export type the recipes record for it exported in
|
|
1714
|
+
# place, most telling first (a const bound to an arrow function or a
|
|
1715
|
+
# function expression is a function; an import is a re-export).
|
|
1716
|
+
BINDING_TYPES = {"function": "function", "class": "class", "enum": "enum", "arrow": "function",
|
|
1717
|
+
"const": "const", "interface": "interface", "type": "type", "import": "re-export"}
|
|
1718
|
+
|
|
1719
|
+
|
|
1720
|
+
def _binding_rules(language: str) -> list[dict]:
|
|
1721
|
+
"""A JS/TS file's top-level bindings (outside every block and class
|
|
1722
|
+
body), one rule per form, whose id ends in it: the types a local
|
|
1723
|
+
`export { ... }` list's items take. TypeScript only kinds stay out of
|
|
1724
|
+
a javascript rule, which ast-grep would refuse."""
|
|
1725
|
+
top = {"not": {"inside": {"any": [{"kind": "statement_block"}, {"kind": "class_body"}], "stopBy": "end"}}}
|
|
1726
|
+
named = {"has": {"field": "name", "pattern": "$NAME"}}
|
|
1727
|
+
ts = language != "javascript"
|
|
1728
|
+
kinds = {
|
|
1729
|
+
"function": ["function_declaration", "generator_function_declaration"] + (["function_signature"] if ts else []),
|
|
1730
|
+
"class": ["class_declaration"] + (["abstract_class_declaration"] if ts else []),
|
|
1731
|
+
"interface": ["interface_declaration"] if ts else [],
|
|
1732
|
+
"type": ["type_alias_declaration"] if ts else [],
|
|
1733
|
+
"enum": ["enum_declaration"] if ts else [],
|
|
1734
|
+
}
|
|
1735
|
+
rules = [{"id": f"{BINDING_RULE}-{form}", "language": language,
|
|
1736
|
+
"rule": {"any": [{"kind": kind} for kind in found], **named, **top}}
|
|
1737
|
+
for form, found in kinds.items() if found]
|
|
1738
|
+
declarator = {"kind": "variable_declarator",
|
|
1739
|
+
"has": {"field": "name", "kind": "identifier", "pattern": "$NAME"}, **top}
|
|
1740
|
+
value = {"has": {"field": "value", "any": [{"kind": "arrow_function"}, {"kind": "function_expression"},
|
|
1741
|
+
{"kind": "generator_function"}]}}
|
|
1742
|
+
rules.append({"id": f"{BINDING_RULE}-arrow", "language": language, "rule": dict(declarator, all=[value])})
|
|
1743
|
+
rules.append({"id": f"{BINDING_RULE}-const", "language": language, "rule": declarator})
|
|
1744
|
+
return rules + [r for r in _trace_rules(language) if r["id"] in IMPORT_RULES]
|
|
1745
|
+
|
|
1746
|
+
|
|
1747
|
+
def _binding_types(matches: list[dict]) -> dict[tuple[str, str], str]:
|
|
1748
|
+
"""(file, name) -> the export type of the top-level binding `name` has
|
|
1749
|
+
in that file, from the matches of _binding_rules."""
|
|
1750
|
+
rank = list(BINDING_TYPES)
|
|
1751
|
+
best: dict[tuple[str, str], str] = {}
|
|
1752
|
+
for m in matches:
|
|
1753
|
+
form = "import" if m["rule"] in IMPORT_RULES else m["rule"].removeprefix(f"{BINDING_RULE}-")
|
|
1754
|
+
if form in BINDING_TYPES and m["name"]:
|
|
1755
|
+
key = (m["file"], m["name"])
|
|
1756
|
+
if key not in best or rank.index(form) < rank.index(best[key]):
|
|
1757
|
+
best[key] = form
|
|
1758
|
+
return {key: BINDING_TYPES[form] for key, form in best.items()}
|
|
1759
|
+
|
|
1760
|
+
|
|
1761
|
+
# --------------------------------------------------------------------------
|
|
1762
|
+
# Files in scope
|
|
1763
|
+
# --------------------------------------------------------------------------
|
|
1764
|
+
|
|
1765
|
+
|
|
1766
|
+
def _hidden(path: str) -> bool:
|
|
1767
|
+
return any(part.startswith(".") for part in path.split("/"))
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
def _git_files(root: Path) -> list[str] | None:
|
|
1771
|
+
"""The files git lists under `root` (tracked, and untracked but not
|
|
1772
|
+
ignored), or None when git is missing or `root` is no work tree."""
|
|
1773
|
+
try:
|
|
1774
|
+
res = _sibling("skf-source-tree.py")._git(
|
|
1775
|
+
root, "ls-files", "-z", "--cached", "--others", "--exclude-standard")
|
|
1776
|
+
except RunnerError:
|
|
1777
|
+
return None
|
|
1778
|
+
if res is None or res[0] != 0:
|
|
1779
|
+
return None
|
|
1780
|
+
files = []
|
|
1781
|
+
for raw in res[1].split(b"\0"):
|
|
1782
|
+
rel = os.fsdecode(raw).replace("\\", "/") if raw else ""
|
|
1783
|
+
if rel and (root / rel).is_file() and not (root / rel).is_symlink():
|
|
1784
|
+
files.append(rel)
|
|
1785
|
+
return files
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
def _walked_files(root: Path) -> list[str]:
|
|
1789
|
+
files = []
|
|
1790
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
1791
|
+
rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/")
|
|
1792
|
+
rel_dir = "" if rel_dir == "." else rel_dir
|
|
1793
|
+
dirnames[:] = sorted(d for d in dirnames
|
|
1794
|
+
if not d.startswith(".") and not os.path.islink(os.path.join(dirpath, d)))
|
|
1795
|
+
for name in filenames:
|
|
1796
|
+
if not name.startswith(".") and not os.path.islink(os.path.join(dirpath, name)):
|
|
1797
|
+
files.append(f"{rel_dir}/{name}" if rel_dir else name)
|
|
1798
|
+
return files
|
|
1799
|
+
|
|
1800
|
+
|
|
1801
|
+
def list_tree(root: Path) -> list[str]:
|
|
1802
|
+
"""The files under `root` a scan may read, as sorted POSIX paths
|
|
1803
|
+
relative to it: those git lists when `root` is in a git work tree (so
|
|
1804
|
+
.gitignore holds, as in ast-grep's own walk), else every file, as when
|
|
1805
|
+
git lists none (a folder the work tree ignores). Hidden entries and
|
|
1806
|
+
symbolic links are left out either way."""
|
|
1807
|
+
listed = _git_files(root) or _walked_files(root)
|
|
1808
|
+
return sorted(p for p in listed if not _hidden(p))
|
|
1809
|
+
|
|
1810
|
+
|
|
1811
|
+
def _file_language(path: str) -> str | None:
|
|
1812
|
+
return EXTENSION_LANGUAGES.get(posixpath.splitext(path)[1].lower())
|
|
1813
|
+
|
|
1814
|
+
|
|
1815
|
+
def _rule_languages(file_language: str) -> tuple[str, ...]:
|
|
1816
|
+
"""The recipe languages that run in a file of `file_language`."""
|
|
1817
|
+
return ("typescript", "tsx") if file_language == "vue" else (file_language,)
|
|
1818
|
+
|
|
1819
|
+
|
|
1820
|
+
def _wanted_languages(values: list[str]) -> tuple[set[str], list[str]]:
|
|
1821
|
+
"""The file languages the --language values select, and the values no
|
|
1822
|
+
recipe language family knows. No value selects every language."""
|
|
1823
|
+
if not values:
|
|
1824
|
+
return set(EXTENSION_LANGUAGES.values()), []
|
|
1825
|
+
wanted: set[str] = set()
|
|
1826
|
+
unknown = []
|
|
1827
|
+
for value in values:
|
|
1828
|
+
family = LANGUAGE_FAMILIES.get(str(value).strip().lower())
|
|
1829
|
+
if family is None:
|
|
1830
|
+
unknown.append(str(value))
|
|
1831
|
+
else:
|
|
1832
|
+
wanted.update(family)
|
|
1833
|
+
return wanted, unknown
|
|
1834
|
+
|
|
1835
|
+
|
|
1836
|
+
def _read_file_list(path: Path) -> list[str]:
|
|
1837
|
+
"""--files-from: a JSON list of paths when the whole text is one, else
|
|
1838
|
+
one path per line, so a first path such as `[slug]/page.tsx` is a path."""
|
|
1839
|
+
try:
|
|
1840
|
+
text = path.read_text(encoding="utf-8")
|
|
1841
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
1842
|
+
raise RunnerError(f"cannot read file list {path}: {exc}") from exc
|
|
1843
|
+
items = None
|
|
1844
|
+
if text.lstrip().startswith("["):
|
|
1845
|
+
try:
|
|
1846
|
+
value = json.loads(text)
|
|
1847
|
+
except ValueError:
|
|
1848
|
+
value = None
|
|
1849
|
+
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
|
1850
|
+
items = value
|
|
1851
|
+
if items is None:
|
|
1852
|
+
items = text.splitlines()
|
|
1853
|
+
return list(dict.fromkeys(rel for rel in (_norm(item) for item in items) if rel))
|
|
1854
|
+
|
|
1855
|
+
|
|
1856
|
+
def _read_brief(path: Path) -> dict:
|
|
1857
|
+
"""The scope fields and language of a skill brief."""
|
|
1858
|
+
yaml = _yaml()
|
|
1859
|
+
try:
|
|
1860
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
1861
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
1862
|
+
raise RunnerError(f"cannot read brief {path}: {exc}") from exc
|
|
1863
|
+
except yaml.YAMLError as exc:
|
|
1864
|
+
raise RunnerError(f"brief {path} is not valid YAML: {exc}") from exc
|
|
1865
|
+
if not isinstance(data, dict):
|
|
1866
|
+
raise RunnerError(f"brief {path} is not a YAML mapping")
|
|
1867
|
+
scope = data.get("scope") if isinstance(data.get("scope"), dict) else {}
|
|
1868
|
+
|
|
1869
|
+
def strings(value: object) -> list[str]:
|
|
1870
|
+
return [v for v in value if isinstance(v, str)] if isinstance(value, list) else []
|
|
1871
|
+
|
|
1872
|
+
return {
|
|
1873
|
+
"language": data.get("language") if isinstance(data.get("language"), str) else None,
|
|
1874
|
+
"type": scope.get("type") if isinstance(scope.get("type"), str) else None,
|
|
1875
|
+
"include": strings(scope.get("include")),
|
|
1876
|
+
"exclude": strings(scope.get("exclude")),
|
|
1877
|
+
"tier_a_include": strings(scope.get("tier_a_include")),
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
# --------------------------------------------------------------------------
|
|
1882
|
+
# Running ast-grep
|
|
1883
|
+
# --------------------------------------------------------------------------
|
|
1884
|
+
|
|
1885
|
+
|
|
1886
|
+
def _batches(files: list[str]) -> Iterator[list[str]]:
|
|
1887
|
+
batch: list[str] = []
|
|
1888
|
+
size = 0
|
|
1889
|
+
for rel in files:
|
|
1890
|
+
if batch and size + len(rel) + 3 > ARGV_BUDGET:
|
|
1891
|
+
yield batch
|
|
1892
|
+
batch, size = [], 0
|
|
1893
|
+
batch.append(rel)
|
|
1894
|
+
size += len(rel) + 3
|
|
1895
|
+
if batch:
|
|
1896
|
+
yield batch
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
def _position(node: object) -> int | None:
|
|
1900
|
+
"""The 1-based line a JSON match (or metavariable) starts on."""
|
|
1901
|
+
if not isinstance(node, dict):
|
|
1902
|
+
return None
|
|
1903
|
+
line = ((node.get("range") or {}).get("start") or {}).get("line")
|
|
1904
|
+
return line + 1 if isinstance(line, int) and not isinstance(line, bool) else None
|
|
1905
|
+
|
|
1906
|
+
|
|
1907
|
+
_SIGNATURE_QUOTES = {"python": "'\"", "rust": '"'} # a Rust `'a` is a lifetime
|
|
1908
|
+
|
|
1909
|
+
|
|
1910
|
+
def _code_end(text: str, language: str, depth: int) -> tuple[int, int]:
|
|
1911
|
+
"""(the bracket depth after `text`, one source line, where its line
|
|
1912
|
+
comment starts, else its length): each ( or [ outside a string opens
|
|
1913
|
+
one, each ) or ] closes one."""
|
|
1914
|
+
quotes = _SIGNATURE_QUOTES.get(language, "'\"`")
|
|
1915
|
+
comment = "#" if language == "python" else "//"
|
|
1916
|
+
quote = None
|
|
1917
|
+
i = 0
|
|
1918
|
+
while i < len(text):
|
|
1919
|
+
ch = text[i]
|
|
1920
|
+
if quote:
|
|
1921
|
+
if ch == "\\":
|
|
1922
|
+
i += 1
|
|
1923
|
+
elif ch == quote:
|
|
1924
|
+
quote = None
|
|
1925
|
+
elif ch in quotes:
|
|
1926
|
+
quote = ch
|
|
1927
|
+
elif text.startswith(comment, i):
|
|
1928
|
+
return depth, i
|
|
1929
|
+
elif ch in "([":
|
|
1930
|
+
depth += 1
|
|
1931
|
+
elif ch in ")]":
|
|
1932
|
+
depth -= 1
|
|
1933
|
+
i += 1
|
|
1934
|
+
return depth, len(text)
|
|
1935
|
+
|
|
1936
|
+
|
|
1937
|
+
def _declaration_signature(rows: list[str], start: int, language: str) -> str:
|
|
1938
|
+
"""The declaration a match names, on one line: the source line holding
|
|
1939
|
+
`$NAME`, and, while a ( or [ it opens is still open at a line's end (or
|
|
1940
|
+
a generic's < that ends the line, outside Python), the lines after it
|
|
1941
|
+
(at most SIGNATURE_MAX_LINES), each without its line comment, as is a
|
|
1942
|
+
one-line declaration. A parameter list a formatter split one parameter
|
|
1943
|
+
per line, its trailing comma included, reads as its one-line form does:
|
|
1944
|
+
`def f(a: int, b: str) -> R:`, and so does a destructured parameter
|
|
1945
|
+
(`({ a, b }: Props) => {`) or a split generic (`f<T, U>(a: T)`). A {
|
|
1946
|
+
adds no depth: it opens a body, an interface or an object as often as a
|
|
1947
|
+
parameter, and a ( or [ around it already holds the line open."""
|
|
1948
|
+
out, depth, angle = "", 0, 0
|
|
1949
|
+
for row in rows[start:start + SIGNATURE_MAX_LINES]:
|
|
1950
|
+
text = row.strip()
|
|
1951
|
+
if out and angle and text.startswith(">"):
|
|
1952
|
+
angle -= 1
|
|
1953
|
+
depth, end = _code_end(text, language, depth)
|
|
1954
|
+
piece = text[:end].rstrip()
|
|
1955
|
+
if language != "python" and piece.endswith("<"):
|
|
1956
|
+
angle += 1
|
|
1957
|
+
if not out:
|
|
1958
|
+
out = piece
|
|
1959
|
+
elif piece:
|
|
1960
|
+
if piece[0] in ")]}>" and out.endswith(","):
|
|
1961
|
+
out = out[:-1]
|
|
1962
|
+
out += piece if out[-1] in "([<" or piece[0] in ")]>" else " " + piece
|
|
1963
|
+
if depth <= 0 and angle <= 0:
|
|
1964
|
+
break
|
|
1965
|
+
return out
|
|
1966
|
+
|
|
1967
|
+
|
|
1968
|
+
def _parse_match(raw: object) -> dict | None:
|
|
1969
|
+
"""One `--json=stream` match: its rule, language, file, `$NAME`'s line
|
|
1970
|
+
(the match's own first line when it binds no `$NAME`), `$NAME`,
|
|
1971
|
+
`$SOURCE`, the source line holding `$NAME` (from `lines`, the whole
|
|
1972
|
+
lines the match spans, so a list item gives its full line), the
|
|
1973
|
+
declaration signature that line opens (_declaration_signature) and the
|
|
1974
|
+
match text."""
|
|
1975
|
+
first = _position(raw)
|
|
1976
|
+
if first is None:
|
|
1977
|
+
return None
|
|
1978
|
+
single = (raw.get("metaVariables") or {}).get("single") or {}
|
|
1979
|
+
name_var, source_var = single.get("NAME"), single.get("SOURCE")
|
|
1980
|
+
name, line = None, first
|
|
1981
|
+
if isinstance(name_var, dict):
|
|
1982
|
+
name = str(name_var.get("text") or "")
|
|
1983
|
+
line = _position(name_var) or first
|
|
1984
|
+
text = str(raw.get("text") or "")
|
|
1985
|
+
rows = str(raw.get("lines") or text).split("\n")
|
|
1986
|
+
row = min(max(line - first, 0), len(rows) - 1)
|
|
1987
|
+
language = str(raw.get("language") or "").lower()
|
|
1988
|
+
return {
|
|
1989
|
+
"rule": str(raw.get("ruleId") or ""),
|
|
1990
|
+
"language": language,
|
|
1991
|
+
"file": _norm(raw.get("file")),
|
|
1992
|
+
"line": line,
|
|
1993
|
+
"start_line": first,
|
|
1994
|
+
"name": name,
|
|
1995
|
+
"source": (str(source_var.get("text") or "") or None) if isinstance(source_var, dict) else None,
|
|
1996
|
+
"signature_line": rows[row].strip(),
|
|
1997
|
+
"signature": _declaration_signature(rows, row, language),
|
|
1998
|
+
"text": text,
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
|
|
2002
|
+
def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str],
|
|
2003
|
+
errors: list[dict], clock: _Clock | None = None) -> list[dict]:
|
|
2004
|
+
"""ast-grep's matches of `rules` in `files` (relative to `root`), in
|
|
2005
|
+
batches. The rules go in a file as JSON documents (YAML reads JSON),
|
|
2006
|
+
never on the command line. A call that times out, exits non-zero (its
|
|
2007
|
+
rules carry no severity, so a scan that read every file exits 0, matches
|
|
2008
|
+
or not) or prints other than JSON lines adds to `errors`: its batch's
|
|
2009
|
+
files may be unread. The matches it printed are kept. When `clock` (the
|
|
2010
|
+
run's --timeout) runs out, the call running is stopped and no other
|
|
2011
|
+
starts: one `time-limit` item names every file left unread."""
|
|
2012
|
+
if not rules or not files:
|
|
2013
|
+
return []
|
|
2014
|
+
rule_file = folder / "rules.yml"
|
|
2015
|
+
rule_file.write_text("\n---\n".join(json.dumps(r, ensure_ascii=False) for r in rules) + "\n",
|
|
2016
|
+
encoding="utf-8")
|
|
2017
|
+
config = folder / "sgconfig.yml"
|
|
2018
|
+
config.write_text(SGCONFIG, encoding="utf-8")
|
|
2019
|
+
found = []
|
|
2020
|
+
batches = list(_batches(files))
|
|
2021
|
+
for index, batch in enumerate(batches):
|
|
2022
|
+
left = None if clock is None else clock.left()
|
|
2023
|
+
if left is not None and left <= 0:
|
|
2024
|
+
unread = [rel for rest in batches[index:] for rel in rest]
|
|
2025
|
+
errors.append({"reason": "time-limit", "files": len(unread), "first_file": unread[0],
|
|
2026
|
+
"detail": f"the --timeout of {clock.seconds:g}s ran out before ast-grep read them",
|
|
2027
|
+
"unread": unread})
|
|
2028
|
+
break
|
|
2029
|
+
limit = SCAN_TIMEOUT_SEC if left is None else min(SCAN_TIMEOUT_SEC, left)
|
|
2030
|
+
cmd = [exe, "scan", "--config", str(config), "-r", str(rule_file), "--json=stream", "--", *batch]
|
|
2031
|
+
failure = None
|
|
2032
|
+
try:
|
|
2033
|
+
res = subprocess.run(cmd, cwd=root, capture_output=True, timeout=limit, check=False)
|
|
2034
|
+
except subprocess.TimeoutExpired:
|
|
2035
|
+
if clock is not None and clock.left() <= 0:
|
|
2036
|
+
failure = ("time-limit", f"the --timeout of {clock.seconds:g}s ran out while ast-grep read them")
|
|
2037
|
+
else:
|
|
2038
|
+
failure = ("ast-grep-timeout", f"no answer in {limit:g}s")
|
|
2039
|
+
except (OSError, ValueError) as exc:
|
|
2040
|
+
failure = ("ast-grep-error", f"{type(exc).__name__}: {exc}")
|
|
2041
|
+
else:
|
|
2042
|
+
stdout = res.stdout.decode("utf-8", errors="replace")
|
|
2043
|
+
if res.returncode != 0:
|
|
2044
|
+
stderr = res.stderr.decode("utf-8", errors="replace")
|
|
2045
|
+
detail = next((ln.strip() for ln in stderr.splitlines() if ln.strip()), "")
|
|
2046
|
+
failure = ("ast-grep-error", f"exit {res.returncode}: {detail}")
|
|
2047
|
+
for raw in stdout.splitlines():
|
|
2048
|
+
if not raw.strip():
|
|
2049
|
+
continue
|
|
2050
|
+
try:
|
|
2051
|
+
match = _parse_match(json.loads(raw))
|
|
2052
|
+
except ValueError:
|
|
2053
|
+
failure = failure or ("ast-grep-error", "output is not JSON lines")
|
|
2054
|
+
break
|
|
2055
|
+
if match is not None:
|
|
2056
|
+
found.append(match)
|
|
2057
|
+
if failure is not None:
|
|
2058
|
+
error = {"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1]}
|
|
2059
|
+
if failure[0] == "time-limit":
|
|
2060
|
+
error["unread"] = list(batch)
|
|
2061
|
+
errors.append(error)
|
|
2062
|
+
return found
|
|
2063
|
+
|
|
2064
|
+
|
|
2065
|
+
def _native_ast_grep(exe: str) -> str | None:
|
|
2066
|
+
"""The executable a scan runs for `exe`, the ast-grep PATH gives: `exe`
|
|
2067
|
+
itself, or for an npm `.cmd` or `.bat` shim (Windows) the native
|
|
2068
|
+
ast-grep.exe it wraps, which @ast-grep/cli's postinstall puts in its
|
|
2069
|
+
package folder (node_modules/@ast-grep/cli beside a global install's
|
|
2070
|
+
shim, ../@ast-grep/cli from node_modules/.bin), else its platform
|
|
2071
|
+
package's. None when neither holds one: a shim runs through cmd.exe,
|
|
2072
|
+
which would read the `&`, `^` and `%` of the repository paths a scan
|
|
2073
|
+
passes as its own syntax."""
|
|
2074
|
+
if not exe.lower().endswith((".cmd", ".bat")):
|
|
2075
|
+
return exe
|
|
2076
|
+
shim_dir = Path(exe).parent
|
|
2077
|
+
for scope in (shim_dir / "node_modules" / "@ast-grep", shim_dir.parent / "@ast-grep"):
|
|
2078
|
+
for package in [scope / "cli", *sorted(scope.glob("cli-win32-*"))]:
|
|
2079
|
+
native = package / "ast-grep.exe"
|
|
2080
|
+
if native.is_file():
|
|
2081
|
+
return str(native)
|
|
2082
|
+
return None
|
|
2083
|
+
|
|
2084
|
+
|
|
2085
|
+
def _ast_grep() -> tuple[str | None, str | None, str | None]:
|
|
2086
|
+
"""(the ast-grep executable a scan runs, its version, the npm shim PATH
|
|
2087
|
+
gives when no native binary backs it), each None when unknown."""
|
|
2088
|
+
exe = _sibling("skf-source-tree.py")._resolve_outside_cwd("ast-grep")
|
|
2089
|
+
if exe is None:
|
|
2090
|
+
return None, None, None
|
|
2091
|
+
native = _native_ast_grep(exe)
|
|
2092
|
+
if native is None:
|
|
2093
|
+
return None, None, exe
|
|
2094
|
+
try:
|
|
2095
|
+
res = subprocess.run([native, "--version"], capture_output=True, text=True, timeout=30, check=False)
|
|
2096
|
+
except (OSError, subprocess.SubprocessError):
|
|
2097
|
+
return native, None, None
|
|
2098
|
+
words = res.stdout.split()
|
|
2099
|
+
return native, (words[-1] if res.returncode == 0 and words else None), None
|
|
2100
|
+
|
|
2101
|
+
|
|
2102
|
+
# --------------------------------------------------------------------------
|
|
2103
|
+
# Merging the matches into exports
|
|
2104
|
+
# --------------------------------------------------------------------------
|
|
2105
|
+
|
|
2106
|
+
_WORD_RE = re.compile(r"[A-Za-z_$][\w$]*")
|
|
2107
|
+
|
|
2108
|
+
|
|
2109
|
+
def _keyword_type(text: str, name: str, choices: list[str]) -> str:
|
|
2110
|
+
"""The last of `choices` that stands as a word before `name` in the
|
|
2111
|
+
match text (`export declare const enum E` -> enum)."""
|
|
2112
|
+
picked = choices[0]
|
|
2113
|
+
for word in _WORD_RE.findall(text):
|
|
2114
|
+
if word == name:
|
|
2115
|
+
break
|
|
2116
|
+
if word in choices:
|
|
2117
|
+
picked = word
|
|
2118
|
+
return picked
|
|
2119
|
+
|
|
2120
|
+
|
|
2121
|
+
def _export_record(match: dict, form: dict) -> dict:
|
|
2122
|
+
export_type = form["metadata"]["export_type"]
|
|
2123
|
+
if isinstance(export_type, list):
|
|
2124
|
+
export_type = _keyword_type(match["text"], match["name"], export_type)
|
|
2125
|
+
params, return_type = (signature_parts(match["signature"], match["language"], match["name"])
|
|
2126
|
+
if export_type == "function" else (None, None))
|
|
2127
|
+
return {
|
|
2128
|
+
"export_name": match["name"],
|
|
2129
|
+
"source_file": match["file"],
|
|
2130
|
+
"source_line": match["line"],
|
|
2131
|
+
"signature_line": match["signature_line"],
|
|
2132
|
+
"signature": match["signature"],
|
|
2133
|
+
"params": params,
|
|
2134
|
+
"return_type": return_type,
|
|
2135
|
+
"citation": f"[AST:{match['file']}:L{match['line']}]",
|
|
2136
|
+
"ast_recipe": match["rule"],
|
|
2137
|
+
"ast_node_type": form["rule"]["kind"],
|
|
2138
|
+
"export_type": export_type,
|
|
2139
|
+
"language": match["language"],
|
|
2140
|
+
"from": match["source"],
|
|
2141
|
+
"from_file": None,
|
|
2142
|
+
"confidence": "T1",
|
|
2143
|
+
"extraction_method": "ast-grep",
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
# --------------------------------------------------------------------------
|
|
2148
|
+
# A function's parameters and return type, read from its one-line signature
|
|
2149
|
+
# --------------------------------------------------------------------------
|
|
2150
|
+
|
|
2151
|
+
_OPENERS = {"(": ")", "[": "]", "{": "}", "<": ">"}
|
|
2152
|
+
|
|
2153
|
+
|
|
2154
|
+
def _split_top(text: str, sep: str, angle: bool = True, quotes: str = "'\"`") -> list[str]:
|
|
2155
|
+
"""`text` split at each `sep` outside brackets and strings (`<` counts
|
|
2156
|
+
as a bracket when `angle`, and the `>` of `=>` or `->` closes none).
|
|
2157
|
+
`quotes` are the characters that open a string: only `"` in Rust, whose
|
|
2158
|
+
`'a` is a lifetime."""
|
|
2159
|
+
parts, depth, quote, start, i = [], 0, None, 0, 0
|
|
2160
|
+
while i < len(text):
|
|
2161
|
+
ch = text[i]
|
|
2162
|
+
if quote:
|
|
2163
|
+
if ch == "\\":
|
|
2164
|
+
i += 1
|
|
2165
|
+
elif ch == quote:
|
|
2166
|
+
quote = None
|
|
2167
|
+
elif ch in quotes:
|
|
2168
|
+
quote = ch
|
|
2169
|
+
elif ch in "([{" or (angle and ch == "<"):
|
|
2170
|
+
depth += 1
|
|
2171
|
+
elif ch in ")]}" or (angle and ch == ">" and text[i - 1:i] not in ("=", "-")):
|
|
2172
|
+
depth -= 1
|
|
2173
|
+
elif depth == 0 and text.startswith(sep, i) and not (sep == "=" and text[i + 1:i + 2] in ("=", ">")):
|
|
2174
|
+
parts.append(text[start:i])
|
|
2175
|
+
start = i + len(sep)
|
|
2176
|
+
i = start
|
|
2177
|
+
continue
|
|
2178
|
+
i += 1
|
|
2179
|
+
parts.append(text[start:])
|
|
2180
|
+
return parts
|
|
2181
|
+
|
|
2182
|
+
|
|
2183
|
+
def _close(text: str, start: int, angle: bool = True, quotes: str = "'\"`") -> int | None:
|
|
2184
|
+
"""The index of the bracket that closes the one at `start`, else None
|
|
2185
|
+
(strings as _split_top reads them)."""
|
|
2186
|
+
depth, quote, i = 0, None, start
|
|
2187
|
+
while i < len(text):
|
|
2188
|
+
ch = text[i]
|
|
2189
|
+
if quote:
|
|
2190
|
+
if ch == "\\":
|
|
2191
|
+
i += 1
|
|
2192
|
+
elif ch == quote:
|
|
2193
|
+
quote = None
|
|
2194
|
+
elif ch in quotes:
|
|
2195
|
+
quote = ch
|
|
2196
|
+
elif ch in "([{" or (angle and ch == "<"):
|
|
2197
|
+
depth += 1
|
|
2198
|
+
elif ch in ")]}" or (angle and ch == ">" and text[i - 1:i] not in ("=", "-")):
|
|
2199
|
+
depth -= 1
|
|
2200
|
+
if depth == 0:
|
|
2201
|
+
return i
|
|
2202
|
+
i += 1
|
|
2203
|
+
return None
|
|
2204
|
+
|
|
2205
|
+
|
|
2206
|
+
def _param(name: str | None, type_: str | None, default: str | None, optional: bool) -> dict:
|
|
2207
|
+
return {"name": name or None, "type": type_ or None, "default": default or None, "optional": optional}
|
|
2208
|
+
|
|
2209
|
+
|
|
2210
|
+
def _python_parts(signature: str) -> tuple[list[dict] | None, str | None]:
|
|
2211
|
+
"""A Python def's parameters and return annotation, through its own
|
|
2212
|
+
parser, each as the source writes it."""
|
|
2213
|
+
head = signature.strip()
|
|
2214
|
+
if head.endswith(":"):
|
|
2215
|
+
head = head[:-1]
|
|
2216
|
+
source = head + ": pass"
|
|
2217
|
+
try:
|
|
2218
|
+
tree = ast.parse(source)
|
|
2219
|
+
except SyntaxError:
|
|
2220
|
+
return None, None
|
|
2221
|
+
node = tree.body[0] if tree.body else None
|
|
2222
|
+
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
2223
|
+
return None, None
|
|
2224
|
+
a = node.args
|
|
2225
|
+
positional = a.posonlyargs + a.args
|
|
2226
|
+
defaults = [None] * (len(positional) - len(a.defaults)) + list(a.defaults)
|
|
2227
|
+
params = []
|
|
2228
|
+
|
|
2229
|
+
def text(value):
|
|
2230
|
+
# the source's own text (`"x"`, `0x10`), never ast.unparse's normal form (`'x'`, `16`)
|
|
2231
|
+
return (ast.get_source_segment(source, value) or ast.unparse(value)) if value is not None else None
|
|
2232
|
+
|
|
2233
|
+
for arg, default in zip(positional, defaults):
|
|
2234
|
+
params.append(_param(arg.arg, text(arg.annotation), text(default), default is not None))
|
|
2235
|
+
if a.vararg is not None:
|
|
2236
|
+
params.append(_param("*" + a.vararg.arg, text(a.vararg.annotation), None, True))
|
|
2237
|
+
for arg, default in zip(a.kwonlyargs, a.kw_defaults):
|
|
2238
|
+
params.append(_param(arg.arg, text(arg.annotation), text(default), default is not None))
|
|
2239
|
+
if a.kwarg is not None:
|
|
2240
|
+
params.append(_param("**" + a.kwarg.arg, text(a.kwarg.annotation), None, True))
|
|
2241
|
+
return params, text(node.returns)
|
|
2242
|
+
|
|
2243
|
+
|
|
2244
|
+
def _js_param(part: str) -> dict | None:
|
|
2245
|
+
"""One JS/TS parameter: `name`, `name?: T`, `name: T = d`, `...rest: T[]`
|
|
2246
|
+
or a destructuring pattern; None for TypeScript's `this` parameter."""
|
|
2247
|
+
part = part.strip()
|
|
2248
|
+
default = None
|
|
2249
|
+
pieces = _split_top(part, "=")
|
|
2250
|
+
if len(pieces) > 1:
|
|
2251
|
+
part, default = pieces[0].strip(), "=".join(pieces[1:]).strip()
|
|
2252
|
+
typed = _split_top(part, ":")
|
|
2253
|
+
name, type_ = typed[0].strip(), (":".join(typed[1:]).strip() if len(typed) > 1 else None)
|
|
2254
|
+
optional = default is not None or name.startswith("...")
|
|
2255
|
+
if name.endswith("?"):
|
|
2256
|
+
name, optional = name[:-1].rstrip(), True
|
|
2257
|
+
if name == "this":
|
|
2258
|
+
return None
|
|
2259
|
+
return _param(name, type_, default, optional)
|
|
2260
|
+
|
|
2261
|
+
|
|
2262
|
+
def _js_parts(signature: str, name: str) -> tuple[list[dict] | None, str | None]:
|
|
2263
|
+
"""An exported JS/TS function's parameters and return type: a function
|
|
2264
|
+
declaration (`function name<T>(...): R {`), or a const bound to an arrow
|
|
2265
|
+
function (`const name = async <T>(...): R => ...`, `const name = x => ...`)."""
|
|
2266
|
+
# `$` is an identifier character in JS, so `\b` would never bound `$fetch`
|
|
2267
|
+
at = re.search(r"(?<![\w$])" + re.escape(name) + r"(?![\w$])", signature)
|
|
2268
|
+
if at is None:
|
|
2269
|
+
return None, None
|
|
2270
|
+
rest = signature[at.end():]
|
|
2271
|
+
arrow = re.match(r"\s*(?::[^=]*?)?=\s*(?:async\s+)?", rest)
|
|
2272
|
+
if arrow and not re.match(r"\s*\(", rest):
|
|
2273
|
+
rest = rest[arrow.end():]
|
|
2274
|
+
bare = re.match(r"([A-Za-z_$][\w$]*)\s*=>", rest)
|
|
2275
|
+
if bare:
|
|
2276
|
+
return [_param(bare.group(1), None, None, False)], None
|
|
2277
|
+
else:
|
|
2278
|
+
arrow = None
|
|
2279
|
+
rest = rest.lstrip()
|
|
2280
|
+
if rest.startswith("<"):
|
|
2281
|
+
end = _close(rest, 0)
|
|
2282
|
+
if end is None:
|
|
2283
|
+
return None, None
|
|
2284
|
+
rest = rest[end + 1:].lstrip()
|
|
2285
|
+
if not rest.startswith("("):
|
|
2286
|
+
return None, None
|
|
2287
|
+
end = _close(rest, 0)
|
|
2288
|
+
if end is None:
|
|
2289
|
+
return None, None
|
|
2290
|
+
inner = rest[1:end].strip()
|
|
2291
|
+
params = [p for p in (_js_param(part) for part in _split_top(inner, ",") if part.strip()) if p is not None]
|
|
2292
|
+
tail = rest[end + 1:].strip()
|
|
2293
|
+
if not tail.startswith(":"):
|
|
2294
|
+
return params, None
|
|
2295
|
+
tail = tail[1:].strip()
|
|
2296
|
+
if arrow is not None:
|
|
2297
|
+
pieces = _split_top(tail, "=>")
|
|
2298
|
+
if len(pieces) < 2:
|
|
2299
|
+
return None, None
|
|
2300
|
+
returns = pieces[0].strip()
|
|
2301
|
+
else:
|
|
2302
|
+
returns = tail[:-1].rstrip() if tail.endswith(("{", ";")) else tail
|
|
2303
|
+
return (params, returns) if returns else (None, None)
|
|
2304
|
+
|
|
2305
|
+
|
|
2306
|
+
def _rust_parts(signature: str, name: str) -> tuple[list[dict] | None, str | None]:
|
|
2307
|
+
"""A Rust fn's parameters (`self` forms included) and its `->` type."""
|
|
2308
|
+
at = re.search(r"\bfn\s+" + re.escape(name) + r"\b", signature)
|
|
2309
|
+
if at is None:
|
|
2310
|
+
return None, None
|
|
2311
|
+
rest = signature[at.end():].lstrip()
|
|
2312
|
+
if rest.startswith("<"):
|
|
2313
|
+
end = _close(rest, 0, quotes='"')
|
|
2314
|
+
if end is None:
|
|
2315
|
+
return None, None
|
|
2316
|
+
rest = rest[end + 1:].lstrip()
|
|
2317
|
+
if not rest.startswith("("):
|
|
2318
|
+
return None, None
|
|
2319
|
+
end = _close(rest, 0, angle=False, quotes='"')
|
|
2320
|
+
if end is None:
|
|
2321
|
+
return None, None
|
|
2322
|
+
params = []
|
|
2323
|
+
for part in _split_top(rest[1:end], ",", quotes='"'):
|
|
2324
|
+
part = part.strip()
|
|
2325
|
+
if not part:
|
|
2326
|
+
continue
|
|
2327
|
+
typed = _split_top(part, ":", quotes='"')
|
|
2328
|
+
if len(typed) == 1:
|
|
2329
|
+
params.append(_param(part, None, None, False)) # self, &self, &mut self
|
|
2330
|
+
else:
|
|
2331
|
+
params.append(_param(typed[0].strip(), ":".join(typed[1:]).strip(), None, False))
|
|
2332
|
+
tail = rest[end + 1:].strip()
|
|
2333
|
+
if not tail.startswith("->"):
|
|
2334
|
+
return params, None
|
|
2335
|
+
returns = re.split(r"\s+where\s+", tail[2:].strip(), maxsplit=1)[0].strip()
|
|
2336
|
+
returns = returns[:-1].rstrip() if returns.endswith(("{", ";")) else returns
|
|
2337
|
+
return (params, returns) if returns else (None, None)
|
|
2338
|
+
|
|
2339
|
+
|
|
2340
|
+
_GO_TYPE_WORDS = frozenset({"chan", "func", "map", "struct", "interface"})
|
|
2341
|
+
|
|
2342
|
+
|
|
2343
|
+
def _go_parts(signature: str, name: str) -> tuple[list[dict] | None, str | None]:
|
|
2344
|
+
"""A Go func's parameters (a type shared by the names before it, as in
|
|
2345
|
+
`a, b int`, given to each) and its result list or type."""
|
|
2346
|
+
at = re.search(r"\bfunc\s+" + re.escape(name) + r"\b", signature)
|
|
2347
|
+
if at is None:
|
|
2348
|
+
return None, None
|
|
2349
|
+
rest = signature[at.end():].lstrip()
|
|
2350
|
+
if rest.startswith("["):
|
|
2351
|
+
end = _close(rest, 0, angle=False, quotes='"`')
|
|
2352
|
+
if end is None:
|
|
2353
|
+
return None, None
|
|
2354
|
+
rest = rest[end + 1:].lstrip()
|
|
2355
|
+
if not rest.startswith("("):
|
|
2356
|
+
return None, None
|
|
2357
|
+
end = _close(rest, 0, angle=False, quotes='"`')
|
|
2358
|
+
if end is None:
|
|
2359
|
+
return None, None
|
|
2360
|
+
parts = [p.strip() for p in _split_top(rest[1:end], ",", angle=False, quotes='"`') if p.strip()]
|
|
2361
|
+
# `a, b int` names its parameters; `chan int` alone is a type
|
|
2362
|
+
named = any(re.match(r"[A-Za-z_]\w*\s+\S", p) and p.split(None, 1)[0] not in _GO_TYPE_WORDS for p in parts)
|
|
2363
|
+
params: list[dict] = []
|
|
2364
|
+
pending: list[str] = []
|
|
2365
|
+
for part in parts:
|
|
2366
|
+
if not named:
|
|
2367
|
+
params.append(_param(None, part, None, part.startswith("...")))
|
|
2368
|
+
continue
|
|
2369
|
+
words = part.split(None, 1)
|
|
2370
|
+
if len(words) == 1:
|
|
2371
|
+
pending.append(words[0])
|
|
2372
|
+
continue
|
|
2373
|
+
type_ = words[1].strip()
|
|
2374
|
+
for waiting in pending + [words[0]]:
|
|
2375
|
+
params.append(_param(waiting, type_, None, type_.startswith("...")))
|
|
2376
|
+
pending = []
|
|
2377
|
+
if pending:
|
|
2378
|
+
return None, None
|
|
2379
|
+
tail = rest[end + 1:].strip()
|
|
2380
|
+
tail = tail[:-1].rstrip() if tail.endswith("{") else tail
|
|
2381
|
+
return params, (tail or None)
|
|
2382
|
+
|
|
2383
|
+
|
|
2384
|
+
def signature_parts(signature: str, language: str, name: str) -> tuple[list[dict] | None, str | None]:
|
|
2385
|
+
"""(params, return_type) of an exported function, read from the
|
|
2386
|
+
one-line `signature` its recipe matched: each parameter as {name, type,
|
|
2387
|
+
default, optional}, and the declared return type, null when the
|
|
2388
|
+
declaration states none. (None, None) when the signature is a form this
|
|
2389
|
+
reader does not parse, such as a declaration cut at its line cap."""
|
|
2390
|
+
if language == "python":
|
|
2391
|
+
return _python_parts(signature)
|
|
2392
|
+
if language in ("typescript", "tsx", "javascript"):
|
|
2393
|
+
return _js_parts(signature, name)
|
|
2394
|
+
if language == "rust":
|
|
2395
|
+
return _rust_parts(signature, name)
|
|
2396
|
+
if language == "go":
|
|
2397
|
+
return _go_parts(signature, name)
|
|
2398
|
+
return None, None
|
|
2399
|
+
|
|
2400
|
+
|
|
2401
|
+
def _preferred(at_line: list[dict], forms: dict, order: dict[str, int]) -> dict:
|
|
2402
|
+
"""The match an export records among several recipes' matches of one
|
|
2403
|
+
name on one line: one that no other lists under prefer_over, then the
|
|
2404
|
+
first in the recipe file."""
|
|
2405
|
+
ids = {m["rule"] for m in at_line}
|
|
2406
|
+
beaten = {loser for m in at_line
|
|
2407
|
+
for loser in forms[(m["rule"], m["language"])]["metadata"].get("prefer_over", [])
|
|
2408
|
+
if loser in ids and loser != m["rule"]}
|
|
2409
|
+
left = [m for m in at_line if m["rule"] not in beaten] or at_line
|
|
2410
|
+
return min(left, key=lambda m: (order[m["rule"]], m["language"]))
|
|
2411
|
+
|
|
2412
|
+
|
|
2413
|
+
def merge_exports(matches: list[dict], recipes: list[dict], recipe_set: str,
|
|
2414
|
+
head_cap: int | None) -> tuple[list[dict], list[dict]]:
|
|
2415
|
+
"""(exports, per-recipe stats) from the matches of one recipe set: at
|
|
2416
|
+
most `head_cap` matches of each recipe, in file and line order, merged
|
|
2417
|
+
by (file, name) at the first line that names it."""
|
|
2418
|
+
forms = _forms(recipes)
|
|
2419
|
+
order: dict[str, int] = {}
|
|
2420
|
+
languages: dict[str, list[str]] = {}
|
|
2421
|
+
for recipe in recipes:
|
|
2422
|
+
if recipe_set in recipe["metadata"]["sets"]:
|
|
2423
|
+
order.setdefault(recipe["id"], len(order))
|
|
2424
|
+
langs = languages.setdefault(recipe["id"], [])
|
|
2425
|
+
langs.extend(lang for lang in recipe["metadata"]["languages"] if lang not in langs)
|
|
2426
|
+
per_recipe: dict[str, list[dict]] = {rid: [] for rid in order}
|
|
2427
|
+
for m in matches:
|
|
2428
|
+
if m["rule"] in per_recipe and m["name"] and not m["name"].startswith("_") and (m["rule"], m["language"]) in forms:
|
|
2429
|
+
per_recipe[m["rule"]].append(m)
|
|
2430
|
+
stats, kept = [], []
|
|
2431
|
+
for rid, found in per_recipe.items():
|
|
2432
|
+
found.sort(key=lambda m: (m["file"], m["line"], m["name"]))
|
|
2433
|
+
cut = head_cap is not None and len(found) > head_cap
|
|
2434
|
+
stats.append({"id": rid, "languages": languages[rid], "matches": len(found), "truncated": cut})
|
|
2435
|
+
kept.extend(found[:head_cap] if cut else found)
|
|
2436
|
+
groups: dict[tuple[str, str], list[dict]] = {}
|
|
2437
|
+
for m in kept:
|
|
2438
|
+
groups.setdefault((m["file"], m["name"]), []).append(m)
|
|
2439
|
+
exports = []
|
|
2440
|
+
for group in groups.values():
|
|
2441
|
+
first_line = min(m["line"] for m in group)
|
|
2442
|
+
chosen = _preferred([m for m in group if m["line"] == first_line], forms, order)
|
|
2443
|
+
exports.append(_export_record(chosen, forms[(chosen["rule"], chosen["language"])]))
|
|
2444
|
+
exports.sort(key=lambda e: (e["source_file"], e["source_line"], e["export_name"]))
|
|
2445
|
+
return exports, stats
|
|
2446
|
+
|
|
2447
|
+
|
|
2448
|
+
def _syntax_issues(matches: list[dict]) -> list[dict]:
|
|
2449
|
+
"""One `syntax-errors` issue per file with an outermost ERROR node."""
|
|
2450
|
+
per_file: dict[str, tuple[int, int]] = {}
|
|
2451
|
+
for m in matches:
|
|
2452
|
+
if m["rule"] == SYNTAX_ERROR_RULE:
|
|
2453
|
+
count, line = per_file.get(m["file"], (0, m["start_line"]))
|
|
2454
|
+
per_file[m["file"]] = (count + 1, min(line, m["start_line"]))
|
|
2455
|
+
return [{"file": f, "issue": "syntax-errors", "count": c, "line": ln}
|
|
2456
|
+
for f, (c, ln) in sorted(per_file.items())]
|
|
2457
|
+
|
|
2458
|
+
|
|
2459
|
+
def _type_list_items(exe: str, root: Path, folder: Path, matches: list[dict], exports: list[dict],
|
|
2460
|
+
errors: list[dict], clock: _Clock | None = None) -> None:
|
|
2461
|
+
"""Give each export a local `export { ... }` list names (an
|
|
2462
|
+
export_specifier with no `from`) the type of the top-level binding it
|
|
2463
|
+
names in its file, as BINDING_TYPES has it; the recipe's own
|
|
2464
|
+
export_type stays when no such binding has the name (a destructured
|
|
2465
|
+
one)."""
|
|
2466
|
+
items = [e for e in exports if e["ast_node_type"] == "export_specifier" and e["from"] is None]
|
|
2467
|
+
if not items:
|
|
2468
|
+
return
|
|
2469
|
+
local: dict[tuple[str, str, str], str] = {}
|
|
2470
|
+
for m in matches:
|
|
2471
|
+
if m["name"] and m["source"] is None:
|
|
2472
|
+
local.setdefault((m["file"], m["rule"], m["name"]), _specifier_local(m["text"]))
|
|
2473
|
+
files = sorted({e["source_file"] for e in items})
|
|
2474
|
+
languages = sorted({lang for f in files for lang in _rule_languages(_file_language(f) or "")}
|
|
2475
|
+
& set(RECIPE_LANGUAGES))
|
|
2476
|
+
rules = [rule for language in languages for rule in _binding_rules(language)]
|
|
2477
|
+
types = _binding_types(_scan(exe, root, folder, rules, files, errors, clock))
|
|
2478
|
+
for e in items:
|
|
2479
|
+
name = local.get((e["source_file"], e["ast_recipe"], e["export_name"]), e["export_name"])
|
|
2480
|
+
e["export_type"] = types.get((e["source_file"], name), e["export_type"])
|
|
2481
|
+
|
|
2482
|
+
|
|
2483
|
+
# --------------------------------------------------------------------------
|
|
2484
|
+
# Comments and strings, blanked for the text readers below
|
|
2485
|
+
# --------------------------------------------------------------------------
|
|
2486
|
+
|
|
2487
|
+
_RUST_RAW_OPEN_RE = re.compile(r'r(#*)"')
|
|
2488
|
+
_RUST_CHAR_RE = re.compile(r"'(?:\\(?:x[0-9A-Fa-f]{2}|u\{[0-9A-Fa-f]{1,6}\}|.)|[^\\'\n])'")
|
|
2489
|
+
# What a JavaScript `/` follows when it opens a regular expression literal
|
|
2490
|
+
# rather than dividing: punctuation no operand ends with, or a keyword.
|
|
2491
|
+
_JS_REGEX_AFTER = frozenset("(,=:[!&|?{};")
|
|
2492
|
+
_JS_REGEX_AFTER_WORDS = frozenset({
|
|
2493
|
+
"return", "typeof", "case", "do", "else", "in", "of", "instanceof", "new", "delete", "void", "throw",
|
|
2494
|
+
"yield", "await",
|
|
2495
|
+
})
|
|
2496
|
+
|
|
2497
|
+
|
|
2498
|
+
def _js_regex_end(text: str, done: list[str], i: int) -> int | None:
|
|
2499
|
+
"""The index of the `/` that closes the regular expression literal the
|
|
2500
|
+
`/` at `i` opens, or None when that `/` divides. `done` is the text up
|
|
2501
|
+
to `i` with its comments and strings blanked; a literal opens a line or
|
|
2502
|
+
follows _JS_REGEX_AFTER or a word of _JS_REGEX_AFTER_WORDS (a `}` before
|
|
2503
|
+
a JSX `/>` aside), and ends on its line."""
|
|
2504
|
+
k = i - 1
|
|
2505
|
+
while k >= 0 and done[k] in " \t\r":
|
|
2506
|
+
k -= 1
|
|
2507
|
+
before = done[k] if k >= 0 else "\n"
|
|
2508
|
+
if before == "}" and text.startswith(">", i + 1):
|
|
2509
|
+
return None
|
|
2510
|
+
if before != "\n" and before not in _JS_REGEX_AFTER:
|
|
2511
|
+
end = k + 1
|
|
2512
|
+
while k >= 0 and (done[k].isalnum() or done[k] in "_$"):
|
|
2513
|
+
k -= 1
|
|
2514
|
+
if "".join(done[k + 1:end]) not in _JS_REGEX_AFTER_WORDS or (k >= 0 and done[k] == "."):
|
|
2515
|
+
return None
|
|
2516
|
+
in_class, j = False, i + 1
|
|
2517
|
+
while j < len(text) and text[j] != "\n":
|
|
2518
|
+
ch = text[j]
|
|
2519
|
+
if ch == "\\":
|
|
2520
|
+
j += 2
|
|
2521
|
+
continue
|
|
2522
|
+
if in_class:
|
|
2523
|
+
in_class = ch != "]"
|
|
2524
|
+
elif ch == "[":
|
|
2525
|
+
in_class = True
|
|
2526
|
+
elif ch == "/":
|
|
2527
|
+
return j
|
|
2528
|
+
j += 1
|
|
2529
|
+
return None
|
|
2530
|
+
|
|
2531
|
+
|
|
2532
|
+
def _blank(text: str, language: str) -> str:
|
|
2533
|
+
"""`text` with its comments, and the insides of its strings, turned into
|
|
2534
|
+
spaces, its lines kept: a pattern run over it finds code only.
|
|
2535
|
+
JavaScript/TypeScript (`js`): `//`, `/* */` and '...', "..." and `...`
|
|
2536
|
+
strings, a string followed within its line (a template literal spanning
|
|
2537
|
+
lines is not tracked), and the insides of a regular expression literal
|
|
2538
|
+
(`/\\/*$/` opens no comment). Go: `//`, `/* */`, "..." and '...' within a
|
|
2539
|
+
line and `...` raw strings across lines. Rust: `//`, nested `/* */`,
|
|
2540
|
+
"..." and r#"..."# strings across lines, and char literals."""
|
|
2541
|
+
out = list(text)
|
|
2542
|
+
n = len(text)
|
|
2543
|
+
|
|
2544
|
+
def blank(start: int, end: int) -> None:
|
|
2545
|
+
for k in range(start, min(end, n)):
|
|
2546
|
+
if out[k] != "\n":
|
|
2547
|
+
out[k] = " "
|
|
2548
|
+
|
|
2549
|
+
quotes = "\"'`" if language in ("js", "go") else '"'
|
|
2550
|
+
i = 0
|
|
2551
|
+
while i < n:
|
|
2552
|
+
if text.startswith("//", i):
|
|
2553
|
+
end = text.find("\n", i)
|
|
2554
|
+
end = n if end < 0 else end
|
|
2555
|
+
blank(i, end)
|
|
2556
|
+
i = end
|
|
2557
|
+
continue
|
|
2558
|
+
if text.startswith("/*", i):
|
|
2559
|
+
depth, j = 1, i + 2
|
|
2560
|
+
while j < n and depth:
|
|
2561
|
+
if language == "rust" and text.startswith("/*", j):
|
|
2562
|
+
depth, j = depth + 1, j + 2
|
|
2563
|
+
elif text.startswith("*/", j):
|
|
2564
|
+
depth, j = depth - 1, j + 2
|
|
2565
|
+
else:
|
|
2566
|
+
j += 1
|
|
2567
|
+
blank(i, j)
|
|
2568
|
+
i = j
|
|
2569
|
+
continue
|
|
2570
|
+
ch = text[i]
|
|
2571
|
+
if language == "js" and ch == "/":
|
|
2572
|
+
end = _js_regex_end(text, out, i)
|
|
2573
|
+
if end is not None:
|
|
2574
|
+
blank(i + 1, end)
|
|
2575
|
+
i = end + 1
|
|
2576
|
+
continue
|
|
2577
|
+
if language == "rust":
|
|
2578
|
+
raw = _RUST_RAW_OPEN_RE.match(text, i) if ch == "r" else None
|
|
2579
|
+
if raw and not (i and (text[i - 1].isalnum() or text[i - 1] == "_")):
|
|
2580
|
+
close = '"' + raw.group(1)
|
|
2581
|
+
end = text.find(close, raw.end())
|
|
2582
|
+
end = n if end < 0 else end
|
|
2583
|
+
blank(raw.end(), end)
|
|
2584
|
+
i = end + len(close)
|
|
2585
|
+
continue
|
|
2586
|
+
char = _RUST_CHAR_RE.match(text, i) if ch == "'" else None
|
|
2587
|
+
if char:
|
|
2588
|
+
i = char.end()
|
|
2589
|
+
continue
|
|
2590
|
+
if ch in quotes:
|
|
2591
|
+
across = language == "rust" or (language == "go" and ch == "`")
|
|
2592
|
+
escapes = not (language == "go" and ch == "`")
|
|
2593
|
+
j = i + 1
|
|
2594
|
+
while j < n and text[j] != ch and (across or text[j] != "\n"):
|
|
2595
|
+
j += 2 if escapes and text[j] == "\\" else 1
|
|
2596
|
+
blank(i + 1, j)
|
|
2597
|
+
i = j + 1 if j < n and text[j] == ch else j
|
|
2598
|
+
continue
|
|
2599
|
+
i += 1
|
|
2600
|
+
return "".join(out)
|
|
2601
|
+
|
|
2602
|
+
|
|
2603
|
+
def _read_text(root: Path, rel: str) -> str | None:
|
|
2604
|
+
try:
|
|
2605
|
+
return (root / rel).read_text(encoding="utf-8")
|
|
2606
|
+
except (OSError, UnicodeDecodeError):
|
|
2607
|
+
return None
|
|
2608
|
+
|
|
2609
|
+
|
|
2610
|
+
# --------------------------------------------------------------------------
|
|
2611
|
+
# Entry points: JavaScript and TypeScript
|
|
2612
|
+
# --------------------------------------------------------------------------
|
|
2613
|
+
|
|
2614
|
+
_JS_DECLARATION_RE = re.compile(
|
|
2615
|
+
r"^export\s+(?:declare\s+)?(default\s+)?(?:abstract\s+)?(?:async\s+)?"
|
|
2616
|
+
r"(?:const\s+enum|const|let|var|function\s*\*?|class|interface|type|enum|namespace|module)"
|
|
2617
|
+
r"\s+([A-Za-z_$][\w$]*)",
|
|
2618
|
+
re.M,
|
|
2619
|
+
)
|
|
2620
|
+
_JS_IMPORT_EQUALS_RE = re.compile(r"^export\s+import()\s+([A-Za-z_$][\w$]*)\s*=", re.M)
|
|
2621
|
+
_JS_DEFAULT_NAME_RE = re.compile(r"^export\s+default\s+([A-Za-z_$][\w$]*)\s*;?\s*$", re.M)
|
|
2622
|
+
_JS_DEFAULT_EXPORT_RE = re.compile(r"export\s+default\b")
|
|
2623
|
+
|
|
2624
|
+
|
|
2625
|
+
def _js_text_names(text: str) -> tuple[list[tuple[str, int, bool]], list[tuple[str, int]]]:
|
|
2626
|
+
"""The names a JS/TS file's column-0 `export` declarations bind, read
|
|
2627
|
+
from its text for the forms no recipe matches (`export let`, `export
|
|
2628
|
+
declare const`, `export namespace`), as (name, line, is the default
|
|
2629
|
+
export), and its `export default <name>;` statements, as (name, line)."""
|
|
2630
|
+
code = _blank(text, "js")
|
|
2631
|
+
declared = [(m.group(2), _line_of(code, m.start(2)), bool(m.group(1)))
|
|
2632
|
+
for rx in (_JS_DECLARATION_RE, _JS_IMPORT_EQUALS_RE) for m in rx.finditer(code)]
|
|
2633
|
+
defaults = [(m.group(1), _line_of(code, m.start(1))) for m in _JS_DEFAULT_NAME_RE.finditer(code)]
|
|
2634
|
+
return declared, defaults
|
|
2635
|
+
|
|
2636
|
+
|
|
2637
|
+
def resolve_js_module(spec: str | None, from_file: str, tree: set[str]) -> str | None:
|
|
2638
|
+
"""The file a relative import or re-export names: the path, then with
|
|
2639
|
+
each JS/TS extension, then its index file, a `.js` specifier naming its
|
|
2640
|
+
TypeScript source first. None for a package name or a path outside the
|
|
2641
|
+
root."""
|
|
2642
|
+
if not spec or not spec.startswith("."):
|
|
2643
|
+
return None
|
|
2644
|
+
base = posixpath.normpath(posixpath.join(posixpath.dirname(from_file), spec))
|
|
2645
|
+
if base == ".." or base.startswith("../"):
|
|
2646
|
+
return None
|
|
2647
|
+
base = "" if base == "." else base
|
|
2648
|
+
stem, ext = posixpath.splitext(base)
|
|
2649
|
+
candidates = [base] if base else []
|
|
2650
|
+
candidates += [stem + e for e in JS_SOURCE_OF.get(ext, ())]
|
|
2651
|
+
candidates += [base + e for e in JS_RESOLVE_EXTENSIONS]
|
|
2652
|
+
candidates += [posixpath.join(base, "index" + e) for e in JS_RESOLVE_EXTENSIONS]
|
|
2653
|
+
return next((c for c in candidates if c in tree), None)
|
|
2654
|
+
|
|
2655
|
+
|
|
2656
|
+
def _targets(value: object) -> list[str]:
|
|
2657
|
+
"""Every target path of a package.json `exports` value, conditions and
|
|
2658
|
+
fallbacks included."""
|
|
2659
|
+
if isinstance(value, str):
|
|
2660
|
+
return [value]
|
|
2661
|
+
if isinstance(value, list):
|
|
2662
|
+
return [t for item in value for t in _targets(item)]
|
|
2663
|
+
if isinstance(value, dict):
|
|
2664
|
+
return [t for item in value.values() for t in _targets(item)]
|
|
2665
|
+
return []
|
|
2666
|
+
|
|
2667
|
+
|
|
2668
|
+
def exports_subpaths(exports: object) -> dict[str, list[str]]:
|
|
2669
|
+
"""A package.json `exports` value as {subpath: its target paths}."""
|
|
2670
|
+
if isinstance(exports, dict):
|
|
2671
|
+
keys = [k for k in exports if isinstance(k, str)]
|
|
2672
|
+
if keys and all(k.startswith(".") for k in keys):
|
|
2673
|
+
return {k: _targets(exports[k]) for k in keys}
|
|
2674
|
+
targets = _targets(exports)
|
|
2675
|
+
return {".": targets} if targets else {}
|
|
2676
|
+
|
|
2677
|
+
|
|
2678
|
+
def _source_stems(rel: str) -> list[str]:
|
|
2679
|
+
"""The stems a build output's source can have, in the order tried: the
|
|
2680
|
+
path without its extension, then with one more of its leading build
|
|
2681
|
+
folders swapped for `src/` or dropped at each try (`dist/esm/index.js`
|
|
2682
|
+
-> dist/esm/index, src/esm/index, esm/index, src/index, index)."""
|
|
2683
|
+
stem = next((rel[: -len(s)] for s in JS_TARGET_SUFFIXES if rel.endswith(s)), rel)
|
|
2684
|
+
parts = stem.split("/")
|
|
2685
|
+
stems = [stem]
|
|
2686
|
+
for depth in range(1, len(parts)):
|
|
2687
|
+
if parts[depth - 1] not in BUILD_DIRS:
|
|
2688
|
+
break
|
|
2689
|
+
rest = "/".join(parts[depth:])
|
|
2690
|
+
stems += [f"src/{rest}", rest]
|
|
2691
|
+
return stems
|
|
2692
|
+
|
|
2693
|
+
|
|
2694
|
+
def _package_target(pkg: str, target: str, tree: set[str]) -> tuple[str | None, str]:
|
|
2695
|
+
"""(the file a package.json target names, `file`), or, for a build
|
|
2696
|
+
output that is not committed, (its source, `source-guess`): each of its
|
|
2697
|
+
source stems with each source extension (`dist/esm/index.js` ->
|
|
2698
|
+
src/index.ts, `build/lib/util.js` -> lib/util.ts)."""
|
|
2699
|
+
|
|
2700
|
+
def join(rel: str) -> str:
|
|
2701
|
+
return f"{pkg}/{rel}" if pkg else rel
|
|
2702
|
+
|
|
2703
|
+
rel = posixpath.normpath(_norm(target))
|
|
2704
|
+
if rel in (".", "..") or rel.startswith("../"):
|
|
2705
|
+
return None, "unresolved"
|
|
2706
|
+
if join(rel) in tree:
|
|
2707
|
+
return join(rel), "file"
|
|
2708
|
+
for stem in _source_stems(rel):
|
|
2709
|
+
for ext in JS_SOURCE_EXTENSIONS:
|
|
2710
|
+
if join(stem + ext) in tree:
|
|
2711
|
+
return join(stem + ext), "source-guess"
|
|
2712
|
+
return None, "unresolved"
|
|
2713
|
+
|
|
2714
|
+
|
|
2715
|
+
def _pattern_files(pkg: str, pattern: str, tree: set[str]) -> dict[str, str]:
|
|
2716
|
+
"""{what the `*` stands for: file} of the files a path pattern with one
|
|
2717
|
+
`*`, which spans folders as in Node, names in package folder `pkg`."""
|
|
2718
|
+
head, _, tail = (f"{pkg}/{pattern}" if pkg else pattern).partition("*")
|
|
2719
|
+
return {path[len(head): len(path) - len(tail)]: path for path in sorted(tree)
|
|
2720
|
+
if path.startswith(head) and path.endswith(tail) and len(path) > len(head) + len(tail)}
|
|
2721
|
+
|
|
2722
|
+
|
|
2723
|
+
def _package_pattern(pkg: str, target: str, tree: set[str]) -> tuple[dict[str, str], str]:
|
|
2724
|
+
"""A package.json target with one `*` (a subpath pattern's): ({what the
|
|
2725
|
+
`*` stands for: file} of the files it names, `file`), else, for a
|
|
2726
|
+
build output that is not committed, those of the first source stem
|
|
2727
|
+
that names any (`source-guess`)."""
|
|
2728
|
+
rel = posixpath.normpath(_norm(target))
|
|
2729
|
+
if rel.count("*") != 1 or rel.startswith("../"):
|
|
2730
|
+
return {}, "unresolved"
|
|
2731
|
+
found = _pattern_files(pkg, rel, tree)
|
|
2732
|
+
if found:
|
|
2733
|
+
return found, "file"
|
|
2734
|
+
for stem in _source_stems(rel):
|
|
2735
|
+
for ext in JS_SOURCE_EXTENSIONS:
|
|
2736
|
+
for key, file in _pattern_files(pkg, stem + ext, tree).items():
|
|
2737
|
+
found.setdefault(key, file)
|
|
2738
|
+
if found:
|
|
2739
|
+
return found, "source-guess"
|
|
2740
|
+
return {}, "unresolved"
|
|
2741
|
+
|
|
2742
|
+
|
|
2743
|
+
def _subpath_excluded(subpath: str, excluded: list[str]) -> bool:
|
|
2744
|
+
"""Whether a subpath (or subpath pattern) the exports map maps to `null`
|
|
2745
|
+
takes `subpath` out of the package's surface."""
|
|
2746
|
+
for pattern in excluded:
|
|
2747
|
+
head, star, tail = pattern.partition("*")
|
|
2748
|
+
if subpath == pattern or (star and subpath.startswith(head) and subpath.endswith(tail)
|
|
2749
|
+
and len(subpath) >= len(head) + len(tail)):
|
|
2750
|
+
return True
|
|
2751
|
+
return False
|
|
2752
|
+
|
|
2753
|
+
|
|
2754
|
+
def _owners(files: list[str], manifest: str, tree: list[str]) -> list[str]:
|
|
2755
|
+
"""The folders ('' for the root) of the nearest `manifest` above each
|
|
2756
|
+
file."""
|
|
2757
|
+
dirs = {posixpath.dirname(p) for p in tree if posixpath.basename(p) == manifest}
|
|
2758
|
+
owners = set()
|
|
2759
|
+
for rel in files:
|
|
2760
|
+
folder = posixpath.dirname(rel)
|
|
2761
|
+
while True:
|
|
2762
|
+
if folder in dirs:
|
|
2763
|
+
owners.add(folder)
|
|
2764
|
+
break
|
|
2765
|
+
if not folder:
|
|
2766
|
+
break
|
|
2767
|
+
folder = posixpath.dirname(folder)
|
|
2768
|
+
return sorted(owners)
|
|
2769
|
+
|
|
2770
|
+
|
|
2771
|
+
def _js_entries(root: Path, files: list[str], tree: list[str], tree_set: set[str],
|
|
2772
|
+
warnings: list[str]) -> tuple[list[dict], list[dict], list[dict]]:
|
|
2773
|
+
"""(entry points, exports-map facts, unresolved subpaths) of the
|
|
2774
|
+
packages that own the in-scope JS/TS files: the files each subpath of
|
|
2775
|
+
an `exports` map names (a pattern subpath, one with `*`, one entry per
|
|
2776
|
+
file its target matches, less those a `null` target takes out), else
|
|
2777
|
+
the package's types/module/main/browser fields, else its index file."""
|
|
2778
|
+
js_files = [f for f in files if _file_language(f) in JS_FAMILY]
|
|
2779
|
+
packages = _owners(js_files, "package.json", tree) or ([""] if js_files else [])
|
|
2780
|
+
entries: list[dict] = []
|
|
2781
|
+
maps: list[dict] = []
|
|
2782
|
+
unresolved: list[dict] = []
|
|
2783
|
+
for pkg in packages:
|
|
2784
|
+
found, facts, missing = _js_package_entries(root, pkg, tree_set, warnings)
|
|
2785
|
+
entries += found
|
|
2786
|
+
maps += facts
|
|
2787
|
+
unresolved += missing
|
|
2788
|
+
return entries, maps, unresolved
|
|
2789
|
+
|
|
2790
|
+
|
|
2791
|
+
def _js_package_entries(root: Path, pkg: str, tree_set: set[str],
|
|
2792
|
+
warnings: list[str]) -> tuple[list[dict], list[dict], list[dict]]:
|
|
2793
|
+
"""(entry points, exports-map facts, unresolved subpaths) of the one
|
|
2794
|
+
package in folder `pkg` ("" for the root), by _js_entries's rules, its
|
|
2795
|
+
package.json read from `root`."""
|
|
2796
|
+
label = pkg or "."
|
|
2797
|
+
entries: list[dict] = []
|
|
2798
|
+
maps: list[dict] = []
|
|
2799
|
+
unresolved: list[dict] = []
|
|
2800
|
+
|
|
2801
|
+
def add(file: str, how: str, subpath: str) -> None:
|
|
2802
|
+
if not any(e["file"] == file and e["subpath"] == subpath for e in entries):
|
|
2803
|
+
entries.append({"language": _file_language(file) or "javascript", "file": file,
|
|
2804
|
+
"package": label, "subpath": subpath, "resolution": how})
|
|
2805
|
+
|
|
2806
|
+
manifest: dict = {}
|
|
2807
|
+
manifest_path = f"{pkg}/package.json" if pkg else "package.json"
|
|
2808
|
+
if manifest_path in tree_set:
|
|
2809
|
+
try:
|
|
2810
|
+
loaded = json.loads((root / manifest_path).read_text(encoding="utf-8"))
|
|
2811
|
+
manifest = loaded if isinstance(loaded, dict) else {}
|
|
2812
|
+
except (OSError, UnicodeDecodeError, ValueError) as exc:
|
|
2813
|
+
warnings.append(f"{manifest_path}: cannot read ({exc}); its entry points are the index files")
|
|
2814
|
+
subpaths = exports_subpaths(manifest.get("exports")) if "exports" in manifest else {}
|
|
2815
|
+
if subpaths:
|
|
2816
|
+
code = [s for s in subpaths if s != "." and "*" not in s
|
|
2817
|
+
and any(t.endswith(JS_TARGET_SUFFIXES) for t in subpaths[s])]
|
|
2818
|
+
maps.append({"package": label, "subpaths": sorted(subpaths),
|
|
2819
|
+
"non_root_subpaths": len(code),
|
|
2820
|
+
"wildcard_subpaths": sum(1 for s in subpaths if "*" in s)})
|
|
2821
|
+
else:
|
|
2822
|
+
fields = [manifest[f] for f in PACKAGE_ENTRY_FIELDS if isinstance(manifest.get(f), str)]
|
|
2823
|
+
subpaths = {".": fields} if fields else {}
|
|
2824
|
+
excluded = [s for s, targets in subpaths.items() if not targets]
|
|
2825
|
+
for subpath, targets in subpaths.items():
|
|
2826
|
+
code_targets = [t for t in targets if t.endswith(JS_TARGET_SUFFIXES)]
|
|
2827
|
+
found = 0
|
|
2828
|
+
for target in code_targets:
|
|
2829
|
+
if "*" in target:
|
|
2830
|
+
matched, how = _package_pattern(pkg, target, tree_set)
|
|
2831
|
+
found += len(matched)
|
|
2832
|
+
for key, file in matched.items():
|
|
2833
|
+
named = subpath.replace("*", key, 1)
|
|
2834
|
+
if not _subpath_excluded(named, excluded):
|
|
2835
|
+
add(file, how, named)
|
|
2836
|
+
continue
|
|
2837
|
+
file, how = _package_target(pkg, target, tree_set)
|
|
2838
|
+
if file:
|
|
2839
|
+
add(file, how, subpath)
|
|
2840
|
+
found += 1
|
|
2841
|
+
if code_targets and not found:
|
|
2842
|
+
unresolved.append({"package": label, "subpath": subpath, "targets": code_targets})
|
|
2843
|
+
if "." in subpaths or not subpaths:
|
|
2844
|
+
if not any(e["subpath"] == "." for e in entries):
|
|
2845
|
+
folder = f"{pkg}/" if pkg else ""
|
|
2846
|
+
conventional = next((f"{folder}{d}index{e}" for d in ("", "src/") for e in JS_SOURCE_EXTENSIONS
|
|
2847
|
+
if f"{folder}{d}index{e}" in tree_set), None)
|
|
2848
|
+
if conventional:
|
|
2849
|
+
entries.append({"language": _file_language(conventional), "file": conventional,
|
|
2850
|
+
"package": label, "subpath": ".", "resolution": "conventional"})
|
|
2851
|
+
unresolved = [u for u in unresolved if u["subpath"] != "."]
|
|
2852
|
+
return entries, maps, unresolved
|
|
2853
|
+
|
|
2854
|
+
|
|
2855
|
+
def _specifier_local(text: str) -> str:
|
|
2856
|
+
"""The local (or imported) side of an export or import specifier's text:
|
|
2857
|
+
`a as b` -> a, `type T` -> T, `"a-b" as c` -> a-b."""
|
|
2858
|
+
item = text.strip()
|
|
2859
|
+
if item.startswith("type ") or item.startswith("typeof "):
|
|
2860
|
+
item = item.split(None, 1)[1]
|
|
2861
|
+
return re.split(r"\s+as\s+", item, maxsplit=1)[0].strip().strip("'\"")
|
|
2862
|
+
|
|
2863
|
+
|
|
2864
|
+
class _Surface:
|
|
2865
|
+
"""The public names one language family's entry points declare:
|
|
2866
|
+
{name: info}, the first entry point naming it wins; the names, and the
|
|
2867
|
+
files, a namespace export (a module) makes reachable, which are not
|
|
2868
|
+
internal; and the re-exports the trace could not follow."""
|
|
2869
|
+
|
|
2870
|
+
def __init__(self) -> None:
|
|
2871
|
+
self.names: dict[str, dict] = {}
|
|
2872
|
+
self.reachable: set[str] = set()
|
|
2873
|
+
self.reachable_files: set[str] = set()
|
|
2874
|
+
self.unresolved: list[dict] = []
|
|
2875
|
+
|
|
2876
|
+
def add(self, name: str, entry: str, via: str, file: str | None, line: int | None,
|
|
2877
|
+
source: str | None = None, local: str | None = None) -> dict | None:
|
|
2878
|
+
if not name or name.startswith("_"):
|
|
2879
|
+
return None
|
|
2880
|
+
if name not in self.names:
|
|
2881
|
+
self.names[name] = {"name": name, "entry": entry, "via": via, "from": source,
|
|
2882
|
+
"local": local if local and local != name else None,
|
|
2883
|
+
"file": file, "line": line}
|
|
2884
|
+
return self.names[name]
|
|
2885
|
+
|
|
2886
|
+
def unresolve(self, entry: str, name: str, source: str | None, file: str) -> None:
|
|
2887
|
+
"""A re-export in `file`, met tracing `entry`, that names no file here."""
|
|
2888
|
+
item = {"entry": entry, "name": name, "from": source, "file": file}
|
|
2889
|
+
if item not in self.unresolved:
|
|
2890
|
+
self.unresolved.append(item)
|
|
2891
|
+
|
|
2892
|
+
|
|
2893
|
+
def _js_facts(matches: list[dict], text: str, forms: dict, barrel_ids: set[str]) -> dict:
|
|
2894
|
+
"""What one JS/TS module exports, read from its scan matches (the
|
|
2895
|
+
standard export recipes' and the trace rules') and, for the forms no
|
|
2896
|
+
recipe matches, its text: {"names": {exposed name: fact}, "stars":
|
|
2897
|
+
[(source, line)], "imports": {local name: (imported, source)}}, the
|
|
2898
|
+
imported name being `default` for a default import and `*` for a
|
|
2899
|
+
namespace import. A fact is {"how", "line", "local", "source"}:
|
|
2900
|
+
`declaration` (the module's own binding `local`), `re-export` (name
|
|
2901
|
+
`local` of module `source`: a named re-export, or a list item or the
|
|
2902
|
+
default export of an imported binding) or `namespace` (module `source`
|
|
2903
|
+
as a whole). The default export is the name `default`, `local` naming
|
|
2904
|
+
what it declares. Of two facts for one name, a declaration beats a list
|
|
2905
|
+
item, which beats a name read from the text, then the first line wins."""
|
|
2906
|
+
imports: dict[str, tuple[str, str | None]] = {}
|
|
2907
|
+
stars: list[tuple[str | None, int]] = []
|
|
2908
|
+
ordered = sorted(matches, key=lambda m: (m["line"], m["name"] or ""))
|
|
2909
|
+
for m in ordered:
|
|
2910
|
+
if m["rule"] in IMPORT_RULES and m["name"]:
|
|
2911
|
+
imported = {IMPORT_DEFAULT_RULE: "default", IMPORT_NAMESPACE_RULE: "*"}.get(m["rule"])
|
|
2912
|
+
imports.setdefault(m["name"], (imported or _specifier_local(m["text"]), m["source"]))
|
|
2913
|
+
elif m["rule"] == STAR_RULE:
|
|
2914
|
+
stars.append((m["source"], m["line"]))
|
|
2915
|
+
ranked: dict[str, tuple[tuple[int, int], dict]] = {}
|
|
2916
|
+
|
|
2917
|
+
def put(name: str, rank: int, how: str, line: int | None, local: str | None, source: str | None) -> None:
|
|
2918
|
+
key = (rank, line or 0)
|
|
2919
|
+
if name and (name not in ranked or key < ranked[name][0]):
|
|
2920
|
+
ranked[name] = (key, {"how": how, "line": line, "local": local, "source": source})
|
|
2921
|
+
|
|
2922
|
+
def bound(name: str, local: str, line: int | None, rank: int) -> None:
|
|
2923
|
+
"""`name` exported for the module's binding `local`, which an import may bind."""
|
|
2924
|
+
imported, source = imports.get(local, (None, None))
|
|
2925
|
+
if imported == "*":
|
|
2926
|
+
put(name, rank, "namespace", line, None, source)
|
|
2927
|
+
elif imported is not None:
|
|
2928
|
+
put(name, rank, "re-export", line, imported, source)
|
|
2929
|
+
else:
|
|
2930
|
+
put(name, rank, "declaration", line, local, None)
|
|
2931
|
+
|
|
2932
|
+
for m in ordered:
|
|
2933
|
+
if m["rule"] not in barrel_ids or not m["name"]:
|
|
2934
|
+
continue
|
|
2935
|
+
if forms[(m["rule"], m["language"])]["rule"]["kind"] == "export_specifier":
|
|
2936
|
+
if m["source"]:
|
|
2937
|
+
put(m["name"], 1, "re-export", m["line"], _specifier_local(m["text"]), m["source"])
|
|
2938
|
+
else:
|
|
2939
|
+
bound(m["name"], _specifier_local(m["text"]), m["line"], 1)
|
|
2940
|
+
elif m["source"]:
|
|
2941
|
+
put(m["name"], 0, "namespace", m["line"], None, m["source"])
|
|
2942
|
+
elif _JS_DEFAULT_EXPORT_RE.match(m["text"]):
|
|
2943
|
+
put("default", 0, "declaration", m["line"], m["name"], None)
|
|
2944
|
+
else:
|
|
2945
|
+
put(m["name"], 0, "declaration", m["line"], m["name"], None)
|
|
2946
|
+
declared, defaults = _js_text_names(text)
|
|
2947
|
+
for name, line, is_default in declared:
|
|
2948
|
+
put("default" if is_default else name, 2, "declaration", line, name, None)
|
|
2949
|
+
for local, line in defaults:
|
|
2950
|
+
bound("default", local, line, 2)
|
|
2951
|
+
return {"names": {name: fact for name, (_, fact) in ranked.items()}, "stars": stars, "imports": imports}
|
|
2952
|
+
|
|
2953
|
+
|
|
2954
|
+
def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict,
|
|
2955
|
+
barrel_ids: set[str], scan: Callable[[list[str]], dict[str, list[dict]]],
|
|
2956
|
+
surface: _Surface) -> None:
|
|
2957
|
+
"""Add the names the JS/TS entry points export: those each declares,
|
|
2958
|
+
re-exports or exports as a namespace, and those its `export *`
|
|
2959
|
+
statements pass on through every module they chain to (a default
|
|
2960
|
+
export aside). A re-exported name is traced to the module that declares
|
|
2961
|
+
it, through each re-export, imported binding and `export *` between (a
|
|
2962
|
+
cycle ends the trace); a namespace export makes reachable the names its
|
|
2963
|
+
module passes on, traced the same way, and those of the namespaces
|
|
2964
|
+
among them. The modules the exports lead to are scanned a level at a
|
|
2965
|
+
time."""
|
|
2966
|
+
entry_files = sorted({e["file"] for e in entries})
|
|
2967
|
+
by_file = dict(scan(entry_files))
|
|
2968
|
+
cache: dict[str, dict] = {}
|
|
2969
|
+
|
|
2970
|
+
def facts(file: str) -> dict:
|
|
2971
|
+
if file not in cache:
|
|
2972
|
+
cache[file] = _js_facts(by_file.get(file, []), _read_text(root, file) or "", forms, barrel_ids)
|
|
2973
|
+
return cache[file]
|
|
2974
|
+
|
|
2975
|
+
def target(source: str | None, file: str) -> str | None:
|
|
2976
|
+
return resolve_js_module(source, file, tree_set)
|
|
2977
|
+
|
|
2978
|
+
seen, level = set(entry_files), entry_files
|
|
2979
|
+
while level:
|
|
2980
|
+
found: set[str] = set()
|
|
2981
|
+
for file in level:
|
|
2982
|
+
module = facts(file)
|
|
2983
|
+
sources = [s for s, _ in module["stars"]] + [fact["source"] for fact in module["names"].values()]
|
|
2984
|
+
found.update(t for t in (target(s, file) for s in sources) if t is not None and t not in seen)
|
|
2985
|
+
seen |= found
|
|
2986
|
+
level = sorted(found)
|
|
2987
|
+
by_file.update(scan(level))
|
|
2988
|
+
|
|
2989
|
+
closures: dict[str, list[str]] = {}
|
|
2990
|
+
passed: dict[str, dict[str, str]] = {}
|
|
2991
|
+
|
|
2992
|
+
def closure(file: str) -> list[str]:
|
|
2993
|
+
"""`file`, then each module its `export *` statements chain to."""
|
|
2994
|
+
if file not in closures:
|
|
2995
|
+
order: list[str] = []
|
|
2996
|
+
stack, visited = [file], set()
|
|
2997
|
+
while stack:
|
|
2998
|
+
module = stack.pop()
|
|
2999
|
+
if module in visited:
|
|
3000
|
+
continue
|
|
3001
|
+
visited.add(module)
|
|
3002
|
+
order.append(module)
|
|
3003
|
+
stack += reversed([t for t in (target(s, module) for s, _ in facts(module)["stars"]) if t])
|
|
3004
|
+
closures[file] = order
|
|
3005
|
+
return closures[file]
|
|
3006
|
+
|
|
3007
|
+
def passed_on(file: str) -> dict[str, str]:
|
|
3008
|
+
"""{name: the module that exports it} of what `export * from` the
|
|
3009
|
+
module `file` passes on: its own names, then its chain's."""
|
|
3010
|
+
if file not in passed:
|
|
3011
|
+
out: dict[str, str] = {}
|
|
3012
|
+
for module in closure(file):
|
|
3013
|
+
for name in facts(module)["names"]:
|
|
3014
|
+
if name != "default":
|
|
3015
|
+
out.setdefault(name, module)
|
|
3016
|
+
passed[file] = out
|
|
3017
|
+
return passed[file]
|
|
3018
|
+
|
|
3019
|
+
def resolve(file: str, name: str, chain: set[str]) -> tuple[str, str, int | None, dict | None]:
|
|
3020
|
+
"""(module, its name there, line, fact) of what `name`, as module
|
|
3021
|
+
`file` exports it, comes from: a declaration or a namespace export,
|
|
3022
|
+
or a re-export from a module no file here holds, where the trace
|
|
3023
|
+
stops. The fact and the line are None where no module exports it;
|
|
3024
|
+
`chain` collects each name met."""
|
|
3025
|
+
visited: set[tuple[str, str]] = set()
|
|
3026
|
+
while (file, name) not in visited:
|
|
3027
|
+
visited.add((file, name))
|
|
3028
|
+
chain.add(name)
|
|
3029
|
+
fact = facts(file)["names"].get(name)
|
|
3030
|
+
if fact is None:
|
|
3031
|
+
owner = passed_on(file).get(name) if name != "default" else None
|
|
3032
|
+
if owner is None or owner == file:
|
|
3033
|
+
return file, name, None, None
|
|
3034
|
+
file = owner
|
|
3035
|
+
continue
|
|
3036
|
+
follow = target(fact["source"], file) if fact["how"] == "re-export" else None
|
|
3037
|
+
if follow is None:
|
|
3038
|
+
return file, fact["local"] or name, fact["line"], fact
|
|
3039
|
+
file, name = follow, fact["local"]
|
|
3040
|
+
return file, name, None, None
|
|
3041
|
+
|
|
3042
|
+
reached: set[str] = set()
|
|
3043
|
+
|
|
3044
|
+
def reach(file: str | None) -> None:
|
|
3045
|
+
"""Make reachable what the namespace module `file` passes on."""
|
|
3046
|
+
if file is None or file in reached:
|
|
3047
|
+
return
|
|
3048
|
+
reached.add(file)
|
|
3049
|
+
for name, owner in passed_on(file).items():
|
|
3050
|
+
chain: set[str] = set()
|
|
3051
|
+
where, local, _, fact = resolve(owner, name, chain)
|
|
3052
|
+
surface.reachable.update(chain | {local})
|
|
3053
|
+
if fact is not None and fact["how"] == "namespace":
|
|
3054
|
+
reach(target(fact["source"], where))
|
|
3055
|
+
|
|
3056
|
+
def publish(entry: str, name: str, via: str, module: str, source: str | None) -> None:
|
|
3057
|
+
"""Add public `name`, which `module` exports, traced to what it is."""
|
|
3058
|
+
chain: set[str] = set()
|
|
3059
|
+
where, local, line, fact = resolve(module, name, chain)
|
|
3060
|
+
surface.reachable.update(chain)
|
|
3061
|
+
how = fact["how"] if fact is not None else None
|
|
3062
|
+
if how == "namespace":
|
|
3063
|
+
namespace = target(fact["source"], where)
|
|
3064
|
+
surface.add(name, entry, "namespace", namespace, None, source)
|
|
3065
|
+
if namespace is None:
|
|
3066
|
+
surface.unresolve(entry, name, fact["source"], where)
|
|
3067
|
+
reach(namespace)
|
|
3068
|
+
elif how == "re-export":
|
|
3069
|
+
surface.add(name, entry, via, None, None, source, local)
|
|
3070
|
+
surface.unresolve(entry, name, fact["source"], where)
|
|
3071
|
+
else:
|
|
3072
|
+
surface.add(name, entry, via, where, line, source, local)
|
|
3073
|
+
|
|
3074
|
+
for entry in entry_files:
|
|
3075
|
+
for name, fact in facts(entry)["names"].items():
|
|
3076
|
+
if fact["how"] == "declaration":
|
|
3077
|
+
surface.add(name, entry, "declaration", entry, fact["line"], local=fact["local"])
|
|
3078
|
+
else:
|
|
3079
|
+
publish(entry, name, "re-export", entry, fact["source"])
|
|
3080
|
+
for entry in entry_files:
|
|
3081
|
+
for source, _ in facts(entry)["stars"]:
|
|
3082
|
+
start = target(source, entry)
|
|
3083
|
+
if start is None:
|
|
3084
|
+
surface.unresolve(entry, "*", source, entry)
|
|
3085
|
+
continue
|
|
3086
|
+
for name, owner in passed_on(start).items():
|
|
3087
|
+
publish(entry, name, "star", owner, source)
|
|
3088
|
+
for module in closure(start):
|
|
3089
|
+
for inner, _ in facts(module)["stars"]:
|
|
3090
|
+
if target(inner, module) is None:
|
|
3091
|
+
surface.unresolve(entry, "*", inner, module)
|
|
3092
|
+
|
|
3093
|
+
|
|
3094
|
+
# --------------------------------------------------------------------------
|
|
3095
|
+
# Entry points: Python
|
|
3096
|
+
# --------------------------------------------------------------------------
|
|
3097
|
+
|
|
3098
|
+
|
|
3099
|
+
def _type_checking(node: ast.stmt) -> bool:
|
|
3100
|
+
"""An `if TYPE_CHECKING:` (or `typing.TYPE_CHECKING`) block: its body
|
|
3101
|
+
never runs, so what it imports is no attribute of the module."""
|
|
3102
|
+
test = node.test if isinstance(node, ast.If) else None
|
|
3103
|
+
return (isinstance(test, ast.Name) and test.id == "TYPE_CHECKING") or (
|
|
3104
|
+
isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING"
|
|
3105
|
+
and isinstance(test.value, ast.Name) and test.value.id in ("typing", "typing_extensions"))
|
|
3106
|
+
|
|
3107
|
+
|
|
3108
|
+
def _module_statements(body: list) -> Iterator[ast.stmt]:
|
|
3109
|
+
"""A module's statements, those inside its module-level if, try, with,
|
|
3110
|
+
for, while and match blocks included, never a function's or a class's,
|
|
3111
|
+
nor those of an `if TYPE_CHECKING:` body (its `else` runs)."""
|
|
3112
|
+
for node in body:
|
|
3113
|
+
yield node
|
|
3114
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
3115
|
+
continue
|
|
3116
|
+
for field in ("orelse",) if _type_checking(node) else ("body", "orelse", "finalbody"):
|
|
3117
|
+
inner = getattr(node, field, None)
|
|
3118
|
+
if isinstance(inner, list):
|
|
3119
|
+
yield from _module_statements(inner)
|
|
3120
|
+
for handler in getattr(node, "handlers", None) or []:
|
|
3121
|
+
yield from _module_statements(handler.body)
|
|
3122
|
+
for case in getattr(node, "cases", None) or []:
|
|
3123
|
+
yield from _module_statements(case.body)
|
|
3124
|
+
|
|
3125
|
+
|
|
3126
|
+
def _target_names(node: ast.AST) -> list[str]:
|
|
3127
|
+
if isinstance(node, ast.Name):
|
|
3128
|
+
return [node.id]
|
|
3129
|
+
if isinstance(node, (ast.Tuple, ast.List)):
|
|
3130
|
+
return [n for elt in node.elts for n in _target_names(elt)]
|
|
3131
|
+
if isinstance(node, ast.Starred):
|
|
3132
|
+
return _target_names(node.value)
|
|
3133
|
+
return []
|
|
3134
|
+
|
|
3135
|
+
|
|
3136
|
+
def _literal_names(node: ast.AST | None) -> list[str] | None:
|
|
3137
|
+
"""The strings of a literal `__all__` value (a list, a tuple, a sum of
|
|
3138
|
+
them or one string); None when it holds no literal part."""
|
|
3139
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
3140
|
+
return [node.value]
|
|
3141
|
+
if isinstance(node, (ast.List, ast.Tuple)):
|
|
3142
|
+
return [e.value for e in node.elts if isinstance(e, ast.Constant) and isinstance(e.value, str)]
|
|
3143
|
+
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
|
|
3144
|
+
left, right = _literal_names(node.left), _literal_names(node.right)
|
|
3145
|
+
if left is None and right is None:
|
|
3146
|
+
return None
|
|
3147
|
+
return (left or []) + (right or [])
|
|
3148
|
+
return None
|
|
3149
|
+
|
|
3150
|
+
|
|
3151
|
+
def _py_bindings(text: str) -> tuple[list[str] | None, dict[str, dict], list[dict]]:
|
|
3152
|
+
"""A Python module's (__all__ or None, the names it binds at module
|
|
3153
|
+
level, its star imports). A binding: {"line", "kind": "definition" |
|
|
3154
|
+
"import" | "module-import"}; a definition's "form" (`def` for a def or
|
|
3155
|
+
async def, `class`, `variable` for an assignment); an import's
|
|
3156
|
+
"module", "level" and "imported" name. Raises SyntaxError."""
|
|
3157
|
+
tree = ast.parse(text)
|
|
3158
|
+
all_names: list[str] | None = None
|
|
3159
|
+
bound: dict[str, dict] = {}
|
|
3160
|
+
stars: list[dict] = []
|
|
3161
|
+
for node in _module_statements(tree.body):
|
|
3162
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
3163
|
+
form = "class" if isinstance(node, ast.ClassDef) else "def"
|
|
3164
|
+
bound.setdefault(node.name, {"line": node.lineno, "kind": "definition", "form": form})
|
|
3165
|
+
elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
|
|
3166
|
+
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
|
3167
|
+
names = [n for t in targets for n in _target_names(t)]
|
|
3168
|
+
if "__all__" in names:
|
|
3169
|
+
listed = _literal_names(node.value)
|
|
3170
|
+
if isinstance(node, ast.AugAssign):
|
|
3171
|
+
all_names = (all_names or []) + (listed or [])
|
|
3172
|
+
elif listed is not None:
|
|
3173
|
+
all_names = listed
|
|
3174
|
+
if not isinstance(node, ast.AugAssign):
|
|
3175
|
+
for name in names:
|
|
3176
|
+
bound.setdefault(name, {"line": node.lineno, "kind": "definition", "form": "variable"})
|
|
3177
|
+
elif isinstance(node, ast.ImportFrom):
|
|
3178
|
+
for alias in node.names:
|
|
3179
|
+
if alias.name == "*":
|
|
3180
|
+
stars.append({"module": node.module, "level": node.level, "line": node.lineno})
|
|
3181
|
+
else:
|
|
3182
|
+
bound.setdefault(alias.asname or alias.name, {
|
|
3183
|
+
"line": node.lineno, "kind": "import", "module": node.module,
|
|
3184
|
+
"level": node.level, "imported": alias.name})
|
|
3185
|
+
elif isinstance(node, ast.Import):
|
|
3186
|
+
for alias in node.names:
|
|
3187
|
+
bound.setdefault(alias.asname or alias.name.split(".")[0],
|
|
3188
|
+
{"line": node.lineno, "kind": "module-import"})
|
|
3189
|
+
elif (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
|
|
3190
|
+
and isinstance(node.value.func, ast.Attribute)
|
|
3191
|
+
and isinstance(node.value.func.value, ast.Name) and node.value.func.value.id == "__all__"
|
|
3192
|
+
and node.value.func.attr in ("extend", "append") and node.value.args):
|
|
3193
|
+
all_names = (all_names or []) + (_literal_names(node.value.args[0]) or [])
|
|
3194
|
+
return all_names, bound, stars
|
|
3195
|
+
|
|
3196
|
+
|
|
3197
|
+
_PY_ALL_RE = re.compile(r"^__all__\s*\+?=\s*[\[(]([^\])]*)[\])]", re.M)
|
|
3198
|
+
_PY_DEF_RE = re.compile(r"^(?:async\s+)?(def|class)\s+(\w+)", re.M)
|
|
3199
|
+
_PY_FROM_RE = re.compile(r"^from\s+(\.*)([\w.]*)\s+import\s+(\([^)]*\)|[^\n]+)", re.M)
|
|
3200
|
+
|
|
3201
|
+
|
|
3202
|
+
def _py_text_bindings(text: str) -> tuple[list[str] | None, dict[str, dict], list[dict]]:
|
|
3203
|
+
"""_py_bindings read from text, for a module this Python cannot parse
|
|
3204
|
+
(newer syntax): __all__, column-0 def and class, from-imports."""
|
|
3205
|
+
all_names = None
|
|
3206
|
+
for m in _PY_ALL_RE.finditer(text):
|
|
3207
|
+
all_names = (all_names or []) + re.findall(r"""['"]([^'"]+)['"]""", m.group(1))
|
|
3208
|
+
bound: dict[str, dict] = {}
|
|
3209
|
+
stars: list[dict] = []
|
|
3210
|
+
for m in _PY_DEF_RE.finditer(text):
|
|
3211
|
+
bound.setdefault(m.group(2), {"line": _line_of(text, m.start(2)), "kind": "definition", "form": m.group(1)})
|
|
3212
|
+
for m in _PY_FROM_RE.finditer(text):
|
|
3213
|
+
line, level, module = _line_of(text, m.start()), len(m.group(1)), m.group(2) or None
|
|
3214
|
+
for item in m.group(3).strip("() \t").split(","):
|
|
3215
|
+
parts = item.split()
|
|
3216
|
+
if parts == ["*"]:
|
|
3217
|
+
stars.append({"module": module, "level": level, "line": line})
|
|
3218
|
+
elif parts and re.fullmatch(r"\w+", parts[0]):
|
|
3219
|
+
name = parts[2] if len(parts) == 3 and parts[1] == "as" else parts[0]
|
|
3220
|
+
bound.setdefault(name, {"line": line, "kind": "import", "module": module,
|
|
3221
|
+
"level": level, "imported": parts[0]})
|
|
3222
|
+
return all_names, bound, stars
|
|
3223
|
+
|
|
3224
|
+
|
|
3225
|
+
def _py_module_file(pkg_dir: str, level: int, module: str | None, tree: set[str]) -> str | None:
|
|
3226
|
+
"""The file of the module `from <level dots><module> import` names, from
|
|
3227
|
+
a module of the package in `pkg_dir`: relative imports, or an absolute
|
|
3228
|
+
one within that package."""
|
|
3229
|
+
if level > 0:
|
|
3230
|
+
base = pkg_dir
|
|
3231
|
+
for _ in range(level - 1):
|
|
3232
|
+
if not base:
|
|
3233
|
+
return None
|
|
3234
|
+
base = posixpath.dirname(base)
|
|
3235
|
+
else:
|
|
3236
|
+
top = posixpath.basename(pkg_dir)
|
|
3237
|
+
if not top or not module or module.split(".")[0] != top:
|
|
3238
|
+
return None
|
|
3239
|
+
base = posixpath.dirname(pkg_dir)
|
|
3240
|
+
path = "/".join(p for p in (base, *(module.split(".") if module else [])) if p)
|
|
3241
|
+
candidates = ([f"{path}.py"] if module else []) + [f"{path}/__init__.py" if path else "__init__.py"]
|
|
3242
|
+
return next((c for c in candidates if c in tree), None)
|
|
3243
|
+
|
|
3244
|
+
|
|
3245
|
+
def _py_submodule(folder: str, name: str, tree: set[str]) -> str | None:
|
|
3246
|
+
base = f"{folder}/{name}" if folder else name
|
|
3247
|
+
return next((c for c in (f"{base}.py", f"{base}/__init__.py") if c in tree), None)
|
|
3248
|
+
|
|
3249
|
+
|
|
3250
|
+
def _py_entries(files: list[str], tree_set: set[str]) -> list[dict]:
|
|
3251
|
+
"""The top __init__.py of each package that holds an in-scope file."""
|
|
3252
|
+
tops = set()
|
|
3253
|
+
for rel in files:
|
|
3254
|
+
if _file_language(rel) != "python":
|
|
3255
|
+
continue
|
|
3256
|
+
folder = posixpath.dirname(rel)
|
|
3257
|
+
if (f"{folder}/__init__.py" if folder else "__init__.py") not in tree_set:
|
|
3258
|
+
continue
|
|
3259
|
+
while folder and (f"{posixpath.dirname(folder)}/__init__.py" if posixpath.dirname(folder)
|
|
3260
|
+
else "__init__.py") in tree_set:
|
|
3261
|
+
folder = posixpath.dirname(folder)
|
|
3262
|
+
tops.add(f"{folder}/__init__.py" if folder else "__init__.py")
|
|
3263
|
+
return [{"language": "python", "file": f, "package": posixpath.dirname(f) or ".", "subpath": None,
|
|
3264
|
+
"resolution": "file"} for f in sorted(tops)]
|
|
3265
|
+
|
|
3266
|
+
|
|
3267
|
+
def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _Surface,
|
|
3268
|
+
warnings: list[str]) -> None:
|
|
3269
|
+
"""Add the names each package __init__.py exports: its __all__ when it
|
|
3270
|
+
has one, else the public names it defines or imports from its package
|
|
3271
|
+
and those its star imports give, through every module they chain to.
|
|
3272
|
+
Each name is traced to the module that defines it, through each import
|
|
3273
|
+
and star import between (a cycle ends the trace); a submodule is a
|
|
3274
|
+
namespace, which makes reachable the names it exports, traced the same
|
|
3275
|
+
way, and those of the submodules among them."""
|
|
3276
|
+
parsed: dict[str, tuple] = {}
|
|
3277
|
+
|
|
3278
|
+
def bindings(rel: str) -> tuple[list[str] | None, dict[str, dict], list[dict]]:
|
|
3279
|
+
if rel not in parsed:
|
|
3280
|
+
text = _read_text(root, rel) or ""
|
|
3281
|
+
try:
|
|
3282
|
+
parsed[rel] = _py_bindings(text)
|
|
3283
|
+
except SyntaxError as exc:
|
|
3284
|
+
warnings.append(f"{rel}: this Python cannot parse it ({exc.msg}, line {exc.lineno}); "
|
|
3285
|
+
"its names are read from its text")
|
|
3286
|
+
parsed[rel] = _py_text_bindings(text)
|
|
3287
|
+
return parsed[rel]
|
|
3288
|
+
|
|
3289
|
+
def module_file(rel: str, level: int, module: str | None, top: str) -> str | None:
|
|
3290
|
+
"""The module `from <level dots><module> import` names in module
|
|
3291
|
+
`rel`: relative to its package, or absolute within the package in
|
|
3292
|
+
folder `top`."""
|
|
3293
|
+
return _py_module_file(posixpath.dirname(rel) if level else top, level, module, tree_set)
|
|
3294
|
+
|
|
3295
|
+
def packaged(rel: str, binding: dict, top: str) -> bool:
|
|
3296
|
+
"""A binding module `rel` defines, or imports from the package in
|
|
3297
|
+
folder `top`: an import from another package is not its own."""
|
|
3298
|
+
if binding["kind"] == "definition":
|
|
3299
|
+
return True
|
|
3300
|
+
return binding["kind"] == "import" and (
|
|
3301
|
+
binding["level"] > 0 or module_file(rel, 0, binding["module"], top) is not None)
|
|
3302
|
+
|
|
3303
|
+
star_cache: dict[tuple[str, str], dict[str, str]] = {}
|
|
3304
|
+
|
|
3305
|
+
def star_names(rel: str, top: str, missing: list[tuple[str, str]] | None = None) -> dict[str, str]:
|
|
3306
|
+
"""{name: the module that binds it} of what `from <rel> import *`
|
|
3307
|
+
gives: its __all__, else the public names it defines or imports
|
|
3308
|
+
(from its own package) and those its own star imports give.
|
|
3309
|
+
`missing` collects the (module, source) of each relative star
|
|
3310
|
+
import that names no file."""
|
|
3311
|
+
if missing is None and (rel, top) in star_cache:
|
|
3312
|
+
return star_cache[(rel, top)]
|
|
3313
|
+
out: dict[str, str] = {}
|
|
3314
|
+
stack, visited = [rel], set()
|
|
3315
|
+
while stack:
|
|
3316
|
+
module = stack.pop()
|
|
3317
|
+
if module in visited:
|
|
3318
|
+
continue
|
|
3319
|
+
visited.add(module)
|
|
3320
|
+
listed, bound, stars = bindings(module)
|
|
3321
|
+
if listed is not None:
|
|
3322
|
+
for name in listed:
|
|
3323
|
+
out.setdefault(name, module)
|
|
3324
|
+
continue
|
|
3325
|
+
for name, binding in bound.items():
|
|
3326
|
+
if packaged(module, binding, top) and not name.startswith("_"):
|
|
3327
|
+
out.setdefault(name, module)
|
|
3328
|
+
chained = []
|
|
3329
|
+
for star in stars:
|
|
3330
|
+
found = module_file(module, star["level"], star["module"], top)
|
|
3331
|
+
if found is not None:
|
|
3332
|
+
chained.append(found)
|
|
3333
|
+
elif star["level"] and missing is not None:
|
|
3334
|
+
missing.append((module, "." * star["level"] + (star["module"] or "")))
|
|
3335
|
+
stack += reversed(chained)
|
|
3336
|
+
star_cache[(rel, top)] = out
|
|
3337
|
+
return out
|
|
3338
|
+
|
|
3339
|
+
def resolve(rel: str, name: str, top: str, chain: set[str]) -> tuple[str | None, str, int | None, str]:
|
|
3340
|
+
"""(module, its name there, line, what it is) that `name`, bound in
|
|
3341
|
+
module `rel`, comes from, `what` being `definition`, `submodule`
|
|
3342
|
+
(the module is it), `module` (an `import x` binding), `import` (from
|
|
3343
|
+
outside the package: no module) or `missing` (bound nowhere the
|
|
3344
|
+
trace reaches). `chain` collects each name met."""
|
|
3345
|
+
visited: set[tuple[str, str]] = set()
|
|
3346
|
+
while (rel, name) not in visited:
|
|
3347
|
+
visited.add((rel, name))
|
|
3348
|
+
chain.add(name)
|
|
3349
|
+
_, bound, stars = bindings(rel)
|
|
3350
|
+
binding = bound.get(name)
|
|
3351
|
+
package = posixpath.dirname(rel) if rel.endswith("__init__.py") else None
|
|
3352
|
+
if binding is None or binding["kind"] == "module-import":
|
|
3353
|
+
sub = _py_submodule(package, name, tree_set) if package is not None else None
|
|
3354
|
+
if sub:
|
|
3355
|
+
return sub, name, None, "submodule"
|
|
3356
|
+
if binding is not None:
|
|
3357
|
+
return rel, name, binding["line"], "module"
|
|
3358
|
+
owner = None
|
|
3359
|
+
for star in stars:
|
|
3360
|
+
found = module_file(rel, star["level"], star["module"], top)
|
|
3361
|
+
owner = star_names(found, top).get(name) if found is not None else None
|
|
3362
|
+
if owner is not None:
|
|
3363
|
+
break
|
|
3364
|
+
if owner is None:
|
|
3365
|
+
return rel, name, None, "missing"
|
|
3366
|
+
rel = owner
|
|
3367
|
+
continue
|
|
3368
|
+
if binding["kind"] == "definition":
|
|
3369
|
+
return rel, name, binding["line"], "definition"
|
|
3370
|
+
found = module_file(rel, binding["level"], binding["module"], top)
|
|
3371
|
+
if found is None:
|
|
3372
|
+
return None, binding["imported"], None, "import"
|
|
3373
|
+
if found.endswith("__init__.py"):
|
|
3374
|
+
sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
|
|
3375
|
+
if sub and sub != rel:
|
|
3376
|
+
return sub, binding["imported"], None, "submodule"
|
|
3377
|
+
rel, name = found, binding["imported"]
|
|
3378
|
+
return rel, name, None, "missing"
|
|
3379
|
+
|
|
3380
|
+
reached: set[str] = set()
|
|
3381
|
+
|
|
3382
|
+
def reach(rel: str, top: str) -> None:
|
|
3383
|
+
"""Make reachable what the submodule `rel` exports."""
|
|
3384
|
+
if rel in reached:
|
|
3385
|
+
return
|
|
3386
|
+
reached.add(rel)
|
|
3387
|
+
for name, owner in star_names(rel, top).items():
|
|
3388
|
+
chain: set[str] = set()
|
|
3389
|
+
where, local, _, what = resolve(owner, name, top, chain)
|
|
3390
|
+
surface.reachable.update(chain | {local})
|
|
3391
|
+
if what == "submodule" and where is not None:
|
|
3392
|
+
reach(where, top)
|
|
3393
|
+
|
|
3394
|
+
for entry in [e["file"] for e in entries]:
|
|
3395
|
+
top = posixpath.dirname(entry)
|
|
3396
|
+
all_names, bound, stars = bindings(entry)
|
|
3397
|
+
names: dict[str, str | None] # name -> the star import that gives it, if one does
|
|
3398
|
+
if all_names is not None:
|
|
3399
|
+
names = dict.fromkeys(all_names)
|
|
3400
|
+
else:
|
|
3401
|
+
names = {name: None for name, binding in bound.items() if packaged(entry, binding, top)}
|
|
3402
|
+
missing: list[tuple[str, str]] = []
|
|
3403
|
+
for star in stars:
|
|
3404
|
+
source = "." * star["level"] + (star["module"] or "")
|
|
3405
|
+
found = module_file(entry, star["level"], star["module"], top)
|
|
3406
|
+
if found is None:
|
|
3407
|
+
if star["level"] > 0:
|
|
3408
|
+
missing.append((entry, source))
|
|
3409
|
+
continue
|
|
3410
|
+
for name in star_names(found, top, missing):
|
|
3411
|
+
names.setdefault(name, source)
|
|
3412
|
+
for module, source in missing:
|
|
3413
|
+
surface.unresolve(entry, "*", source, module)
|
|
3414
|
+
for name, star_source in names.items():
|
|
3415
|
+
binding = bound.get(name)
|
|
3416
|
+
source = star_source
|
|
3417
|
+
if binding is not None and binding["kind"] == "import":
|
|
3418
|
+
source = "." * binding["level"] + (binding["module"] or "")
|
|
3419
|
+
chain: set[str] = set()
|
|
3420
|
+
where, local, line, what = resolve(entry, name, top, chain)
|
|
3421
|
+
surface.reachable.update(chain)
|
|
3422
|
+
if what == "submodule":
|
|
3423
|
+
surface.add(name, entry, "namespace", where, None, source, local)
|
|
3424
|
+
reach(where, top)
|
|
3425
|
+
elif what == "import":
|
|
3426
|
+
surface.add(name, entry, "re-export", None, None, source, local)
|
|
3427
|
+
surface.unresolve(entry, name, source, entry)
|
|
3428
|
+
elif binding is not None and binding["kind"] == "definition":
|
|
3429
|
+
surface.add(name, entry, "declaration", entry, binding["line"])
|
|
3430
|
+
elif binding is not None and binding["kind"] == "import":
|
|
3431
|
+
surface.add(name, entry, "re-export", where, line, source, local)
|
|
3432
|
+
elif star_source is not None or (what in ("definition", "module") and where != entry):
|
|
3433
|
+
surface.add(name, entry, "star", where, line, source, local)
|
|
3434
|
+
else:
|
|
3435
|
+
# __all__ lists it but nothing binds it, or an `import x`
|
|
3436
|
+
surface.add(name, entry, "declaration", None, None)
|
|
3437
|
+
|
|
3438
|
+
|
|
3439
|
+
# --------------------------------------------------------------------------
|
|
3440
|
+
# Entry points: Rust
|
|
3441
|
+
# --------------------------------------------------------------------------
|
|
3442
|
+
|
|
3443
|
+
_RUST_ITEM_RE = re.compile(
|
|
3444
|
+
r'^pub\s+((?:(?:async|const|unsafe|default|extern(?:\s+"[^"]*")?)\s+)*)'
|
|
3445
|
+
r"(fn|struct|enum|trait|type|const|static|union|mod)\s+(?:mut\s+)?((?:r#)?[A-Za-z_]\w*)",
|
|
3446
|
+
re.M,
|
|
3447
|
+
)
|
|
3448
|
+
_RUST_USE_RE = re.compile(r"^pub\s+use\s+([^;]*);", re.M)
|
|
3449
|
+
_RUST_MOD_RE = re.compile(r"^(?:pub(?:\([^)]*\))?\s+)?mod\s+((?:r#)?[A-Za-z_]\w*)\s*([;{])", re.M)
|
|
3450
|
+
_RUST_TOKEN_RE = re.compile(r"r#\w+|\w+|::|[{},*]")
|
|
3451
|
+
|
|
3452
|
+
|
|
3453
|
+
def _rust_items(code: str) -> list[tuple[str, str, int]]:
|
|
3454
|
+
"""(name, kind, line) of each unrestricted column-0 `pub` item."""
|
|
3455
|
+
return [(m.group(3), m.group(2), _line_of(code, m.start(3))) for m in _RUST_ITEM_RE.finditer(code)]
|
|
3456
|
+
|
|
3457
|
+
|
|
3458
|
+
def _rust_use_leaves(tree_text: str) -> list[tuple[list[str], str | None, bool]]:
|
|
3459
|
+
"""Each (path, alias, glob) a `pub use` tree brings in:
|
|
3460
|
+
`a::{b, c::D as E, f::*}` -> (a, b), (a, c, D; E), (a, f; glob)."""
|
|
3461
|
+
tokens = _RUST_TOKEN_RE.findall(tree_text)
|
|
3462
|
+
leaves: list[tuple[list[str], str | None, bool]] = []
|
|
3463
|
+
|
|
3464
|
+
def parse(i: int, prefix: list[str]) -> int:
|
|
3465
|
+
segments: list[str] = []
|
|
3466
|
+
while i < len(tokens):
|
|
3467
|
+
token = tokens[i]
|
|
3468
|
+
if token == "::":
|
|
3469
|
+
i += 1
|
|
3470
|
+
elif token == "{":
|
|
3471
|
+
i += 1
|
|
3472
|
+
while i < len(tokens) and tokens[i] != "}":
|
|
3473
|
+
i = parse(i, prefix + segments)
|
|
3474
|
+
if i < len(tokens) and tokens[i] == ",":
|
|
3475
|
+
i += 1
|
|
3476
|
+
return i + 1
|
|
3477
|
+
elif token == "*":
|
|
3478
|
+
leaves.append((prefix + segments, None, True))
|
|
3479
|
+
return i + 1
|
|
3480
|
+
elif token in (",", "}"):
|
|
3481
|
+
break
|
|
3482
|
+
elif token == "as" and segments and i + 1 < len(tokens):
|
|
3483
|
+
leaves.append((prefix + segments, tokens[i + 1], False))
|
|
3484
|
+
return i + 2
|
|
3485
|
+
else:
|
|
3486
|
+
segments.append(token)
|
|
3487
|
+
i += 1
|
|
3488
|
+
if segments:
|
|
3489
|
+
leaves.append((prefix + segments, None, False))
|
|
3490
|
+
return i
|
|
3491
|
+
|
|
3492
|
+
parse(0, [])
|
|
3493
|
+
return leaves
|
|
3494
|
+
|
|
3495
|
+
|
|
3496
|
+
def _rust_file(lib: str, path: list[str], tree: set[str]) -> str | None:
|
|
3497
|
+
"""The file of the crate module at `path` (its names from the crate
|
|
3498
|
+
root, whose file is `lib`): <name>.rs or <name>/mod.rs at each level;
|
|
3499
|
+
None for a module no such file holds (an inline one, a `#[path]` one)."""
|
|
3500
|
+
base, current = posixpath.dirname(lib), lib
|
|
3501
|
+
for seg in path:
|
|
3502
|
+
stem = f"{base}/{seg}" if base else seg
|
|
3503
|
+
current = next((c for c in (f"{stem}.rs", f"{stem}/mod.rs") if c in tree), None)
|
|
3504
|
+
if current is None:
|
|
3505
|
+
return None
|
|
3506
|
+
base = stem
|
|
3507
|
+
return current
|
|
3508
|
+
|
|
3509
|
+
|
|
3510
|
+
def _rust_use_path(segments: list[str], here: list[str], mods: set[str]) -> list[str] | None:
|
|
3511
|
+
"""A `use` path written in the module at `here` as a path from the
|
|
3512
|
+
crate root: `crate::` starts at the root, `self::` at `here`, each
|
|
3513
|
+
`super::` one module up, and a module `here` declares at `here`; None
|
|
3514
|
+
for another crate."""
|
|
3515
|
+
segs = [s.removeprefix("r#") for s in segments]
|
|
3516
|
+
if not segs:
|
|
3517
|
+
return None
|
|
3518
|
+
if segs[0] == "crate":
|
|
3519
|
+
return segs[1:]
|
|
3520
|
+
base = list(here)
|
|
3521
|
+
if segs[0] == "self":
|
|
3522
|
+
return base + segs[1:]
|
|
3523
|
+
if segs[0] == "super":
|
|
3524
|
+
while segs and segs[0] == "super":
|
|
3525
|
+
if not base:
|
|
3526
|
+
return None
|
|
3527
|
+
base, segs = base[:-1], segs[1:]
|
|
3528
|
+
return base + segs
|
|
3529
|
+
return base + segs if segs[0] in mods else None
|
|
3530
|
+
|
|
3531
|
+
|
|
3532
|
+
def _rust_entries(root: Path, files: list[str], tree: list[str], tree_set: set[str]) -> list[dict]:
|
|
3533
|
+
"""The library root (`[lib] path`, else src/lib.rs) of each crate that
|
|
3534
|
+
holds an in-scope file."""
|
|
3535
|
+
rust_files = [f for f in files if _file_language(f) == "rust"]
|
|
3536
|
+
entries = []
|
|
3537
|
+
for crate in _owners(rust_files, "Cargo.toml", tree):
|
|
3538
|
+
lib = "src/lib.rs"
|
|
3539
|
+
try:
|
|
3540
|
+
manifest = tomllib.loads((root / (f"{crate}/Cargo.toml" if crate else "Cargo.toml")).read_text(encoding="utf-8"))
|
|
3541
|
+
path = (manifest.get("lib") or {}).get("path")
|
|
3542
|
+
lib = _norm(path) if isinstance(path, str) and path.strip() else lib
|
|
3543
|
+
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError, AttributeError):
|
|
3544
|
+
pass
|
|
3545
|
+
file = posixpath.normpath(f"{crate}/{lib}" if crate else lib)
|
|
3546
|
+
if file in tree_set:
|
|
3547
|
+
entries.append({"language": "rust", "file": file, "package": crate or ".", "subpath": None,
|
|
3548
|
+
"resolution": "file"})
|
|
3549
|
+
return entries
|
|
3550
|
+
|
|
3551
|
+
|
|
3552
|
+
def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _Surface) -> None:
|
|
3553
|
+
"""Add the names each crate root exports: its unrestricted column-0
|
|
3554
|
+
`pub` items (a `pub mod` is a namespace) and its `pub use` items, each
|
|
3555
|
+
traced to the module file that declares it through the `pub use` items
|
|
3556
|
+
between (a cycle ends the trace); a glob `pub use m::*` adds what `m`
|
|
3557
|
+
exports, its globs included. The crate root, and each module a `pub
|
|
3558
|
+
mod` or a `pub use` of a reachable module names, in turn, are reachable
|
|
3559
|
+
files: every recipe match in them is reachable (the recipe keeps to the
|
|
3560
|
+
`pub mod` blocks within a file), and so is each name they export."""
|
|
3561
|
+
parsed: dict[str, dict] = {}
|
|
3562
|
+
|
|
3563
|
+
def module(rel: str) -> dict:
|
|
3564
|
+
"""A module file's unrestricted column-0 `pub` items, the modules it
|
|
3565
|
+
declares ({name: declared with `;`, its body in a file of its own})
|
|
3566
|
+
and its `pub use` leaves, with their lines."""
|
|
3567
|
+
if rel not in parsed:
|
|
3568
|
+
code = _blank(_read_text(root, rel) or "", "rust")
|
|
3569
|
+
uses = []
|
|
3570
|
+
for use in _RUST_USE_RE.finditer(code):
|
|
3571
|
+
for path, alias, glob in _rust_use_leaves(use.group(1)):
|
|
3572
|
+
if path and path[-1] == "self" and not glob:
|
|
3573
|
+
path = path[:-1]
|
|
3574
|
+
if path and alias != "_":
|
|
3575
|
+
uses.append((path, alias, glob, _line_of(code, use.start())))
|
|
3576
|
+
parsed[rel] = {"items": _rust_items(code), "uses": uses,
|
|
3577
|
+
"mods": {m.group(1).removeprefix("r#"): m.group(2) == ";"
|
|
3578
|
+
for m in _RUST_MOD_RE.finditer(code)}}
|
|
3579
|
+
return parsed[rel]
|
|
3580
|
+
|
|
3581
|
+
def exported(lib: str, here: list[str]) -> list[str]:
|
|
3582
|
+
"""The names the module at `here` exports: its `pub` items, its
|
|
3583
|
+
`pub use` names and what its globs pass on, in that order."""
|
|
3584
|
+
out: dict[str, None] = {}
|
|
3585
|
+
stack, visited = [list(here)], set()
|
|
3586
|
+
while stack:
|
|
3587
|
+
path = stack.pop()
|
|
3588
|
+
file = _rust_file(lib, path, tree_set)
|
|
3589
|
+
if tuple(path) in visited or file is None:
|
|
3590
|
+
continue
|
|
3591
|
+
visited.add(tuple(path))
|
|
3592
|
+
info = module(file)
|
|
3593
|
+
out.update(dict.fromkeys(name for name, _, _ in info["items"]))
|
|
3594
|
+
globs = []
|
|
3595
|
+
for use, alias, glob, _ in info["uses"]:
|
|
3596
|
+
if not glob:
|
|
3597
|
+
out.setdefault(alias or use[-1])
|
|
3598
|
+
elif (full := _rust_use_path(use, path, set(info["mods"]))) is not None:
|
|
3599
|
+
globs.append(full)
|
|
3600
|
+
stack += reversed(globs)
|
|
3601
|
+
return list(out)
|
|
3602
|
+
|
|
3603
|
+
def resolve(lib: str, here: list[str], name: str, chain: set[str]) -> tuple:
|
|
3604
|
+
"""(file, its name there, line, module path, stop) of what `name`,
|
|
3605
|
+
as the module at `here` exports it, is: an item of that file, or a
|
|
3606
|
+
module (its path, and its file when one holds it). `stop` is the
|
|
3607
|
+
(`use` path, file) where the trace ends in another crate, the file
|
|
3608
|
+
None with it; the line is None where the trace ends at a module
|
|
3609
|
+
that does not export the name, or at a cycle. `chain` collects each
|
|
3610
|
+
name met."""
|
|
3611
|
+
visited: set[tuple[tuple[str, ...], str]] = set()
|
|
3612
|
+
while (tuple(here), name) not in visited:
|
|
3613
|
+
visited.add((tuple(here), name))
|
|
3614
|
+
chain.add(name)
|
|
3615
|
+
file = _rust_file(lib, here, tree_set)
|
|
3616
|
+
if file is None:
|
|
3617
|
+
return None, name, None, None, None
|
|
3618
|
+
info = module(file)
|
|
3619
|
+
item = next(((kind, line) for n, kind, line in info["items"] if n == name), None)
|
|
3620
|
+
if item is not None:
|
|
3621
|
+
if item[0] != "mod":
|
|
3622
|
+
return file, name, item[1], None, None
|
|
3623
|
+
sub = [*here, name.removeprefix("r#")]
|
|
3624
|
+
sub_file = _rust_file(lib, sub, tree_set)
|
|
3625
|
+
return sub_file or file, name, None if sub_file else item[1], sub, None
|
|
3626
|
+
mods = set(info["mods"])
|
|
3627
|
+
use = next(((path, line) for path, alias, glob, line in info["uses"]
|
|
3628
|
+
if not glob and (alias or path[-1]) == name), None)
|
|
3629
|
+
if use is not None:
|
|
3630
|
+
path = use[0]
|
|
3631
|
+
full = _rust_use_path(path, here, mods)
|
|
3632
|
+
if full is None:
|
|
3633
|
+
return None, path[-1], None, None, ("::".join(path), file)
|
|
3634
|
+
if not path[-1][:1].isupper() and _rust_file(lib, full, tree_set) is not None:
|
|
3635
|
+
return _rust_file(lib, full, tree_set), path[-1], None, full, None
|
|
3636
|
+
here, name = full[:-1], path[-1]
|
|
3637
|
+
continue
|
|
3638
|
+
owner = next((full for path, _, glob, _ in info["uses"] if glob
|
|
3639
|
+
for full in [_rust_use_path(path, here, mods)]
|
|
3640
|
+
if full is not None and name in exported(lib, full)), None)
|
|
3641
|
+
if owner is None:
|
|
3642
|
+
return file, name, None, None, None
|
|
3643
|
+
here = owner
|
|
3644
|
+
return _rust_file(lib, here, tree_set), name, None, None, None
|
|
3645
|
+
|
|
3646
|
+
def reach(lib: str, here: list[str], reached: set[tuple[str, ...]]) -> None:
|
|
3647
|
+
"""Make reachable the module at `here`: its file and the names it
|
|
3648
|
+
exports, then the modules its `pub mod` and `pub use` items name."""
|
|
3649
|
+
if tuple(here) in reached:
|
|
3650
|
+
return
|
|
3651
|
+
reached.add(tuple(here))
|
|
3652
|
+
file = _rust_file(lib, here, tree_set)
|
|
3653
|
+
if file is None:
|
|
3654
|
+
parent = _rust_file(lib, here[:-1], tree_set)
|
|
3655
|
+
if here and parent is not None and module(parent)["mods"].get(here[-1]):
|
|
3656
|
+
# a `pub mod name;` whose file is not name.rs or name/mod.rs
|
|
3657
|
+
surface.unresolve(lib, here[-1], "::".join(["crate", *here]), parent)
|
|
3658
|
+
return
|
|
3659
|
+
surface.reachable_files.add(file)
|
|
3660
|
+
info = module(file)
|
|
3661
|
+
for name, kind, _ in info["items"]:
|
|
3662
|
+
surface.reachable.add(name)
|
|
3663
|
+
if kind == "mod":
|
|
3664
|
+
reach(lib, [*here, name.removeprefix("r#")], reached)
|
|
3665
|
+
for path, alias, glob, _ in info["uses"]:
|
|
3666
|
+
if glob:
|
|
3667
|
+
full = _rust_use_path(path, here, set(info["mods"]))
|
|
3668
|
+
if full is not None:
|
|
3669
|
+
reach(lib, full, reached)
|
|
3670
|
+
continue
|
|
3671
|
+
chain: set[str] = set()
|
|
3672
|
+
sub = resolve(lib, here, alias or path[-1], chain)[3]
|
|
3673
|
+
surface.reachable.update(chain)
|
|
3674
|
+
if sub is not None:
|
|
3675
|
+
reach(lib, sub, reached)
|
|
3676
|
+
|
|
3677
|
+
def publish(lib: str, here: list[str], name: str, via: str, source: str | None,
|
|
3678
|
+
reached: set[tuple[str, ...]]) -> None:
|
|
3679
|
+
"""Add public `name`, which the module at `here` exports, traced to
|
|
3680
|
+
what it is."""
|
|
3681
|
+
chain: set[str] = set()
|
|
3682
|
+
file, local, line, sub, stop = resolve(lib, here, name, chain)
|
|
3683
|
+
surface.reachable.update(chain)
|
|
3684
|
+
if stop is not None:
|
|
3685
|
+
surface.add(name, lib, via, None, None, source, local)
|
|
3686
|
+
surface.unresolve(lib, name, *stop)
|
|
3687
|
+
elif sub is not None:
|
|
3688
|
+
surface.add(name, lib, "namespace", file, line, "::".join(sub))
|
|
3689
|
+
reach(lib, sub, reached)
|
|
3690
|
+
else:
|
|
3691
|
+
surface.add(name, lib, via, file, line, source, local)
|
|
3692
|
+
|
|
3693
|
+
for lib in [e["file"] for e in entries]:
|
|
3694
|
+
surface.reachable_files.add(lib)
|
|
3695
|
+
reached: set[tuple[str, ...]] = set()
|
|
3696
|
+
info = module(lib)
|
|
3697
|
+
for name, kind, line in info["items"]:
|
|
3698
|
+
if kind != "mod":
|
|
3699
|
+
surface.add(name, lib, "declaration", lib, line)
|
|
3700
|
+
continue
|
|
3701
|
+
sub = [name.removeprefix("r#")]
|
|
3702
|
+
sub_file = _rust_file(lib, sub, tree_set)
|
|
3703
|
+
surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line)
|
|
3704
|
+
reach(lib, sub, reached)
|
|
3705
|
+
for path, alias, glob, _ in info["uses"]:
|
|
3706
|
+
if not glob:
|
|
3707
|
+
publish(lib, [], alias or path[-1], "re-export", "::".join(path[:-1]) or None, reached)
|
|
3708
|
+
continue
|
|
3709
|
+
full = _rust_use_path(path, [], set(info["mods"]))
|
|
3710
|
+
if full is None or _rust_file(lib, full, tree_set) is None:
|
|
3711
|
+
# another crate's module, or one no file of its own holds
|
|
3712
|
+
surface.unresolve(lib, "*", "::".join(path), lib)
|
|
3713
|
+
continue
|
|
3714
|
+
for name in exported(lib, full):
|
|
3715
|
+
publish(lib, full, name, "star", "::".join(path), reached)
|
|
3716
|
+
|
|
3717
|
+
|
|
3718
|
+
# --------------------------------------------------------------------------
|
|
3719
|
+
# Entry points: Go
|
|
3720
|
+
# --------------------------------------------------------------------------
|
|
3721
|
+
|
|
3722
|
+
_GO_FUNC_RE = re.compile(r"^func\s+([^\W\d]\w*)", re.M)
|
|
3723
|
+
_GO_DECL_RE = re.compile(r"^(?:type|var|const)\s+([^\W\d]\w*)", re.M)
|
|
3724
|
+
_GO_GROUP_RE = re.compile(r"^(?:type|var|const)\s*\(\s*$")
|
|
3725
|
+
_GO_SPEC_RE = re.compile(r"^\t([^\W\d]\w*(?:[ \t]*,[ \t]*[^\W\d]\w*)*)")
|
|
3726
|
+
_GO_MAIN_RE = re.compile(r"^package\s+main\b", re.M)
|
|
3727
|
+
|
|
3728
|
+
|
|
3729
|
+
def _go_names(code: str) -> list[tuple[str, int]]:
|
|
3730
|
+
"""(name, line) of each capitalized top-level func (not a method),
|
|
3731
|
+
type, var and const, grouped declarations included."""
|
|
3732
|
+
found = [(m.group(1), _line_of(code, m.start(1))) for rx in (_GO_FUNC_RE, _GO_DECL_RE)
|
|
3733
|
+
for m in rx.finditer(code)]
|
|
3734
|
+
lines = code.split("\n")
|
|
3735
|
+
i = 0
|
|
3736
|
+
while i < len(lines):
|
|
3737
|
+
if _GO_GROUP_RE.match(lines[i]):
|
|
3738
|
+
i += 1
|
|
3739
|
+
while i < len(lines) and not lines[i].startswith(")"):
|
|
3740
|
+
spec = _GO_SPEC_RE.match(lines[i])
|
|
3741
|
+
if spec:
|
|
3742
|
+
found += [(n, i + 1) for n in re.split(r"[ \t]*,[ \t]*", spec.group(1))]
|
|
3743
|
+
i += 1
|
|
3744
|
+
i += 1
|
|
3745
|
+
return sorted((n, ln) for n, ln in found if n[0].isupper())
|
|
3746
|
+
|
|
3747
|
+
|
|
3748
|
+
def _go_surface(root: Path, files: list[str], surface: _Surface, entries: list[dict]) -> None:
|
|
3749
|
+
"""Add the capitalized top-level names of the in-scope Go files outside
|
|
3750
|
+
_test.go files, package main and internal/ folders."""
|
|
3751
|
+
packages: set[str] = set()
|
|
3752
|
+
for rel in files:
|
|
3753
|
+
if (_file_language(rel) != "go" or rel.endswith("_test.go")
|
|
3754
|
+
or "internal" in rel.split("/")[:-1]):
|
|
3755
|
+
continue
|
|
3756
|
+
code = _blank(_read_text(root, rel) or "", "go")
|
|
3757
|
+
if _GO_MAIN_RE.search(code):
|
|
3758
|
+
continue
|
|
3759
|
+
packages.add(posixpath.dirname(rel) or ".")
|
|
3760
|
+
for name, line in _go_names(code):
|
|
3761
|
+
surface.add(name, rel, "declaration", rel, line)
|
|
3762
|
+
entries.extend({"language": "go", "file": p, "package": p, "subpath": None, "resolution": "package"}
|
|
3763
|
+
for p in sorted(packages))
|
|
3764
|
+
|
|
3765
|
+
|
|
3766
|
+
# --------------------------------------------------------------------------
|
|
3767
|
+
# The run
|
|
3768
|
+
# --------------------------------------------------------------------------
|
|
3769
|
+
|
|
3770
|
+
|
|
3771
|
+
def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries: list[dict],
|
|
3772
|
+
in_scope: set[str]) -> tuple[dict[str, str], dict[tuple[str, str], dict], list[dict],
|
|
3773
|
+
list[tuple[str, dict]], list[tuple[str, dict]]]:
|
|
3774
|
+
"""The entry-point diff, one language family at a time: (status by
|
|
3775
|
+
family, public names {(family, name): info}, internal export records,
|
|
3776
|
+
and (family, info) of the extraction gaps and of the names outside
|
|
3777
|
+
scope). A family whose entry points declare names diffs them with its
|
|
3778
|
+
exports: an export no entry point names, nor traces back to, nor makes
|
|
3779
|
+
reachable (a name, or any export of a reachable file), is internal. A
|
|
3780
|
+
declared name no recipe found is a gap when a file in scope defines it
|
|
3781
|
+
(or the trace found no file), and is outside scope when a file out of
|
|
3782
|
+
scope does; a module is no recipe's target, so neither. In a family
|
|
3783
|
+
with no entry point, or only empty ones, every export is public."""
|
|
3784
|
+
statuses: dict[str, str] = {}
|
|
3785
|
+
public: dict[tuple[str, str], dict] = {}
|
|
3786
|
+
internal: list[dict] = []
|
|
3787
|
+
gaps: list[tuple[str, dict]] = []
|
|
3788
|
+
outside: list[tuple[str, dict]] = []
|
|
3789
|
+
entry_families = {FAMILY_OF[e["language"]] for e in entries}
|
|
3790
|
+
families = {FAMILY_OF[e["language"]] for e in exports} | entry_families
|
|
3791
|
+
for family in sorted(families | {f for f, s in surfaces.items() if s.names}):
|
|
3792
|
+
records = [e for e in exports if FAMILY_OF[e["language"]] == family]
|
|
3793
|
+
declared = surfaces[family].names
|
|
3794
|
+
if not declared:
|
|
3795
|
+
statuses[family] = "empty-barrel" if family in entry_families else "no-entry-point"
|
|
3796
|
+
for e in records:
|
|
3797
|
+
public.setdefault((family, e["export_name"]), {
|
|
3798
|
+
"name": e["export_name"], "entry": None, "via": "declaration", "from": None,
|
|
3799
|
+
"local": None, "file": e["source_file"], "line": e["source_line"]})
|
|
3800
|
+
continue
|
|
3801
|
+
statuses[family] = "barrel"
|
|
3802
|
+
surface = surfaces[family]
|
|
3803
|
+
names = {e["export_name"] for e in records}
|
|
3804
|
+
traced = {info["local"] for info in declared.values() if info["local"]} | surface.reachable
|
|
3805
|
+
public.update(((family, name), info) for name, info in declared.items())
|
|
3806
|
+
seen: set[str] = set()
|
|
3807
|
+
for e in records:
|
|
3808
|
+
name = e["export_name"]
|
|
3809
|
+
if (name not in declared and name not in traced and name not in seen
|
|
3810
|
+
and e["source_file"] not in surface.reachable_files):
|
|
3811
|
+
seen.add(name)
|
|
3812
|
+
internal.append(e)
|
|
3813
|
+
for name, info in declared.items():
|
|
3814
|
+
if name in names or (info["local"] or name) in names or info["via"] == "namespace":
|
|
3815
|
+
continue
|
|
3816
|
+
(gaps if info["file"] is None or info["file"] in in_scope else outside).append((family, info))
|
|
3817
|
+
return statuses, public, internal, gaps, outside
|
|
3818
|
+
|
|
3819
|
+
|
|
3820
|
+
def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> int | None:
|
|
3821
|
+
if value is not None:
|
|
3822
|
+
return value or None
|
|
3823
|
+
return HEAD_CAPS.get((tier, scope_type), DEFAULT_HEAD_CAP)
|
|
3824
|
+
|
|
3825
|
+
|
|
3826
|
+
def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, str | None]:
|
|
3827
|
+
"""skf-detect-workspaces.py's verdict on the source root."""
|
|
3828
|
+
manifests = {}
|
|
3829
|
+
for name in WORKSPACE_MANIFESTS:
|
|
3830
|
+
if name in tree_set:
|
|
3831
|
+
text = _read_text(root, name)
|
|
3832
|
+
if text is not None:
|
|
3833
|
+
manifests[name] = text
|
|
3834
|
+
result = _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
|
|
3835
|
+
return bool(result.get("is_monorepo")), result.get("manifest_kind")
|
|
3836
|
+
|
|
3837
|
+
|
|
3838
|
+
def _select_files(root: Path, candidates: list[str], listed: bool, includes: list[str], excludes: list[str],
|
|
3839
|
+
wanted: set[str], glob_match: Callable[[str, str], bool], issues: list[dict],
|
|
3840
|
+
warnings: list[str]) -> tuple[list[str], dict[str, int]]:
|
|
3841
|
+
"""(the files in scope, by extension the in-scope source files no recipe
|
|
3842
|
+
reads). A file a --files-from list names that is missing, or that no
|
|
3843
|
+
recipe reads, is an issue; an include glob no candidate matches, a
|
|
3844
|
+
warning."""
|
|
3845
|
+
in_scope: list[str] = []
|
|
3846
|
+
without_recipes: Counter = Counter()
|
|
3847
|
+
used = dict.fromkeys(includes, False)
|
|
3848
|
+
for rel in candidates:
|
|
3849
|
+
hits = [g for g in includes if glob_match(rel, g)]
|
|
3850
|
+
used.update(dict.fromkeys(hits, True))
|
|
3851
|
+
if (includes and not hits) or any(glob_match(rel, g) for g in excludes):
|
|
3852
|
+
continue
|
|
3853
|
+
if listed and not (root / rel).is_file():
|
|
3854
|
+
issues.append({"file": rel, "issue": "missing"})
|
|
3855
|
+
continue
|
|
3856
|
+
language = _file_language(rel)
|
|
3857
|
+
if language is None or language not in wanted:
|
|
3858
|
+
extension = posixpath.splitext(rel)[1].lower()
|
|
3859
|
+
if listed:
|
|
3860
|
+
issues.append({"file": rel, "issue": "no-recipes"})
|
|
3861
|
+
elif extension in NO_RECIPE_EXTENSIONS:
|
|
3862
|
+
without_recipes[extension] += 1
|
|
3863
|
+
continue
|
|
3864
|
+
in_scope.append(rel)
|
|
3865
|
+
warnings += [f"scope.include pattern {g!r} matches no file" for g, hit in used.items() if not hit]
|
|
3866
|
+
return in_scope, dict(sorted(without_recipes.items()))
|
|
3867
|
+
|
|
3868
|
+
|
|
3869
|
+
def run_full(args: argparse.Namespace) -> tuple[dict, int]:
|
|
3870
|
+
"""Run the recipes over the source tree; (result, exit code)."""
|
|
3871
|
+
clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
|
|
3872
|
+
root = Path(args.source_root)
|
|
3873
|
+
if not root.is_dir():
|
|
3874
|
+
raise RunnerError(f"source root not found: {args.source_root}")
|
|
3875
|
+
recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
|
|
3876
|
+
recipes = load_recipe_file(recipes_path)
|
|
3877
|
+
brief = _read_brief(Path(args.brief)) if args.brief else {}
|
|
3878
|
+
includes = [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g]
|
|
3879
|
+
excludes = [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g]
|
|
3880
|
+
tier_a = [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g]
|
|
3881
|
+
scope_type = args.scope_type or brief.get("type")
|
|
3882
|
+
language_values = args.language or ([brief["language"]] if brief.get("language") else [])
|
|
3883
|
+
wanted, unknown_languages = _wanted_languages(language_values)
|
|
3884
|
+
recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
|
|
3885
|
+
head_cap = _head_cap(args.head_cap, args.tier, scope_type)
|
|
3886
|
+
glob_match = _sibling("skf-resolve-authoritative-files.py").glob_match
|
|
3887
|
+
|
|
3888
|
+
warnings: list[str] = [f"no recipe reads {value} files" for value in unknown_languages]
|
|
3889
|
+
errors: list[dict] = []
|
|
3890
|
+
issues: list[dict] = []
|
|
3891
|
+
tree = list_tree(root)
|
|
3892
|
+
tree_set = set(tree)
|
|
3893
|
+
listed = _read_file_list(Path(args.files_from)) if args.files_from else None
|
|
3894
|
+
in_scope, without_recipes = _select_files(root, tree if listed is None else listed, listed is not None,
|
|
3895
|
+
includes, excludes, wanted, glob_match, issues, warnings)
|
|
3896
|
+
readable = []
|
|
3897
|
+
for rel in in_scope:
|
|
3898
|
+
try:
|
|
3899
|
+
(root / rel).read_bytes().decode("utf-8")
|
|
3900
|
+
except OSError:
|
|
3901
|
+
issues.append({"file": rel, "issue": "unreadable"})
|
|
3902
|
+
continue
|
|
3903
|
+
except UnicodeDecodeError:
|
|
3904
|
+
issues.append({"file": rel, "issue": "not-utf8"})
|
|
3905
|
+
continue
|
|
3906
|
+
readable.append(rel)
|
|
3907
|
+
|
|
3908
|
+
# The entry points and the arms need no ast-grep.
|
|
3909
|
+
in_scope_set = set(in_scope)
|
|
3910
|
+
surfaces = {family: _Surface() for family in FAMILIES}
|
|
3911
|
+
js_entries, maps, unresolved_subpaths = _js_entries(root, in_scope, tree, tree_set, warnings)
|
|
3912
|
+
py_entries = _py_entries(in_scope, tree_set)
|
|
3913
|
+
rust_entries = _rust_entries(root, in_scope, tree, tree_set)
|
|
3914
|
+
go_entries: list[dict] = []
|
|
3915
|
+
_go_surface(root, in_scope, surfaces["go"], go_entries)
|
|
3916
|
+
entries = js_entries + py_entries + rust_entries + go_entries
|
|
3917
|
+
for entry in entries:
|
|
3918
|
+
# a Go entry is a package folder, which the files in scope give
|
|
3919
|
+
entry["in_scope"] = entry["language"] == "go" or entry["file"] in in_scope_set
|
|
3920
|
+
if not entry["in_scope"] and entry["subpath"] is not None:
|
|
3921
|
+
warnings.append(f"public entry point {entry['file']} (subpath '{entry['subpath']}' of package "
|
|
3922
|
+
f"{entry['package']}) is outside the files in scope: widen scope.include to document "
|
|
3923
|
+
"the names it exports (entry_point_diff.outside_scope), which effective_denominator "
|
|
3924
|
+
"leaves out")
|
|
3925
|
+
monorepo, monorepo_kind = _monorepo(root, tree, tree_set)
|
|
3926
|
+
arms = {
|
|
3927
|
+
"monorepo": monorepo,
|
|
3928
|
+
"monorepo_kind": monorepo_kind,
|
|
3929
|
+
"specific_modules": scope_type == "specific-modules" and not monorepo,
|
|
3930
|
+
"multi_subpath_exports": not monorepo and any(m["non_root_subpaths"] > 1 for m in maps),
|
|
3931
|
+
}
|
|
3932
|
+
|
|
3933
|
+
exe, version, shim = _ast_grep()
|
|
3934
|
+
result: dict = {
|
|
3935
|
+
"mode": "full",
|
|
3936
|
+
"status": "ok",
|
|
3937
|
+
"source_root": args.source_root,
|
|
3938
|
+
"recipes_file": recipes_path.as_posix(),
|
|
3939
|
+
"recipe_set": recipe_set,
|
|
3940
|
+
"ast_grep": {"path": exe, "version": version},
|
|
3941
|
+
"scope": {"include": includes, "exclude": excludes, "tier_a_include": tier_a or None,
|
|
3942
|
+
"type": scope_type, "languages": sorted(wanted),
|
|
3943
|
+
"files_from": args.files_from},
|
|
3944
|
+
"files_in_scope": len(in_scope),
|
|
3945
|
+
"files_by_language": dict(sorted(Counter(_file_language(f) for f in in_scope).items())),
|
|
3946
|
+
"files_without_recipes": without_recipes,
|
|
3947
|
+
"file_issues": issues,
|
|
3948
|
+
"head_cap": head_cap,
|
|
3949
|
+
"truncated": False,
|
|
3950
|
+
"recipes": [],
|
|
3951
|
+
"exports": [],
|
|
3952
|
+
"aggregates": {"exports": 0, "by_type": {}, "t1": 0, "t1_low": 0},
|
|
3953
|
+
"entry_points": {"status": None, "by_language": {}, "files": entries, "exports_maps": maps,
|
|
3954
|
+
"unresolved": unresolved_subpaths},
|
|
3955
|
+
"entry_point_diff": None,
|
|
3956
|
+
"reexport_targets": [],
|
|
3957
|
+
"counts": None,
|
|
3958
|
+
"arms": arms,
|
|
3959
|
+
"errors": errors,
|
|
3960
|
+
"warnings": warnings,
|
|
3961
|
+
}
|
|
3962
|
+
if exe is None:
|
|
3963
|
+
result["status"] = "no-ast-grep"
|
|
3964
|
+
result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
|
|
3965
|
+
if shim is None:
|
|
3966
|
+
warnings.append("ast-grep is not on PATH: no recipe ran; extract by source reading")
|
|
3967
|
+
else:
|
|
3968
|
+
warnings.append(f"ast-grep on PATH is the npm shim {shim}, with no native ast-grep.exe beside it: "
|
|
3969
|
+
"a shim runs through cmd.exe, which would read the file paths a scan passes as "
|
|
3970
|
+
"commands, so no recipe ran; extract by source reading, or install a native "
|
|
3971
|
+
"ast-grep (pip install ast-grep-cli)")
|
|
3972
|
+
return result, 3
|
|
3973
|
+
|
|
3974
|
+
forms = _forms(recipes)
|
|
3975
|
+
set_recipes = [r for r in recipes if recipe_set in r["metadata"]["sets"]]
|
|
3976
|
+
barrel_recipes = [r for r in recipes if "standard" in r["metadata"]["sets"]
|
|
3977
|
+
and r["rule"]["kind"] in ("export_statement", "export_specifier")
|
|
3978
|
+
and set(r["metadata"]["languages"]) & set(JS_FAMILY)]
|
|
3979
|
+
barrel_ids = {r["id"] for r in barrel_recipes}
|
|
3980
|
+
languages = {lang for f in readable for lang in _rule_languages(_file_language(f))}
|
|
3981
|
+
with tempfile.TemporaryDirectory(prefix="skf-extract-", ignore_cleanup_errors=True) as tmp:
|
|
3982
|
+
folder = Path(tmp)
|
|
3983
|
+
rules = _recipe_rules(set_recipes, languages) + [_syntax_error_rule(lang) for lang in sorted(languages)]
|
|
3984
|
+
matches = _scan(exe, root, folder, rules, readable, errors, clock)
|
|
3985
|
+
issues += _syntax_issues(matches)
|
|
3986
|
+
exports, stats = merge_exports(matches, recipes, recipe_set, head_cap)
|
|
3987
|
+
_type_list_items(exe, root, folder, matches, exports, errors, clock)
|
|
3988
|
+
|
|
3989
|
+
scanned: dict[str, list[dict]] = {}
|
|
3990
|
+
|
|
3991
|
+
def scan_barrels(files: list[str]) -> dict[str, list[dict]]:
|
|
3992
|
+
todo = [f for f in files if f not in scanned and _read_text(root, f) is not None]
|
|
3993
|
+
langs = {lang for f in todo for lang in _rule_languages(_file_language(f) or "")}
|
|
3994
|
+
barrel_rules = _recipe_rules(barrel_recipes, langs)
|
|
3995
|
+
barrel_rules += [r for lang in sorted(langs & set(RECIPE_LANGUAGES)) for r in _trace_rules(lang)]
|
|
3996
|
+
for f in todo:
|
|
3997
|
+
scanned[f] = []
|
|
3998
|
+
for m in _scan(exe, root, folder, barrel_rules, todo, errors, clock):
|
|
3999
|
+
scanned.setdefault(m["file"], []).append(m)
|
|
4000
|
+
return {f: scanned.get(f, []) for f in files}
|
|
4001
|
+
|
|
4002
|
+
if js_entries:
|
|
4003
|
+
_js_surface(root, js_entries, tree_set, forms, barrel_ids, scan_barrels, surfaces["javascript"])
|
|
4004
|
+
_py_surface(root, py_entries, tree_set, surfaces["python"], warnings)
|
|
4005
|
+
_rust_surface(root, rust_entries, tree_set, surfaces["rust"])
|
|
4006
|
+
|
|
4007
|
+
for record in exports:
|
|
4008
|
+
if record["from"] and record["language"] in JS_FAMILY:
|
|
4009
|
+
record["from_file"] = resolve_js_module(record["from"], record["source_file"], tree_set)
|
|
4010
|
+
statuses, public, internal, gaps, outside = entry_point_diff(exports, surfaces, entries, in_scope_set)
|
|
4011
|
+
if "barrel" in statuses.values():
|
|
4012
|
+
status = "barrel"
|
|
4013
|
+
else:
|
|
4014
|
+
status = "empty-barrel" if "empty-barrel" in statuses.values() else "no-entry-point"
|
|
4015
|
+
|
|
4016
|
+
basis_globs = tier_a or includes
|
|
4017
|
+
basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
|
|
4018
|
+
denominator_files = {f for f in tree if (not basis_globs or any(glob_match(f, g) for g in basis_globs))
|
|
4019
|
+
and not any(glob_match(f, g) for g in excludes)}
|
|
4020
|
+
|
|
4021
|
+
by_type: Counter = Counter(e["export_type"] for e in exports)
|
|
4022
|
+
ordered = sorted(public.items(), key=lambda item: (item[0][1], item[0][0]))
|
|
4023
|
+
result.update({
|
|
4024
|
+
"truncated": any(s["truncated"] for s in stats),
|
|
4025
|
+
"recipes": stats,
|
|
4026
|
+
"exports": exports,
|
|
4027
|
+
"aggregates": {"exports": len(exports), "by_type": dict(sorted(by_type.items())),
|
|
4028
|
+
"t1": len(exports), "t1_low": 0},
|
|
4029
|
+
"entry_points": {"status": status, "by_language": statuses, "files": entries, "exports_maps": maps,
|
|
4030
|
+
"unresolved": unresolved_subpaths + [u for s in surfaces.values() for u in s.unresolved]},
|
|
4031
|
+
"entry_point_diff": {
|
|
4032
|
+
"public": [dict(info, language=family) for (family, _), info in ordered],
|
|
4033
|
+
"internal": [{"name": e["export_name"], "language": FAMILY_OF[e["language"]],
|
|
4034
|
+
"source_file": e["source_file"], "source_line": e["source_line"]} for e in internal],
|
|
4035
|
+
"extraction_gaps": [{"name": info["name"], "language": family, "entry": info["entry"],
|
|
4036
|
+
"file": info["file"], "line": info["line"]}
|
|
4037
|
+
for family, info in sorted(gaps, key=lambda g: (g[1]["name"], g[0]))],
|
|
4038
|
+
"outside_scope": [{"name": info["name"], "language": family, "entry": info["entry"],
|
|
4039
|
+
"file": info["file"], "line": info["line"]}
|
|
4040
|
+
for family, info in sorted(outside, key=lambda g: (g[1]["name"], g[0]))],
|
|
4041
|
+
},
|
|
4042
|
+
"reexport_targets": [
|
|
4043
|
+
{"name": info["name"], "language": family, "local": info["local"], "from": info["from"],
|
|
4044
|
+
"entry": info["entry"], "file": info["file"], "line": info["line"], "via": info["via"]}
|
|
4045
|
+
for (family, _), info in ordered if info["via"] in ("re-export", "star", "namespace")],
|
|
4046
|
+
"counts": {
|
|
4047
|
+
"exports_public_api": len(public),
|
|
4048
|
+
"exports_internal": len(internal),
|
|
4049
|
+
"effective_denominator": sum(1 for info in public.values() if info["file"] in denominator_files),
|
|
4050
|
+
"effective_denominator_basis": basis,
|
|
4051
|
+
"denominator_files": len(denominator_files),
|
|
4052
|
+
},
|
|
4053
|
+
})
|
|
4054
|
+
result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
|
|
4055
|
+
if errors:
|
|
4056
|
+
result["status"] = "incomplete"
|
|
4057
|
+
return result, 1
|
|
4058
|
+
return result, 0
|
|
4059
|
+
|
|
4060
|
+
|
|
4061
|
+
def _main_full(args: argparse.Namespace) -> int:
|
|
4062
|
+
if not args.source_root:
|
|
4063
|
+
sys.stderr.write("error: --mode full needs --source-root\n")
|
|
4064
|
+
return 2
|
|
4065
|
+
if args.head_cap is not None and args.head_cap < 0:
|
|
4066
|
+
sys.stderr.write("error: --head-cap must be 0 (no cap) or more\n")
|
|
4067
|
+
return 2
|
|
4068
|
+
if args.timeout is not None and not args.timeout > 0:
|
|
4069
|
+
sys.stderr.write("error: --timeout must be more than 0 seconds\n")
|
|
4070
|
+
return 2
|
|
4071
|
+
try:
|
|
4072
|
+
result, code = run_full(args)
|
|
4073
|
+
except RunnerError as exc:
|
|
4074
|
+
sys.stderr.write(f"error: {exc}\n")
|
|
4075
|
+
return 2
|
|
4076
|
+
except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
|
|
4077
|
+
detail = " ".join(str(exc).split())
|
|
4078
|
+
sys.stderr.write(f"error: the run failed: {type(exc).__name__}: {detail}\n")
|
|
4079
|
+
return 2
|
|
4080
|
+
text = json.dumps(result, indent=2) + "\n"
|
|
4081
|
+
if args.output:
|
|
4082
|
+
try:
|
|
4083
|
+
Path(args.output).write_text(text, encoding="utf-8")
|
|
4084
|
+
except OSError as exc:
|
|
4085
|
+
sys.stderr.write(f"error: cannot write {args.output}: {exc}\n")
|
|
4086
|
+
return 2
|
|
4087
|
+
else:
|
|
4088
|
+
sys.stdout.write(text)
|
|
4089
|
+
return code
|
|
4090
|
+
|
|
4091
|
+
|
|
4092
|
+
# --------------------------------------------------------------------------
|
|
4093
|
+
# Entries mode: the files quick mode reads, found in the listing
|
|
4094
|
+
# --------------------------------------------------------------------------
|
|
4095
|
+
|
|
4096
|
+
# The quick languages entries mode finds entry files for, by family.
|
|
4097
|
+
ENTRY_LANGUAGES = {"js": "javascript", "ts": "javascript", "javascript": "javascript",
|
|
4098
|
+
"typescript": "javascript", "python": "python", "rust": "rust", "go": "go",
|
|
4099
|
+
"java": "java", "kotlin": "kotlin"}
|
|
4100
|
+
# The manifest of each family at the package folder, the first one listed read
|
|
4101
|
+
# (Python: the first that names the package, read in this order).
|
|
4102
|
+
ENTRY_MANIFESTS = {"javascript": ("package.json",), "python": ("pyproject.toml", "setup.py", "setup.cfg"),
|
|
4103
|
+
"rust": ("Cargo.toml",), "go": ("go.mod",), "java": ("pom.xml",),
|
|
4104
|
+
"kotlin": ("build.gradle.kts", "build.gradle")}
|
|
4105
|
+
# A glob-picked family (Go, Java, Kotlin) reads at most this many files, as
|
|
4106
|
+
# the fetch's `--limit 5` did; no family reads more than ENTRY_FILE_LIMIT.
|
|
4107
|
+
ENTRY_GLOB_LIMIT = 5
|
|
4108
|
+
ENTRY_FILE_LIMIT = 25
|
|
4109
|
+
# Top folders of a Python tree that hold no package code (test* folders too).
|
|
4110
|
+
PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
|
|
4111
|
+
|
|
4112
|
+
|
|
4113
|
+
def _scope_folder(scope: str | None) -> str:
|
|
4114
|
+
"""The package folder a --scope names, "" for the repository root."""
|
|
4115
|
+
folder = _norm(scope or "").strip("/")
|
|
4116
|
+
return "" if folder == "." else folder
|
|
4117
|
+
|
|
4118
|
+
|
|
4119
|
+
def _import_name(name: object) -> str:
|
|
4120
|
+
"""A distribution name the way Python imports it: lower case, each run
|
|
4121
|
+
of `-` and `.` read as `_` (`zope.interface` -> zope_interface)."""
|
|
4122
|
+
return re.sub(r"[-.]+", "_", name.strip().lower()) if isinstance(name, str) else ""
|
|
4123
|
+
|
|
4124
|
+
|
|
4125
|
+
def _py_quick_entries(folder: str, tree: list[str], tree_set: set[str],
|
|
4126
|
+
name: str | None) -> tuple[list[str], list[str]]:
|
|
4127
|
+
"""(entry files, the paths tried when there is none) of the Python
|
|
4128
|
+
package in `folder`: the top __init__.py of each package full mode finds
|
|
4129
|
+
(_py_entries) among its .py files, less those under a top folder named
|
|
4130
|
+
test*, docs, examples or benchmarks, the one whose folder is the
|
|
4131
|
+
manifest's import name preferred (Pillow's src/PIL/ is kept, as no
|
|
4132
|
+
folder is named pillow); with no package at all, the single module
|
|
4133
|
+
`<name>.py` or `src/<name>.py` (six.py)."""
|
|
4134
|
+
prefix = f"{folder}/" if folder else ""
|
|
4135
|
+
files = []
|
|
4136
|
+
for path in tree:
|
|
4137
|
+
if not path.startswith(prefix) or not path.endswith(".py"):
|
|
4138
|
+
continue
|
|
4139
|
+
rest = path[len(prefix):]
|
|
4140
|
+
top = rest.split("/", 1)[0].lower() if "/" in rest else ""
|
|
4141
|
+
if top.startswith(("test", ".")) or top in PY_NON_CORE_TOPS:
|
|
4142
|
+
continue
|
|
4143
|
+
files.append(path)
|
|
4144
|
+
tops = [entry["file"] for entry in _py_entries(files, tree_set)]
|
|
4145
|
+
wanted = _import_name(name)
|
|
4146
|
+
if tops:
|
|
4147
|
+
named = [top for top in tops if posixpath.basename(posixpath.dirname(top)).lower() == wanted]
|
|
4148
|
+
return named or tops, []
|
|
4149
|
+
tried = [f"{prefix}{wanted}.py", f"{prefix}src/{wanted}.py"] if wanted else []
|
|
4150
|
+
return [path for path in tried if path in tree_set][:1], tried
|
|
4151
|
+
|
|
4152
|
+
|
|
4153
|
+
def _py_manifest_name(manifest: str, content: str) -> str | None:
|
|
4154
|
+
"""The distribution name a Python manifest gives: pyproject.toml's
|
|
4155
|
+
[project] or [tool.poetry] `name`, setup.py's `name=`, setup.cfg's
|
|
4156
|
+
[metadata] `name`."""
|
|
4157
|
+
if manifest.endswith("setup.py"):
|
|
4158
|
+
name = parse_setup_py(content).get("name")
|
|
4159
|
+
elif manifest.endswith("setup.cfg"):
|
|
4160
|
+
name = parse_setup_cfg(content).get("name")
|
|
4161
|
+
else:
|
|
4162
|
+
try:
|
|
4163
|
+
data = tomllib.loads(content)
|
|
4164
|
+
except tomllib.TOMLDecodeError:
|
|
4165
|
+
return None
|
|
4166
|
+
project = data.get("project") if isinstance(data.get("project"), dict) else {}
|
|
4167
|
+
tool = data.get("tool") if isinstance(data.get("tool"), dict) else {}
|
|
4168
|
+
poetry = tool.get("poetry") if isinstance(tool.get("poetry"), dict) else {}
|
|
4169
|
+
name = project.get("name") or poetry.get("name")
|
|
4170
|
+
return name.strip() if isinstance(name, str) and name.strip() else None
|
|
4171
|
+
|
|
4172
|
+
|
|
4173
|
+
def _pom_group(content: str) -> str | None:
|
|
4174
|
+
"""The groupId of a pom.xml (parse_pom_xml's), else the one its <parent> gives."""
|
|
4175
|
+
group = (parse_pom_xml(content).get("_extra") or {}).get("group_id")
|
|
4176
|
+
if group:
|
|
4177
|
+
return group
|
|
4178
|
+
try:
|
|
4179
|
+
project = ET.fromstring(content)
|
|
4180
|
+
except ET.ParseError:
|
|
4181
|
+
return None
|
|
4182
|
+
for child in project:
|
|
4183
|
+
if _strip_ns(child.tag) == "parent":
|
|
4184
|
+
for item in child:
|
|
4185
|
+
if _strip_ns(item.tag) == "groupId" and (item.text or "").strip():
|
|
4186
|
+
return item.text.strip()
|
|
4187
|
+
return None
|
|
4188
|
+
|
|
4189
|
+
|
|
4190
|
+
def _rust_lib_path(content: str | None) -> str:
|
|
4191
|
+
"""The crate root a Cargo.toml names (`[lib] path`), else src/lib.rs."""
|
|
4192
|
+
try:
|
|
4193
|
+
path = (tomllib.loads(content or "").get("lib") or {}).get("path")
|
|
4194
|
+
except (tomllib.TOMLDecodeError, AttributeError):
|
|
4195
|
+
path = None
|
|
4196
|
+
return _norm(path) if isinstance(path, str) and path.strip() else "src/lib.rs"
|
|
4197
|
+
|
|
4198
|
+
|
|
4199
|
+
def find_entries(language: str, tree: list[str], root: Path, scope: str | None = None) -> dict:
|
|
4200
|
+
"""The entry files quick mode reads for the one package at `scope` (the
|
|
4201
|
+
repository root by default), found in the listing `tree` with full
|
|
4202
|
+
mode's rules, its manifest read from `root` (the folder it was fetched
|
|
4203
|
+
into). See "Entries mode" above."""
|
|
4204
|
+
lowered = (language or "").lower()
|
|
4205
|
+
if lowered not in LANGUAGE_DISPATCH:
|
|
4206
|
+
return {"_error": f"unknown language: {language!r}; expected one of {sorted(LANGUAGE_DISPATCH)}"}
|
|
4207
|
+
family = ENTRY_LANGUAGES.get(lowered)
|
|
4208
|
+
folder = _scope_folder(scope)
|
|
4209
|
+
prefix = f"{folder}/" if folder else ""
|
|
4210
|
+
label = folder or "."
|
|
4211
|
+
tree_set = set(tree)
|
|
4212
|
+
warnings: list[str] = []
|
|
4213
|
+
out = {"mode": "entries", "language": lowered, "scope": label, "manifest": None,
|
|
4214
|
+
"entry_files": [], "unresolved": [], "warnings": warnings}
|
|
4215
|
+
if family is None:
|
|
4216
|
+
warnings.append(f"no entry-point rule for {lowered}: read its source by eye")
|
|
4217
|
+
return out
|
|
4218
|
+
listed = [prefix + name for name in ENTRY_MANIFESTS[family] if prefix + name in tree_set]
|
|
4219
|
+
contents: dict[str, str] = {}
|
|
4220
|
+
for path in listed if family == "python" else listed[:1]:
|
|
4221
|
+
try:
|
|
4222
|
+
contents[path] = (root / path).read_bytes().decode("utf-8-sig", errors="replace")
|
|
4223
|
+
except OSError:
|
|
4224
|
+
warnings.append(f"{path} is in the listing but not under --source-root: fetch it first")
|
|
4225
|
+
manifest = listed[0] if listed else None
|
|
4226
|
+
py_name = None
|
|
4227
|
+
if family == "python":
|
|
4228
|
+
named = ((path, _py_manifest_name(path, text)) for path, text in contents.items())
|
|
4229
|
+
manifest, py_name = next(((path, name) for path, name in named if name), (manifest, None))
|
|
4230
|
+
out["manifest"] = manifest
|
|
4231
|
+
content = contents.get(manifest) if manifest else None
|
|
4232
|
+
files: list[str] = []
|
|
4233
|
+
tried: list[str] = []
|
|
4234
|
+
if family == "javascript":
|
|
4235
|
+
found, _, out["unresolved"] = _js_package_entries(root, folder, tree_set, warnings)
|
|
4236
|
+
files = [entry["file"] for entry in found]
|
|
4237
|
+
tried = [f"{prefix}index.*", f"{prefix}src/index.*"]
|
|
4238
|
+
elif family == "python":
|
|
4239
|
+
files, tried = _py_quick_entries(folder, tree, tree_set, py_name)
|
|
4240
|
+
elif family == "rust":
|
|
4241
|
+
crate = [path for path in tree if path.startswith(prefix) and path.endswith(".rs")]
|
|
4242
|
+
files = [entry["file"] for entry in _rust_entries(root, crate, tree, tree_set) if entry["package"] == label]
|
|
4243
|
+
tried = [posixpath.normpath(prefix + _rust_lib_path(content))]
|
|
4244
|
+
elif family == "go":
|
|
4245
|
+
files = sorted(path for path in tree if posixpath.dirname(path) == folder and path.endswith(".go")
|
|
4246
|
+
and not path.endswith("_test.go"))[:ENTRY_GLOB_LIMIT]
|
|
4247
|
+
tried = [f"{prefix}*.go"]
|
|
4248
|
+
elif family == "java":
|
|
4249
|
+
group = _pom_group(content) if content else None
|
|
4250
|
+
base = f"{prefix}src/main/java/{group.replace('.', '/')}" if group else None
|
|
4251
|
+
if base:
|
|
4252
|
+
files = sorted(path for path in tree
|
|
4253
|
+
if posixpath.dirname(path) == base and path.endswith(".java"))[:ENTRY_GLOB_LIMIT]
|
|
4254
|
+
tried = [f"{base}/*.java" if base else f"{prefix}src/main/java/<groupId>/*.java"]
|
|
4255
|
+
else:
|
|
4256
|
+
base = f"{prefix}src/main/kotlin/"
|
|
4257
|
+
files = sorted(path for path in tree if path.startswith(base) and path.endswith(".kt"))[:ENTRY_GLOB_LIMIT]
|
|
4258
|
+
tried = [f"{base}**/*.kt"]
|
|
4259
|
+
if len(files) > ENTRY_FILE_LIMIT:
|
|
4260
|
+
warnings.append(f"{len(files)} entry files: quick mode reads the first {ENTRY_FILE_LIMIT}")
|
|
4261
|
+
files = files[:ENTRY_FILE_LIMIT]
|
|
4262
|
+
if not files and not out["unresolved"]:
|
|
4263
|
+
out["unresolved"] = [{"package": label, "subpath": None, "targets": tried}]
|
|
4264
|
+
out["entry_files"] = files
|
|
4265
|
+
return out
|
|
4266
|
+
|
|
4267
|
+
|
|
4268
|
+
# --------------------------------------------------------------------------
|
|
4269
|
+
# CLI
|
|
4270
|
+
# --------------------------------------------------------------------------
|
|
4271
|
+
|
|
4272
|
+
|
|
4273
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
4274
|
+
parser = argparse.ArgumentParser(
|
|
4275
|
+
description=(
|
|
4276
|
+
"Extract a package's public-API surface: from manifest and entry-point content on stdin or "
|
|
4277
|
+
"in files (quick, a pure parser), or by running the ast-grep recipes over a source tree (full); "
|
|
4278
|
+
"or find the entry files quick mode reads in a repository listing (entries)."
|
|
4279
|
+
),
|
|
4280
|
+
)
|
|
4281
|
+
parser.add_argument(
|
|
4282
|
+
"--mode",
|
|
4283
|
+
default="quick",
|
|
4284
|
+
choices=("quick", "full", "entries"),
|
|
4285
|
+
help="quick (default): parse the stdin payload, or the --manifest-file and --entry-file files; "
|
|
4286
|
+
"full: run the recipes over --source-root; entries: list one package's entry files from "
|
|
4287
|
+
"--tree-file.",
|
|
4288
|
+
)
|
|
4289
|
+
quick = parser.add_argument_group("quick mode, files read from disk")
|
|
4290
|
+
quick.add_argument("--manifest-file", metavar="PATH",
|
|
4291
|
+
help="the package manifest (relative to --source-root when given); needs --language")
|
|
4292
|
+
quick.add_argument("--entry-file", action="append", metavar="PATH",
|
|
4293
|
+
help="an entry-point file (relative to --source-root when given), repeatable; needs --language")
|
|
4294
|
+
quick.add_argument("--follow-file", action="append", metavar="PATH",
|
|
4295
|
+
help="a module an unlisted[] record named (a --fetch-list path), read as an entry less "
|
|
4296
|
+
"its JS/TS default export, which `export *` does not pass on; repeatable")
|
|
4297
|
+
quick.add_argument("--follow", action="store_true",
|
|
4298
|
+
help=f"read each module an unlisted[] record names as one more --follow-file, round "
|
|
4299
|
+
f"after round (at most {FOLLOW_ROUNDS}), until none is left; needs --tree-file and "
|
|
4300
|
+
f"--source-root")
|
|
4301
|
+
quick.add_argument("--repo", metavar="OWNER/REPO",
|
|
4302
|
+
help="with --follow: fetch each module from this GitHub repository into --source-root "
|
|
4303
|
+
"first (skf-github-fetch.py)")
|
|
4304
|
+
quick.add_argument("--ref", metavar="REF",
|
|
4305
|
+
help="with --repo: the branch, tag or commit to fetch at (default: the default branch)")
|
|
4306
|
+
listing = parser.add_argument_group("entries mode, and the listing quick mode follows chains in")
|
|
4307
|
+
listing.add_argument("--tree-file", metavar="PATH",
|
|
4308
|
+
help="the repository's file listing (skf-github-probe.py tree output, JSON, or one "
|
|
4309
|
+
"path per line): entries mode finds the entry files in it, and quick mode "
|
|
4310
|
+
"resolves each unlisted[] statement's module_file against it")
|
|
4311
|
+
listing.add_argument("--scope", metavar="FOLDER",
|
|
4312
|
+
help="entries mode: the package folder to read (default: the repository root)")
|
|
4313
|
+
listing.add_argument("--fetch-list", metavar="FILE",
|
|
4314
|
+
help="write one path per line for skf-github-fetch.py --patterns-file: the entry "
|
|
4315
|
+
"files (entries mode), or each unlisted[] module_file still to read (quick mode)")
|
|
4316
|
+
full = parser.add_argument_group("full mode")
|
|
4317
|
+
full.add_argument("--source-root",
|
|
4318
|
+
help="the source tree to extract from (quick mode: the folder the file paths are relative to)")
|
|
4319
|
+
full.add_argument("--brief", help="a skill-brief.yaml: its scope and language")
|
|
4320
|
+
full.add_argument("--include", action="append", metavar="GLOB",
|
|
4321
|
+
help="an include glob, repeatable (replaces the brief's scope.include)")
|
|
4322
|
+
full.add_argument("--exclude", action="append", metavar="GLOB",
|
|
4323
|
+
help="an exclude glob, repeatable (replaces the brief's scope.exclude)")
|
|
4324
|
+
full.add_argument("--tier-a-include", action="append", metavar="GLOB",
|
|
4325
|
+
help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include)")
|
|
4326
|
+
full.add_argument("--scope-type", help="the scope type (replaces the brief's scope.type)")
|
|
4327
|
+
full.add_argument("--language", action="append",
|
|
4328
|
+
help="a language family to extract, repeatable (replaces the brief's language); quick "
|
|
4329
|
+
"mode: the files' language, once")
|
|
4330
|
+
full.add_argument("--files-from", metavar="FILE",
|
|
4331
|
+
help="read only these files: one path per line, or a JSON list")
|
|
4332
|
+
full.add_argument("--recipe-set", choices=RECIPE_SETS,
|
|
4333
|
+
help="the recipe set to run (default: from the scope type)")
|
|
4334
|
+
full.add_argument("--recipes", metavar="FILE", help="the recipe file (default: the module's data file)")
|
|
4335
|
+
full.add_argument("--tier", choices=TIERS, help="the forge tier, for the default head cap")
|
|
4336
|
+
full.add_argument("--head-cap", type=int, metavar="N",
|
|
4337
|
+
help="the most matches kept per recipe (0: no cap; default: by tier and scope type)")
|
|
4338
|
+
full.add_argument("--timeout", type=float, metavar="SECONDS",
|
|
4339
|
+
help=f"stop the ast-grep runs after this many seconds and still write the JSON, "
|
|
4340
|
+
f"status incomplete (default {DEFAULT_TIMEOUT_SEC})")
|
|
4341
|
+
full.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
|
|
4342
|
+
return parser
|
|
4343
|
+
|
|
4344
|
+
|
|
4345
|
+
FULL_MODE_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include", "scope_type", "language",
|
|
4346
|
+
"files_from", "recipe_set", "recipes", "tier", "head_cap", "timeout", "output")
|
|
4347
|
+
# The full mode flags quick mode takes with its file inputs, and entries mode always.
|
|
4348
|
+
QUICK_FILE_FLAGS = ("source_root", "language")
|
|
4349
|
+
# The flags of entries mode; quick mode takes --tree-file and --fetch-list with its file inputs.
|
|
4350
|
+
LISTING_FLAGS = ("tree_file", "scope", "fetch_list")
|
|
4351
|
+
# Quick mode's own flags with its file inputs: the chains it follows itself.
|
|
4352
|
+
FOLLOW_FLAGS = ("follow", "repo", "ref")
|
|
4353
|
+
FOLLOW_ROUNDS = 5
|
|
4354
|
+
|
|
4355
|
+
|
|
4356
|
+
class QuickInputError(Exception):
|
|
4357
|
+
"""A --manifest-file or --entry-file quick mode cannot read (exit 2)."""
|
|
4358
|
+
|
|
4359
|
+
|
|
4360
|
+
def _read_quick_file(root: Path | None, value: str, flag: str) -> dict:
|
|
4361
|
+
"""{"path", "content"} of a --manifest-file or --entry-file: the path as
|
|
4362
|
+
given, written with `/`, and the file's text. Raises QuickInputError."""
|
|
4363
|
+
given = Path(value)
|
|
4364
|
+
path = given
|
|
4365
|
+
if root is not None:
|
|
4366
|
+
if given.is_absolute():
|
|
4367
|
+
raise QuickInputError(f"{flag} must be relative to --source-root: {value}")
|
|
4368
|
+
path = root / given
|
|
4369
|
+
if not path.resolve().is_relative_to(root.resolve()):
|
|
4370
|
+
raise QuickInputError(f"{flag} leads outside --source-root: {value}")
|
|
4371
|
+
try:
|
|
4372
|
+
data = path.read_bytes()
|
|
4373
|
+
except OSError as exc:
|
|
4374
|
+
raise QuickInputError(f"cannot read {flag} {value}: {exc.strerror or exc}") from exc
|
|
4375
|
+
return {"path": given.as_posix(), "content": data.decode("utf-8-sig", errors="replace")}
|
|
4376
|
+
|
|
4377
|
+
|
|
4378
|
+
def _read_listing(value: str) -> tuple[list[str], bool]:
|
|
4379
|
+
"""(paths, whether the listing was cut short) of a --tree-file, read by
|
|
4380
|
+
skf-detect-language.py's read_tree_file(). Raises QuickInputError."""
|
|
4381
|
+
try:
|
|
4382
|
+
return _sibling("skf-detect-language.py").read_tree_file(value)
|
|
4383
|
+
except (RunnerError, ValueError) as exc: # ValueError: the sibling's TreeListingError
|
|
4384
|
+
raise QuickInputError(str(exc)) from exc
|
|
4385
|
+
|
|
4386
|
+
|
|
4387
|
+
def _write_fetch_list(path: str, files: list[str]) -> None:
|
|
4388
|
+
"""Write `files` one per line (LF on every platform) for --patterns-file. Raises QuickInputError."""
|
|
4389
|
+
try:
|
|
4390
|
+
Path(path).write_text("".join(f"{f}\n" for f in files), encoding="utf-8", newline="\n")
|
|
4391
|
+
except OSError as exc:
|
|
4392
|
+
raise QuickInputError(f"cannot write --fetch-list {path}: {exc.strerror or exc}") from exc
|
|
4393
|
+
|
|
4394
|
+
|
|
4395
|
+
def _quick_files_payload(args: argparse.Namespace) -> dict:
|
|
4396
|
+
"""The quick payload of --manifest-file and --entry-file, read from
|
|
4397
|
+
disk, with the --tree-file listing as its `tree`. Raises
|
|
4398
|
+
QuickInputError."""
|
|
4399
|
+
root = None
|
|
4400
|
+
if args.source_root is not None:
|
|
4401
|
+
root = Path(args.source_root)
|
|
4402
|
+
if not root.is_dir():
|
|
4403
|
+
raise QuickInputError(f"--source-root is not a folder: {args.source_root}")
|
|
4404
|
+
manifest = {"path": "", "content": ""}
|
|
4405
|
+
if args.manifest_file is not None:
|
|
4406
|
+
manifest = _read_quick_file(root, args.manifest_file, "--manifest-file")
|
|
4407
|
+
entries: list[dict] = []
|
|
4408
|
+
for value in args.entry_file or []:
|
|
4409
|
+
entry = _read_quick_file(root, value, "--entry-file")
|
|
4410
|
+
if all(e["path"] != entry["path"] for e in entries):
|
|
4411
|
+
entries.append(entry)
|
|
4412
|
+
for value in getattr(args, "follow_file", None) or []:
|
|
4413
|
+
entry = _read_quick_file(root, value, "--follow-file")
|
|
4414
|
+
if all(e["path"] != entry["path"] for e in entries):
|
|
4415
|
+
entries.append({**entry, "followed": True})
|
|
4416
|
+
payload = {"language": args.language[0], "manifest": manifest, "entries": entries, "mode": "quick"}
|
|
4417
|
+
if getattr(args, "tree_file", None) is not None:
|
|
4418
|
+
payload["tree"] = _read_listing(args.tree_file)[0]
|
|
4419
|
+
return payload
|
|
4420
|
+
|
|
4421
|
+
|
|
4422
|
+
def _follow_reader(args: argparse.Namespace):
|
|
4423
|
+
"""--follow's reader: paths -> (those now under --source-root, {path:
|
|
4424
|
+
why not} for the rest), each fetched there first with --repo. Raises
|
|
4425
|
+
QuickInputError."""
|
|
4426
|
+
root = Path(args.source_root)
|
|
4427
|
+
if args.repo is None:
|
|
4428
|
+
def read_local(paths: list[str]) -> tuple[list[str], dict[str, str]]:
|
|
4429
|
+
got = [p for p in paths if (root / p).is_file() and (root / p).resolve().is_relative_to(root.resolve())]
|
|
4430
|
+
return got, {p: "it is not under --source-root" for p in paths if p not in got}
|
|
4431
|
+
return read_local
|
|
4432
|
+
try:
|
|
4433
|
+
fetch = _sibling("skf-github-fetch.py")
|
|
4434
|
+
except RunnerError as exc:
|
|
4435
|
+
raise QuickInputError(str(exc)) from exc
|
|
4436
|
+
try:
|
|
4437
|
+
# No pattern: the call checks --repo, --ref and the listing, and fetches nothing.
|
|
4438
|
+
checked = fetch.run(args.repo, args.ref, args.tree_file, str(root), [])
|
|
4439
|
+
except fetch.UsageError as exc:
|
|
4440
|
+
raise QuickInputError(str(exc)) from exc
|
|
4441
|
+
owner, repo = checked["repo"].split("/", 1)
|
|
4442
|
+
fetch._start_clock(fetch.DEFAULT_TIMEOUT_SEC)
|
|
4443
|
+
|
|
4444
|
+
def read_remote(paths: list[str]) -> tuple[list[str], dict[str, str]]:
|
|
4445
|
+
out = fetch.fetch(owner, repo, checked["ref"], paths, root.absolute())
|
|
4446
|
+
return out["fetched"], {f["path"]: f["detail"] for f in out["failed"]}
|
|
4447
|
+
return read_remote
|
|
4448
|
+
|
|
4449
|
+
|
|
4450
|
+
def _follow(args: argparse.Namespace, result: dict) -> dict:
|
|
4451
|
+
"""--follow: read each module_file `unlisted[]` names as one more
|
|
4452
|
+
--follow-file and parse the files again, round after round, until no
|
|
4453
|
+
module is left untried or FOLLOW_ROUNDS have run. Raises
|
|
4454
|
+
QuickInputError."""
|
|
4455
|
+
read = _follow_reader(args)
|
|
4456
|
+
tried = set(args.follow_file or [])
|
|
4457
|
+
followed: list[str] = []
|
|
4458
|
+
missed: dict[str, str] = {}
|
|
4459
|
+
for _ in range(FOLLOW_ROUNDS):
|
|
4460
|
+
named = [path for path in dict.fromkeys(item["module_file"] for item in result["unlisted"])
|
|
4461
|
+
if path and path not in tried]
|
|
4462
|
+
if not named:
|
|
4463
|
+
break
|
|
4464
|
+
tried.update(named)
|
|
4465
|
+
got, failed = read(named)
|
|
4466
|
+
missed.update(failed)
|
|
4467
|
+
if got:
|
|
4468
|
+
followed += got
|
|
4469
|
+
files = argparse.Namespace(**{**vars(args), "follow_file": [*(args.follow_file or []), *followed]})
|
|
4470
|
+
result = extract(_quick_files_payload(files))
|
|
4471
|
+
left = [path for path in dict.fromkeys(item["module_file"] for item in result["unlisted"])
|
|
4472
|
+
if path and path not in tried]
|
|
4473
|
+
result["warnings"] += [f"--follow could not read {path} ({why}), so the names it passes on are missing"
|
|
4474
|
+
for path, why in missed.items()]
|
|
4475
|
+
if left:
|
|
4476
|
+
result["warnings"].append(f"--follow stopped after {FOLLOW_ROUNDS} rounds with modules left to read: "
|
|
4477
|
+
f"{', '.join(left)}")
|
|
4478
|
+
result["followed"] = followed
|
|
4479
|
+
return result
|
|
4480
|
+
|
|
4481
|
+
|
|
4482
|
+
def _main_entries(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
4483
|
+
"""`--mode entries`: the entry files of one package, as JSON on stdout."""
|
|
4484
|
+
if args.manifest_file is not None or args.entry_file or args.follow_file:
|
|
4485
|
+
parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
|
|
4486
|
+
given = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
|
|
4487
|
+
if getattr(args, flag) is not None and flag not in QUICK_FILE_FLAGS]
|
|
4488
|
+
if given:
|
|
4489
|
+
parser.error(f"{', '.join(given)}: full mode only (--mode full)")
|
|
4490
|
+
if not args.language or len(args.language) != 1:
|
|
4491
|
+
parser.error("--mode entries takes one --language")
|
|
4492
|
+
if args.tree_file is None or args.source_root is None:
|
|
4493
|
+
parser.error("--mode entries needs --tree-file and --source-root")
|
|
4494
|
+
root = Path(args.source_root)
|
|
4495
|
+
try:
|
|
4496
|
+
if root.exists() and not root.is_dir():
|
|
4497
|
+
raise QuickInputError(f"--source-root is not a folder: {args.source_root}")
|
|
4498
|
+
tree, truncated = _read_listing(args.tree_file)
|
|
4499
|
+
result = find_entries(args.language[0], tree, root, args.scope)
|
|
4500
|
+
if "_error" in result:
|
|
4501
|
+
print(json.dumps(result, indent=2))
|
|
4502
|
+
return 1
|
|
4503
|
+
if truncated:
|
|
4504
|
+
result["warnings"].append("the listing was cut short (a very large tree): an entry file may be "
|
|
4505
|
+
"missing from it")
|
|
4506
|
+
if args.fetch_list is not None:
|
|
4507
|
+
_write_fetch_list(args.fetch_list, result["entry_files"])
|
|
4508
|
+
except QuickInputError as exc:
|
|
4509
|
+
sys.stderr.write(f"error: {exc}\n")
|
|
4510
|
+
return 2
|
|
4511
|
+
print(json.dumps(result, indent=2))
|
|
4512
|
+
return 0
|
|
4513
|
+
|
|
4514
|
+
|
|
4515
|
+
def main(argv: list[str]) -> int:
|
|
4516
|
+
parser = _build_parser()
|
|
4517
|
+
args = parser.parse_args(argv)
|
|
4518
|
+
files = args.manifest_file is not None or bool(args.entry_file) or bool(args.follow_file)
|
|
4519
|
+
listing = [f"--{flag.replace('_', '-')}" for flag in LISTING_FLAGS if getattr(args, flag) is not None]
|
|
4520
|
+
following = [f"--{flag}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
|
|
4521
|
+
if following and (args.mode != "quick" or not files):
|
|
4522
|
+
parser.error(f"{', '.join(following)}: quick mode only, with --manifest-file or --entry-file")
|
|
4523
|
+
if args.mode == "full":
|
|
4524
|
+
if files:
|
|
4525
|
+
parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
|
|
4526
|
+
if listing:
|
|
4527
|
+
parser.error(f"{', '.join(listing)}: entries or quick mode only")
|
|
4528
|
+
return _main_full(args)
|
|
4529
|
+
if args.mode == "entries":
|
|
4530
|
+
return _main_entries(args, parser)
|
|
4531
|
+
if args.scope is not None:
|
|
4532
|
+
parser.error("--scope: entries mode only (--mode entries)")
|
|
4533
|
+
if listing and not files:
|
|
4534
|
+
parser.error(f"{', '.join(listing)}: quick mode takes them only with --manifest-file or --entry-file")
|
|
4535
|
+
given = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
|
|
4536
|
+
if getattr(args, flag) is not None and not (files and flag in QUICK_FILE_FLAGS)]
|
|
4537
|
+
if given:
|
|
4538
|
+
hint = ""
|
|
4539
|
+
if any(flag in given for flag in ("--source-root", "--language")):
|
|
4540
|
+
hint = " (quick mode takes --source-root and --language only with --manifest-file or --entry-file)"
|
|
4541
|
+
parser.error(f"{', '.join(given)}: full mode only (--mode full){hint}")
|
|
4542
|
+
|
|
4543
|
+
if files:
|
|
4544
|
+
if not args.language or len(args.language) != 1:
|
|
4545
|
+
parser.error("--manifest-file and --entry-file take one --language")
|
|
4546
|
+
if (args.repo is not None or args.ref is not None) and not args.follow:
|
|
4547
|
+
parser.error("--repo and --ref fetch the modules --follow reads, so pass --follow")
|
|
4548
|
+
if args.ref is not None and args.repo is None:
|
|
4549
|
+
parser.error("--ref is the ref of --repo, so pass --repo")
|
|
4550
|
+
if args.follow and (args.tree_file is None or args.source_root is None):
|
|
4551
|
+
parser.error("--follow finds each module in --tree-file and reads it under --source-root: pass both")
|
|
4552
|
+
try:
|
|
4553
|
+
payload = _quick_files_payload(args)
|
|
4554
|
+
result = extract(payload)
|
|
4555
|
+
if args.follow and "_error" not in result:
|
|
4556
|
+
result = _follow(args, result)
|
|
4557
|
+
if args.fetch_list is not None and "_error" not in result:
|
|
4558
|
+
follow = [item["module_file"] for item in result["unlisted"] if item["module_file"]]
|
|
4559
|
+
_write_fetch_list(args.fetch_list, list(dict.fromkeys(follow)))
|
|
4560
|
+
except QuickInputError as exc:
|
|
4561
|
+
sys.stderr.write(f"error: {exc}\n")
|
|
4562
|
+
return 2
|
|
4563
|
+
print(json.dumps(result, indent=2))
|
|
4564
|
+
return 1 if "_error" in result else 0
|
|
566
4565
|
|
|
567
4566
|
raw = sys.stdin.read()
|
|
568
4567
|
if not raw.strip():
|