bmad-module-skill-forge 2.2.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +49 -49
- package/docs/404.md +10 -8
- package/docs/_data/pinned.yaml +22 -13
- package/docs/_internal/STABILITY.md +58 -35
- package/docs/agents.md +39 -53
- package/docs/architecture.md +65 -51
- package/docs/bmad-synergy.md +53 -51
- package/docs/campaign.md +127 -75
- package/docs/concepts.md +34 -28
- package/docs/examples.md +70 -68
- package/docs/forge-auto.md +34 -31
- package/docs/getting-started.md +118 -57
- package/docs/how-it-works.md +25 -21
- package/docs/index.md +15 -15
- package/docs/skill-model.md +70 -59
- package/docs/troubleshooting.md +143 -18
- package/docs/verifying-a-skill.md +87 -57
- package/docs/why-skf.md +24 -22
- package/docs/workflows.md +274 -110
- package/package.json +18 -10
- package/src/forger/preferences.yaml +8 -1
- package/src/knowledge/agentskills-spec.md +2 -0
- package/src/knowledge/ccc-bridge.md +40 -24
- package/src/knowledge/confidence-tiers.md +9 -5
- package/src/knowledge/overview.md +3 -3
- package/src/knowledge/progressive-capability.md +8 -8
- package/src/knowledge/provenance-tracking.md +9 -5
- package/src/knowledge/qmd-registry.md +3 -3
- package/src/knowledge/skf-knowledge-index.csv +2 -2
- package/src/knowledge/skill-lifecycle.md +18 -11
- package/src/knowledge/split-body-strategy.md +0 -3
- package/src/knowledge/tool-resolution.md +4 -2
- package/src/knowledge/version-paths.md +101 -15
- package/src/module-help.csv +3 -3
- package/src/module.yaml +10 -0
- package/src/shared/data/ast-grep-recipes.yaml +608 -0
- package/src/shared/data/ide-context-files.json +30 -0
- package/src/shared/health-check.md +31 -59
- package/src/shared/references/description-guard-protocol.md +12 -9
- package/src/shared/references/feasibility-report-schema.md +19 -13
- package/src/shared/references/headless-gate-convention.md +55 -3
- package/src/shared/references/output-contract-schema.md +19 -8
- package/src/shared/references/pipeline-contracts.md +94 -29
- package/src/shared/references/tessl-review.md +77 -0
- package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
- package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
- package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
- package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
- package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
- package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
- package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
- package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
- package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
- package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
- package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
- package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
- package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
- package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
- package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
- package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
- package/src/shared/scripts/skf-atomic-write.py +233 -64
- package/src/shared/scripts/skf-build-change-manifest.py +2753 -28
- package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
- package/src/shared/scripts/skf-check-unit-records.py +184 -0
- package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
- package/src/shared/scripts/skf-classify-changed-files.py +772 -0
- package/src/shared/scripts/skf-comention-pairs.py +1333 -123
- package/src/shared/scripts/skf-compare-file-hashes.py +267 -105
- package/src/shared/scripts/skf-count-imports.py +1867 -0
- package/src/shared/scripts/skf-count-tokens.py +1 -1
- package/src/shared/scripts/skf-derive-assembly-shape.py +21 -3
- package/src/shared/scripts/skf-description-guard.py +99 -11
- package/src/shared/scripts/skf-detect-docs.py +694 -2
- package/src/shared/scripts/skf-detect-language.py +486 -107
- package/src/shared/scripts/skf-detect-registry.py +887 -0
- package/src/shared/scripts/skf-detect-scripts-assets.py +197 -55
- package/src/shared/scripts/skf-detect-tools.py +209 -27
- package/src/shared/scripts/skf-detect-workspaces.py +506 -30
- package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
- package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
- package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
- package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
- package/src/shared/scripts/skf-extract-public-api.py +5247 -96
- package/src/shared/scripts/skf-extraction-inventory.py +1280 -0
- package/src/shared/scripts/skf-extraction-snapshot.py +664 -0
- package/src/shared/scripts/skf-fetch-temporal.py +674 -0
- package/src/shared/scripts/skf-find-cycles.py +97 -29
- package/src/shared/scripts/skf-find-test-report.py +476 -0
- package/src/shared/scripts/skf-forge-tier-rw.py +469 -205
- package/src/shared/scripts/skf-github-fetch.py +365 -0
- package/src/shared/scripts/skf-github-probe.py +700 -0
- package/src/shared/scripts/skf-hash-content.py +180 -20
- package/src/shared/scripts/skf-language-corpora.py +1 -1
- package/src/shared/scripts/skf-load-provenance.py +161 -27
- package/src/shared/scripts/skf-manifest-ops.py +1 -1
- package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
- package/src/shared/scripts/skf-merge-doc-urls.py +143 -20
- package/src/shared/scripts/skf-names-present.py +221 -0
- package/src/shared/scripts/skf-pair-intersect.py +119 -28
- package/src/shared/scripts/skf-parse-gaps.py +1384 -0
- package/src/shared/scripts/skf-preapply.py +1 -1
- package/src/shared/scripts/skf-preflight.py +176 -28
- package/src/shared/scripts/skf-promote-staged.py +430 -0
- package/src/shared/scripts/skf-provenance-gap-dispatch.py +64 -8
- package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
- package/src/shared/scripts/skf-quick-batch.py +774 -0
- package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
- package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
- package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
- package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
- package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
- package/src/shared/scripts/skf-resolve-authoritative-files.py +155 -14
- package/src/shared/scripts/skf-resolve-package.py +523 -106
- package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
- package/src/shared/scripts/skf-run-lock.py +577 -0
- package/src/shared/scripts/skf-scan-manifests.py +998 -57
- package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
- package/src/shared/scripts/skf-severity-classify.py +712 -108
- package/src/shared/scripts/skf-shape-detect.py +298 -204
- package/src/shared/scripts/skf-shard-body.py +9 -8
- package/src/shared/scripts/skf-skill-inventory.py +2412 -89
- package/src/shared/scripts/skf-skills-module.py +472 -0
- package/src/shared/scripts/skf-source-tree.py +1796 -0
- package/src/shared/scripts/skf-structural-diff.py +787 -121
- package/src/shared/scripts/skf-tessl-review.py +706 -0
- package/src/shared/scripts/skf-update-active-symlink.py +120 -67
- package/src/shared/scripts/skf-update-run-state.py +624 -0
- package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
- package/src/shared/scripts/skf-validate-brief-schema.py +23 -2
- package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
- package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
- package/src/shared/scripts/skf-validate-output.py +712 -64
- package/src/shared/scripts/skf-validate-pins.py +1 -1
- package/src/shared/scripts/skf-verify-no-trace.py +116 -10
- package/src/shared/scripts/skf-verify-provenance-completeness.py +3945 -152
- package/src/shared/scripts/skf-write-skill-brief.py +406 -25
- package/src/shared/tool-requirements.yaml +201 -0
- package/src/skf-analyze-source/SKILL.md +41 -36
- package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
- package/src/skf-analyze-source/customize.toml +36 -24
- package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
- package/src/skf-analyze-source/references/continue.md +2 -13
- package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
- package/src/skf-analyze-source/references/generate-briefs.md +121 -55
- package/src/skf-analyze-source/references/headless-contract.md +42 -15
- package/src/skf-analyze-source/references/identify-units.md +96 -110
- package/src/skf-analyze-source/references/init.md +73 -71
- package/src/skf-analyze-source/references/map-and-detect.md +123 -105
- package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
- package/src/skf-analyze-source/references/recommend.md +51 -70
- package/src/skf-analyze-source/references/scan-project.md +60 -84
- package/src/skf-analyze-source/references/scan-root.md +39 -0
- package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
- package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
- package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
- package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
- package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
- package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
- package/src/skf-audit-skill/SKILL.md +43 -51
- package/src/skf-audit-skill/assets/drift-report-template.md +28 -3
- package/src/skf-audit-skill/customize.toml +41 -20
- package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
- package/src/skf-audit-skill/references/doc-drift.md +141 -0
- package/src/skf-audit-skill/references/headless-contract.md +29 -0
- package/src/skf-audit-skill/references/health-check.md +2 -0
- package/src/skf-audit-skill/references/init.md +149 -210
- package/src/skf-audit-skill/references/re-index.md +103 -110
- package/src/skf-audit-skill/references/report.md +96 -33
- package/src/skf-audit-skill/references/semantic-diff.md +34 -25
- package/src/skf-audit-skill/references/severity-classify.md +79 -65
- package/src/skf-audit-skill/references/severity-rules.md +15 -6
- package/src/skf-audit-skill/references/structural-diff.md +80 -90
- package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
- package/src/skf-audit-skill/scripts/render-drift-tables.py +712 -0
- package/src/skf-brief-skill/SKILL.md +37 -26
- package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
- package/src/skf-brief-skill/assets/scope-templates.md +18 -25
- package/src/skf-brief-skill/assets/skill-brief-schema.md +30 -23
- package/src/skf-brief-skill/customize.toml +48 -20
- package/src/skf-brief-skill/references/analyze-target.md +144 -127
- package/src/skf-brief-skill/references/confirm-brief.md +11 -9
- package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
- package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
- package/src/skf-brief-skill/references/gather-intent.md +87 -182
- package/src/skf-brief-skill/references/headless-args.md +84 -16
- package/src/skf-brief-skill/references/health-check.md +1 -1
- package/src/skf-brief-skill/references/invocation-contract.md +18 -13
- package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
- package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
- package/src/skf-brief-skill/references/scope-definition.md +123 -47
- package/src/skf-brief-skill/references/step-auto-brief.md +26 -100
- package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
- package/src/skf-brief-skill/references/version-resolution.md +7 -10
- package/src/skf-brief-skill/references/write-brief.md +66 -57
- package/src/skf-campaign/SKILL.md +31 -34
- package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
- package/src/skf-campaign/customize.toml +47 -16
- package/src/skf-campaign/references/campaign-contracts.md +39 -29
- package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
- package/src/skf-campaign/references/health-check.md +7 -16
- package/src/skf-campaign/references/step-01-setup.md +46 -84
- package/src/skf-campaign/references/step-02-strategy.md +31 -30
- package/src/skf-campaign/references/step-03-pins.md +26 -18
- package/src/skf-campaign/references/step-04-provenance.md +24 -23
- package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
- package/src/skf-campaign/references/step-06-batch.md +41 -34
- package/src/skf-campaign/references/step-07-capstone.md +40 -25
- package/src/skf-campaign/references/step-08-verify.md +28 -29
- package/src/skf-campaign/references/step-09-refine.md +29 -30
- package/src/skf-campaign/references/step-10-export.md +48 -36
- package/src/skf-campaign/references/step-11-maintenance.md +31 -34
- package/src/skf-campaign/references/step-resume.md +37 -68
- package/src/skf-campaign/scripts/campaign-deps.py +57 -18
- package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
- package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
- package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
- package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
- package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
- package/src/skf-campaign/scripts/campaign-report.py +204 -37
- package/src/skf-campaign/scripts/campaign-state.py +1128 -0
- package/src/skf-campaign/scripts/campaign-status.py +10 -74
- package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
- package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
- package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
- package/src/skf-campaign/templates/campaign-report-template.md +5 -1
- package/src/skf-campaign/templates/kickoff-template.md +0 -14
- package/src/skf-create-skill/SKILL.md +27 -20
- package/src/skf-create-skill/assets/compile-assembly-rules.md +11 -8
- package/src/skf-create-skill/assets/skill-sections.md +21 -11
- package/src/skf-create-skill/customize.toml +20 -8
- package/src/skf-create-skill/references/authoritative-files-protocol.md +36 -58
- package/src/skf-create-skill/references/batch-mode.md +81 -0
- package/src/skf-create-skill/references/ccc-index-check.md +27 -0
- package/src/skf-create-skill/references/compile.md +119 -70
- package/src/skf-create-skill/references/component-extraction.md +113 -91
- package/src/skf-create-skill/references/enrich.md +41 -23
- package/src/skf-create-skill/references/entry-points-by-hand.md +37 -0
- package/src/skf-create-skill/references/extract.md +204 -162
- package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
- package/src/skf-create-skill/references/extraction-patterns-tracing.md +11 -11
- package/src/skf-create-skill/references/extraction-patterns.md +588 -201
- package/src/skf-create-skill/references/generate-artifacts.md +145 -93
- package/src/skf-create-skill/references/health-check.md +1 -1
- package/src/skf-create-skill/references/load-brief.md +53 -36
- package/src/skf-create-skill/references/relabel-rule.md +12 -0
- package/src/skf-create-skill/references/report.md +37 -57
- package/src/skf-create-skill/references/source-resolution-protocols.md +58 -169
- package/src/skf-create-skill/references/step-auto-shard.md +20 -93
- package/src/skf-create-skill/references/step-doc-rot.md +44 -60
- package/src/skf-create-skill/references/step-doc-sources.md +28 -19
- package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
- package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
- package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
- package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
- package/src/skf-create-skill/references/validate.md +173 -99
- package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
- package/src/skf-create-stack-skill/SKILL.md +29 -56
- package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
- package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
- package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
- package/src/skf-create-stack-skill/customize.toml +23 -12
- package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
- package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
- package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
- package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
- package/src/skf-create-stack-skill/references/generate-output.md +138 -128
- package/src/skf-create-stack-skill/references/init.md +90 -32
- package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
- package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
- package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
- package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
- package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
- package/src/skf-create-stack-skill/references/report.md +39 -33
- package/src/skf-create-stack-skill/references/validate.md +20 -39
- package/src/skf-drop-skill/SKILL.md +28 -51
- package/src/skf-drop-skill/customize.toml +44 -30
- package/src/skf-drop-skill/references/execute.md +83 -146
- package/src/skf-drop-skill/references/health-check.md +2 -0
- package/src/skf-drop-skill/references/invocation-contract.md +64 -0
- package/src/skf-drop-skill/references/report.md +46 -30
- package/src/skf-drop-skill/references/select.md +122 -124
- package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
- package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
- package/src/skf-export-skill/SKILL.md +40 -62
- package/src/skf-export-skill/assets/managed-section-format.md +14 -50
- package/src/skf-export-skill/assets/snippet-format.md +3 -1
- package/src/skf-export-skill/customize.toml +42 -13
- package/src/skf-export-skill/references/generate-snippet.md +54 -46
- package/src/skf-export-skill/references/health-check.md +2 -0
- package/src/skf-export-skill/references/invocation-contract.md +15 -0
- package/src/skf-export-skill/references/load-skill.md +88 -62
- package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
- package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
- package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
- package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
- package/src/skf-export-skill/references/package.md +15 -73
- package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
- package/src/skf-export-skill/references/result-envelope.md +31 -16
- package/src/skf-export-skill/references/summary.md +41 -17
- package/src/skf-export-skill/references/token-report.md +4 -12
- package/src/skf-export-skill/references/update-context.md +118 -237
- package/src/skf-forger/SKILL.md +61 -37
- package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
- package/src/skf-forger/customize.toml +43 -0
- package/src/skf-forger/references/pipeline-mode.md +73 -18
- package/src/skf-forger/scripts/forge-status.py +299 -0
- package/src/skf-forger/scripts/parse-pipeline.py +322 -54
- package/src/skf-forger/scripts/pipeline-gate.py +428 -0
- package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
- package/src/skf-quick-skill/SKILL.md +54 -53
- package/src/skf-quick-skill/assets/skill-template.md +4 -2
- package/src/skf-quick-skill/customize.toml +51 -20
- package/src/skf-quick-skill/references/batch-mode.md +74 -21
- package/src/skf-quick-skill/references/compile.md +30 -31
- package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
- package/src/skf-quick-skill/references/finalize.md +69 -18
- package/src/skf-quick-skill/references/halt-contract.md +41 -29
- package/src/skf-quick-skill/references/health-check.md +5 -3
- package/src/skf-quick-skill/references/quick-extract.md +114 -45
- package/src/skf-quick-skill/references/registry-resolution.md +9 -64
- package/src/skf-quick-skill/references/resolve-target.md +116 -66
- package/src/skf-quick-skill/references/write-and-validate.md +79 -40
- package/src/skf-refine-architecture/SKILL.md +43 -53
- package/src/skf-refine-architecture/customize.toml +49 -20
- package/src/skf-refine-architecture/references/compile.md +97 -54
- package/src/skf-refine-architecture/references/exit-codes.md +30 -0
- package/src/skf-refine-architecture/references/finding-storage.md +21 -0
- package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
- package/src/skf-refine-architecture/references/improvements.md +14 -20
- package/src/skf-refine-architecture/references/init.md +98 -57
- package/src/skf-refine-architecture/references/issue-detection.md +32 -39
- package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
- package/src/skf-refine-architecture/references/report.md +65 -54
- package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
- package/src/skf-rename-skill/SKILL.md +19 -54
- package/src/skf-rename-skill/customize.toml +32 -26
- package/src/skf-rename-skill/references/execute.md +230 -184
- package/src/skf-rename-skill/references/exit-codes.md +7 -7
- package/src/skf-rename-skill/references/health-check.md +1 -7
- package/src/skf-rename-skill/references/invocation-contract.md +25 -0
- package/src/skf-rename-skill/references/report.md +63 -52
- package/src/skf-rename-skill/references/select.md +197 -115
- package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
- package/src/skf-setup/SKILL.md +26 -26
- package/src/skf-setup/customize.toml +14 -10
- package/src/skf-setup/references/auto-index.md +50 -63
- package/src/skf-setup/references/ccc-index.md +29 -90
- package/src/skf-setup/references/detect-and-tier.md +58 -67
- package/src/skf-setup/references/health-check.md +2 -0
- package/src/skf-setup/references/invocation-contract.md +14 -0
- package/src/skf-setup/references/report.md +43 -166
- package/src/skf-setup/references/tier-rules.md +5 -5
- package/src/skf-setup/references/write-config.md +34 -57
- package/src/skf-test-skill/SKILL.md +17 -57
- package/src/skf-test-skill/assets/output-section-formats.md +33 -34
- package/src/skf-test-skill/customize.toml +27 -13
- package/src/skf-test-skill/references/coherence-check.md +134 -62
- package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
- package/src/skf-test-skill/references/coverage-check.md +209 -264
- package/src/skf-test-skill/references/detect-mode.md +5 -5
- package/src/skf-test-skill/references/external-validators.md +94 -82
- package/src/skf-test-skill/references/health-check.md +6 -7
- package/src/skf-test-skill/references/init.md +164 -91
- package/src/skf-test-skill/references/invocation-contract.md +66 -0
- package/src/skf-test-skill/references/migration-section-rules.md +5 -2
- package/src/skf-test-skill/references/report.md +164 -142
- package/src/skf-test-skill/references/score.md +80 -107
- package/src/skf-test-skill/references/scoring-rules.md +42 -155
- package/src/skf-test-skill/references/source-access-protocol.md +18 -94
- package/src/skf-test-skill/references/step-hard-gate.md +51 -33
- package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
- package/src/skf-test-skill/scripts/build-result-context.py +380 -0
- package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
- package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
- package/src/skf-test-skill/scripts/compute-score.py +223 -45
- package/src/skf-test-skill/scripts/gap-ledger.py +1666 -0
- package/src/skf-test-skill/scripts/hard-gate.py +203 -0
- package/src/skf-test-skill/scripts/load-coverage-inputs.py +1198 -0
- package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
- package/src/skf-test-skill/scripts/reconcile-coverage.py +547 -79
- package/src/skf-test-skill/scripts/score-signatures.py +479 -0
- package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
- package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
- package/src/skf-test-skill/scripts/verify-declared-numerator.py +133 -56
- package/src/skf-test-skill/templates/test-report-template.md +21 -15
- package/src/skf-update-skill/SKILL.md +38 -25
- package/src/skf-update-skill/customize.toml +27 -12
- package/src/skf-update-skill/references/detect-changes.md +253 -223
- package/src/skf-update-skill/references/gap-driven.md +336 -0
- package/src/skf-update-skill/references/health-check.md +31 -5
- package/src/skf-update-skill/references/init.md +254 -84
- package/src/skf-update-skill/references/invocation-contract.md +12 -0
- package/src/skf-update-skill/references/manual-section-rules.md +6 -23
- package/src/skf-update-skill/references/merge.md +182 -49
- package/src/skf-update-skill/references/re-extract.md +53 -262
- package/src/skf-update-skill/references/report.md +102 -35
- package/src/skf-update-skill/references/write.md +357 -155
- package/src/skf-update-skill/scripts/skf-new-file-diff.py +183 -46
- package/src/skf-verify-stack/SKILL.md +23 -42
- package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
- package/src/skf-verify-stack/customize.toml +55 -22
- package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
- package/src/skf-verify-stack/references/coverage.md +67 -64
- package/src/skf-verify-stack/references/exit-codes.md +21 -8
- package/src/skf-verify-stack/references/init.md +107 -62
- package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
- package/src/skf-verify-stack/references/integrations.md +106 -65
- package/src/skf-verify-stack/references/report.md +59 -42
- package/src/skf-verify-stack/references/requirements.md +32 -9
- package/src/skf-verify-stack/references/synthesize.md +53 -39
- package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
- package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
- package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
- package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
- package/tools/cli/commands/install.js +11 -4
- package/tools/cli/commands/status.js +15 -3
- package/tools/cli/commands/update.js +11 -4
- package/tools/cli/lib/tool-check.js +320 -0
- package/tools/cli/lib/version-check.js +22 -9
- package/tools/cli/skf-cli.js +4 -1
- package/.gitattributes +0 -16
- package/.nvmrc +0 -1
- package/docs/_internal/RELEASING.md +0 -679
- package/release-audits/v1.0.0-launch-audit.md +0 -728
- package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
- package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
- package/src/skf-campaign/manifest.yaml +0 -15
- package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
- package/src/skf-create-skill/references/ecosystem-check.md +0 -74
- package/src/skf-drop-skill/references/headless-contract.md +0 -18
- package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
- package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
- package/src/skf-update-skill/references/validate.md +0 -124
- package/tools/validate-docs-drift.js +0 -233
- package/tools/validate-docs-links.js +0 -665
|
@@ -0,0 +1,1867 @@
|
|
|
1
|
+
# /// script
|
|
2
|
+
# requires-python = ">=3.11"
|
|
3
|
+
# dependencies = []
|
|
4
|
+
# ///
|
|
5
|
+
"""SKF Count Imports: count, for each dependency, the source files that import it.
|
|
6
|
+
|
|
7
|
+
Two workflows need the same fact, which has one right answer per tree:
|
|
8
|
+
|
|
9
|
+
1. **skf-create-stack-skill** ranks the project's dependencies by how many
|
|
10
|
+
files import them (2 or more files: included by default), and hands
|
|
11
|
+
each library's file list to `skf-pair-intersect.py`.
|
|
12
|
+
2. **skf-analyze-source** maps which units import which (Imports From and
|
|
13
|
+
Imported By), gives co-imported units to `skf-pair-intersect.py` and
|
|
14
|
+
the unit-to-unit edges to `skf-find-cycles.py`, and reads the graph
|
|
15
|
+
through `summary`, which leaves out the file lists it does not need.
|
|
16
|
+
|
|
17
|
+
A grep with no module boundary counts `react-dom` and `react-router` as
|
|
18
|
+
`react`, and a Python distribution is often imported under another name
|
|
19
|
+
(`PyYAML` as `yaml`, `Pillow` as `PIL`), so hand counts drift from run to
|
|
20
|
+
run. This helper reads each source file once, parses its import statements
|
|
21
|
+
per language, matches them on module boundaries and maps distribution names
|
|
22
|
+
to import names.
|
|
23
|
+
|
|
24
|
+
Subcommands:
|
|
25
|
+
count <root> [--deps <json-file-or-'-'>] [--units <json-file-or-'-'>]
|
|
26
|
+
[--threshold N] [--exclude GLOB]... [--format full|libraries]
|
|
27
|
+
[--relative-to DIR]
|
|
28
|
+
|
|
29
|
+
Walk <root>, read every source file of the languages the targets need,
|
|
30
|
+
and count the files that import each target. At least one of --deps
|
|
31
|
+
and --units is required; only one of them may read stdin ('-').
|
|
32
|
+
|
|
33
|
+
--deps accepts any of:
|
|
34
|
+
- the `skf-scan-manifests.py scan` envelope, as it is: every
|
|
35
|
+
`manifests[].deps[]` entry becomes a dependency of that manifest's
|
|
36
|
+
ecosystem (a dependency tagged `scope: dev` stays dev only when no
|
|
37
|
+
manifest lists it as a runtime dependency);
|
|
38
|
+
- {"dependencies": [<dep>, ...]} or a bare [<dep>, ...] list, where
|
|
39
|
+
<dep> is a name string or
|
|
40
|
+
{"name": "...", "ecosystem": "npm|python|...", "modules": [...],
|
|
41
|
+
"scope": "runtime|dev"}. `ecosystem` limits the match to that
|
|
42
|
+
ecosystem's source files (omitted: every language, with the name
|
|
43
|
+
and its Python-style guesses). `modules` gives the import names
|
|
44
|
+
explicitly and skips the name rules below.
|
|
45
|
+
|
|
46
|
+
--units accepts {"units": [<unit>, ...]} or a bare list, where <unit> is
|
|
47
|
+
{"name": "...", "path": "<dir relative to root>", "modules": [...],
|
|
48
|
+
"manifest_name": "...", "ecosystem": "..."}.
|
|
49
|
+
A unit is an internal part of the tree. Each scanned file belongs to
|
|
50
|
+
the unit with the longest `path` that contains it. A file imports a
|
|
51
|
+
unit when one of its imports matches the unit's import names, or when
|
|
52
|
+
a relative import (JS/TS `./` and `../`, Python leading dots, Ruby
|
|
53
|
+
`require_relative`) resolves into the unit's `path`. A unit's own
|
|
54
|
+
files never count as importing it. A unit's import names are its
|
|
55
|
+
`modules` when given; without them, what the unit declares: the
|
|
56
|
+
packages its own .java, .kt, .groovy and .scala files declare for a
|
|
57
|
+
Maven or Gradle unit, the namespaces its composer.json autoloads for
|
|
58
|
+
a Composer unit, the targets its Package.swift declares for a Swift
|
|
59
|
+
unit, else its `manifest_name` by the dependency rules below (an npm
|
|
60
|
+
package by its name, a crate with `-` as `_`, a Go module by its
|
|
61
|
+
path); none for a unit with neither, which only relative imports
|
|
62
|
+
reach. With --units, a dependency whose import names are a unit's is
|
|
63
|
+
that unit, not an external dependency: it is left out.
|
|
64
|
+
|
|
65
|
+
--threshold N a dependency is `above_threshold` when file_count >= N
|
|
66
|
+
(default 2: two or more importing files)
|
|
67
|
+
--exclude GLOB another path to leave out, added to the defaults below
|
|
68
|
+
(repeatable)
|
|
69
|
+
--format `full` (default) prints the envelope below;
|
|
70
|
+
`libraries` prints [{"name", "files": [<path>, ...]}]
|
|
71
|
+
for every dependency, then every unit: the input of
|
|
72
|
+
`skf-pair-intersect.py intersect --libraries -`.
|
|
73
|
+
--relative-to DIR
|
|
74
|
+
write each `files[].path` relative to DIR, a folder
|
|
75
|
+
that holds <root>, instead of to <root>: a caller that
|
|
76
|
+
scans one package of a project keeps project-relative
|
|
77
|
+
paths. Excludes and a unit's `path` stay relative to
|
|
78
|
+
<root>.
|
|
79
|
+
|
|
80
|
+
Emit JSON (`full`):
|
|
81
|
+
{
|
|
82
|
+
"threshold": 2,
|
|
83
|
+
"files_scanned": N,
|
|
84
|
+
"dependencies": [
|
|
85
|
+
{"name": "react", "ecosystem": "npm",
|
|
86
|
+
"import_names": ["react"], "resolution": "exact",
|
|
87
|
+
"files": [{"path": "src/App.tsx", "line": 1}, ...],
|
|
88
|
+
"file_count": N, "above_threshold": true},
|
|
89
|
+
...
|
|
90
|
+
],
|
|
91
|
+
"unresolved": [
|
|
92
|
+
{"name": "...", "ecosystem": "...", "import_names": [...],
|
|
93
|
+
"reason": "guess-unmatched|no-import-name|unsupported-ecosystem"},
|
|
94
|
+
...
|
|
95
|
+
],
|
|
96
|
+
"units": [
|
|
97
|
+
{"name": "...", "path": "...", "import_names": [...],
|
|
98
|
+
"files": [{"path": "...", "line": N}, ...], "file_count": N,
|
|
99
|
+
"imported_by": [...], "imports_from": [...],
|
|
100
|
+
"external_deps": [...]},
|
|
101
|
+
...
|
|
102
|
+
],
|
|
103
|
+
"edges": [["<importing unit>", "<imported unit>"], ...],
|
|
104
|
+
"edge_files": [
|
|
105
|
+
{"from": "<importing unit>", "to": "<imported unit>",
|
|
106
|
+
"files": [{"path": "...", "line": N}, ...]},
|
|
107
|
+
...
|
|
108
|
+
],
|
|
109
|
+
"warnings": ["..."]
|
|
110
|
+
}
|
|
111
|
+
`units`, `edges` and `edge_files` are present only with --units;
|
|
112
|
+
`warnings` only when a file could not be read. A dependency entry
|
|
113
|
+
carries `scope: "dev"` when its input said so. `files` lists each
|
|
114
|
+
importing file once, with the first line that imports the target,
|
|
115
|
+
sorted by path; an `edge_files` entry lists the files of the
|
|
116
|
+
importing unit that import the other one, sorted by `from`, then
|
|
117
|
+
`to`. A unit's `external_deps` are the dependencies its files import.
|
|
118
|
+
`dependencies[]` is sorted by file_count (highest first), then name.
|
|
119
|
+
`unresolved[]` (sorted by name) holds the dependencies this helper
|
|
120
|
+
cannot count with confidence, for the model to judge: a guessed import
|
|
121
|
+
name that matched no file (`guess-unmatched`), no import name to try
|
|
122
|
+
(`no-import-name`), or an ecosystem it has no import rules for
|
|
123
|
+
(`unsupported-ecosystem`). `edges` is the `skf-find-cycles.py find
|
|
124
|
+
--edges` input as it is (that helper reads only the `edges` key).
|
|
125
|
+
|
|
126
|
+
summary <imports-file-or-'-'> [--manifests <json-file-or-'-'>]
|
|
127
|
+
[--max-files N]
|
|
128
|
+
|
|
129
|
+
Print the unit graph of a saved `count --units` envelope without the
|
|
130
|
+
file lists a reader of the graph does not need (on a large monorepo
|
|
131
|
+
the envelope runs to hundreds of KB, the summary to a few):
|
|
132
|
+
{
|
|
133
|
+
"units": [
|
|
134
|
+
{"name": "...", "imports_from": [...], "imported_by": [...],
|
|
135
|
+
"file_count": N, "external_dep_count": N},
|
|
136
|
+
...
|
|
137
|
+
],
|
|
138
|
+
"edge_files": [
|
|
139
|
+
{"from": "...", "to": "...", "file_count": N,
|
|
140
|
+
"files": [{"path": "...", "line": N}, ...]},
|
|
141
|
+
...
|
|
142
|
+
],
|
|
143
|
+
"umbrella_candidates": [...],
|
|
144
|
+
"manifests": [
|
|
145
|
+
{"path": "...", "name": "...", "private": true | false | null,
|
|
146
|
+
"internal_deps": [...]},
|
|
147
|
+
...
|
|
148
|
+
],
|
|
149
|
+
"warnings": ["..."]
|
|
150
|
+
}
|
|
151
|
+
A unit keeps its `file_count` (the files outside it that import it)
|
|
152
|
+
and the number of its `external_deps`; an `edge_files` entry keeps
|
|
153
|
+
its file count and its first --max-files files (default 5), so the
|
|
154
|
+
full list stays in the envelope for the edge that needs it.
|
|
155
|
+
`umbrella_candidates` and `manifests` come from --manifests, the
|
|
156
|
+
`skf-scan-manifests.py scan` envelope (present only with it), each
|
|
157
|
+
manifest with the fields the cohesion triggers read. `warnings` is
|
|
158
|
+
the envelope's, present only when it has some. Only one of the two
|
|
159
|
+
inputs may read stdin.
|
|
160
|
+
|
|
161
|
+
Resolution of a dependency's import names (`resolution`):
|
|
162
|
+
exact the ecosystem's own rule: an npm package is imported by its name,
|
|
163
|
+
a Rust crate by its name with `-` as `_`, a Go module by its path;
|
|
164
|
+
a Composer package by the namespaces its installed
|
|
165
|
+
`vendor/<name>/composer.json` autoloads
|
|
166
|
+
mapped an entry of the built-in distribution-to-import tables (`PyYAML`
|
|
167
|
+
as `yaml`, `Pillow` as `PIL`, `beautifulsoup4` as `bs4`,
|
|
168
|
+
`com.google.guava:guava` as `com.google.common`, ...)
|
|
169
|
+
guessed a naming rule that is often but not always right: a Python
|
|
170
|
+
distribution as its name with `-` as `_`, a namespace family
|
|
171
|
+
(`google-cloud-storage` as `google.cloud.storage`), `@types/react`
|
|
172
|
+
as `react`, a Symfony package as its namespace
|
|
173
|
+
(`symfony/framework-bundle` as `Symfony\\Bundle\\FrameworkBundle`),
|
|
174
|
+
a Maven group as its package, a gem as its name. A guess that
|
|
175
|
+
matches no file goes to `unresolved[]` instead of counting 0
|
|
176
|
+
explicit the `modules` the caller gave
|
|
177
|
+
|
|
178
|
+
Matching is boundary-anchored: an import binds to the dependency whose
|
|
179
|
+
import name equals the imported module or is a prefix of it at a module
|
|
180
|
+
separator (`/` for JS, Go and Ruby, `.` for Python and the JVM, `::` for
|
|
181
|
+
Rust, `\\` for PHP, whole names for Swift). `react` matches `react` and
|
|
182
|
+
`react/jsx-runtime`, never `react-dom`; `yaml` matches `yaml.loader`, never
|
|
183
|
+
`yamlordereddict`. When several import names prefix the same import, the
|
|
184
|
+
longest wins, so `opentelemetry.sdk.trace` counts for `opentelemetry-sdk`
|
|
185
|
+
and not for `opentelemetry-api`.
|
|
186
|
+
|
|
187
|
+
Import statements read per language (lines that start a comment are
|
|
188
|
+
skipped; Python files are parsed with `ast`, and one that does not parse or
|
|
189
|
+
is over 1,000,000 characters (`PYTHON_AST_MAX_CHARS`) is read line by line,
|
|
190
|
+
outside comments and triple-quoted strings):
|
|
191
|
+
js .js .jsx .mjs .cjs .ts .tsx .mts .cts .vue .svelte .astro:
|
|
192
|
+
`import ... from 'm'`, `import 'm'`, `export ... from 'm'`,
|
|
193
|
+
`import('m')`, `require('m')`
|
|
194
|
+
python .py .pyi: `import m`, `from m import n` (also read as `m.n`),
|
|
195
|
+
`importlib.import_module('m')`, `__import__('m')`
|
|
196
|
+
rust .rs: `use m::...`, `extern crate m`, and qualified paths `m::x`
|
|
197
|
+
go .go: `import "m"` and `import ( ... )` blocks
|
|
198
|
+
jvm .java .kt .kts .groovy .scala: `import m.x`, `import static m.x`
|
|
199
|
+
ruby .rb: `require 'm'`, `require_relative 'p'`
|
|
200
|
+
php .php: `use M\\X` (case-insensitive, as PHP namespaces are)
|
|
201
|
+
swift .swift: `import M`
|
|
202
|
+
Ecosystems map to languages: npm to js, python to python, rust to rust,
|
|
203
|
+
go to go, maven and gradle to jvm, ruby to ruby, composer to php, swift to
|
|
204
|
+
swift.
|
|
205
|
+
|
|
206
|
+
Excluded paths, matched against the path relative to <root> (the list in
|
|
207
|
+
`DEFAULT_EXCLUDES`, which `references/manifest-patterns.md` in
|
|
208
|
+
skf-create-stack-skill documents): a pattern ending in `/` names a directory
|
|
209
|
+
at any depth, a pattern with no `/` matches a file name at any depth, any
|
|
210
|
+
other pattern matches the whole relative path (`**` spans directories, `*`
|
|
211
|
+
and `?` stay inside one). A leading `/` or `./` anchors a pattern to <root>
|
|
212
|
+
(`./examples/` is only the top-level `examples`). Symbolic links are not
|
|
213
|
+
followed.
|
|
214
|
+
|
|
215
|
+
CLI examples:
|
|
216
|
+
uv run skf-scan-manifests.py scan . > manifests.json
|
|
217
|
+
uv run skf-count-imports.py count . --deps manifests.json
|
|
218
|
+
echo '["react", "PyYAML"]' | uv run skf-count-imports.py count . --deps -
|
|
219
|
+
uv run skf-count-imports.py count . --units units.json --format libraries \\
|
|
220
|
+
| uv run skf-pair-intersect.py intersect --libraries -
|
|
221
|
+
uv run skf-count-imports.py count . --units units.json \\
|
|
222
|
+
| uv run skf-find-cycles.py find --edges -
|
|
223
|
+
uv run skf-count-imports.py count . --units units.json --deps manifests.json > imports.json
|
|
224
|
+
uv run skf-count-imports.py summary imports.json --manifests manifests.json
|
|
225
|
+
|
|
226
|
+
Exit codes:
|
|
227
|
+
0 success (including: no dependency imported, no source file found)
|
|
228
|
+
1 user error (root not a directory, unreadable or malformed JSON input,
|
|
229
|
+
a bad dependency or unit entry, both inputs on stdin, bad --threshold,
|
|
230
|
+
a --relative-to that is not a folder holding <root>, a summary input
|
|
231
|
+
that is not a `count --units` envelope or a manifest scan, bad
|
|
232
|
+
--max-files)
|
|
233
|
+
"""
|
|
234
|
+
|
|
235
|
+
from __future__ import annotations
|
|
236
|
+
|
|
237
|
+
import argparse
|
|
238
|
+
import ast
|
|
239
|
+
import json
|
|
240
|
+
import os
|
|
241
|
+
import posixpath
|
|
242
|
+
import re
|
|
243
|
+
import sys
|
|
244
|
+
import warnings
|
|
245
|
+
from pathlib import Path
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
DEFAULT_THRESHOLD = 2
|
|
249
|
+
|
|
250
|
+
# A Python file up to this many characters is parsed with `ast`. A larger one
|
|
251
|
+
# (a generated table or protobuf module) is read line by line, because its
|
|
252
|
+
# syntax tree can take 200 times its size in memory.
|
|
253
|
+
PYTHON_AST_MAX_CHARS = 1_000_000
|
|
254
|
+
|
|
255
|
+
# Source file extensions read for each language.
|
|
256
|
+
LANGUAGE_EXTENSIONS: dict[str, tuple[str, ...]] = {
|
|
257
|
+
"js": (
|
|
258
|
+
".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte", ".astro",
|
|
259
|
+
),
|
|
260
|
+
"python": (".py", ".pyi"),
|
|
261
|
+
"rust": (".rs",),
|
|
262
|
+
"go": (".go",),
|
|
263
|
+
"jvm": (".java", ".kt", ".kts", ".groovy", ".scala"),
|
|
264
|
+
"ruby": (".rb",),
|
|
265
|
+
"php": (".php",),
|
|
266
|
+
"swift": (".swift",),
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
# `skf-scan-manifests.py` ecosystem label -> language whose imports it reads.
|
|
270
|
+
ECOSYSTEM_LANGUAGE: dict[str, str] = {
|
|
271
|
+
"npm": "js",
|
|
272
|
+
"python": "python",
|
|
273
|
+
"rust": "rust",
|
|
274
|
+
"go": "go",
|
|
275
|
+
"maven": "jvm",
|
|
276
|
+
"gradle": "jvm",
|
|
277
|
+
"ruby": "ruby",
|
|
278
|
+
"composer": "php",
|
|
279
|
+
"swift": "swift",
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
# Module separator per language; None matches whole module names only.
|
|
283
|
+
SEPARATORS: dict[str, str | None] = {
|
|
284
|
+
"js": "/",
|
|
285
|
+
"python": ".",
|
|
286
|
+
"rust": "::",
|
|
287
|
+
"go": "/",
|
|
288
|
+
"jvm": ".",
|
|
289
|
+
"ruby": "/",
|
|
290
|
+
"php": "\\",
|
|
291
|
+
"swift": None,
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
# Paths never counted, documented in
|
|
295
|
+
# src/skf-create-stack-skill/references/manifest-patterns.md "Import Counting"
|
|
296
|
+
# (test/test-skf-count-imports.py keeps the two lists equal).
|
|
297
|
+
DEFAULT_EXCLUDES: tuple[str, ...] = (
|
|
298
|
+
# hidden files and directories
|
|
299
|
+
".*/",
|
|
300
|
+
".*",
|
|
301
|
+
# tests
|
|
302
|
+
"test/",
|
|
303
|
+
"tests/",
|
|
304
|
+
"__tests__/",
|
|
305
|
+
"spec/",
|
|
306
|
+
"*.test.*",
|
|
307
|
+
"*.spec.*",
|
|
308
|
+
"*_test.*",
|
|
309
|
+
"test_*.py",
|
|
310
|
+
"*_spec.rb",
|
|
311
|
+
"conftest.py",
|
|
312
|
+
# config and build scripts
|
|
313
|
+
"*.config.*",
|
|
314
|
+
"setup.py",
|
|
315
|
+
"noxfile.py",
|
|
316
|
+
"*.gradle.kts",
|
|
317
|
+
"Package.swift",
|
|
318
|
+
# build output and vendored code
|
|
319
|
+
"dist/",
|
|
320
|
+
"build/",
|
|
321
|
+
"out/",
|
|
322
|
+
"target/",
|
|
323
|
+
"node_modules/",
|
|
324
|
+
"vendor/",
|
|
325
|
+
"venv/",
|
|
326
|
+
"Pods/",
|
|
327
|
+
"__pycache__/",
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
# --------------------------------------------------------------------------
|
|
331
|
+
# Distribution-to-import tables
|
|
332
|
+
# --------------------------------------------------------------------------
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
# Python distributions whose import name differs from the distribution name,
|
|
336
|
+
# keyed by the PEP 503 normalised name (lower case, runs of -_. as -).
|
|
337
|
+
PYTHON_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
|
|
338
|
+
"absl-py": ("absl",),
|
|
339
|
+
"apache-airflow": ("airflow",),
|
|
340
|
+
"attrs": ("attr", "attrs"),
|
|
341
|
+
"beautifulsoup4": ("bs4",),
|
|
342
|
+
"discord-py": ("discord",),
|
|
343
|
+
"django-cors-headers": ("corsheaders",),
|
|
344
|
+
"django-environ": ("environ",),
|
|
345
|
+
"django-filter": ("django_filters",),
|
|
346
|
+
"djangorestframework": ("rest_framework",),
|
|
347
|
+
"dnspython": ("dns",),
|
|
348
|
+
"faiss-cpu": ("faiss",),
|
|
349
|
+
"faiss-gpu": ("faiss",),
|
|
350
|
+
"gitpython": ("git",),
|
|
351
|
+
"google-api-python-client": ("googleapiclient",),
|
|
352
|
+
"google-auth": ("google.auth",),
|
|
353
|
+
"google-genai": ("google.genai",),
|
|
354
|
+
"google-generativeai": ("google.generativeai",),
|
|
355
|
+
"grpcio": ("grpc",),
|
|
356
|
+
"grpcio-status": ("grpc_status",),
|
|
357
|
+
"grpcio-tools": ("grpc_tools",),
|
|
358
|
+
"ipython": ("IPython",),
|
|
359
|
+
"msgpack-python": ("msgpack",),
|
|
360
|
+
"mysql-connector-python": ("mysql.connector",),
|
|
361
|
+
"mysqlclient": ("MySQLdb",),
|
|
362
|
+
"opencv-contrib-python": ("cv2",),
|
|
363
|
+
"opencv-contrib-python-headless": ("cv2",),
|
|
364
|
+
"opencv-python": ("cv2",),
|
|
365
|
+
"opencv-python-headless": ("cv2",),
|
|
366
|
+
"opentelemetry-api": ("opentelemetry",),
|
|
367
|
+
"paho-mqtt": ("paho.mqtt",),
|
|
368
|
+
"pdfminer-six": ("pdfminer",),
|
|
369
|
+
"pillow": ("PIL",),
|
|
370
|
+
"protobuf": ("google.protobuf",),
|
|
371
|
+
"psycopg2-binary": ("psycopg2",),
|
|
372
|
+
"pycairo": ("cairo",),
|
|
373
|
+
"pycryptodome": ("Crypto",),
|
|
374
|
+
"pycryptodomex": ("Cryptodome",),
|
|
375
|
+
"pygithub": ("github",),
|
|
376
|
+
"pygobject": ("gi",),
|
|
377
|
+
"pyinstaller": ("PyInstaller",),
|
|
378
|
+
"pyjwt": ("jwt",),
|
|
379
|
+
"pymongo": ("pymongo", "bson", "gridfs"),
|
|
380
|
+
"pymupdf": ("fitz", "pymupdf"),
|
|
381
|
+
"pynacl": ("nacl",),
|
|
382
|
+
"pyopenssl": ("OpenSSL",),
|
|
383
|
+
"pyqt5": ("PyQt5",),
|
|
384
|
+
"pyqt6": ("PyQt6",),
|
|
385
|
+
"pyserial": ("serial",),
|
|
386
|
+
"pyside2": ("PySide2",),
|
|
387
|
+
"pyside6": ("PySide6",),
|
|
388
|
+
"pysocks": ("socks",),
|
|
389
|
+
"python-dateutil": ("dateutil",),
|
|
390
|
+
"python-docx": ("docx",),
|
|
391
|
+
"python-dotenv": ("dotenv",),
|
|
392
|
+
"python-gitlab": ("gitlab",),
|
|
393
|
+
"python-jose": ("jose",),
|
|
394
|
+
"python-json-logger": ("pythonjsonlogger",),
|
|
395
|
+
"python-levenshtein": ("Levenshtein",),
|
|
396
|
+
"python-magic": ("magic",),
|
|
397
|
+
"python-multipart": ("multipart", "python_multipart"),
|
|
398
|
+
"python-pptx": ("pptx",),
|
|
399
|
+
"python-slugify": ("slugify",),
|
|
400
|
+
"python-socketio": ("socketio",),
|
|
401
|
+
"python-telegram-bot": ("telegram",),
|
|
402
|
+
"pywavelets": ("pywt",),
|
|
403
|
+
"pywin32": ("win32api", "win32con", "win32com", "win32gui", "pythoncom", "pywintypes"),
|
|
404
|
+
"pyyaml": ("yaml",),
|
|
405
|
+
"pyzmq": ("zmq",),
|
|
406
|
+
"ruamel-yaml": ("ruamel.yaml",),
|
|
407
|
+
"scikit-image": ("skimage",),
|
|
408
|
+
"scikit-learn": ("sklearn",),
|
|
409
|
+
"setuptools": ("setuptools", "pkg_resources"),
|
|
410
|
+
"tensorflow-cpu": ("tensorflow",),
|
|
411
|
+
"tensorflow-gpu": ("tensorflow",),
|
|
412
|
+
"websocket-client": ("websocket",),
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
# Python namespace families: normalised prefix -> (import namespace, the
|
|
416
|
+
# character that replaces `-` in the rest of the name). A rule, not a table
|
|
417
|
+
# of names, so the names it builds are guesses.
|
|
418
|
+
PYTHON_NAMESPACE_FAMILIES: dict[str, tuple[str, str]] = {
|
|
419
|
+
"google-cloud-": ("google.cloud.", "_"),
|
|
420
|
+
"azure-": ("azure.", "."),
|
|
421
|
+
"opentelemetry-": ("opentelemetry.", "."),
|
|
422
|
+
"backports-": ("backports.", "."),
|
|
423
|
+
"jaraco-": ("jaraco.", "."),
|
|
424
|
+
"sphinxcontrib-": ("sphinxcontrib.", "."),
|
|
425
|
+
"zope-": ("zope.", "."),
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
# Maven and Gradle coordinates (group:artifact) whose package is not the group.
|
|
429
|
+
JVM_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
|
|
430
|
+
"com.fasterxml.jackson.core:jackson-annotations": ("com.fasterxml.jackson.annotation",),
|
|
431
|
+
"com.fasterxml.jackson.core:jackson-core": ("com.fasterxml.jackson.core",),
|
|
432
|
+
"com.fasterxml.jackson.core:jackson-databind": ("com.fasterxml.jackson.databind",),
|
|
433
|
+
"com.google.code.gson:gson": ("com.google.gson",),
|
|
434
|
+
"com.google.guava:guava": ("com.google.common",),
|
|
435
|
+
"com.h2database:h2": ("org.h2",),
|
|
436
|
+
"com.squareup.okhttp3:okhttp": ("okhttp3",),
|
|
437
|
+
"com.squareup.retrofit2:retrofit": ("retrofit2",),
|
|
438
|
+
"commons-codec:commons-codec": ("org.apache.commons.codec",),
|
|
439
|
+
"commons-io:commons-io": ("org.apache.commons.io",),
|
|
440
|
+
"jakarta.servlet:jakarta.servlet-api": ("jakarta.servlet",),
|
|
441
|
+
"javax.servlet:javax.servlet-api": ("javax.servlet",),
|
|
442
|
+
"junit:junit": ("org.junit", "junit.framework"),
|
|
443
|
+
"mysql:mysql-connector-java": ("com.mysql",),
|
|
444
|
+
"org.apache.commons:commons-collections4": ("org.apache.commons.collections4",),
|
|
445
|
+
"org.apache.commons:commons-lang3": ("org.apache.commons.lang3",),
|
|
446
|
+
"org.jetbrains.kotlinx:kotlinx-coroutines-core": ("kotlinx.coroutines",),
|
|
447
|
+
"org.jetbrains.kotlinx:kotlinx-serialization-json": ("kotlinx.serialization",),
|
|
448
|
+
"org.junit.jupiter:junit-jupiter": ("org.junit.jupiter",),
|
|
449
|
+
"org.junit.jupiter:junit-jupiter-api": ("org.junit.jupiter.api",),
|
|
450
|
+
"org.postgresql:postgresql": ("org.postgresql",),
|
|
451
|
+
"org.projectlombok:lombok": ("lombok",),
|
|
452
|
+
"org.slf4j:slf4j-api": ("org.slf4j",),
|
|
453
|
+
"org.xerial:sqlite-jdbc": ("org.sqlite",),
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
# Gems required under another name, keyed by lower-case gem name.
|
|
457
|
+
RUBY_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
|
|
458
|
+
"actioncable": ("action_cable",),
|
|
459
|
+
"actionmailer": ("action_mailer",),
|
|
460
|
+
"actionpack": ("action_controller", "action_dispatch"),
|
|
461
|
+
"actionview": ("action_view",),
|
|
462
|
+
"activejob": ("active_job",),
|
|
463
|
+
"activemodel": ("active_model",),
|
|
464
|
+
"activerecord": ("active_record",),
|
|
465
|
+
"activestorage": ("active_storage",),
|
|
466
|
+
"activesupport": ("active_support",),
|
|
467
|
+
"concurrent-ruby": ("concurrent",),
|
|
468
|
+
"dotenv-rails": ("dotenv",),
|
|
469
|
+
"google-protobuf": ("google/protobuf",),
|
|
470
|
+
"railties": ("rails",),
|
|
471
|
+
"rest-client": ("rest-client", "restclient"),
|
|
472
|
+
"ruby-openai": ("openai",),
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
# Composer packages and the namespace they autoload, keyed by lower-case name.
|
|
476
|
+
# An installed vendor/<name>/composer.json wins over this table.
|
|
477
|
+
COMPOSER_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
|
|
478
|
+
"aws/aws-sdk-php": ("Aws",),
|
|
479
|
+
"doctrine/dbal": ("Doctrine\\DBAL",),
|
|
480
|
+
"doctrine/orm": ("Doctrine\\ORM",),
|
|
481
|
+
"firebase/php-jwt": ("Firebase\\JWT",),
|
|
482
|
+
"guzzlehttp/guzzle": ("GuzzleHttp",),
|
|
483
|
+
"guzzlehttp/promises": ("GuzzleHttp\\Promise",),
|
|
484
|
+
"guzzlehttp/psr7": ("GuzzleHttp\\Psr7",),
|
|
485
|
+
"laravel/framework": ("Illuminate",),
|
|
486
|
+
"league/flysystem": ("League\\Flysystem",),
|
|
487
|
+
"monolog/monolog": ("Monolog",),
|
|
488
|
+
"nesbot/carbon": ("Carbon",),
|
|
489
|
+
"phpunit/phpunit": ("PHPUnit",),
|
|
490
|
+
"predis/predis": ("Predis",),
|
|
491
|
+
"psr/http-message": ("Psr\\Http\\Message",),
|
|
492
|
+
"psr/log": ("Psr\\Log",),
|
|
493
|
+
"ramsey/uuid": ("Ramsey\\Uuid",),
|
|
494
|
+
"stripe/stripe-php": ("Stripe",),
|
|
495
|
+
"twig/twig": ("Twig",),
|
|
496
|
+
"vlucas/phpdotenv": ("Dotenv",),
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
# Swift packages (as skf-scan-manifests.py names them: the last URL segment)
|
|
500
|
+
# and the modules they ship, keyed by lower-case name.
|
|
501
|
+
SWIFT_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
|
|
502
|
+
"async-http-client": ("AsyncHTTPClient",),
|
|
503
|
+
"firebase-ios-sdk": (
|
|
504
|
+
"FirebaseAnalytics",
|
|
505
|
+
"FirebaseAuth",
|
|
506
|
+
"FirebaseCore",
|
|
507
|
+
"FirebaseCrashlytics",
|
|
508
|
+
"FirebaseDatabase",
|
|
509
|
+
"FirebaseFirestore",
|
|
510
|
+
"FirebaseFunctions",
|
|
511
|
+
"FirebaseMessaging",
|
|
512
|
+
"FirebaseRemoteConfig",
|
|
513
|
+
"FirebaseStorage",
|
|
514
|
+
),
|
|
515
|
+
"grpc-swift": ("GRPC",),
|
|
516
|
+
"realm-swift": ("RealmSwift", "Realm"),
|
|
517
|
+
"rxswift": ("RxSwift", "RxCocoa", "RxRelay"),
|
|
518
|
+
"snapkit": ("SnapKit",),
|
|
519
|
+
"swift-algorithms": ("Algorithms",),
|
|
520
|
+
"swift-argument-parser": ("ArgumentParser",),
|
|
521
|
+
"swift-async-algorithms": ("AsyncAlgorithms",),
|
|
522
|
+
"swift-atomics": ("Atomics",),
|
|
523
|
+
"swift-collections": ("Collections", "DequeModule", "OrderedCollections"),
|
|
524
|
+
"swift-crypto": ("Crypto",),
|
|
525
|
+
"swift-log": ("Logging",),
|
|
526
|
+
"swift-markdown": ("Markdown",),
|
|
527
|
+
"swift-metrics": ("Metrics",),
|
|
528
|
+
"swift-nio": (
|
|
529
|
+
"NIO",
|
|
530
|
+
"NIOConcurrencyHelpers",
|
|
531
|
+
"NIOCore",
|
|
532
|
+
"NIOEmbedded",
|
|
533
|
+
"NIOFoundationCompat",
|
|
534
|
+
"NIOHTTP1",
|
|
535
|
+
"NIOPosix",
|
|
536
|
+
"NIOTLS",
|
|
537
|
+
"NIOWebSocket",
|
|
538
|
+
),
|
|
539
|
+
"swift-nio-ssl": ("NIOSSL",),
|
|
540
|
+
"swift-numerics": ("Numerics", "RealModule", "ComplexModule"),
|
|
541
|
+
"swift-protobuf": ("SwiftProtobuf",),
|
|
542
|
+
"swift-syntax": ("SwiftSyntax", "SwiftParser"),
|
|
543
|
+
"swift-system": ("SystemPackage",),
|
|
544
|
+
"swiftyjson": ("SwiftyJSON",),
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
# --------------------------------------------------------------------------
|
|
549
|
+
# Input
|
|
550
|
+
# --------------------------------------------------------------------------
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
class InputError(Exception):
|
|
554
|
+
"""A malformed input: exit code 1."""
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _read_json_source(source: str, what: str) -> object:
|
|
558
|
+
"""Read JSON from a file path, or from stdin when source is '-'."""
|
|
559
|
+
if source == "-":
|
|
560
|
+
try:
|
|
561
|
+
text = sys.stdin.read()
|
|
562
|
+
except OSError as exc:
|
|
563
|
+
raise InputError(f"failed to read {what} JSON from stdin: {exc}") from exc
|
|
564
|
+
else:
|
|
565
|
+
path = Path(source)
|
|
566
|
+
if not path.exists() or path.is_dir():
|
|
567
|
+
raise InputError(f"{what} file not found: {path}")
|
|
568
|
+
try:
|
|
569
|
+
text = path.read_text(encoding="utf-8")
|
|
570
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
571
|
+
raise InputError(f"failed to read {what} file {path}: {exc}") from exc
|
|
572
|
+
try:
|
|
573
|
+
return json.loads(text)
|
|
574
|
+
except json.JSONDecodeError as exc:
|
|
575
|
+
raise InputError(f"malformed JSON in {what} input: {exc}") from exc
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _pep503(name: str) -> str:
|
|
579
|
+
return re.sub(r"[-_.]+", "-", name).lower()
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _dedupe_key(ecosystem: str | None, name: str) -> tuple[str, str]:
|
|
583
|
+
"""Key under which two spellings of one dependency collapse."""
|
|
584
|
+
if ecosystem == "python":
|
|
585
|
+
return ecosystem, _pep503(name)
|
|
586
|
+
if ecosystem == "rust":
|
|
587
|
+
return ecosystem, name.lower().replace("_", "-")
|
|
588
|
+
if ecosystem == "composer":
|
|
589
|
+
return ecosystem, name.lower()
|
|
590
|
+
return ecosystem or "", name
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def _string_list(value: object, where: str) -> list[str]:
|
|
594
|
+
if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value):
|
|
595
|
+
raise InputError(f"{where} must be a list of non-empty strings")
|
|
596
|
+
return list(dict.fromkeys(value))
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _dependency_entry(raw: object, where: str) -> dict:
|
|
600
|
+
if isinstance(raw, str):
|
|
601
|
+
raw = {"name": raw}
|
|
602
|
+
if not isinstance(raw, dict):
|
|
603
|
+
raise InputError(f"{where} must be a name string or an object")
|
|
604
|
+
name = raw.get("name")
|
|
605
|
+
if not isinstance(name, str) or not name.strip():
|
|
606
|
+
raise InputError(f"{where} needs a non-empty `name`")
|
|
607
|
+
ecosystem = raw.get("ecosystem")
|
|
608
|
+
if ecosystem is not None and (not isinstance(ecosystem, str) or not ecosystem):
|
|
609
|
+
raise InputError(f"{where} `ecosystem` must be a non-empty string")
|
|
610
|
+
scope = raw.get("scope")
|
|
611
|
+
if scope not in (None, "runtime", "dev"):
|
|
612
|
+
raise InputError(f"{where} `scope` must be 'runtime' or 'dev'; got {scope!r}")
|
|
613
|
+
entry = {"name": name.strip(), "ecosystem": ecosystem, "scope": scope or "runtime"}
|
|
614
|
+
if raw.get("modules") is not None:
|
|
615
|
+
entry["modules"] = _string_list(raw["modules"], f"{where} `modules`")
|
|
616
|
+
return entry
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def parse_dependencies(payload: object) -> list[dict]:
|
|
620
|
+
"""Read the --deps payload into deduplicated dependency entries.
|
|
621
|
+
|
|
622
|
+
Accepts the skf-scan-manifests.py envelope, {"dependencies": [...]} or a
|
|
623
|
+
bare list. Two entries of one ecosystem whose names normalise alike
|
|
624
|
+
(PyYAML and pyyaml) collapse into the first; the merged entry is dev only
|
|
625
|
+
when every occurrence is dev.
|
|
626
|
+
"""
|
|
627
|
+
raw_entries: list[tuple[object, str]] = []
|
|
628
|
+
if isinstance(payload, dict) and "manifests" in payload:
|
|
629
|
+
manifests = payload["manifests"]
|
|
630
|
+
if not isinstance(manifests, list):
|
|
631
|
+
raise InputError("`manifests` must be a list")
|
|
632
|
+
for m_idx, manifest in enumerate(manifests):
|
|
633
|
+
if not isinstance(manifest, dict):
|
|
634
|
+
raise InputError(f"manifests[{m_idx}] is not an object")
|
|
635
|
+
ecosystem = manifest.get("ecosystem")
|
|
636
|
+
deps = manifest.get("deps") or []
|
|
637
|
+
if not isinstance(deps, list):
|
|
638
|
+
raise InputError(f"manifests[{m_idx}].deps must be a list")
|
|
639
|
+
for d_idx, dep in enumerate(deps):
|
|
640
|
+
if not isinstance(dep, dict):
|
|
641
|
+
raise InputError(f"manifests[{m_idx}].deps[{d_idx}] is not an object")
|
|
642
|
+
name = dep.get("name")
|
|
643
|
+
# the scanner writes `<unparsable>` for a line it could not read
|
|
644
|
+
if not isinstance(name, str) or not name or name.startswith("<"):
|
|
645
|
+
continue
|
|
646
|
+
raw_entries.append(
|
|
647
|
+
(
|
|
648
|
+
{"name": name, "ecosystem": ecosystem, "scope": dep.get("scope")},
|
|
649
|
+
f"manifests[{m_idx}].deps[{d_idx}]",
|
|
650
|
+
)
|
|
651
|
+
)
|
|
652
|
+
else:
|
|
653
|
+
if isinstance(payload, dict) and "dependencies" in payload:
|
|
654
|
+
payload = payload["dependencies"]
|
|
655
|
+
if not isinstance(payload, list):
|
|
656
|
+
raise InputError(
|
|
657
|
+
"--deps must be the skf-scan-manifests.py envelope, "
|
|
658
|
+
'{"dependencies": [...]} or a JSON array'
|
|
659
|
+
)
|
|
660
|
+
raw_entries = [(raw, f"dependencies[{idx}]") for idx, raw in enumerate(payload)]
|
|
661
|
+
|
|
662
|
+
merged: dict[tuple[str, str], dict] = {}
|
|
663
|
+
for raw, where in raw_entries:
|
|
664
|
+
entry = _dependency_entry(raw, where)
|
|
665
|
+
key = _dedupe_key(entry["ecosystem"], entry["name"])
|
|
666
|
+
seen = merged.get(key)
|
|
667
|
+
if seen is None:
|
|
668
|
+
merged[key] = entry
|
|
669
|
+
continue
|
|
670
|
+
if entry["scope"] == "runtime":
|
|
671
|
+
seen["scope"] = "runtime"
|
|
672
|
+
if "modules" in entry:
|
|
673
|
+
seen["modules"] = list(dict.fromkeys(seen.get("modules", []) + entry["modules"]))
|
|
674
|
+
return list(merged.values())
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _normalise_unit_path(path: str) -> str:
|
|
678
|
+
"""A unit path as a root-relative POSIX path; the root itself is ''."""
|
|
679
|
+
path = path.replace("\\", "/").strip().strip("/")
|
|
680
|
+
if not path:
|
|
681
|
+
return ""
|
|
682
|
+
path = posixpath.normpath(path)
|
|
683
|
+
return "" if path == "." else path
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def parse_units(payload: object) -> list[dict]:
|
|
687
|
+
"""Read the --units payload: unique names and unique paths."""
|
|
688
|
+
if isinstance(payload, dict) and "units" in payload:
|
|
689
|
+
payload = payload["units"]
|
|
690
|
+
if not isinstance(payload, list):
|
|
691
|
+
raise InputError('--units must be {"units": [...]} or a JSON array')
|
|
692
|
+
units: list[dict] = []
|
|
693
|
+
names: set[str] = set()
|
|
694
|
+
paths: set[str] = set()
|
|
695
|
+
for idx, raw in enumerate(payload):
|
|
696
|
+
where = f"units[{idx}]"
|
|
697
|
+
if not isinstance(raw, dict):
|
|
698
|
+
raise InputError(f"{where} is not an object")
|
|
699
|
+
name = raw.get("name")
|
|
700
|
+
if not isinstance(name, str) or not name.strip():
|
|
701
|
+
raise InputError(f"{where} needs a non-empty `name`")
|
|
702
|
+
path = raw.get("path")
|
|
703
|
+
if not isinstance(path, str):
|
|
704
|
+
raise InputError(f"{where} needs a `path` string (relative to the root)")
|
|
705
|
+
path = _normalise_unit_path(path)
|
|
706
|
+
if path == ".." or path.startswith("../"):
|
|
707
|
+
raise InputError(f"{where} `path` leaves the root: {raw.get('path')!r}")
|
|
708
|
+
ecosystem = raw.get("ecosystem")
|
|
709
|
+
if ecosystem is not None and (not isinstance(ecosystem, str) or not ecosystem):
|
|
710
|
+
raise InputError(f"{where} `ecosystem` must be a non-empty string")
|
|
711
|
+
manifest_name = raw.get("manifest_name")
|
|
712
|
+
if manifest_name is not None and not isinstance(manifest_name, str):
|
|
713
|
+
raise InputError(f"{where} `manifest_name` must be a string")
|
|
714
|
+
name = name.strip()
|
|
715
|
+
if name in names:
|
|
716
|
+
raise InputError(f"{where} repeats unit name {name!r}")
|
|
717
|
+
if path in paths:
|
|
718
|
+
raise InputError(f"{where} repeats unit path {path!r}")
|
|
719
|
+
names.add(name)
|
|
720
|
+
paths.add(path)
|
|
721
|
+
unit = {
|
|
722
|
+
"name": name,
|
|
723
|
+
"path": path,
|
|
724
|
+
"ecosystem": ecosystem,
|
|
725
|
+
"manifest_name": (manifest_name or "").strip() or None,
|
|
726
|
+
"modules": None, # None: derived from what the unit declares
|
|
727
|
+
}
|
|
728
|
+
if raw.get("modules") is not None:
|
|
729
|
+
unit["modules"] = _string_list(raw["modules"], f"{where} `modules`")
|
|
730
|
+
units.append(unit)
|
|
731
|
+
return units
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
# --------------------------------------------------------------------------
|
|
735
|
+
# Import names
|
|
736
|
+
# --------------------------------------------------------------------------
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def _studly(part: str) -> str:
|
|
740
|
+
return "".join(p[:1].upper() + p[1:] for p in re.split(r"[-_.]+", part) if p)
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _npm_import_names(name: str) -> tuple[list[str], str]:
|
|
744
|
+
if name.startswith("@types/") and len(name) > len("@types/"):
|
|
745
|
+
# a guess: @types/node types built-ins, @types/jest types globals
|
|
746
|
+
inner = name[len("@types/"):]
|
|
747
|
+
# DefinitelyTyped spells @scope/pkg as scope__pkg
|
|
748
|
+
if "__" in inner:
|
|
749
|
+
scope, _, pkg = inner.partition("__")
|
|
750
|
+
return [f"@{scope}/{pkg}"], "guessed"
|
|
751
|
+
return [inner], "guessed"
|
|
752
|
+
return [name], "exact"
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def _python_import_names(name: str) -> tuple[list[str], str]:
|
|
756
|
+
key = _pep503(name)
|
|
757
|
+
if key.startswith("types-") and len(key) > len("types-"):
|
|
758
|
+
# typeshed stubs: types-PyYAML types the `yaml` module
|
|
759
|
+
return _python_import_names(key[len("types-"):])
|
|
760
|
+
mapped = PYTHON_IMPORT_NAMES.get(key)
|
|
761
|
+
if mapped:
|
|
762
|
+
return list(mapped), "mapped"
|
|
763
|
+
for prefix, (namespace, joiner) in PYTHON_NAMESPACE_FAMILIES.items():
|
|
764
|
+
if key.startswith(prefix) and len(key) > len(prefix):
|
|
765
|
+
# a guess: google-cloud-pubsub is imported as google.cloud.pubsub_v1
|
|
766
|
+
return [namespace + key[len(prefix):].replace("-", joiner)], "guessed"
|
|
767
|
+
guess = name.replace("-", "_")
|
|
768
|
+
if not re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*", guess):
|
|
769
|
+
return [], "guessed"
|
|
770
|
+
return list(dict.fromkeys([guess, guess.lower()])), "guessed"
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def _jvm_import_names(name: str) -> tuple[list[str], str]:
|
|
774
|
+
mapped = JVM_IMPORT_NAMES.get(name)
|
|
775
|
+
if mapped:
|
|
776
|
+
return list(mapped), "mapped"
|
|
777
|
+
group = name.split(":", 1)[0]
|
|
778
|
+
if re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*", group):
|
|
779
|
+
return [group], "guessed"
|
|
780
|
+
return [], "guessed"
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
def _ruby_import_names(name: str) -> tuple[list[str], str]:
|
|
784
|
+
mapped = RUBY_IMPORT_NAMES.get(name.lower())
|
|
785
|
+
if mapped:
|
|
786
|
+
return list(mapped), "mapped"
|
|
787
|
+
return list(dict.fromkeys([name, name.replace("-", "/")])), "guessed"
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
_COMPOSER_NAME = re.compile(r"[A-Za-z0-9][\w.-]*/[A-Za-z0-9][\w.-]*")
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def _composer_autoload(manifest: Path) -> list[str]:
|
|
794
|
+
"""Namespaces a composer.json autoloads (its psr-4 and psr-0 prefixes)."""
|
|
795
|
+
try:
|
|
796
|
+
data = json.loads(manifest.read_text(encoding="utf-8"))
|
|
797
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
798
|
+
return []
|
|
799
|
+
autoload = data.get("autoload") if isinstance(data, dict) else None
|
|
800
|
+
if not isinstance(autoload, dict):
|
|
801
|
+
return []
|
|
802
|
+
namespaces: set[str] = set()
|
|
803
|
+
for kind in ("psr-4", "psr-0"):
|
|
804
|
+
table = autoload.get(kind)
|
|
805
|
+
if isinstance(table, dict):
|
|
806
|
+
namespaces.update(p.strip("\\") for p in table if p.strip("\\"))
|
|
807
|
+
return sorted(namespaces)
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def _composer_installed_namespaces(root: Path, name: str) -> list[str]:
|
|
811
|
+
"""Namespaces the installed vendor/<name>/composer.json autoloads."""
|
|
812
|
+
if not _COMPOSER_NAME.fullmatch(name):
|
|
813
|
+
return []
|
|
814
|
+
return _composer_autoload(root / "vendor" / name / "composer.json")
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
_SWIFT_TARGET = re.compile(r'\.(?:target|binaryTarget)\s*\(\s*name\s*:\s*"(?P<name>[^"]+)"')
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _swift_targets(manifest: Path) -> list[str]:
|
|
821
|
+
"""The targets a Package.swift declares: the modules code imports it by."""
|
|
822
|
+
try:
|
|
823
|
+
text = manifest.read_text(encoding="utf-8")
|
|
824
|
+
except (OSError, UnicodeDecodeError):
|
|
825
|
+
return []
|
|
826
|
+
return sorted({m.group("name") for m in _SWIFT_TARGET.finditer(text)})
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def _symfony_namespace(package: str) -> str:
|
|
830
|
+
"""The namespace a symfony/<package> usually autoloads, by Symfony's naming."""
|
|
831
|
+
if package.endswith("-bundle"):
|
|
832
|
+
return "Symfony\\Bundle\\" + _studly(package)
|
|
833
|
+
if package.endswith("-bridge"):
|
|
834
|
+
return "Symfony\\Bridge\\" + _studly(package[: -len("-bridge")])
|
|
835
|
+
if package.endswith("-contracts"):
|
|
836
|
+
return "Symfony\\Contracts\\" + _studly(package[: -len("-contracts")])
|
|
837
|
+
if package.startswith("ux-"):
|
|
838
|
+
return "Symfony\\UX\\" + _studly(package[len("ux-"):])
|
|
839
|
+
if package.startswith("polyfill-"):
|
|
840
|
+
return "Symfony\\Polyfill\\" + _studly(package[len("polyfill-"):])
|
|
841
|
+
if package.startswith("security-"):
|
|
842
|
+
return "Symfony\\Component\\Security\\" + _studly(package[len("security-"):])
|
|
843
|
+
return "Symfony\\Component\\" + _studly(package)
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def _composer_import_names(name: str, root: Path) -> tuple[list[str], str]:
|
|
847
|
+
installed = _composer_installed_namespaces(root, name)
|
|
848
|
+
if installed:
|
|
849
|
+
return installed, "exact"
|
|
850
|
+
key = name.lower()
|
|
851
|
+
mapped = COMPOSER_IMPORT_NAMES.get(key)
|
|
852
|
+
if mapped:
|
|
853
|
+
return list(mapped), "mapped"
|
|
854
|
+
vendor, _, package = key.partition("/")
|
|
855
|
+
if vendor == "symfony" and package:
|
|
856
|
+
return [_symfony_namespace(package)], "guessed"
|
|
857
|
+
if vendor and package:
|
|
858
|
+
return [f"{_studly(vendor)}\\{_studly(package)}", _studly(vendor)], "guessed"
|
|
859
|
+
return [], "guessed"
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def _swift_import_names(name: str) -> tuple[list[str], str]:
|
|
863
|
+
mapped = SWIFT_IMPORT_NAMES.get(name.lower())
|
|
864
|
+
if mapped:
|
|
865
|
+
return list(mapped), "mapped"
|
|
866
|
+
base = name[len("swift-"):] if name.lower().startswith("swift-") else name
|
|
867
|
+
names = [name] if re.fullmatch(r"[A-Za-z_]\w*", name) else []
|
|
868
|
+
camel = _studly(base)
|
|
869
|
+
if camel:
|
|
870
|
+
names.append(camel)
|
|
871
|
+
return list(dict.fromkeys(names)), "guessed"
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def import_names(name: str, ecosystem: str | None, root: Path) -> tuple[list[str], str]:
|
|
875
|
+
"""Return (import names, resolution) for a dependency of an ecosystem.
|
|
876
|
+
|
|
877
|
+
With no ecosystem the name is tried as it is and with the Python guesses,
|
|
878
|
+
in every language.
|
|
879
|
+
"""
|
|
880
|
+
if ecosystem == "npm":
|
|
881
|
+
return _npm_import_names(name)
|
|
882
|
+
if ecosystem == "python":
|
|
883
|
+
return _python_import_names(name)
|
|
884
|
+
if ecosystem == "rust":
|
|
885
|
+
return [name.replace("-", "_")], "exact"
|
|
886
|
+
if ecosystem == "go":
|
|
887
|
+
return [name], "exact"
|
|
888
|
+
if ecosystem in ("maven", "gradle"):
|
|
889
|
+
return _jvm_import_names(name)
|
|
890
|
+
if ecosystem == "ruby":
|
|
891
|
+
return _ruby_import_names(name)
|
|
892
|
+
if ecosystem == "composer":
|
|
893
|
+
return _composer_import_names(name, root)
|
|
894
|
+
if ecosystem == "swift":
|
|
895
|
+
return _swift_import_names(name)
|
|
896
|
+
guesses, _ = _python_import_names(name)
|
|
897
|
+
return list(dict.fromkeys([name, *guesses])), "guessed"
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
# --------------------------------------------------------------------------
|
|
901
|
+
# Exclusion globs
|
|
902
|
+
# --------------------------------------------------------------------------
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
def _glob_regex(pattern: str) -> re.Pattern[str]:
|
|
906
|
+
"""Translate a glob to a regex: `*` and `?` stay in one path segment,
|
|
907
|
+
`**/` spans zero or more directories and `**` any characters."""
|
|
908
|
+
out: list[str] = []
|
|
909
|
+
i = 0
|
|
910
|
+
while i < len(pattern):
|
|
911
|
+
char = pattern[i]
|
|
912
|
+
if pattern.startswith("**/", i):
|
|
913
|
+
out.append("(?:.*/)?")
|
|
914
|
+
i += 3
|
|
915
|
+
continue
|
|
916
|
+
if pattern.startswith("**", i):
|
|
917
|
+
out.append(".*")
|
|
918
|
+
i += 2
|
|
919
|
+
continue
|
|
920
|
+
if char == "*":
|
|
921
|
+
out.append("[^/]*")
|
|
922
|
+
elif char == "?":
|
|
923
|
+
out.append("[^/]")
|
|
924
|
+
elif char == "[" and pattern.find("]", i + 1) != -1:
|
|
925
|
+
end = pattern.find("]", i + 1)
|
|
926
|
+
body = pattern[i + 1 : end]
|
|
927
|
+
if body.startswith("!"):
|
|
928
|
+
body = "^" + body[1:]
|
|
929
|
+
out.append("[" + body.replace("\\", "\\\\") + "]")
|
|
930
|
+
i = end + 1
|
|
931
|
+
continue
|
|
932
|
+
else:
|
|
933
|
+
out.append(re.escape(char))
|
|
934
|
+
i += 1
|
|
935
|
+
return re.compile("".join(out))
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
class Excludes:
|
|
939
|
+
"""The exclusion globs, split by what each one is matched against."""
|
|
940
|
+
|
|
941
|
+
def __init__(self, patterns: list[str]) -> None:
|
|
942
|
+
self.dir_names: list[re.Pattern[str]] = []
|
|
943
|
+
self.dir_paths: list[re.Pattern[str]] = []
|
|
944
|
+
self.file_names: list[re.Pattern[str]] = []
|
|
945
|
+
self.file_paths: list[re.Pattern[str]] = []
|
|
946
|
+
for raw in patterns:
|
|
947
|
+
pattern = raw.strip().replace("\\", "/")
|
|
948
|
+
# a leading / or ./ anchors the pattern to the root
|
|
949
|
+
anchored = pattern.startswith(("/", "./"))
|
|
950
|
+
is_dir = pattern.endswith("/")
|
|
951
|
+
body = posixpath.normpath(pattern.lstrip("/") or ".")
|
|
952
|
+
if body == ".":
|
|
953
|
+
continue
|
|
954
|
+
if is_dir:
|
|
955
|
+
target = self.dir_paths if anchored or "/" in body else self.dir_names
|
|
956
|
+
else:
|
|
957
|
+
target = self.file_paths if anchored or "/" in body else self.file_names
|
|
958
|
+
target.append(_glob_regex(body))
|
|
959
|
+
|
|
960
|
+
def skip_dir(self, name: str, rel: str) -> bool:
|
|
961
|
+
return any(r.fullmatch(name) for r in self.dir_names) or any(
|
|
962
|
+
r.fullmatch(rel) for r in self.dir_paths
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
def skip_file(self, name: str, rel: str) -> bool:
|
|
966
|
+
return any(r.fullmatch(name) for r in self.file_names) or any(
|
|
967
|
+
r.fullmatch(rel) for r in self.file_paths
|
|
968
|
+
)
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
# --------------------------------------------------------------------------
|
|
972
|
+
# Import extraction
|
|
973
|
+
# --------------------------------------------------------------------------
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
# A record is (line, kind, value): kind "module" holds an imported module
|
|
977
|
+
# name; kind "path" holds a relative import resolved to a root-relative path.
|
|
978
|
+
Record = tuple[int, str, str]
|
|
979
|
+
|
|
980
|
+
_JS_IMPORT = re.compile(
|
|
981
|
+
r"""(?<![\w$.])(?:
|
|
982
|
+
(?:from|import)\s*(?P<q1>['"])(?P<s1>[^'"\n]+)(?P=q1)
|
|
983
|
+
| (?:import|require)\s*\(\s*(?P<q2>['"`])(?P<s2>[^'"`\n]+)(?P=q2)
|
|
984
|
+
)""",
|
|
985
|
+
re.VERBOSE,
|
|
986
|
+
)
|
|
987
|
+
_PY_FROM = re.compile(r"from\s+(?P<mod>\.+[\w.]*|[A-Za-z_][\w.]*)\s+import\s*(?P<names>.*)")
|
|
988
|
+
_PY_DYNAMIC = re.compile(
|
|
989
|
+
r"""(?<!\w)(?:import_module|__import__)\s*\(\s*(['"])(?P<mod>[A-Za-z_][\w.]*)\1"""
|
|
990
|
+
)
|
|
991
|
+
_PY_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*")
|
|
992
|
+
_RS_USE = re.compile(
|
|
993
|
+
r"^\s*(?:pub(?:\s*\([^)]*\))?\s+)?use\s+(?:::)?(?P<path>[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)"
|
|
994
|
+
)
|
|
995
|
+
_RS_EXTERN = re.compile(r"^\s*extern\s+crate\s+(?P<path>[A-Za-z_]\w*)")
|
|
996
|
+
_RS_PATH = re.compile(r"(?<![\w:])(?P<path>[A-Za-z_]\w*(?:::[A-Za-z_]\w*)+)")
|
|
997
|
+
_GO_IMPORT_ONE = re.compile(r'^\s*import\s+(?:[\w.]+\s+)?"(?P<path>[^"]+)"')
|
|
998
|
+
_GO_IMPORT_OPEN = re.compile(r"^\s*import\s*\(")
|
|
999
|
+
_GO_QUOTED = re.compile(r'(?:^|[\s(;])(?:[\w.]+\s+)?"(?P<path>[^"]+)"')
|
|
1000
|
+
_JVM_IMPORT = re.compile(
|
|
1001
|
+
r"^\s*import\s+(?:static\s+)?(?P<path>[A-Za-z_]\w*(?:\.(?:[A-Za-z_]\w*|\*))*)"
|
|
1002
|
+
)
|
|
1003
|
+
_RB_REQUIRE = re.compile(
|
|
1004
|
+
r"""^\s*(?P<kind>require_relative|require)\s*\(?\s*(['"])(?P<path>[^'"]+)\2"""
|
|
1005
|
+
)
|
|
1006
|
+
_PHP_USE = re.compile(r"^\s*use\s+(?P<body>[^;]*)")
|
|
1007
|
+
_SWIFT_IMPORT = re.compile(
|
|
1008
|
+
r"^\s*(?:@\w+(?:\([^)]*\))?\s+)*import\s+"
|
|
1009
|
+
r"(?:(?:typealias|struct|class|enum|protocol|let|var|func)\s+)?(?P<mod>[A-Za-z_]\w*)"
|
|
1010
|
+
)
|
|
1011
|
+
|
|
1012
|
+
|
|
1013
|
+
def _lines(text: str) -> list[str]:
|
|
1014
|
+
"""The lines of a file as editors and `ast` number them (str.splitlines
|
|
1015
|
+
also breaks at form feeds and Unicode line separators)."""
|
|
1016
|
+
return text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def _resolve_relative(base_dir: str, spec: str) -> str | None:
|
|
1020
|
+
"""Join a relative import to its file's directory; None if it leaves the root."""
|
|
1021
|
+
joined = posixpath.normpath(posixpath.join(base_dir, spec) if base_dir else spec)
|
|
1022
|
+
if joined == ".." or joined.startswith("../"):
|
|
1023
|
+
return None
|
|
1024
|
+
return "" if joined == "." else joined
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def _js_records(text: str, rel: str) -> list[Record]:
|
|
1028
|
+
records: list[Record] = []
|
|
1029
|
+
base = posixpath.dirname(rel)
|
|
1030
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1031
|
+
if line.lstrip().startswith(("//", "/*", "*")):
|
|
1032
|
+
continue
|
|
1033
|
+
if "import" not in line and "require" not in line and "from" not in line:
|
|
1034
|
+
continue
|
|
1035
|
+
for m in _JS_IMPORT.finditer(line):
|
|
1036
|
+
spec = m.group("s1") or m.group("s2")
|
|
1037
|
+
if not spec or "${" in spec:
|
|
1038
|
+
continue
|
|
1039
|
+
if spec in (".", "..") or spec.startswith(("./", "../")):
|
|
1040
|
+
resolved = _resolve_relative(base, spec)
|
|
1041
|
+
if resolved is not None:
|
|
1042
|
+
records.append((number, "path", resolved))
|
|
1043
|
+
elif not spec.startswith("/"):
|
|
1044
|
+
records.append((number, "module", spec))
|
|
1045
|
+
return records
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
def _py_names(text: str) -> tuple[list[str], bool]:
|
|
1049
|
+
"""Names of a `from m import ...` list, and whether its `)` was seen."""
|
|
1050
|
+
closed = True
|
|
1051
|
+
if text.startswith("("):
|
|
1052
|
+
text = text[1:]
|
|
1053
|
+
closed = False
|
|
1054
|
+
if not closed and ")" in text:
|
|
1055
|
+
text = text.split(")", 1)[0]
|
|
1056
|
+
closed = True
|
|
1057
|
+
names = []
|
|
1058
|
+
for item in text.split(","):
|
|
1059
|
+
item = item.strip().split(" as ", 1)[0].strip()
|
|
1060
|
+
if item and item != "*" and re.fullmatch(r"[A-Za-z_]\w*", item):
|
|
1061
|
+
names.append(item)
|
|
1062
|
+
return names, closed
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
def _py_from_records(number: int, mod: str, names: list[str], base: str) -> list[Record]:
|
|
1066
|
+
if not mod.startswith("."):
|
|
1067
|
+
return [(number, "module", mod)] + [(number, "module", f"{mod}.{n}") for n in names]
|
|
1068
|
+
level = len(mod) - len(mod.lstrip("."))
|
|
1069
|
+
rest = mod[level:]
|
|
1070
|
+
target = base
|
|
1071
|
+
for _ in range(level - 1):
|
|
1072
|
+
if not target:
|
|
1073
|
+
return []
|
|
1074
|
+
target = posixpath.dirname(target)
|
|
1075
|
+
if rest:
|
|
1076
|
+
sub = rest.replace(".", "/")
|
|
1077
|
+
target = posixpath.join(target, sub) if target else sub
|
|
1078
|
+
records: list[Record] = [(number, "path", target)]
|
|
1079
|
+
for n in names:
|
|
1080
|
+
records.append((number, "path", posixpath.join(target, n) if target else n))
|
|
1081
|
+
return records
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def _py_statements(tree: ast.Module):
|
|
1085
|
+
"""Every statement of a module, the ones in nested blocks included."""
|
|
1086
|
+
queue: list[ast.AST] = list(tree.body)
|
|
1087
|
+
for node in queue:
|
|
1088
|
+
yield node
|
|
1089
|
+
for field in ("body", "orelse", "finalbody", "handlers", "cases"):
|
|
1090
|
+
queue.extend(getattr(node, field, ()))
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def _python_records(text: str, rel: str) -> list[Record]:
|
|
1094
|
+
base = posixpath.dirname(rel)
|
|
1095
|
+
if "import" not in text:
|
|
1096
|
+
return []
|
|
1097
|
+
if len(text) > PYTHON_AST_MAX_CHARS:
|
|
1098
|
+
return _python_line_records(text, base)
|
|
1099
|
+
try:
|
|
1100
|
+
with warnings.catch_warnings():
|
|
1101
|
+
# an invalid escape sequence warns; the file is only read
|
|
1102
|
+
warnings.simplefilter("ignore")
|
|
1103
|
+
tree = ast.parse(text)
|
|
1104
|
+
except (SyntaxError, ValueError, RecursionError, MemoryError):
|
|
1105
|
+
# Python 2, a syntax error, syntax newer than this interpreter, or
|
|
1106
|
+
# nesting too deep to parse
|
|
1107
|
+
return _python_line_records(text, base)
|
|
1108
|
+
# an import statement sits in a block; a dynamic import is a call in any expression
|
|
1109
|
+
dynamic = "import_module" in text or "__import__" in text
|
|
1110
|
+
records: list[Record] = []
|
|
1111
|
+
for node in ast.walk(tree) if dynamic else _py_statements(tree):
|
|
1112
|
+
if isinstance(node, ast.Import):
|
|
1113
|
+
records.extend((node.lineno, "module", alias.name) for alias in node.names)
|
|
1114
|
+
elif isinstance(node, ast.ImportFrom):
|
|
1115
|
+
mod = "." * node.level + (node.module or "")
|
|
1116
|
+
names = [alias.name for alias in node.names if alias.name != "*"]
|
|
1117
|
+
records.extend(_py_from_records(node.lineno, mod, names, base))
|
|
1118
|
+
elif isinstance(node, ast.Call) and node.args:
|
|
1119
|
+
func, arg = node.func, node.args[0]
|
|
1120
|
+
called = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None)
|
|
1121
|
+
if (
|
|
1122
|
+
called in ("import_module", "__import__")
|
|
1123
|
+
and isinstance(arg, ast.Constant)
|
|
1124
|
+
and isinstance(arg.value, str)
|
|
1125
|
+
and _PY_DOTTED.fullmatch(arg.value)
|
|
1126
|
+
):
|
|
1127
|
+
records.append((node.lineno, "module", arg.value))
|
|
1128
|
+
return sorted(records, key=lambda record: record[0])
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
def _py_code(line: str, quote: str | None) -> tuple[str, str | None]:
|
|
1132
|
+
"""The code of one line, and the triple quote still open at its end.
|
|
1133
|
+
|
|
1134
|
+
`quote` is the triple quote open when the line starts. Triple-quoted text
|
|
1135
|
+
and the comment are left out; a `#` inside a string literal is kept.
|
|
1136
|
+
"""
|
|
1137
|
+
code: list[str] = []
|
|
1138
|
+
i, end = 0, len(line)
|
|
1139
|
+
while i < end:
|
|
1140
|
+
if quote:
|
|
1141
|
+
if line.startswith(quote, i):
|
|
1142
|
+
quote = None
|
|
1143
|
+
i += 3
|
|
1144
|
+
else:
|
|
1145
|
+
i += 2 if line[i] == "\\" else 1
|
|
1146
|
+
continue
|
|
1147
|
+
char = line[i]
|
|
1148
|
+
if char == "#":
|
|
1149
|
+
break
|
|
1150
|
+
if line.startswith(('"""', "'''"), i):
|
|
1151
|
+
quote = line[i : i + 3]
|
|
1152
|
+
i += 3
|
|
1153
|
+
elif char in "'\"":
|
|
1154
|
+
close = i + 1
|
|
1155
|
+
while close < end and line[close] != char:
|
|
1156
|
+
close += 2 if line[close] == "\\" else 1
|
|
1157
|
+
code.append(line[i : close + 1])
|
|
1158
|
+
i = close + 1
|
|
1159
|
+
else:
|
|
1160
|
+
code.append(char)
|
|
1161
|
+
i += 1
|
|
1162
|
+
return "".join(code), quote
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def _python_line_records(text: str, base: str) -> list[Record]:
|
|
1166
|
+
"""The imports of a file `ast` cannot parse, read line by line."""
|
|
1167
|
+
records: list[Record] = []
|
|
1168
|
+
quote: str | None = None
|
|
1169
|
+
pending: tuple[int, str] | None = None # (line, module) of an open `from m import (`
|
|
1170
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1171
|
+
code, quote = _py_code(line, quote)
|
|
1172
|
+
if pending is not None:
|
|
1173
|
+
names, closed = _py_names("(" + code)
|
|
1174
|
+
records.extend(_py_from_records(pending[0], pending[1], names, base)[1:])
|
|
1175
|
+
if closed:
|
|
1176
|
+
pending = None
|
|
1177
|
+
continue
|
|
1178
|
+
for statement in code.split(";"):
|
|
1179
|
+
statement = statement.strip()
|
|
1180
|
+
if statement.startswith("import "):
|
|
1181
|
+
for item in statement[len("import "):].split(","):
|
|
1182
|
+
item = item.strip().split(" as ", 1)[0].strip()
|
|
1183
|
+
if _PY_DOTTED.fullmatch(item):
|
|
1184
|
+
records.append((number, "module", item))
|
|
1185
|
+
elif statement.startswith("from "):
|
|
1186
|
+
m = _PY_FROM.match(statement)
|
|
1187
|
+
if not m:
|
|
1188
|
+
continue
|
|
1189
|
+
names, closed = _py_names(m.group("names").strip())
|
|
1190
|
+
records.extend(_py_from_records(number, m.group("mod"), names, base))
|
|
1191
|
+
if not closed:
|
|
1192
|
+
pending = (number, m.group("mod"))
|
|
1193
|
+
if "import_module" in code or "__import__" in code:
|
|
1194
|
+
for m in _PY_DYNAMIC.finditer(code):
|
|
1195
|
+
records.append((number, "module", m.group("mod")))
|
|
1196
|
+
return records
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
def _rust_records(text: str, rel: str) -> list[Record]:
|
|
1200
|
+
records: list[Record] = []
|
|
1201
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1202
|
+
if line.lstrip().startswith(("//", "/*", "*")):
|
|
1203
|
+
continue
|
|
1204
|
+
for regex in (_RS_USE, _RS_EXTERN):
|
|
1205
|
+
m = regex.match(line)
|
|
1206
|
+
if m:
|
|
1207
|
+
records.append((number, "module", m.group("path")))
|
|
1208
|
+
if "::" in line:
|
|
1209
|
+
for m in _RS_PATH.finditer(line):
|
|
1210
|
+
records.append((number, "module", m.group("path")))
|
|
1211
|
+
return records
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def _go_records(text: str, rel: str) -> list[Record]:
|
|
1215
|
+
records: list[Record] = []
|
|
1216
|
+
in_block = False
|
|
1217
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1218
|
+
code = line.split("//", 1)[0]
|
|
1219
|
+
if in_block:
|
|
1220
|
+
if code.strip().startswith(")"):
|
|
1221
|
+
in_block = False
|
|
1222
|
+
continue
|
|
1223
|
+
m = _GO_QUOTED.search(code)
|
|
1224
|
+
if m:
|
|
1225
|
+
records.append((number, "module", m.group("path")))
|
|
1226
|
+
if ")" in code.split('"')[-1]:
|
|
1227
|
+
in_block = False
|
|
1228
|
+
continue
|
|
1229
|
+
if _GO_IMPORT_OPEN.match(code):
|
|
1230
|
+
rest = code.split("(", 1)[1]
|
|
1231
|
+
for m in _GO_QUOTED.finditer(rest):
|
|
1232
|
+
records.append((number, "module", m.group("path")))
|
|
1233
|
+
in_block = ")" not in rest.split('"')[-1]
|
|
1234
|
+
continue
|
|
1235
|
+
m = _GO_IMPORT_ONE.match(code)
|
|
1236
|
+
if m:
|
|
1237
|
+
records.append((number, "module", m.group("path")))
|
|
1238
|
+
return records
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
def _jvm_records(text: str, rel: str) -> list[Record]:
|
|
1242
|
+
records: list[Record] = []
|
|
1243
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1244
|
+
m = _JVM_IMPORT.match(line)
|
|
1245
|
+
if m:
|
|
1246
|
+
path = m.group("path")
|
|
1247
|
+
if path.endswith(".*"):
|
|
1248
|
+
path = path[:-2]
|
|
1249
|
+
records.append((number, "module", path))
|
|
1250
|
+
return records
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
def _ruby_records(text: str, rel: str) -> list[Record]:
|
|
1254
|
+
records: list[Record] = []
|
|
1255
|
+
base = posixpath.dirname(rel)
|
|
1256
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1257
|
+
m = _RB_REQUIRE.match(line)
|
|
1258
|
+
if not m:
|
|
1259
|
+
continue
|
|
1260
|
+
path = m.group("path")
|
|
1261
|
+
if m.group("kind") == "require_relative" or path.startswith(("./", "../")):
|
|
1262
|
+
resolved = _resolve_relative(base, path)
|
|
1263
|
+
if resolved is not None:
|
|
1264
|
+
records.append((number, "path", resolved))
|
|
1265
|
+
else:
|
|
1266
|
+
records.append((number, "module", path))
|
|
1267
|
+
return records
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
def _php_records(text: str, rel: str) -> list[Record]:
|
|
1271
|
+
records: list[Record] = []
|
|
1272
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1273
|
+
m = _PHP_USE.match(line)
|
|
1274
|
+
if not m:
|
|
1275
|
+
continue
|
|
1276
|
+
body = m.group("body").strip()
|
|
1277
|
+
if body.startswith(("function ", "const ")):
|
|
1278
|
+
body = body.split(None, 1)[1] if " " in body else ""
|
|
1279
|
+
if "{" in body:
|
|
1280
|
+
prefix = body.split("{", 1)[0].strip().strip("\\")
|
|
1281
|
+
items = [prefix] if prefix else []
|
|
1282
|
+
else:
|
|
1283
|
+
items = [
|
|
1284
|
+
item.strip().split(" as ", 1)[0].strip().strip("\\") for item in body.split(",")
|
|
1285
|
+
]
|
|
1286
|
+
for item in items:
|
|
1287
|
+
if re.fullmatch(r"[A-Za-z_]\w*(?:\\[A-Za-z_]\w*)*", item):
|
|
1288
|
+
records.append((number, "module", item))
|
|
1289
|
+
return records
|
|
1290
|
+
|
|
1291
|
+
|
|
1292
|
+
def _swift_records(text: str, rel: str) -> list[Record]:
|
|
1293
|
+
records: list[Record] = []
|
|
1294
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
1295
|
+
m = _SWIFT_IMPORT.match(line)
|
|
1296
|
+
if m:
|
|
1297
|
+
records.append((number, "module", m.group("mod")))
|
|
1298
|
+
return records
|
|
1299
|
+
|
|
1300
|
+
|
|
1301
|
+
EXTRACTORS = {
|
|
1302
|
+
"js": _js_records,
|
|
1303
|
+
"python": _python_records,
|
|
1304
|
+
"rust": _rust_records,
|
|
1305
|
+
"go": _go_records,
|
|
1306
|
+
"jvm": _jvm_records,
|
|
1307
|
+
"ruby": _ruby_records,
|
|
1308
|
+
"php": _php_records,
|
|
1309
|
+
"swift": _swift_records,
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
def extract_records(language: str, text: str, rel: str) -> list[Record]:
|
|
1314
|
+
"""Import records of one source file, in line order."""
|
|
1315
|
+
return EXTRACTORS[language](text.lstrip("\ufeff"), rel)
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def _key(language: str, name: str) -> str:
|
|
1319
|
+
return name.lower() if language == "php" else name
|
|
1320
|
+
|
|
1321
|
+
|
|
1322
|
+
def candidates(language: str, module: str) -> list[str]:
|
|
1323
|
+
"""Every prefix of `module` at the language's separator, longest first."""
|
|
1324
|
+
module = _key(language, module)
|
|
1325
|
+
sep = SEPARATORS[language]
|
|
1326
|
+
if sep is None:
|
|
1327
|
+
return [module]
|
|
1328
|
+
parts = module.split(sep)
|
|
1329
|
+
return [sep.join(parts[:i]) for i in range(len(parts), 0, -1)]
|
|
1330
|
+
|
|
1331
|
+
|
|
1332
|
+
# --------------------------------------------------------------------------
|
|
1333
|
+
# Counting
|
|
1334
|
+
# --------------------------------------------------------------------------
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def _walk(root: Path, excludes: Excludes, extensions: dict[str, str], start: str = ""):
|
|
1338
|
+
"""Yield (rel, path, language) for every source file left after exclusions,
|
|
1339
|
+
only those under the `start` folder (relative to `root`) when one is given."""
|
|
1340
|
+
for dirpath, dirnames, filenames in os.walk(root / start if start else root):
|
|
1341
|
+
rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/")
|
|
1342
|
+
rel_dir = "" if rel_dir == "." else rel_dir
|
|
1343
|
+
kept = []
|
|
1344
|
+
for name in sorted(dirnames):
|
|
1345
|
+
rel = f"{rel_dir}/{name}" if rel_dir else name
|
|
1346
|
+
if os.path.islink(os.path.join(dirpath, name)) or excludes.skip_dir(name, rel):
|
|
1347
|
+
continue
|
|
1348
|
+
kept.append(name)
|
|
1349
|
+
dirnames[:] = kept
|
|
1350
|
+
for name in sorted(filenames):
|
|
1351
|
+
language = extensions.get(os.path.splitext(name)[1])
|
|
1352
|
+
if language is None:
|
|
1353
|
+
continue
|
|
1354
|
+
rel = f"{rel_dir}/{name}" if rel_dir else name
|
|
1355
|
+
path = os.path.join(dirpath, name)
|
|
1356
|
+
if os.path.islink(path) or excludes.skip_file(name, rel):
|
|
1357
|
+
continue
|
|
1358
|
+
yield rel, path, language
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def _files(first_lines: dict[str, int]) -> list[dict]:
|
|
1362
|
+
return [{"path": p, "line": first_lines[p]} for p in sorted(first_lines)]
|
|
1363
|
+
|
|
1364
|
+
|
|
1365
|
+
def _languages(ecosystem: str | None) -> list[str]:
|
|
1366
|
+
"""The languages whose files a target of this ecosystem is matched in."""
|
|
1367
|
+
if ecosystem in ECOSYSTEM_LANGUAGE:
|
|
1368
|
+
return [ECOSYSTEM_LANGUAGE[ecosystem]]
|
|
1369
|
+
return list(LANGUAGE_EXTENSIONS)
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
_JVM_PACKAGE = re.compile(r"^\s*package\s+(?P<name>[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", re.M)
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def _declared_jvm_packages(root: Path, unit_path: str, owns, excludes: Excludes) -> list[str]:
|
|
1376
|
+
"""The packages a unit's own .java, .kt, .groovy and .scala files declare;
|
|
1377
|
+
`owns(rel)` says whether a file belongs to the unit, not to one inside it."""
|
|
1378
|
+
extensions = {ext: "jvm" for ext in LANGUAGE_EXTENSIONS["jvm"]}
|
|
1379
|
+
packages: set[str] = set()
|
|
1380
|
+
for rel, path, _ in _walk(root, excludes, extensions, unit_path):
|
|
1381
|
+
if not owns(rel):
|
|
1382
|
+
continue
|
|
1383
|
+
try:
|
|
1384
|
+
with open(path, "rb") as handle:
|
|
1385
|
+
text = handle.read().decode("utf-8", errors="replace")
|
|
1386
|
+
except OSError:
|
|
1387
|
+
continue
|
|
1388
|
+
m = _JVM_PACKAGE.search(text)
|
|
1389
|
+
if m:
|
|
1390
|
+
packages.add(m.group("name"))
|
|
1391
|
+
return sorted(packages)
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def _unit_import_names(root: Path, unit: dict, owns, excludes: Excludes) -> list[str]:
|
|
1395
|
+
"""A unit's import names: its `modules`, else what it declares (the
|
|
1396
|
+
packages of its JVM files, its composer.json autoload, its Package.swift
|
|
1397
|
+
targets), else its `manifest_name` by the dependency rules."""
|
|
1398
|
+
if unit["modules"] is not None:
|
|
1399
|
+
return list(unit["modules"])
|
|
1400
|
+
ecosystem, base = unit["ecosystem"], root / unit["path"]
|
|
1401
|
+
declared: list[str] = []
|
|
1402
|
+
if ecosystem in ("maven", "gradle"):
|
|
1403
|
+
declared = _declared_jvm_packages(root, unit["path"], owns, excludes)
|
|
1404
|
+
elif ecosystem == "composer":
|
|
1405
|
+
declared = _composer_autoload(base / "composer.json")
|
|
1406
|
+
elif ecosystem == "swift":
|
|
1407
|
+
declared = _swift_targets(base / "Package.swift")
|
|
1408
|
+
if declared:
|
|
1409
|
+
return declared
|
|
1410
|
+
if unit["manifest_name"] and ecosystem in ECOSYSTEM_LANGUAGE:
|
|
1411
|
+
return import_names(unit["manifest_name"], ecosystem, root)[0]
|
|
1412
|
+
return []
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
def count_imports(
|
|
1416
|
+
root: Path,
|
|
1417
|
+
dependencies: list[dict],
|
|
1418
|
+
units: list[dict] | None = None,
|
|
1419
|
+
threshold: int = DEFAULT_THRESHOLD,
|
|
1420
|
+
extra_excludes: list[str] | None = None,
|
|
1421
|
+
) -> dict:
|
|
1422
|
+
"""Count the files under `root` that import each dependency and unit."""
|
|
1423
|
+
units = units or []
|
|
1424
|
+
targets: list[dict] = []
|
|
1425
|
+
unresolved: list[dict] = []
|
|
1426
|
+
excludes = Excludes(list(DEFAULT_EXCLUDES) + list(extra_excludes or []))
|
|
1427
|
+
|
|
1428
|
+
unit_by_path = {unit["path"]: pos for pos, unit in enumerate(units)}
|
|
1429
|
+
|
|
1430
|
+
def unit_of(rel: str) -> int | None:
|
|
1431
|
+
"""The unit with the longest path that contains `rel`."""
|
|
1432
|
+
while True:
|
|
1433
|
+
if rel in unit_by_path:
|
|
1434
|
+
return unit_by_path[rel]
|
|
1435
|
+
if not rel:
|
|
1436
|
+
return None
|
|
1437
|
+
rel = posixpath.dirname(rel)
|
|
1438
|
+
|
|
1439
|
+
unit_names = [
|
|
1440
|
+
_unit_import_names(root, unit, lambda rel, pos=pos: unit_of(rel) == pos, excludes)
|
|
1441
|
+
for pos, unit in enumerate(units)
|
|
1442
|
+
]
|
|
1443
|
+
# a dependency that has one of these (language, import name) keys is the unit
|
|
1444
|
+
internal = {
|
|
1445
|
+
(language, _key(language, name))
|
|
1446
|
+
for pos, unit in enumerate(units)
|
|
1447
|
+
for language in _languages(unit["ecosystem"])
|
|
1448
|
+
for name in unit_names[pos]
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
for dep in dependencies:
|
|
1452
|
+
ecosystem = dep["ecosystem"]
|
|
1453
|
+
if "modules" in dep:
|
|
1454
|
+
names, resolution = list(dep["modules"]), "explicit"
|
|
1455
|
+
elif ecosystem is not None and ecosystem not in ECOSYSTEM_LANGUAGE:
|
|
1456
|
+
unresolved.append(_unresolved(dep, [], "unsupported-ecosystem"))
|
|
1457
|
+
continue
|
|
1458
|
+
else:
|
|
1459
|
+
names, resolution = import_names(dep["name"], ecosystem, root)
|
|
1460
|
+
if not names:
|
|
1461
|
+
unresolved.append(_unresolved(dep, names, "no-import-name"))
|
|
1462
|
+
continue
|
|
1463
|
+
languages = _languages(ecosystem)
|
|
1464
|
+
if any((lang, _key(lang, n)) in internal for lang in languages for n in names):
|
|
1465
|
+
continue
|
|
1466
|
+
targets.append(
|
|
1467
|
+
{
|
|
1468
|
+
"kind": "dependency",
|
|
1469
|
+
"dep": dep,
|
|
1470
|
+
"names": names,
|
|
1471
|
+
"resolution": resolution,
|
|
1472
|
+
"languages": languages,
|
|
1473
|
+
}
|
|
1474
|
+
)
|
|
1475
|
+
|
|
1476
|
+
unit_index: dict[int, int] = {} # position in `units` -> position in `targets`
|
|
1477
|
+
for pos, unit in enumerate(units):
|
|
1478
|
+
unit_index[pos] = len(targets)
|
|
1479
|
+
targets.append(
|
|
1480
|
+
{
|
|
1481
|
+
"kind": "unit",
|
|
1482
|
+
"unit": pos,
|
|
1483
|
+
"names": unit_names[pos],
|
|
1484
|
+
"languages": _languages(unit["ecosystem"]),
|
|
1485
|
+
}
|
|
1486
|
+
)
|
|
1487
|
+
|
|
1488
|
+
# language -> import-name key -> target positions
|
|
1489
|
+
index: dict[str, dict[str, list[int]]] = {lang: {} for lang in LANGUAGE_EXTENSIONS}
|
|
1490
|
+
for t_pos, target in enumerate(targets):
|
|
1491
|
+
for language in target["languages"]:
|
|
1492
|
+
for name in target["names"]:
|
|
1493
|
+
index[language].setdefault(_key(language, name), []).append(t_pos)
|
|
1494
|
+
|
|
1495
|
+
needed = {lang for target in targets for lang in target["languages"] if target["names"]}
|
|
1496
|
+
if units:
|
|
1497
|
+
# relative imports resolve into unit paths in these languages
|
|
1498
|
+
needed.update(("js", "python", "ruby"))
|
|
1499
|
+
extensions = {ext: lang for lang in needed for ext in LANGUAGE_EXTENSIONS[lang]}
|
|
1500
|
+
|
|
1501
|
+
first_lines: list[dict[str, int]] = [{} for _ in targets]
|
|
1502
|
+
# (importing unit, imported unit) -> importing file -> its first line
|
|
1503
|
+
edge_lines: dict[tuple[str, str], dict[str, int]] = {}
|
|
1504
|
+
external: dict[int, set[str]] = {pos: set() for pos in range(len(units))}
|
|
1505
|
+
warnings: list[str] = []
|
|
1506
|
+
files_scanned = 0
|
|
1507
|
+
|
|
1508
|
+
for rel, path, language in _walk(root, excludes, extensions):
|
|
1509
|
+
try:
|
|
1510
|
+
with open(path, "rb") as handle:
|
|
1511
|
+
text = handle.read().decode("utf-8", errors="replace")
|
|
1512
|
+
except OSError as exc:
|
|
1513
|
+
warnings.append(f"{rel}: unreadable ({exc.strerror or exc})")
|
|
1514
|
+
continue
|
|
1515
|
+
files_scanned += 1
|
|
1516
|
+
records = extract_records(language, text, rel)
|
|
1517
|
+
if not records:
|
|
1518
|
+
continue
|
|
1519
|
+
own_unit = unit_of(rel) if units else None
|
|
1520
|
+
lang_index = index[language]
|
|
1521
|
+
for number, kind, value in records:
|
|
1522
|
+
if kind == "module":
|
|
1523
|
+
hits: list[int] = []
|
|
1524
|
+
for candidate in candidates(language, value):
|
|
1525
|
+
if candidate in lang_index:
|
|
1526
|
+
hits = lang_index[candidate]
|
|
1527
|
+
break
|
|
1528
|
+
else:
|
|
1529
|
+
pos = unit_of(value) if units else None
|
|
1530
|
+
hits = [unit_index[pos]] if pos is not None else []
|
|
1531
|
+
for t_pos in hits:
|
|
1532
|
+
target = targets[t_pos]
|
|
1533
|
+
if target["kind"] == "unit":
|
|
1534
|
+
if target["unit"] == own_unit:
|
|
1535
|
+
continue
|
|
1536
|
+
if own_unit is not None:
|
|
1537
|
+
edge = (units[own_unit]["name"], units[target["unit"]]["name"])
|
|
1538
|
+
lines = edge_lines.setdefault(edge, {})
|
|
1539
|
+
if rel not in lines or number < lines[rel]:
|
|
1540
|
+
lines[rel] = number
|
|
1541
|
+
elif own_unit is not None:
|
|
1542
|
+
external[own_unit].add(target["dep"]["name"])
|
|
1543
|
+
seen = first_lines[t_pos].get(rel)
|
|
1544
|
+
if seen is None or number < seen:
|
|
1545
|
+
first_lines[t_pos][rel] = number
|
|
1546
|
+
|
|
1547
|
+
dependencies_out: list[dict] = []
|
|
1548
|
+
for t_pos, target in enumerate(targets):
|
|
1549
|
+
if target["kind"] != "dependency":
|
|
1550
|
+
continue
|
|
1551
|
+
dep = target["dep"]
|
|
1552
|
+
if target["resolution"] == "guessed" and not first_lines[t_pos]:
|
|
1553
|
+
unresolved.append(_unresolved(dep, target["names"], "guess-unmatched"))
|
|
1554
|
+
continue
|
|
1555
|
+
entry = {
|
|
1556
|
+
"name": dep["name"],
|
|
1557
|
+
"ecosystem": dep["ecosystem"],
|
|
1558
|
+
"import_names": target["names"],
|
|
1559
|
+
"resolution": target["resolution"],
|
|
1560
|
+
}
|
|
1561
|
+
if dep["scope"] == "dev":
|
|
1562
|
+
entry["scope"] = "dev"
|
|
1563
|
+
entry["files"] = _files(first_lines[t_pos])
|
|
1564
|
+
entry["file_count"] = len(first_lines[t_pos])
|
|
1565
|
+
entry["above_threshold"] = entry["file_count"] >= threshold
|
|
1566
|
+
dependencies_out.append(entry)
|
|
1567
|
+
dependencies_out.sort(key=lambda d: (-d["file_count"], d["name"], d["ecosystem"] or ""))
|
|
1568
|
+
unresolved.sort(key=lambda d: (d["name"], d["ecosystem"] or ""))
|
|
1569
|
+
|
|
1570
|
+
result: dict = {
|
|
1571
|
+
"threshold": threshold,
|
|
1572
|
+
"files_scanned": files_scanned,
|
|
1573
|
+
"dependencies": dependencies_out,
|
|
1574
|
+
"unresolved": unresolved,
|
|
1575
|
+
}
|
|
1576
|
+
if units:
|
|
1577
|
+
imported_by: dict[str, set[str]] = {u["name"]: set() for u in units}
|
|
1578
|
+
imports_from: dict[str, set[str]] = {u["name"]: set() for u in units}
|
|
1579
|
+
for src, dst in edge_lines:
|
|
1580
|
+
imports_from[src].add(dst)
|
|
1581
|
+
imported_by[dst].add(src)
|
|
1582
|
+
units_out = []
|
|
1583
|
+
for pos, unit in enumerate(units):
|
|
1584
|
+
lines = first_lines[unit_index[pos]]
|
|
1585
|
+
units_out.append(
|
|
1586
|
+
{
|
|
1587
|
+
"name": unit["name"],
|
|
1588
|
+
"path": unit["path"],
|
|
1589
|
+
"import_names": unit_names[pos],
|
|
1590
|
+
"files": _files(lines),
|
|
1591
|
+
"file_count": len(lines),
|
|
1592
|
+
"imported_by": sorted(imported_by[unit["name"]]),
|
|
1593
|
+
"imports_from": sorted(imports_from[unit["name"]]),
|
|
1594
|
+
"external_deps": sorted(external[pos]),
|
|
1595
|
+
}
|
|
1596
|
+
)
|
|
1597
|
+
result["units"] = sorted(units_out, key=lambda u: u["name"])
|
|
1598
|
+
result["edges"] = [list(edge) for edge in sorted(edge_lines)]
|
|
1599
|
+
result["edge_files"] = [
|
|
1600
|
+
{"from": src, "to": dst, "files": _files(edge_lines[(src, dst)])}
|
|
1601
|
+
for src, dst in sorted(edge_lines)
|
|
1602
|
+
]
|
|
1603
|
+
if warnings:
|
|
1604
|
+
result["warnings"] = warnings
|
|
1605
|
+
return result
|
|
1606
|
+
|
|
1607
|
+
|
|
1608
|
+
def _unresolved(dep: dict, names: list[str], reason: str) -> dict:
|
|
1609
|
+
entry = {
|
|
1610
|
+
"name": dep["name"],
|
|
1611
|
+
"ecosystem": dep["ecosystem"],
|
|
1612
|
+
"import_names": names,
|
|
1613
|
+
"reason": reason,
|
|
1614
|
+
}
|
|
1615
|
+
if dep["scope"] == "dev":
|
|
1616
|
+
entry["scope"] = "dev"
|
|
1617
|
+
return entry
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
def prefix_paths(result: dict, prefix: str) -> dict:
|
|
1621
|
+
"""Write every `files[].path` of `result` under `prefix`, a forward-slash folder."""
|
|
1622
|
+
if not prefix:
|
|
1623
|
+
return result
|
|
1624
|
+
|
|
1625
|
+
def moved(files: list[dict]) -> list[dict]:
|
|
1626
|
+
return [{**f, "path": f"{prefix}/{f['path']}"} for f in files]
|
|
1627
|
+
|
|
1628
|
+
for entry in result["dependencies"] + result.get("units", []) + result.get("edge_files", []):
|
|
1629
|
+
entry["files"] = moved(entry["files"])
|
|
1630
|
+
return result
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
def _relative_prefix(root: Path, base: Path) -> str:
|
|
1634
|
+
"""The forward-slash path of `root` inside `base` ('' when they are one folder).
|
|
1635
|
+
|
|
1636
|
+
Raises InputError when `base` is not a folder or does not hold `root`.
|
|
1637
|
+
"""
|
|
1638
|
+
if not base.is_dir():
|
|
1639
|
+
raise InputError(f"--relative-to not a directory: {base}")
|
|
1640
|
+
try:
|
|
1641
|
+
prefix = root.resolve().relative_to(base.resolve()).as_posix()
|
|
1642
|
+
except ValueError:
|
|
1643
|
+
raise InputError(f"root {root} is not inside --relative-to {base}") from None
|
|
1644
|
+
return "" if prefix == "." else prefix
|
|
1645
|
+
|
|
1646
|
+
|
|
1647
|
+
def as_libraries(result: dict) -> list[dict]:
|
|
1648
|
+
"""The skf-pair-intersect.py `--libraries` input: dependencies, then units."""
|
|
1649
|
+
rows = [
|
|
1650
|
+
{"name": d["name"], "files": [f["path"] for f in d["files"]]}
|
|
1651
|
+
for d in result["dependencies"]
|
|
1652
|
+
]
|
|
1653
|
+
rows += [
|
|
1654
|
+
{"name": u["name"], "files": [f["path"] for f in u["files"]]}
|
|
1655
|
+
for u in result.get("units", [])
|
|
1656
|
+
]
|
|
1657
|
+
return rows
|
|
1658
|
+
|
|
1659
|
+
|
|
1660
|
+
SUMMARY_MAX_FILES = 5
|
|
1661
|
+
SUMMARY_IMPORTS = "the imports input (a `count --units` envelope)"
|
|
1662
|
+
SUMMARY_MANIFESTS = "the --manifests input (a skf-scan-manifests.py scan)"
|
|
1663
|
+
|
|
1664
|
+
|
|
1665
|
+
def _object_rows(data: object, key: str, what: str) -> list[dict]:
|
|
1666
|
+
"""`data[key]` when it is a list of objects, else an InputError naming the input."""
|
|
1667
|
+
rows = data.get(key) if isinstance(data, dict) else None
|
|
1668
|
+
if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows):
|
|
1669
|
+
raise InputError(f"{what} holds no `{key}` list of objects")
|
|
1670
|
+
return rows
|
|
1671
|
+
|
|
1672
|
+
|
|
1673
|
+
def summarize(result: object, manifests: object | None = None, max_files: int = SUMMARY_MAX_FILES) -> dict:
|
|
1674
|
+
"""The unit graph of a `count --units` envelope without its file lists, and
|
|
1675
|
+
the fields of a manifest scan the cohesion triggers read (see `summary`)."""
|
|
1676
|
+
units = _object_rows(result, "units", SUMMARY_IMPORTS)
|
|
1677
|
+
edge_files = _object_rows(result, "edge_files", SUMMARY_IMPORTS)
|
|
1678
|
+
try:
|
|
1679
|
+
out: dict = {
|
|
1680
|
+
"units": [
|
|
1681
|
+
{"name": u["name"], "imports_from": u["imports_from"], "imported_by": u["imported_by"],
|
|
1682
|
+
"file_count": u["file_count"], "external_dep_count": len(u["external_deps"])}
|
|
1683
|
+
for u in units
|
|
1684
|
+
],
|
|
1685
|
+
"edge_files": [
|
|
1686
|
+
{"from": e["from"], "to": e["to"], "file_count": len(e["files"]), "files": e["files"][:max_files]}
|
|
1687
|
+
for e in edge_files
|
|
1688
|
+
],
|
|
1689
|
+
}
|
|
1690
|
+
except (KeyError, TypeError) as exc:
|
|
1691
|
+
raise InputError(f"{SUMMARY_IMPORTS} has a malformed units or edge_files entry "
|
|
1692
|
+
f"({type(exc).__name__}: {exc})") from None
|
|
1693
|
+
if manifests is not None:
|
|
1694
|
+
rows = _object_rows(manifests, "manifests", SUMMARY_MANIFESTS)
|
|
1695
|
+
umbrellas = manifests.get("umbrella_candidates")
|
|
1696
|
+
out["umbrella_candidates"] = umbrellas if isinstance(umbrellas, list) else []
|
|
1697
|
+
out["manifests"] = [
|
|
1698
|
+
{"path": m.get("path"), "name": m.get("name"), "private": m.get("private"),
|
|
1699
|
+
"internal_deps": m.get("internal_deps") or []}
|
|
1700
|
+
for m in rows
|
|
1701
|
+
]
|
|
1702
|
+
if result.get("warnings"):
|
|
1703
|
+
out["warnings"] = result["warnings"]
|
|
1704
|
+
return out
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
# --------------------------------------------------------------------------
|
|
1708
|
+
# CLI
|
|
1709
|
+
# --------------------------------------------------------------------------
|
|
1710
|
+
|
|
1711
|
+
|
|
1712
|
+
def _cmd_count(args: argparse.Namespace) -> int:
|
|
1713
|
+
root = Path(args.root)
|
|
1714
|
+
if not root.is_dir():
|
|
1715
|
+
print(f"error: root not a directory: {root}", file=sys.stderr)
|
|
1716
|
+
return 1
|
|
1717
|
+
if args.deps is None and args.units is None:
|
|
1718
|
+
print("error: pass --deps, --units or both", file=sys.stderr)
|
|
1719
|
+
return 1
|
|
1720
|
+
if args.deps == "-" and args.units == "-":
|
|
1721
|
+
print("error: only one of --deps and --units may read stdin", file=sys.stderr)
|
|
1722
|
+
return 1
|
|
1723
|
+
if args.threshold < 0:
|
|
1724
|
+
print(f"error: --threshold must be >= 0; got {args.threshold}", file=sys.stderr)
|
|
1725
|
+
return 1
|
|
1726
|
+
try:
|
|
1727
|
+
prefix = "" if args.relative_to is None else _relative_prefix(root, Path(args.relative_to))
|
|
1728
|
+
dependencies: list[dict] = []
|
|
1729
|
+
units: list[dict] | None = None
|
|
1730
|
+
if args.deps is not None:
|
|
1731
|
+
dependencies = parse_dependencies(_read_json_source(args.deps, "--deps"))
|
|
1732
|
+
if args.units is not None:
|
|
1733
|
+
units = parse_units(_read_json_source(args.units, "--units"))
|
|
1734
|
+
except InputError as exc:
|
|
1735
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
1736
|
+
return 1
|
|
1737
|
+
try:
|
|
1738
|
+
result = count_imports(
|
|
1739
|
+
root,
|
|
1740
|
+
dependencies,
|
|
1741
|
+
units=units,
|
|
1742
|
+
threshold=args.threshold,
|
|
1743
|
+
extra_excludes=args.exclude,
|
|
1744
|
+
)
|
|
1745
|
+
except OSError as exc:
|
|
1746
|
+
print(f"error: filesystem error during scan: {exc}", file=sys.stderr)
|
|
1747
|
+
return 1
|
|
1748
|
+
result = prefix_paths(result, prefix)
|
|
1749
|
+
output = as_libraries(result) if args.format == "libraries" else result
|
|
1750
|
+
json.dump(output, sys.stdout, indent=2)
|
|
1751
|
+
sys.stdout.write("\n")
|
|
1752
|
+
return 0
|
|
1753
|
+
|
|
1754
|
+
|
|
1755
|
+
def _cmd_summary(args: argparse.Namespace) -> int:
|
|
1756
|
+
if args.imports == "-" and args.manifests == "-":
|
|
1757
|
+
print("error: only one of the imports file and --manifests may read stdin", file=sys.stderr)
|
|
1758
|
+
return 1
|
|
1759
|
+
if args.max_files < 0:
|
|
1760
|
+
print(f"error: --max-files must be >= 0; got {args.max_files}", file=sys.stderr)
|
|
1761
|
+
return 1
|
|
1762
|
+
try:
|
|
1763
|
+
result = _read_json_source(args.imports, "imports")
|
|
1764
|
+
manifests = None if args.manifests is None else _read_json_source(args.manifests, "--manifests")
|
|
1765
|
+
output = summarize(result, manifests, args.max_files)
|
|
1766
|
+
except InputError as exc:
|
|
1767
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
1768
|
+
return 1
|
|
1769
|
+
json.dump(output, sys.stdout)
|
|
1770
|
+
sys.stdout.write("\n")
|
|
1771
|
+
return 0
|
|
1772
|
+
|
|
1773
|
+
|
|
1774
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
1775
|
+
parser = argparse.ArgumentParser(
|
|
1776
|
+
prog="skf-count-imports",
|
|
1777
|
+
description=(
|
|
1778
|
+
"Count, per dependency and per unit, the source files that import "
|
|
1779
|
+
"it, with boundary-anchored matching and distribution-to-import names."
|
|
1780
|
+
),
|
|
1781
|
+
)
|
|
1782
|
+
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
1783
|
+
|
|
1784
|
+
p_count = sub.add_parser("count", help="count importing files and emit JSON")
|
|
1785
|
+
p_count.add_argument("root", help="path to the tree to scan")
|
|
1786
|
+
p_count.add_argument(
|
|
1787
|
+
"--deps",
|
|
1788
|
+
default=None,
|
|
1789
|
+
help=(
|
|
1790
|
+
"dependencies JSON file, or '-' for stdin: the skf-scan-manifests.py "
|
|
1791
|
+
'envelope, {"dependencies": [...]} or a list of names or '
|
|
1792
|
+
'{"name", "ecosystem", "modules", "scope"} objects'
|
|
1793
|
+
),
|
|
1794
|
+
)
|
|
1795
|
+
p_count.add_argument(
|
|
1796
|
+
"--units",
|
|
1797
|
+
default=None,
|
|
1798
|
+
help=(
|
|
1799
|
+
"units JSON file, or '-' for stdin: a list of "
|
|
1800
|
+
'{"name", "path", "modules", "manifest_name", "ecosystem"} objects'
|
|
1801
|
+
),
|
|
1802
|
+
)
|
|
1803
|
+
p_count.add_argument(
|
|
1804
|
+
"--threshold",
|
|
1805
|
+
type=int,
|
|
1806
|
+
default=DEFAULT_THRESHOLD,
|
|
1807
|
+
help=f"file count at which a dependency is above_threshold (default {DEFAULT_THRESHOLD})",
|
|
1808
|
+
)
|
|
1809
|
+
p_count.add_argument(
|
|
1810
|
+
"--exclude",
|
|
1811
|
+
action="append",
|
|
1812
|
+
default=[],
|
|
1813
|
+
metavar="GLOB",
|
|
1814
|
+
help=(
|
|
1815
|
+
"another path glob to leave out, added to the defaults (repeatable; "
|
|
1816
|
+
"a leading / or ./ anchors it to the root)"
|
|
1817
|
+
),
|
|
1818
|
+
)
|
|
1819
|
+
p_count.add_argument(
|
|
1820
|
+
"--format",
|
|
1821
|
+
choices=("full", "libraries"),
|
|
1822
|
+
default="full",
|
|
1823
|
+
help="full envelope (default) or the skf-pair-intersect.py --libraries list",
|
|
1824
|
+
)
|
|
1825
|
+
p_count.add_argument(
|
|
1826
|
+
"--relative-to",
|
|
1827
|
+
default=None,
|
|
1828
|
+
metavar="DIR",
|
|
1829
|
+
help=(
|
|
1830
|
+
"write each files[].path relative to DIR, a folder that holds the "
|
|
1831
|
+
"root, instead of to the root"
|
|
1832
|
+
),
|
|
1833
|
+
)
|
|
1834
|
+
p_count.set_defaults(func=_cmd_count)
|
|
1835
|
+
|
|
1836
|
+
p_summary = sub.add_parser(
|
|
1837
|
+
"summary", help="print the unit graph of a saved count --units envelope without its file lists"
|
|
1838
|
+
)
|
|
1839
|
+
p_summary.add_argument("imports", help="the count --units envelope, a JSON file or '-' for stdin")
|
|
1840
|
+
p_summary.add_argument(
|
|
1841
|
+
"--manifests",
|
|
1842
|
+
default=None,
|
|
1843
|
+
help=(
|
|
1844
|
+
"the skf-scan-manifests.py scan envelope, a JSON file or '-' for stdin: adds "
|
|
1845
|
+
"umbrella_candidates and each manifest's path, name, private and internal_deps"
|
|
1846
|
+
),
|
|
1847
|
+
)
|
|
1848
|
+
p_summary.add_argument(
|
|
1849
|
+
"--max-files",
|
|
1850
|
+
type=int,
|
|
1851
|
+
default=SUMMARY_MAX_FILES,
|
|
1852
|
+
metavar="N",
|
|
1853
|
+
help=f"files listed per edge_files entry (default {SUMMARY_MAX_FILES})",
|
|
1854
|
+
)
|
|
1855
|
+
p_summary.set_defaults(func=_cmd_summary)
|
|
1856
|
+
|
|
1857
|
+
return parser
|
|
1858
|
+
|
|
1859
|
+
|
|
1860
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1861
|
+
parser = _build_parser()
|
|
1862
|
+
args = parser.parse_args(argv)
|
|
1863
|
+
return args.func(args)
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
if __name__ == "__main__":
|
|
1867
|
+
raise SystemExit(main())
|