bmad-module-skill-forge 2.2.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +49 -49
  3. package/docs/404.md +10 -8
  4. package/docs/_data/pinned.yaml +22 -13
  5. package/docs/_internal/STABILITY.md +58 -35
  6. package/docs/agents.md +39 -53
  7. package/docs/architecture.md +65 -51
  8. package/docs/bmad-synergy.md +53 -51
  9. package/docs/campaign.md +127 -75
  10. package/docs/concepts.md +34 -28
  11. package/docs/examples.md +70 -68
  12. package/docs/forge-auto.md +34 -31
  13. package/docs/getting-started.md +118 -57
  14. package/docs/how-it-works.md +25 -21
  15. package/docs/index.md +15 -15
  16. package/docs/skill-model.md +70 -59
  17. package/docs/troubleshooting.md +135 -18
  18. package/docs/verifying-a-skill.md +83 -57
  19. package/docs/why-skf.md +24 -22
  20. package/docs/workflows.md +274 -110
  21. package/package.json +18 -10
  22. package/src/forger/preferences.yaml +8 -1
  23. package/src/knowledge/agentskills-spec.md +2 -0
  24. package/src/knowledge/ccc-bridge.md +40 -24
  25. package/src/knowledge/confidence-tiers.md +9 -5
  26. package/src/knowledge/overview.md +3 -3
  27. package/src/knowledge/progressive-capability.md +8 -8
  28. package/src/knowledge/provenance-tracking.md +9 -5
  29. package/src/knowledge/qmd-registry.md +2 -2
  30. package/src/knowledge/skf-knowledge-index.csv +2 -2
  31. package/src/knowledge/skill-lifecycle.md +18 -11
  32. package/src/knowledge/split-body-strategy.md +0 -3
  33. package/src/knowledge/tool-resolution.md +4 -2
  34. package/src/knowledge/version-paths.md +101 -15
  35. package/src/module-help.csv +3 -3
  36. package/src/module.yaml +10 -0
  37. package/src/shared/data/ast-grep-recipes.yaml +608 -0
  38. package/src/shared/data/ide-context-files.json +30 -0
  39. package/src/shared/health-check.md +32 -12
  40. package/src/shared/references/description-guard-protocol.md +12 -9
  41. package/src/shared/references/feasibility-report-schema.md +19 -13
  42. package/src/shared/references/headless-gate-convention.md +55 -3
  43. package/src/shared/references/output-contract-schema.md +19 -8
  44. package/src/shared/references/pipeline-contracts.md +94 -29
  45. package/src/shared/references/tessl-review.md +77 -0
  46. package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
  47. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
  48. package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
  49. package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
  50. package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
  51. package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
  52. package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
  53. package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
  54. package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
  55. package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
  56. package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
  57. package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
  58. package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
  59. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
  60. package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
  61. package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
  62. package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
  63. package/src/shared/scripts/skf-atomic-write.py +30 -14
  64. package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
  65. package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
  66. package/src/shared/scripts/skf-check-unit-records.py +184 -0
  67. package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
  68. package/src/shared/scripts/skf-classify-changed-files.py +694 -0
  69. package/src/shared/scripts/skf-comention-pairs.py +1333 -123
  70. package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
  71. package/src/shared/scripts/skf-count-imports.py +1867 -0
  72. package/src/shared/scripts/skf-count-tokens.py +1 -1
  73. package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
  74. package/src/shared/scripts/skf-description-guard.py +99 -11
  75. package/src/shared/scripts/skf-detect-docs.py +694 -2
  76. package/src/shared/scripts/skf-detect-language.py +486 -107
  77. package/src/shared/scripts/skf-detect-registry.py +887 -0
  78. package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
  79. package/src/shared/scripts/skf-detect-tools.py +209 -27
  80. package/src/shared/scripts/skf-detect-workspaces.py +506 -30
  81. package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
  82. package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
  83. package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
  84. package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
  85. package/src/shared/scripts/skf-extract-public-api.py +4084 -85
  86. package/src/shared/scripts/skf-extraction-inventory.py +746 -0
  87. package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
  88. package/src/shared/scripts/skf-fetch-temporal.py +674 -0
  89. package/src/shared/scripts/skf-find-cycles.py +97 -29
  90. package/src/shared/scripts/skf-find-test-report.py +476 -0
  91. package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
  92. package/src/shared/scripts/skf-github-fetch.py +365 -0
  93. package/src/shared/scripts/skf-github-probe.py +661 -0
  94. package/src/shared/scripts/skf-hash-content.py +166 -12
  95. package/src/shared/scripts/skf-language-corpora.py +1 -1
  96. package/src/shared/scripts/skf-load-provenance.py +159 -27
  97. package/src/shared/scripts/skf-manifest-ops.py +1 -1
  98. package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
  99. package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
  100. package/src/shared/scripts/skf-names-present.py +221 -0
  101. package/src/shared/scripts/skf-pair-intersect.py +119 -28
  102. package/src/shared/scripts/skf-parse-gaps.py +1384 -0
  103. package/src/shared/scripts/skf-preapply.py +1 -1
  104. package/src/shared/scripts/skf-preflight.py +161 -27
  105. package/src/shared/scripts/skf-promote-staged.py +430 -0
  106. package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
  107. package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
  108. package/src/shared/scripts/skf-quick-batch.py +774 -0
  109. package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
  110. package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
  111. package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
  112. package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
  113. package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
  114. package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
  115. package/src/shared/scripts/skf-resolve-package.py +523 -106
  116. package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
  117. package/src/shared/scripts/skf-run-lock.py +577 -0
  118. package/src/shared/scripts/skf-scan-manifests.py +998 -57
  119. package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
  120. package/src/shared/scripts/skf-severity-classify.py +712 -108
  121. package/src/shared/scripts/skf-shape-detect.py +298 -204
  122. package/src/shared/scripts/skf-shard-body.py +9 -8
  123. package/src/shared/scripts/skf-skill-inventory.py +2412 -89
  124. package/src/shared/scripts/skf-skills-module.py +472 -0
  125. package/src/shared/scripts/skf-source-tree.py +1796 -0
  126. package/src/shared/scripts/skf-structural-diff.py +670 -119
  127. package/src/shared/scripts/skf-tessl-review.py +706 -0
  128. package/src/shared/scripts/skf-update-active-symlink.py +5 -5
  129. package/src/shared/scripts/skf-update-run-state.py +624 -0
  130. package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
  131. package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
  132. package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
  133. package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
  134. package/src/shared/scripts/skf-validate-output.py +712 -64
  135. package/src/shared/scripts/skf-validate-pins.py +1 -1
  136. package/src/shared/scripts/skf-verify-no-trace.py +116 -10
  137. package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
  138. package/src/shared/scripts/skf-write-skill-brief.py +367 -19
  139. package/src/shared/tool-requirements.yaml +201 -0
  140. package/src/skf-analyze-source/SKILL.md +41 -36
  141. package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
  142. package/src/skf-analyze-source/customize.toml +36 -24
  143. package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
  144. package/src/skf-analyze-source/references/continue.md +2 -13
  145. package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
  146. package/src/skf-analyze-source/references/generate-briefs.md +121 -55
  147. package/src/skf-analyze-source/references/headless-contract.md +42 -15
  148. package/src/skf-analyze-source/references/identify-units.md +96 -110
  149. package/src/skf-analyze-source/references/init.md +73 -71
  150. package/src/skf-analyze-source/references/map-and-detect.md +123 -105
  151. package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
  152. package/src/skf-analyze-source/references/recommend.md +51 -70
  153. package/src/skf-analyze-source/references/scan-project.md +60 -84
  154. package/src/skf-analyze-source/references/scan-root.md +39 -0
  155. package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
  156. package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
  157. package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
  158. package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
  159. package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
  160. package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
  161. package/src/skf-audit-skill/SKILL.md +43 -51
  162. package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
  163. package/src/skf-audit-skill/customize.toml +41 -20
  164. package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
  165. package/src/skf-audit-skill/references/doc-drift.md +141 -0
  166. package/src/skf-audit-skill/references/headless-contract.md +29 -0
  167. package/src/skf-audit-skill/references/init.md +148 -210
  168. package/src/skf-audit-skill/references/re-index.md +96 -108
  169. package/src/skf-audit-skill/references/report.md +96 -33
  170. package/src/skf-audit-skill/references/semantic-diff.md +27 -18
  171. package/src/skf-audit-skill/references/severity-classify.md +79 -65
  172. package/src/skf-audit-skill/references/severity-rules.md +15 -6
  173. package/src/skf-audit-skill/references/structural-diff.md +75 -90
  174. package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
  175. package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
  176. package/src/skf-brief-skill/SKILL.md +36 -25
  177. package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
  178. package/src/skf-brief-skill/assets/scope-templates.md +17 -24
  179. package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
  180. package/src/skf-brief-skill/customize.toml +48 -20
  181. package/src/skf-brief-skill/references/analyze-target.md +142 -125
  182. package/src/skf-brief-skill/references/confirm-brief.md +10 -8
  183. package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
  184. package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
  185. package/src/skf-brief-skill/references/gather-intent.md +87 -182
  186. package/src/skf-brief-skill/references/headless-args.md +84 -16
  187. package/src/skf-brief-skill/references/health-check.md +1 -1
  188. package/src/skf-brief-skill/references/invocation-contract.md +18 -13
  189. package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
  190. package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
  191. package/src/skf-brief-skill/references/scope-definition.md +123 -47
  192. package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
  193. package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
  194. package/src/skf-brief-skill/references/version-resolution.md +3 -2
  195. package/src/skf-brief-skill/references/write-brief.md +66 -57
  196. package/src/skf-campaign/SKILL.md +31 -34
  197. package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
  198. package/src/skf-campaign/customize.toml +47 -16
  199. package/src/skf-campaign/references/campaign-contracts.md +39 -29
  200. package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
  201. package/src/skf-campaign/references/health-check.md +7 -16
  202. package/src/skf-campaign/references/step-01-setup.md +46 -84
  203. package/src/skf-campaign/references/step-02-strategy.md +31 -30
  204. package/src/skf-campaign/references/step-03-pins.md +26 -18
  205. package/src/skf-campaign/references/step-04-provenance.md +24 -23
  206. package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
  207. package/src/skf-campaign/references/step-06-batch.md +41 -34
  208. package/src/skf-campaign/references/step-07-capstone.md +40 -25
  209. package/src/skf-campaign/references/step-08-verify.md +28 -29
  210. package/src/skf-campaign/references/step-09-refine.md +29 -30
  211. package/src/skf-campaign/references/step-10-export.md +48 -36
  212. package/src/skf-campaign/references/step-11-maintenance.md +31 -34
  213. package/src/skf-campaign/references/step-resume.md +37 -68
  214. package/src/skf-campaign/scripts/campaign-deps.py +57 -18
  215. package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
  216. package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
  217. package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
  218. package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
  219. package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
  220. package/src/skf-campaign/scripts/campaign-report.py +204 -37
  221. package/src/skf-campaign/scripts/campaign-state.py +1128 -0
  222. package/src/skf-campaign/scripts/campaign-status.py +10 -74
  223. package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
  224. package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
  225. package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
  226. package/src/skf-campaign/templates/campaign-report-template.md +5 -1
  227. package/src/skf-campaign/templates/kickoff-template.md +0 -14
  228. package/src/skf-create-skill/SKILL.md +27 -20
  229. package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
  230. package/src/skf-create-skill/assets/skill-sections.md +15 -5
  231. package/src/skf-create-skill/customize.toml +20 -8
  232. package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
  233. package/src/skf-create-skill/references/batch-mode.md +81 -0
  234. package/src/skf-create-skill/references/ccc-index-check.md +27 -0
  235. package/src/skf-create-skill/references/compile.md +107 -29
  236. package/src/skf-create-skill/references/component-extraction.md +113 -91
  237. package/src/skf-create-skill/references/enrich.md +41 -23
  238. package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
  239. package/src/skf-create-skill/references/extract.md +200 -161
  240. package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
  241. package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
  242. package/src/skf-create-skill/references/extraction-patterns.md +584 -196
  243. package/src/skf-create-skill/references/generate-artifacts.md +145 -93
  244. package/src/skf-create-skill/references/health-check.md +1 -1
  245. package/src/skf-create-skill/references/load-brief.md +53 -36
  246. package/src/skf-create-skill/references/relabel-rule.md +12 -0
  247. package/src/skf-create-skill/references/report.md +28 -59
  248. package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
  249. package/src/skf-create-skill/references/step-auto-shard.md +20 -93
  250. package/src/skf-create-skill/references/step-doc-rot.md +44 -60
  251. package/src/skf-create-skill/references/step-doc-sources.md +28 -19
  252. package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
  253. package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
  254. package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
  255. package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
  256. package/src/skf-create-skill/references/validate.md +173 -99
  257. package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
  258. package/src/skf-create-stack-skill/SKILL.md +29 -56
  259. package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
  260. package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
  261. package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
  262. package/src/skf-create-stack-skill/customize.toml +23 -12
  263. package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
  264. package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
  265. package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
  266. package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
  267. package/src/skf-create-stack-skill/references/generate-output.md +138 -128
  268. package/src/skf-create-stack-skill/references/init.md +90 -32
  269. package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
  270. package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
  271. package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
  272. package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
  273. package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
  274. package/src/skf-create-stack-skill/references/report.md +39 -33
  275. package/src/skf-create-stack-skill/references/validate.md +20 -39
  276. package/src/skf-drop-skill/SKILL.md +28 -51
  277. package/src/skf-drop-skill/customize.toml +44 -30
  278. package/src/skf-drop-skill/references/execute.md +79 -146
  279. package/src/skf-drop-skill/references/invocation-contract.md +64 -0
  280. package/src/skf-drop-skill/references/report.md +46 -30
  281. package/src/skf-drop-skill/references/select.md +118 -124
  282. package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
  283. package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
  284. package/src/skf-export-skill/SKILL.md +40 -62
  285. package/src/skf-export-skill/assets/managed-section-format.md +14 -50
  286. package/src/skf-export-skill/assets/snippet-format.md +3 -1
  287. package/src/skf-export-skill/customize.toml +42 -13
  288. package/src/skf-export-skill/references/generate-snippet.md +54 -46
  289. package/src/skf-export-skill/references/health-check.md +2 -0
  290. package/src/skf-export-skill/references/invocation-contract.md +15 -0
  291. package/src/skf-export-skill/references/load-skill.md +88 -62
  292. package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
  293. package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
  294. package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
  295. package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
  296. package/src/skf-export-skill/references/package.md +15 -73
  297. package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
  298. package/src/skf-export-skill/references/result-envelope.md +31 -16
  299. package/src/skf-export-skill/references/summary.md +41 -17
  300. package/src/skf-export-skill/references/token-report.md +4 -12
  301. package/src/skf-export-skill/references/update-context.md +118 -237
  302. package/src/skf-forger/SKILL.md +61 -37
  303. package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
  304. package/src/skf-forger/customize.toml +43 -0
  305. package/src/skf-forger/references/pipeline-mode.md +73 -18
  306. package/src/skf-forger/scripts/forge-status.py +299 -0
  307. package/src/skf-forger/scripts/parse-pipeline.py +322 -54
  308. package/src/skf-forger/scripts/pipeline-gate.py +428 -0
  309. package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
  310. package/src/skf-quick-skill/SKILL.md +54 -53
  311. package/src/skf-quick-skill/assets/skill-template.md +4 -2
  312. package/src/skf-quick-skill/customize.toml +51 -20
  313. package/src/skf-quick-skill/references/batch-mode.md +74 -21
  314. package/src/skf-quick-skill/references/compile.md +30 -31
  315. package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
  316. package/src/skf-quick-skill/references/finalize.md +69 -18
  317. package/src/skf-quick-skill/references/halt-contract.md +41 -29
  318. package/src/skf-quick-skill/references/health-check.md +5 -3
  319. package/src/skf-quick-skill/references/quick-extract.md +114 -45
  320. package/src/skf-quick-skill/references/registry-resolution.md +9 -64
  321. package/src/skf-quick-skill/references/resolve-target.md +116 -66
  322. package/src/skf-quick-skill/references/write-and-validate.md +79 -40
  323. package/src/skf-refine-architecture/SKILL.md +43 -53
  324. package/src/skf-refine-architecture/customize.toml +49 -20
  325. package/src/skf-refine-architecture/references/compile.md +97 -54
  326. package/src/skf-refine-architecture/references/exit-codes.md +30 -0
  327. package/src/skf-refine-architecture/references/finding-storage.md +21 -0
  328. package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
  329. package/src/skf-refine-architecture/references/improvements.md +14 -20
  330. package/src/skf-refine-architecture/references/init.md +98 -57
  331. package/src/skf-refine-architecture/references/issue-detection.md +32 -39
  332. package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
  333. package/src/skf-refine-architecture/references/report.md +65 -54
  334. package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
  335. package/src/skf-rename-skill/SKILL.md +19 -54
  336. package/src/skf-rename-skill/customize.toml +32 -26
  337. package/src/skf-rename-skill/references/execute.md +230 -184
  338. package/src/skf-rename-skill/references/exit-codes.md +7 -7
  339. package/src/skf-rename-skill/references/health-check.md +1 -7
  340. package/src/skf-rename-skill/references/invocation-contract.md +25 -0
  341. package/src/skf-rename-skill/references/report.md +63 -52
  342. package/src/skf-rename-skill/references/select.md +197 -115
  343. package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
  344. package/src/skf-setup/SKILL.md +26 -26
  345. package/src/skf-setup/customize.toml +14 -10
  346. package/src/skf-setup/references/auto-index.md +50 -63
  347. package/src/skf-setup/references/ccc-index.md +29 -90
  348. package/src/skf-setup/references/detect-and-tier.md +58 -67
  349. package/src/skf-setup/references/health-check.md +2 -0
  350. package/src/skf-setup/references/invocation-contract.md +14 -0
  351. package/src/skf-setup/references/report.md +43 -166
  352. package/src/skf-setup/references/tier-rules.md +5 -5
  353. package/src/skf-setup/references/write-config.md +34 -57
  354. package/src/skf-test-skill/SKILL.md +17 -57
  355. package/src/skf-test-skill/assets/output-section-formats.md +33 -34
  356. package/src/skf-test-skill/customize.toml +27 -13
  357. package/src/skf-test-skill/references/coherence-check.md +134 -62
  358. package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
  359. package/src/skf-test-skill/references/coverage-check.md +217 -254
  360. package/src/skf-test-skill/references/detect-mode.md +5 -5
  361. package/src/skf-test-skill/references/external-validators.md +94 -82
  362. package/src/skf-test-skill/references/health-check.md +6 -7
  363. package/src/skf-test-skill/references/init.md +163 -88
  364. package/src/skf-test-skill/references/invocation-contract.md +66 -0
  365. package/src/skf-test-skill/references/migration-section-rules.md +5 -2
  366. package/src/skf-test-skill/references/report.md +162 -142
  367. package/src/skf-test-skill/references/score.md +80 -107
  368. package/src/skf-test-skill/references/scoring-rules.md +41 -155
  369. package/src/skf-test-skill/references/source-access-protocol.md +19 -92
  370. package/src/skf-test-skill/references/step-hard-gate.md +51 -33
  371. package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
  372. package/src/skf-test-skill/scripts/build-result-context.py +287 -0
  373. package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
  374. package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
  375. package/src/skf-test-skill/scripts/compute-score.py +223 -45
  376. package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
  377. package/src/skf-test-skill/scripts/hard-gate.py +203 -0
  378. package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
  379. package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
  380. package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
  381. package/src/skf-test-skill/scripts/score-signatures.py +459 -0
  382. package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
  383. package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
  384. package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
  385. package/src/skf-test-skill/templates/test-report-template.md +21 -15
  386. package/src/skf-update-skill/SKILL.md +38 -25
  387. package/src/skf-update-skill/customize.toml +27 -12
  388. package/src/skf-update-skill/references/detect-changes.md +233 -222
  389. package/src/skf-update-skill/references/gap-driven.md +336 -0
  390. package/src/skf-update-skill/references/health-check.md +31 -5
  391. package/src/skf-update-skill/references/init.md +250 -84
  392. package/src/skf-update-skill/references/invocation-contract.md +12 -0
  393. package/src/skf-update-skill/references/manual-section-rules.md +6 -23
  394. package/src/skf-update-skill/references/merge.md +180 -47
  395. package/src/skf-update-skill/references/re-extract.md +53 -262
  396. package/src/skf-update-skill/references/report.md +100 -35
  397. package/src/skf-update-skill/references/write.md +331 -149
  398. package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
  399. package/src/skf-verify-stack/SKILL.md +23 -42
  400. package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
  401. package/src/skf-verify-stack/customize.toml +55 -22
  402. package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
  403. package/src/skf-verify-stack/references/coverage.md +67 -64
  404. package/src/skf-verify-stack/references/exit-codes.md +21 -8
  405. package/src/skf-verify-stack/references/init.md +107 -62
  406. package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
  407. package/src/skf-verify-stack/references/integrations.md +106 -65
  408. package/src/skf-verify-stack/references/report.md +59 -42
  409. package/src/skf-verify-stack/references/requirements.md +32 -9
  410. package/src/skf-verify-stack/references/synthesize.md +53 -39
  411. package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
  412. package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
  413. package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
  414. package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
  415. package/tools/cli/commands/install.js +11 -4
  416. package/tools/cli/commands/status.js +15 -3
  417. package/tools/cli/commands/update.js +11 -4
  418. package/tools/cli/lib/tool-check.js +320 -0
  419. package/tools/cli/lib/version-check.js +22 -9
  420. package/tools/cli/skf-cli.js +4 -1
  421. package/.gitattributes +0 -16
  422. package/.nvmrc +0 -1
  423. package/docs/_internal/RELEASING.md +0 -679
  424. package/release-audits/v1.0.0-launch-audit.md +0 -728
  425. package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
  426. package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
  427. package/src/skf-campaign/manifest.yaml +0 -15
  428. package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
  429. package/src/skf-create-skill/references/ecosystem-check.md +0 -74
  430. package/src/skf-drop-skill/references/headless-contract.md +0 -18
  431. package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
  432. package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
  433. package/src/skf-update-skill/references/validate.md +0 -124
  434. package/tools/validate-docs-drift.js +0 -233
  435. package/tools/validate-docs-links.js +0 -665
@@ -0,0 +1,1867 @@
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = []
4
+ # ///
5
+ """SKF Count Imports: count, for each dependency, the source files that import it.
6
+
7
+ Two workflows need the same fact, which has one right answer per tree:
8
+
9
+ 1. **skf-create-stack-skill** ranks the project's dependencies by how many
10
+ files import them (2 or more files: included by default), and hands
11
+ each library's file list to `skf-pair-intersect.py`.
12
+ 2. **skf-analyze-source** maps which units import which (Imports From and
13
+ Imported By), gives co-imported units to `skf-pair-intersect.py` and
14
+ the unit-to-unit edges to `skf-find-cycles.py`, and reads the graph
15
+ through `summary`, which leaves out the file lists it does not need.
16
+
17
+ A grep with no module boundary counts `react-dom` and `react-router` as
18
+ `react`, and a Python distribution is often imported under another name
19
+ (`PyYAML` as `yaml`, `Pillow` as `PIL`), so hand counts drift from run to
20
+ run. This helper reads each source file once, parses its import statements
21
+ per language, matches them on module boundaries and maps distribution names
22
+ to import names.
23
+
24
+ Subcommands:
25
+ count <root> [--deps <json-file-or-'-'>] [--units <json-file-or-'-'>]
26
+ [--threshold N] [--exclude GLOB]... [--format full|libraries]
27
+ [--relative-to DIR]
28
+
29
+ Walk <root>, read every source file of the languages the targets need,
30
+ and count the files that import each target. At least one of --deps
31
+ and --units is required; only one of them may read stdin ('-').
32
+
33
+ --deps accepts any of:
34
+ - the `skf-scan-manifests.py scan` envelope, as it is: every
35
+ `manifests[].deps[]` entry becomes a dependency of that manifest's
36
+ ecosystem (a dependency tagged `scope: dev` stays dev only when no
37
+ manifest lists it as a runtime dependency);
38
+ - {"dependencies": [<dep>, ...]} or a bare [<dep>, ...] list, where
39
+ <dep> is a name string or
40
+ {"name": "...", "ecosystem": "npm|python|...", "modules": [...],
41
+ "scope": "runtime|dev"}. `ecosystem` limits the match to that
42
+ ecosystem's source files (omitted: every language, with the name
43
+ and its Python-style guesses). `modules` gives the import names
44
+ explicitly and skips the name rules below.
45
+
46
+ --units accepts {"units": [<unit>, ...]} or a bare list, where <unit> is
47
+ {"name": "...", "path": "<dir relative to root>", "modules": [...],
48
+ "manifest_name": "...", "ecosystem": "..."}.
49
+ A unit is an internal part of the tree. Each scanned file belongs to
50
+ the unit with the longest `path` that contains it. A file imports a
51
+ unit when one of its imports matches the unit's import names, or when
52
+ a relative import (JS/TS `./` and `../`, Python leading dots, Ruby
53
+ `require_relative`) resolves into the unit's `path`. A unit's own
54
+ files never count as importing it. A unit's import names are its
55
+ `modules` when given; without them, what the unit declares: the
56
+ packages its own .java, .kt, .groovy and .scala files declare for a
57
+ Maven or Gradle unit, the namespaces its composer.json autoloads for
58
+ a Composer unit, the targets its Package.swift declares for a Swift
59
+ unit, else its `manifest_name` by the dependency rules below (an npm
60
+ package by its name, a crate with `-` as `_`, a Go module by its
61
+ path); none for a unit with neither, which only relative imports
62
+ reach. With --units, a dependency whose import names are a unit's is
63
+ that unit, not an external dependency: it is left out.
64
+
65
+ --threshold N a dependency is `above_threshold` when file_count >= N
66
+ (default 2: two or more importing files)
67
+ --exclude GLOB another path to leave out, added to the defaults below
68
+ (repeatable)
69
+ --format `full` (default) prints the envelope below;
70
+ `libraries` prints [{"name", "files": [<path>, ...]}]
71
+ for every dependency, then every unit: the input of
72
+ `skf-pair-intersect.py intersect --libraries -`.
73
+ --relative-to DIR
74
+ write each `files[].path` relative to DIR, a folder
75
+ that holds <root>, instead of to <root>: a caller that
76
+ scans one package of a project keeps project-relative
77
+ paths. Excludes and a unit's `path` stay relative to
78
+ <root>.
79
+
80
+ Emit JSON (`full`):
81
+ {
82
+ "threshold": 2,
83
+ "files_scanned": N,
84
+ "dependencies": [
85
+ {"name": "react", "ecosystem": "npm",
86
+ "import_names": ["react"], "resolution": "exact",
87
+ "files": [{"path": "src/App.tsx", "line": 1}, ...],
88
+ "file_count": N, "above_threshold": true},
89
+ ...
90
+ ],
91
+ "unresolved": [
92
+ {"name": "...", "ecosystem": "...", "import_names": [...],
93
+ "reason": "guess-unmatched|no-import-name|unsupported-ecosystem"},
94
+ ...
95
+ ],
96
+ "units": [
97
+ {"name": "...", "path": "...", "import_names": [...],
98
+ "files": [{"path": "...", "line": N}, ...], "file_count": N,
99
+ "imported_by": [...], "imports_from": [...],
100
+ "external_deps": [...]},
101
+ ...
102
+ ],
103
+ "edges": [["<importing unit>", "<imported unit>"], ...],
104
+ "edge_files": [
105
+ {"from": "<importing unit>", "to": "<imported unit>",
106
+ "files": [{"path": "...", "line": N}, ...]},
107
+ ...
108
+ ],
109
+ "warnings": ["..."]
110
+ }
111
+ `units`, `edges` and `edge_files` are present only with --units;
112
+ `warnings` only when a file could not be read. A dependency entry
113
+ carries `scope: "dev"` when its input said so. `files` lists each
114
+ importing file once, with the first line that imports the target,
115
+ sorted by path; an `edge_files` entry lists the files of the
116
+ importing unit that import the other one, sorted by `from`, then
117
+ `to`. A unit's `external_deps` are the dependencies its files import.
118
+ `dependencies[]` is sorted by file_count (highest first), then name.
119
+ `unresolved[]` (sorted by name) holds the dependencies this helper
120
+ cannot count with confidence, for the model to judge: a guessed import
121
+ name that matched no file (`guess-unmatched`), no import name to try
122
+ (`no-import-name`), or an ecosystem it has no import rules for
123
+ (`unsupported-ecosystem`). `edges` is the `skf-find-cycles.py find
124
+ --edges` input as it is (that helper reads only the `edges` key).
125
+
126
+ summary <imports-file-or-'-'> [--manifests <json-file-or-'-'>]
127
+ [--max-files N]
128
+
129
+ Print the unit graph of a saved `count --units` envelope without the
130
+ file lists a reader of the graph does not need (on a large monorepo
131
+ the envelope runs to hundreds of KB, the summary to a few):
132
+ {
133
+ "units": [
134
+ {"name": "...", "imports_from": [...], "imported_by": [...],
135
+ "file_count": N, "external_dep_count": N},
136
+ ...
137
+ ],
138
+ "edge_files": [
139
+ {"from": "...", "to": "...", "file_count": N,
140
+ "files": [{"path": "...", "line": N}, ...]},
141
+ ...
142
+ ],
143
+ "umbrella_candidates": [...],
144
+ "manifests": [
145
+ {"path": "...", "name": "...", "private": true | false | null,
146
+ "internal_deps": [...]},
147
+ ...
148
+ ],
149
+ "warnings": ["..."]
150
+ }
151
+ A unit keeps its `file_count` (the files outside it that import it)
152
+ and the number of its `external_deps`; an `edge_files` entry keeps
153
+ its file count and its first --max-files files (default 5), so the
154
+ full list stays in the envelope for the edge that needs it.
155
+ `umbrella_candidates` and `manifests` come from --manifests, the
156
+ `skf-scan-manifests.py scan` envelope (present only with it), each
157
+ manifest with the fields the cohesion triggers read. `warnings` is
158
+ the envelope's, present only when it has some. Only one of the two
159
+ inputs may read stdin.
160
+
161
+ Resolution of a dependency's import names (`resolution`):
162
+ exact the ecosystem's own rule: an npm package is imported by its name,
163
+ a Rust crate by its name with `-` as `_`, a Go module by its path;
164
+ a Composer package by the namespaces its installed
165
+ `vendor/<name>/composer.json` autoloads
166
+ mapped an entry of the built-in distribution-to-import tables (`PyYAML`
167
+ as `yaml`, `Pillow` as `PIL`, `beautifulsoup4` as `bs4`,
168
+ `com.google.guava:guava` as `com.google.common`, ...)
169
+ guessed a naming rule that is often but not always right: a Python
170
+ distribution as its name with `-` as `_`, a namespace family
171
+ (`google-cloud-storage` as `google.cloud.storage`), `@types/react`
172
+ as `react`, a Symfony package as its namespace
173
+ (`symfony/framework-bundle` as `Symfony\\Bundle\\FrameworkBundle`),
174
+ a Maven group as its package, a gem as its name. A guess that
175
+ matches no file goes to `unresolved[]` instead of counting 0
176
+ explicit the `modules` the caller gave
177
+
178
+ Matching is boundary-anchored: an import binds to the dependency whose
179
+ import name equals the imported module or is a prefix of it at a module
180
+ separator (`/` for JS, Go and Ruby, `.` for Python and the JVM, `::` for
181
+ Rust, `\\` for PHP, whole names for Swift). `react` matches `react` and
182
+ `react/jsx-runtime`, never `react-dom`; `yaml` matches `yaml.loader`, never
183
+ `yamlordereddict`. When several import names prefix the same import, the
184
+ longest wins, so `opentelemetry.sdk.trace` counts for `opentelemetry-sdk`
185
+ and not for `opentelemetry-api`.
186
+
187
+ Import statements read per language (lines that start a comment are
188
+ skipped; Python files are parsed with `ast`, and one that does not parse or
189
+ is over 1,000,000 characters (`PYTHON_AST_MAX_CHARS`) is read line by line,
190
+ outside comments and triple-quoted strings):
191
+ js .js .jsx .mjs .cjs .ts .tsx .mts .cts .vue .svelte .astro:
192
+ `import ... from 'm'`, `import 'm'`, `export ... from 'm'`,
193
+ `import('m')`, `require('m')`
194
+ python .py .pyi: `import m`, `from m import n` (also read as `m.n`),
195
+ `importlib.import_module('m')`, `__import__('m')`
196
+ rust .rs: `use m::...`, `extern crate m`, and qualified paths `m::x`
197
+ go .go: `import "m"` and `import ( ... )` blocks
198
+ jvm .java .kt .kts .groovy .scala: `import m.x`, `import static m.x`
199
+ ruby .rb: `require 'm'`, `require_relative 'p'`
200
+ php .php: `use M\\X` (case-insensitive, as PHP namespaces are)
201
+ swift .swift: `import M`
202
+ Ecosystems map to languages: npm to js, python to python, rust to rust,
203
+ go to go, maven and gradle to jvm, ruby to ruby, composer to php, swift to
204
+ swift.
205
+
206
+ Excluded paths, matched against the path relative to <root> (the list in
207
+ `DEFAULT_EXCLUDES`, which `references/manifest-patterns.md` in
208
+ skf-create-stack-skill documents): a pattern ending in `/` names a directory
209
+ at any depth, a pattern with no `/` matches a file name at any depth, any
210
+ other pattern matches the whole relative path (`**` spans directories, `*`
211
+ and `?` stay inside one). A leading `/` or `./` anchors a pattern to <root>
212
+ (`./examples/` is only the top-level `examples`). Symbolic links are not
213
+ followed.
214
+
215
+ CLI examples:
216
+ uv run skf-scan-manifests.py scan . > manifests.json
217
+ uv run skf-count-imports.py count . --deps manifests.json
218
+ echo '["react", "PyYAML"]' | uv run skf-count-imports.py count . --deps -
219
+ uv run skf-count-imports.py count . --units units.json --format libraries \\
220
+ | uv run skf-pair-intersect.py intersect --libraries -
221
+ uv run skf-count-imports.py count . --units units.json \\
222
+ | uv run skf-find-cycles.py find --edges -
223
+ uv run skf-count-imports.py count . --units units.json --deps manifests.json > imports.json
224
+ uv run skf-count-imports.py summary imports.json --manifests manifests.json
225
+
226
+ Exit codes:
227
+ 0 success (including: no dependency imported, no source file found)
228
+ 1 user error (root not a directory, unreadable or malformed JSON input,
229
+ a bad dependency or unit entry, both inputs on stdin, bad --threshold,
230
+ a --relative-to that is not a folder holding <root>, a summary input
231
+ that is not a `count --units` envelope or a manifest scan, bad
232
+ --max-files)
233
+ """
234
+
235
+ from __future__ import annotations
236
+
237
+ import argparse
238
+ import ast
239
+ import json
240
+ import os
241
+ import posixpath
242
+ import re
243
+ import sys
244
+ import warnings
245
+ from pathlib import Path
246
+
247
+
248
+ DEFAULT_THRESHOLD = 2
249
+
250
+ # A Python file up to this many characters is parsed with `ast`. A larger one
251
+ # (a generated table or protobuf module) is read line by line, because its
252
+ # syntax tree can take 200 times its size in memory.
253
+ PYTHON_AST_MAX_CHARS = 1_000_000
254
+
255
+ # Source file extensions read for each language.
256
+ LANGUAGE_EXTENSIONS: dict[str, tuple[str, ...]] = {
257
+ "js": (
258
+ ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte", ".astro",
259
+ ),
260
+ "python": (".py", ".pyi"),
261
+ "rust": (".rs",),
262
+ "go": (".go",),
263
+ "jvm": (".java", ".kt", ".kts", ".groovy", ".scala"),
264
+ "ruby": (".rb",),
265
+ "php": (".php",),
266
+ "swift": (".swift",),
267
+ }
268
+
269
+ # `skf-scan-manifests.py` ecosystem label -> language whose imports it reads.
270
+ ECOSYSTEM_LANGUAGE: dict[str, str] = {
271
+ "npm": "js",
272
+ "python": "python",
273
+ "rust": "rust",
274
+ "go": "go",
275
+ "maven": "jvm",
276
+ "gradle": "jvm",
277
+ "ruby": "ruby",
278
+ "composer": "php",
279
+ "swift": "swift",
280
+ }
281
+
282
+ # Module separator per language; None matches whole module names only.
283
+ SEPARATORS: dict[str, str | None] = {
284
+ "js": "/",
285
+ "python": ".",
286
+ "rust": "::",
287
+ "go": "/",
288
+ "jvm": ".",
289
+ "ruby": "/",
290
+ "php": "\\",
291
+ "swift": None,
292
+ }
293
+
294
+ # Paths never counted, documented in
295
+ # src/skf-create-stack-skill/references/manifest-patterns.md "Import Counting"
296
+ # (test/test-skf-count-imports.py keeps the two lists equal).
297
+ DEFAULT_EXCLUDES: tuple[str, ...] = (
298
+ # hidden files and directories
299
+ ".*/",
300
+ ".*",
301
+ # tests
302
+ "test/",
303
+ "tests/",
304
+ "__tests__/",
305
+ "spec/",
306
+ "*.test.*",
307
+ "*.spec.*",
308
+ "*_test.*",
309
+ "test_*.py",
310
+ "*_spec.rb",
311
+ "conftest.py",
312
+ # config and build scripts
313
+ "*.config.*",
314
+ "setup.py",
315
+ "noxfile.py",
316
+ "*.gradle.kts",
317
+ "Package.swift",
318
+ # build output and vendored code
319
+ "dist/",
320
+ "build/",
321
+ "out/",
322
+ "target/",
323
+ "node_modules/",
324
+ "vendor/",
325
+ "venv/",
326
+ "Pods/",
327
+ "__pycache__/",
328
+ )
329
+
330
+ # --------------------------------------------------------------------------
331
+ # Distribution-to-import tables
332
+ # --------------------------------------------------------------------------
333
+
334
+
335
+ # Python distributions whose import name differs from the distribution name,
336
+ # keyed by the PEP 503 normalised name (lower case, runs of -_. as -).
337
+ PYTHON_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
338
+ "absl-py": ("absl",),
339
+ "apache-airflow": ("airflow",),
340
+ "attrs": ("attr", "attrs"),
341
+ "beautifulsoup4": ("bs4",),
342
+ "discord-py": ("discord",),
343
+ "django-cors-headers": ("corsheaders",),
344
+ "django-environ": ("environ",),
345
+ "django-filter": ("django_filters",),
346
+ "djangorestframework": ("rest_framework",),
347
+ "dnspython": ("dns",),
348
+ "faiss-cpu": ("faiss",),
349
+ "faiss-gpu": ("faiss",),
350
+ "gitpython": ("git",),
351
+ "google-api-python-client": ("googleapiclient",),
352
+ "google-auth": ("google.auth",),
353
+ "google-genai": ("google.genai",),
354
+ "google-generativeai": ("google.generativeai",),
355
+ "grpcio": ("grpc",),
356
+ "grpcio-status": ("grpc_status",),
357
+ "grpcio-tools": ("grpc_tools",),
358
+ "ipython": ("IPython",),
359
+ "msgpack-python": ("msgpack",),
360
+ "mysql-connector-python": ("mysql.connector",),
361
+ "mysqlclient": ("MySQLdb",),
362
+ "opencv-contrib-python": ("cv2",),
363
+ "opencv-contrib-python-headless": ("cv2",),
364
+ "opencv-python": ("cv2",),
365
+ "opencv-python-headless": ("cv2",),
366
+ "opentelemetry-api": ("opentelemetry",),
367
+ "paho-mqtt": ("paho.mqtt",),
368
+ "pdfminer-six": ("pdfminer",),
369
+ "pillow": ("PIL",),
370
+ "protobuf": ("google.protobuf",),
371
+ "psycopg2-binary": ("psycopg2",),
372
+ "pycairo": ("cairo",),
373
+ "pycryptodome": ("Crypto",),
374
+ "pycryptodomex": ("Cryptodome",),
375
+ "pygithub": ("github",),
376
+ "pygobject": ("gi",),
377
+ "pyinstaller": ("PyInstaller",),
378
+ "pyjwt": ("jwt",),
379
+ "pymongo": ("pymongo", "bson", "gridfs"),
380
+ "pymupdf": ("fitz", "pymupdf"),
381
+ "pynacl": ("nacl",),
382
+ "pyopenssl": ("OpenSSL",),
383
+ "pyqt5": ("PyQt5",),
384
+ "pyqt6": ("PyQt6",),
385
+ "pyserial": ("serial",),
386
+ "pyside2": ("PySide2",),
387
+ "pyside6": ("PySide6",),
388
+ "pysocks": ("socks",),
389
+ "python-dateutil": ("dateutil",),
390
+ "python-docx": ("docx",),
391
+ "python-dotenv": ("dotenv",),
392
+ "python-gitlab": ("gitlab",),
393
+ "python-jose": ("jose",),
394
+ "python-json-logger": ("pythonjsonlogger",),
395
+ "python-levenshtein": ("Levenshtein",),
396
+ "python-magic": ("magic",),
397
+ "python-multipart": ("multipart", "python_multipart"),
398
+ "python-pptx": ("pptx",),
399
+ "python-slugify": ("slugify",),
400
+ "python-socketio": ("socketio",),
401
+ "python-telegram-bot": ("telegram",),
402
+ "pywavelets": ("pywt",),
403
+ "pywin32": ("win32api", "win32con", "win32com", "win32gui", "pythoncom", "pywintypes"),
404
+ "pyyaml": ("yaml",),
405
+ "pyzmq": ("zmq",),
406
+ "ruamel-yaml": ("ruamel.yaml",),
407
+ "scikit-image": ("skimage",),
408
+ "scikit-learn": ("sklearn",),
409
+ "setuptools": ("setuptools", "pkg_resources"),
410
+ "tensorflow-cpu": ("tensorflow",),
411
+ "tensorflow-gpu": ("tensorflow",),
412
+ "websocket-client": ("websocket",),
413
+ }
414
+
415
+ # Python namespace families: normalised prefix -> (import namespace, the
416
+ # character that replaces `-` in the rest of the name). A rule, not a table
417
+ # of names, so the names it builds are guesses.
418
+ PYTHON_NAMESPACE_FAMILIES: dict[str, tuple[str, str]] = {
419
+ "google-cloud-": ("google.cloud.", "_"),
420
+ "azure-": ("azure.", "."),
421
+ "opentelemetry-": ("opentelemetry.", "."),
422
+ "backports-": ("backports.", "."),
423
+ "jaraco-": ("jaraco.", "."),
424
+ "sphinxcontrib-": ("sphinxcontrib.", "."),
425
+ "zope-": ("zope.", "."),
426
+ }
427
+
428
+ # Maven and Gradle coordinates (group:artifact) whose package is not the group.
429
+ JVM_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
430
+ "com.fasterxml.jackson.core:jackson-annotations": ("com.fasterxml.jackson.annotation",),
431
+ "com.fasterxml.jackson.core:jackson-core": ("com.fasterxml.jackson.core",),
432
+ "com.fasterxml.jackson.core:jackson-databind": ("com.fasterxml.jackson.databind",),
433
+ "com.google.code.gson:gson": ("com.google.gson",),
434
+ "com.google.guava:guava": ("com.google.common",),
435
+ "com.h2database:h2": ("org.h2",),
436
+ "com.squareup.okhttp3:okhttp": ("okhttp3",),
437
+ "com.squareup.retrofit2:retrofit": ("retrofit2",),
438
+ "commons-codec:commons-codec": ("org.apache.commons.codec",),
439
+ "commons-io:commons-io": ("org.apache.commons.io",),
440
+ "jakarta.servlet:jakarta.servlet-api": ("jakarta.servlet",),
441
+ "javax.servlet:javax.servlet-api": ("javax.servlet",),
442
+ "junit:junit": ("org.junit", "junit.framework"),
443
+ "mysql:mysql-connector-java": ("com.mysql",),
444
+ "org.apache.commons:commons-collections4": ("org.apache.commons.collections4",),
445
+ "org.apache.commons:commons-lang3": ("org.apache.commons.lang3",),
446
+ "org.jetbrains.kotlinx:kotlinx-coroutines-core": ("kotlinx.coroutines",),
447
+ "org.jetbrains.kotlinx:kotlinx-serialization-json": ("kotlinx.serialization",),
448
+ "org.junit.jupiter:junit-jupiter": ("org.junit.jupiter",),
449
+ "org.junit.jupiter:junit-jupiter-api": ("org.junit.jupiter.api",),
450
+ "org.postgresql:postgresql": ("org.postgresql",),
451
+ "org.projectlombok:lombok": ("lombok",),
452
+ "org.slf4j:slf4j-api": ("org.slf4j",),
453
+ "org.xerial:sqlite-jdbc": ("org.sqlite",),
454
+ }
455
+
456
+ # Gems required under another name, keyed by lower-case gem name.
457
+ RUBY_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
458
+ "actioncable": ("action_cable",),
459
+ "actionmailer": ("action_mailer",),
460
+ "actionpack": ("action_controller", "action_dispatch"),
461
+ "actionview": ("action_view",),
462
+ "activejob": ("active_job",),
463
+ "activemodel": ("active_model",),
464
+ "activerecord": ("active_record",),
465
+ "activestorage": ("active_storage",),
466
+ "activesupport": ("active_support",),
467
+ "concurrent-ruby": ("concurrent",),
468
+ "dotenv-rails": ("dotenv",),
469
+ "google-protobuf": ("google/protobuf",),
470
+ "railties": ("rails",),
471
+ "rest-client": ("rest-client", "restclient"),
472
+ "ruby-openai": ("openai",),
473
+ }
474
+
475
+ # Composer packages and the namespace they autoload, keyed by lower-case name.
476
+ # An installed vendor/<name>/composer.json wins over this table.
477
+ COMPOSER_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
478
+ "aws/aws-sdk-php": ("Aws",),
479
+ "doctrine/dbal": ("Doctrine\\DBAL",),
480
+ "doctrine/orm": ("Doctrine\\ORM",),
481
+ "firebase/php-jwt": ("Firebase\\JWT",),
482
+ "guzzlehttp/guzzle": ("GuzzleHttp",),
483
+ "guzzlehttp/promises": ("GuzzleHttp\\Promise",),
484
+ "guzzlehttp/psr7": ("GuzzleHttp\\Psr7",),
485
+ "laravel/framework": ("Illuminate",),
486
+ "league/flysystem": ("League\\Flysystem",),
487
+ "monolog/monolog": ("Monolog",),
488
+ "nesbot/carbon": ("Carbon",),
489
+ "phpunit/phpunit": ("PHPUnit",),
490
+ "predis/predis": ("Predis",),
491
+ "psr/http-message": ("Psr\\Http\\Message",),
492
+ "psr/log": ("Psr\\Log",),
493
+ "ramsey/uuid": ("Ramsey\\Uuid",),
494
+ "stripe/stripe-php": ("Stripe",),
495
+ "twig/twig": ("Twig",),
496
+ "vlucas/phpdotenv": ("Dotenv",),
497
+ }
498
+
499
+ # Swift packages (as skf-scan-manifests.py names them: the last URL segment)
500
+ # and the modules they ship, keyed by lower-case name.
501
+ SWIFT_IMPORT_NAMES: dict[str, tuple[str, ...]] = {
502
+ "async-http-client": ("AsyncHTTPClient",),
503
+ "firebase-ios-sdk": (
504
+ "FirebaseAnalytics",
505
+ "FirebaseAuth",
506
+ "FirebaseCore",
507
+ "FirebaseCrashlytics",
508
+ "FirebaseDatabase",
509
+ "FirebaseFirestore",
510
+ "FirebaseFunctions",
511
+ "FirebaseMessaging",
512
+ "FirebaseRemoteConfig",
513
+ "FirebaseStorage",
514
+ ),
515
+ "grpc-swift": ("GRPC",),
516
+ "realm-swift": ("RealmSwift", "Realm"),
517
+ "rxswift": ("RxSwift", "RxCocoa", "RxRelay"),
518
+ "snapkit": ("SnapKit",),
519
+ "swift-algorithms": ("Algorithms",),
520
+ "swift-argument-parser": ("ArgumentParser",),
521
+ "swift-async-algorithms": ("AsyncAlgorithms",),
522
+ "swift-atomics": ("Atomics",),
523
+ "swift-collections": ("Collections", "DequeModule", "OrderedCollections"),
524
+ "swift-crypto": ("Crypto",),
525
+ "swift-log": ("Logging",),
526
+ "swift-markdown": ("Markdown",),
527
+ "swift-metrics": ("Metrics",),
528
+ "swift-nio": (
529
+ "NIO",
530
+ "NIOConcurrencyHelpers",
531
+ "NIOCore",
532
+ "NIOEmbedded",
533
+ "NIOFoundationCompat",
534
+ "NIOHTTP1",
535
+ "NIOPosix",
536
+ "NIOTLS",
537
+ "NIOWebSocket",
538
+ ),
539
+ "swift-nio-ssl": ("NIOSSL",),
540
+ "swift-numerics": ("Numerics", "RealModule", "ComplexModule"),
541
+ "swift-protobuf": ("SwiftProtobuf",),
542
+ "swift-syntax": ("SwiftSyntax", "SwiftParser"),
543
+ "swift-system": ("SystemPackage",),
544
+ "swiftyjson": ("SwiftyJSON",),
545
+ }
546
+
547
+
548
+ # --------------------------------------------------------------------------
549
+ # Input
550
+ # --------------------------------------------------------------------------
551
+
552
+
553
+ class InputError(Exception):
554
+ """A malformed input: exit code 1."""
555
+
556
+
557
+ def _read_json_source(source: str, what: str) -> object:
558
+ """Read JSON from a file path, or from stdin when source is '-'."""
559
+ if source == "-":
560
+ try:
561
+ text = sys.stdin.read()
562
+ except OSError as exc:
563
+ raise InputError(f"failed to read {what} JSON from stdin: {exc}") from exc
564
+ else:
565
+ path = Path(source)
566
+ if not path.exists() or path.is_dir():
567
+ raise InputError(f"{what} file not found: {path}")
568
+ try:
569
+ text = path.read_text(encoding="utf-8")
570
+ except (OSError, UnicodeDecodeError) as exc:
571
+ raise InputError(f"failed to read {what} file {path}: {exc}") from exc
572
+ try:
573
+ return json.loads(text)
574
+ except json.JSONDecodeError as exc:
575
+ raise InputError(f"malformed JSON in {what} input: {exc}") from exc
576
+
577
+
578
+ def _pep503(name: str) -> str:
579
+ return re.sub(r"[-_.]+", "-", name).lower()
580
+
581
+
582
+ def _dedupe_key(ecosystem: str | None, name: str) -> tuple[str, str]:
583
+ """Key under which two spellings of one dependency collapse."""
584
+ if ecosystem == "python":
585
+ return ecosystem, _pep503(name)
586
+ if ecosystem == "rust":
587
+ return ecosystem, name.lower().replace("_", "-")
588
+ if ecosystem == "composer":
589
+ return ecosystem, name.lower()
590
+ return ecosystem or "", name
591
+
592
+
593
+ def _string_list(value: object, where: str) -> list[str]:
594
+ if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value):
595
+ raise InputError(f"{where} must be a list of non-empty strings")
596
+ return list(dict.fromkeys(value))
597
+
598
+
599
+ def _dependency_entry(raw: object, where: str) -> dict:
600
+ if isinstance(raw, str):
601
+ raw = {"name": raw}
602
+ if not isinstance(raw, dict):
603
+ raise InputError(f"{where} must be a name string or an object")
604
+ name = raw.get("name")
605
+ if not isinstance(name, str) or not name.strip():
606
+ raise InputError(f"{where} needs a non-empty `name`")
607
+ ecosystem = raw.get("ecosystem")
608
+ if ecosystem is not None and (not isinstance(ecosystem, str) or not ecosystem):
609
+ raise InputError(f"{where} `ecosystem` must be a non-empty string")
610
+ scope = raw.get("scope")
611
+ if scope not in (None, "runtime", "dev"):
612
+ raise InputError(f"{where} `scope` must be 'runtime' or 'dev'; got {scope!r}")
613
+ entry = {"name": name.strip(), "ecosystem": ecosystem, "scope": scope or "runtime"}
614
+ if raw.get("modules") is not None:
615
+ entry["modules"] = _string_list(raw["modules"], f"{where} `modules`")
616
+ return entry
617
+
618
+
619
+ def parse_dependencies(payload: object) -> list[dict]:
620
+ """Read the --deps payload into deduplicated dependency entries.
621
+
622
+ Accepts the skf-scan-manifests.py envelope, {"dependencies": [...]} or a
623
+ bare list. Two entries of one ecosystem whose names normalise alike
624
+ (PyYAML and pyyaml) collapse into the first; the merged entry is dev only
625
+ when every occurrence is dev.
626
+ """
627
+ raw_entries: list[tuple[object, str]] = []
628
+ if isinstance(payload, dict) and "manifests" in payload:
629
+ manifests = payload["manifests"]
630
+ if not isinstance(manifests, list):
631
+ raise InputError("`manifests` must be a list")
632
+ for m_idx, manifest in enumerate(manifests):
633
+ if not isinstance(manifest, dict):
634
+ raise InputError(f"manifests[{m_idx}] is not an object")
635
+ ecosystem = manifest.get("ecosystem")
636
+ deps = manifest.get("deps") or []
637
+ if not isinstance(deps, list):
638
+ raise InputError(f"manifests[{m_idx}].deps must be a list")
639
+ for d_idx, dep in enumerate(deps):
640
+ if not isinstance(dep, dict):
641
+ raise InputError(f"manifests[{m_idx}].deps[{d_idx}] is not an object")
642
+ name = dep.get("name")
643
+ # the scanner writes `<unparsable>` for a line it could not read
644
+ if not isinstance(name, str) or not name or name.startswith("<"):
645
+ continue
646
+ raw_entries.append(
647
+ (
648
+ {"name": name, "ecosystem": ecosystem, "scope": dep.get("scope")},
649
+ f"manifests[{m_idx}].deps[{d_idx}]",
650
+ )
651
+ )
652
+ else:
653
+ if isinstance(payload, dict) and "dependencies" in payload:
654
+ payload = payload["dependencies"]
655
+ if not isinstance(payload, list):
656
+ raise InputError(
657
+ "--deps must be the skf-scan-manifests.py envelope, "
658
+ '{"dependencies": [...]} or a JSON array'
659
+ )
660
+ raw_entries = [(raw, f"dependencies[{idx}]") for idx, raw in enumerate(payload)]
661
+
662
+ merged: dict[tuple[str, str], dict] = {}
663
+ for raw, where in raw_entries:
664
+ entry = _dependency_entry(raw, where)
665
+ key = _dedupe_key(entry["ecosystem"], entry["name"])
666
+ seen = merged.get(key)
667
+ if seen is None:
668
+ merged[key] = entry
669
+ continue
670
+ if entry["scope"] == "runtime":
671
+ seen["scope"] = "runtime"
672
+ if "modules" in entry:
673
+ seen["modules"] = list(dict.fromkeys(seen.get("modules", []) + entry["modules"]))
674
+ return list(merged.values())
675
+
676
+
677
+ def _normalise_unit_path(path: str) -> str:
678
+ """A unit path as a root-relative POSIX path; the root itself is ''."""
679
+ path = path.replace("\\", "/").strip().strip("/")
680
+ if not path:
681
+ return ""
682
+ path = posixpath.normpath(path)
683
+ return "" if path == "." else path
684
+
685
+
686
+ def parse_units(payload: object) -> list[dict]:
687
+ """Read the --units payload: unique names and unique paths."""
688
+ if isinstance(payload, dict) and "units" in payload:
689
+ payload = payload["units"]
690
+ if not isinstance(payload, list):
691
+ raise InputError('--units must be {"units": [...]} or a JSON array')
692
+ units: list[dict] = []
693
+ names: set[str] = set()
694
+ paths: set[str] = set()
695
+ for idx, raw in enumerate(payload):
696
+ where = f"units[{idx}]"
697
+ if not isinstance(raw, dict):
698
+ raise InputError(f"{where} is not an object")
699
+ name = raw.get("name")
700
+ if not isinstance(name, str) or not name.strip():
701
+ raise InputError(f"{where} needs a non-empty `name`")
702
+ path = raw.get("path")
703
+ if not isinstance(path, str):
704
+ raise InputError(f"{where} needs a `path` string (relative to the root)")
705
+ path = _normalise_unit_path(path)
706
+ if path == ".." or path.startswith("../"):
707
+ raise InputError(f"{where} `path` leaves the root: {raw.get('path')!r}")
708
+ ecosystem = raw.get("ecosystem")
709
+ if ecosystem is not None and (not isinstance(ecosystem, str) or not ecosystem):
710
+ raise InputError(f"{where} `ecosystem` must be a non-empty string")
711
+ manifest_name = raw.get("manifest_name")
712
+ if manifest_name is not None and not isinstance(manifest_name, str):
713
+ raise InputError(f"{where} `manifest_name` must be a string")
714
+ name = name.strip()
715
+ if name in names:
716
+ raise InputError(f"{where} repeats unit name {name!r}")
717
+ if path in paths:
718
+ raise InputError(f"{where} repeats unit path {path!r}")
719
+ names.add(name)
720
+ paths.add(path)
721
+ unit = {
722
+ "name": name,
723
+ "path": path,
724
+ "ecosystem": ecosystem,
725
+ "manifest_name": (manifest_name or "").strip() or None,
726
+ "modules": None, # None: derived from what the unit declares
727
+ }
728
+ if raw.get("modules") is not None:
729
+ unit["modules"] = _string_list(raw["modules"], f"{where} `modules`")
730
+ units.append(unit)
731
+ return units
732
+
733
+
734
+ # --------------------------------------------------------------------------
735
+ # Import names
736
+ # --------------------------------------------------------------------------
737
+
738
+
739
+ def _studly(part: str) -> str:
740
+ return "".join(p[:1].upper() + p[1:] for p in re.split(r"[-_.]+", part) if p)
741
+
742
+
743
+ def _npm_import_names(name: str) -> tuple[list[str], str]:
744
+ if name.startswith("@types/") and len(name) > len("@types/"):
745
+ # a guess: @types/node types built-ins, @types/jest types globals
746
+ inner = name[len("@types/"):]
747
+ # DefinitelyTyped spells @scope/pkg as scope__pkg
748
+ if "__" in inner:
749
+ scope, _, pkg = inner.partition("__")
750
+ return [f"@{scope}/{pkg}"], "guessed"
751
+ return [inner], "guessed"
752
+ return [name], "exact"
753
+
754
+
755
+ def _python_import_names(name: str) -> tuple[list[str], str]:
756
+ key = _pep503(name)
757
+ if key.startswith("types-") and len(key) > len("types-"):
758
+ # typeshed stubs: types-PyYAML types the `yaml` module
759
+ return _python_import_names(key[len("types-"):])
760
+ mapped = PYTHON_IMPORT_NAMES.get(key)
761
+ if mapped:
762
+ return list(mapped), "mapped"
763
+ for prefix, (namespace, joiner) in PYTHON_NAMESPACE_FAMILIES.items():
764
+ if key.startswith(prefix) and len(key) > len(prefix):
765
+ # a guess: google-cloud-pubsub is imported as google.cloud.pubsub_v1
766
+ return [namespace + key[len(prefix):].replace("-", joiner)], "guessed"
767
+ guess = name.replace("-", "_")
768
+ if not re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*", guess):
769
+ return [], "guessed"
770
+ return list(dict.fromkeys([guess, guess.lower()])), "guessed"
771
+
772
+
773
+ def _jvm_import_names(name: str) -> tuple[list[str], str]:
774
+ mapped = JVM_IMPORT_NAMES.get(name)
775
+ if mapped:
776
+ return list(mapped), "mapped"
777
+ group = name.split(":", 1)[0]
778
+ if re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*", group):
779
+ return [group], "guessed"
780
+ return [], "guessed"
781
+
782
+
783
+ def _ruby_import_names(name: str) -> tuple[list[str], str]:
784
+ mapped = RUBY_IMPORT_NAMES.get(name.lower())
785
+ if mapped:
786
+ return list(mapped), "mapped"
787
+ return list(dict.fromkeys([name, name.replace("-", "/")])), "guessed"
788
+
789
+
790
+ _COMPOSER_NAME = re.compile(r"[A-Za-z0-9][\w.-]*/[A-Za-z0-9][\w.-]*")
791
+
792
+
793
+ def _composer_autoload(manifest: Path) -> list[str]:
794
+ """Namespaces a composer.json autoloads (its psr-4 and psr-0 prefixes)."""
795
+ try:
796
+ data = json.loads(manifest.read_text(encoding="utf-8"))
797
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
798
+ return []
799
+ autoload = data.get("autoload") if isinstance(data, dict) else None
800
+ if not isinstance(autoload, dict):
801
+ return []
802
+ namespaces: set[str] = set()
803
+ for kind in ("psr-4", "psr-0"):
804
+ table = autoload.get(kind)
805
+ if isinstance(table, dict):
806
+ namespaces.update(p.strip("\\") for p in table if p.strip("\\"))
807
+ return sorted(namespaces)
808
+
809
+
810
+ def _composer_installed_namespaces(root: Path, name: str) -> list[str]:
811
+ """Namespaces the installed vendor/<name>/composer.json autoloads."""
812
+ if not _COMPOSER_NAME.fullmatch(name):
813
+ return []
814
+ return _composer_autoload(root / "vendor" / name / "composer.json")
815
+
816
+
817
+ _SWIFT_TARGET = re.compile(r'\.(?:target|binaryTarget)\s*\(\s*name\s*:\s*"(?P<name>[^"]+)"')
818
+
819
+
820
+ def _swift_targets(manifest: Path) -> list[str]:
821
+ """The targets a Package.swift declares: the modules code imports it by."""
822
+ try:
823
+ text = manifest.read_text(encoding="utf-8")
824
+ except (OSError, UnicodeDecodeError):
825
+ return []
826
+ return sorted({m.group("name") for m in _SWIFT_TARGET.finditer(text)})
827
+
828
+
829
+ def _symfony_namespace(package: str) -> str:
830
+ """The namespace a symfony/<package> usually autoloads, by Symfony's naming."""
831
+ if package.endswith("-bundle"):
832
+ return "Symfony\\Bundle\\" + _studly(package)
833
+ if package.endswith("-bridge"):
834
+ return "Symfony\\Bridge\\" + _studly(package[: -len("-bridge")])
835
+ if package.endswith("-contracts"):
836
+ return "Symfony\\Contracts\\" + _studly(package[: -len("-contracts")])
837
+ if package.startswith("ux-"):
838
+ return "Symfony\\UX\\" + _studly(package[len("ux-"):])
839
+ if package.startswith("polyfill-"):
840
+ return "Symfony\\Polyfill\\" + _studly(package[len("polyfill-"):])
841
+ if package.startswith("security-"):
842
+ return "Symfony\\Component\\Security\\" + _studly(package[len("security-"):])
843
+ return "Symfony\\Component\\" + _studly(package)
844
+
845
+
846
+ def _composer_import_names(name: str, root: Path) -> tuple[list[str], str]:
847
+ installed = _composer_installed_namespaces(root, name)
848
+ if installed:
849
+ return installed, "exact"
850
+ key = name.lower()
851
+ mapped = COMPOSER_IMPORT_NAMES.get(key)
852
+ if mapped:
853
+ return list(mapped), "mapped"
854
+ vendor, _, package = key.partition("/")
855
+ if vendor == "symfony" and package:
856
+ return [_symfony_namespace(package)], "guessed"
857
+ if vendor and package:
858
+ return [f"{_studly(vendor)}\\{_studly(package)}", _studly(vendor)], "guessed"
859
+ return [], "guessed"
860
+
861
+
862
+ def _swift_import_names(name: str) -> tuple[list[str], str]:
863
+ mapped = SWIFT_IMPORT_NAMES.get(name.lower())
864
+ if mapped:
865
+ return list(mapped), "mapped"
866
+ base = name[len("swift-"):] if name.lower().startswith("swift-") else name
867
+ names = [name] if re.fullmatch(r"[A-Za-z_]\w*", name) else []
868
+ camel = _studly(base)
869
+ if camel:
870
+ names.append(camel)
871
+ return list(dict.fromkeys(names)), "guessed"
872
+
873
+
874
+ def import_names(name: str, ecosystem: str | None, root: Path) -> tuple[list[str], str]:
875
+ """Return (import names, resolution) for a dependency of an ecosystem.
876
+
877
+ With no ecosystem the name is tried as it is and with the Python guesses,
878
+ in every language.
879
+ """
880
+ if ecosystem == "npm":
881
+ return _npm_import_names(name)
882
+ if ecosystem == "python":
883
+ return _python_import_names(name)
884
+ if ecosystem == "rust":
885
+ return [name.replace("-", "_")], "exact"
886
+ if ecosystem == "go":
887
+ return [name], "exact"
888
+ if ecosystem in ("maven", "gradle"):
889
+ return _jvm_import_names(name)
890
+ if ecosystem == "ruby":
891
+ return _ruby_import_names(name)
892
+ if ecosystem == "composer":
893
+ return _composer_import_names(name, root)
894
+ if ecosystem == "swift":
895
+ return _swift_import_names(name)
896
+ guesses, _ = _python_import_names(name)
897
+ return list(dict.fromkeys([name, *guesses])), "guessed"
898
+
899
+
900
+ # --------------------------------------------------------------------------
901
+ # Exclusion globs
902
+ # --------------------------------------------------------------------------
903
+
904
+
905
+ def _glob_regex(pattern: str) -> re.Pattern[str]:
906
+ """Translate a glob to a regex: `*` and `?` stay in one path segment,
907
+ `**/` spans zero or more directories and `**` any characters."""
908
+ out: list[str] = []
909
+ i = 0
910
+ while i < len(pattern):
911
+ char = pattern[i]
912
+ if pattern.startswith("**/", i):
913
+ out.append("(?:.*/)?")
914
+ i += 3
915
+ continue
916
+ if pattern.startswith("**", i):
917
+ out.append(".*")
918
+ i += 2
919
+ continue
920
+ if char == "*":
921
+ out.append("[^/]*")
922
+ elif char == "?":
923
+ out.append("[^/]")
924
+ elif char == "[" and pattern.find("]", i + 1) != -1:
925
+ end = pattern.find("]", i + 1)
926
+ body = pattern[i + 1 : end]
927
+ if body.startswith("!"):
928
+ body = "^" + body[1:]
929
+ out.append("[" + body.replace("\\", "\\\\") + "]")
930
+ i = end + 1
931
+ continue
932
+ else:
933
+ out.append(re.escape(char))
934
+ i += 1
935
+ return re.compile("".join(out))
936
+
937
+
938
+ class Excludes:
939
+ """The exclusion globs, split by what each one is matched against."""
940
+
941
+ def __init__(self, patterns: list[str]) -> None:
942
+ self.dir_names: list[re.Pattern[str]] = []
943
+ self.dir_paths: list[re.Pattern[str]] = []
944
+ self.file_names: list[re.Pattern[str]] = []
945
+ self.file_paths: list[re.Pattern[str]] = []
946
+ for raw in patterns:
947
+ pattern = raw.strip().replace("\\", "/")
948
+ # a leading / or ./ anchors the pattern to the root
949
+ anchored = pattern.startswith(("/", "./"))
950
+ is_dir = pattern.endswith("/")
951
+ body = posixpath.normpath(pattern.lstrip("/") or ".")
952
+ if body == ".":
953
+ continue
954
+ if is_dir:
955
+ target = self.dir_paths if anchored or "/" in body else self.dir_names
956
+ else:
957
+ target = self.file_paths if anchored or "/" in body else self.file_names
958
+ target.append(_glob_regex(body))
959
+
960
+ def skip_dir(self, name: str, rel: str) -> bool:
961
+ return any(r.fullmatch(name) for r in self.dir_names) or any(
962
+ r.fullmatch(rel) for r in self.dir_paths
963
+ )
964
+
965
+ def skip_file(self, name: str, rel: str) -> bool:
966
+ return any(r.fullmatch(name) for r in self.file_names) or any(
967
+ r.fullmatch(rel) for r in self.file_paths
968
+ )
969
+
970
+
971
+ # --------------------------------------------------------------------------
972
+ # Import extraction
973
+ # --------------------------------------------------------------------------
974
+
975
+
976
+ # A record is (line, kind, value): kind "module" holds an imported module
977
+ # name; kind "path" holds a relative import resolved to a root-relative path.
978
+ Record = tuple[int, str, str]
979
+
980
+ _JS_IMPORT = re.compile(
981
+ r"""(?<![\w$.])(?:
982
+ (?:from|import)\s*(?P<q1>['"])(?P<s1>[^'"\n]+)(?P=q1)
983
+ | (?:import|require)\s*\(\s*(?P<q2>['"`])(?P<s2>[^'"`\n]+)(?P=q2)
984
+ )""",
985
+ re.VERBOSE,
986
+ )
987
+ _PY_FROM = re.compile(r"from\s+(?P<mod>\.+[\w.]*|[A-Za-z_][\w.]*)\s+import\s*(?P<names>.*)")
988
+ _PY_DYNAMIC = re.compile(
989
+ r"""(?<!\w)(?:import_module|__import__)\s*\(\s*(['"])(?P<mod>[A-Za-z_][\w.]*)\1"""
990
+ )
991
+ _PY_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*")
992
+ _RS_USE = re.compile(
993
+ r"^\s*(?:pub(?:\s*\([^)]*\))?\s+)?use\s+(?:::)?(?P<path>[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)"
994
+ )
995
+ _RS_EXTERN = re.compile(r"^\s*extern\s+crate\s+(?P<path>[A-Za-z_]\w*)")
996
+ _RS_PATH = re.compile(r"(?<![\w:])(?P<path>[A-Za-z_]\w*(?:::[A-Za-z_]\w*)+)")
997
+ _GO_IMPORT_ONE = re.compile(r'^\s*import\s+(?:[\w.]+\s+)?"(?P<path>[^"]+)"')
998
+ _GO_IMPORT_OPEN = re.compile(r"^\s*import\s*\(")
999
+ _GO_QUOTED = re.compile(r'(?:^|[\s(;])(?:[\w.]+\s+)?"(?P<path>[^"]+)"')
1000
+ _JVM_IMPORT = re.compile(
1001
+ r"^\s*import\s+(?:static\s+)?(?P<path>[A-Za-z_]\w*(?:\.(?:[A-Za-z_]\w*|\*))*)"
1002
+ )
1003
+ _RB_REQUIRE = re.compile(
1004
+ r"""^\s*(?P<kind>require_relative|require)\s*\(?\s*(['"])(?P<path>[^'"]+)\2"""
1005
+ )
1006
+ _PHP_USE = re.compile(r"^\s*use\s+(?P<body>[^;]*)")
1007
+ _SWIFT_IMPORT = re.compile(
1008
+ r"^\s*(?:@\w+(?:\([^)]*\))?\s+)*import\s+"
1009
+ r"(?:(?:typealias|struct|class|enum|protocol|let|var|func)\s+)?(?P<mod>[A-Za-z_]\w*)"
1010
+ )
1011
+
1012
+
1013
+ def _lines(text: str) -> list[str]:
1014
+ """The lines of a file as editors and `ast` number them (str.splitlines
1015
+ also breaks at form feeds and Unicode line separators)."""
1016
+ return text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
1017
+
1018
+
1019
+ def _resolve_relative(base_dir: str, spec: str) -> str | None:
1020
+ """Join a relative import to its file's directory; None if it leaves the root."""
1021
+ joined = posixpath.normpath(posixpath.join(base_dir, spec) if base_dir else spec)
1022
+ if joined == ".." or joined.startswith("../"):
1023
+ return None
1024
+ return "" if joined == "." else joined
1025
+
1026
+
1027
+ def _js_records(text: str, rel: str) -> list[Record]:
1028
+ records: list[Record] = []
1029
+ base = posixpath.dirname(rel)
1030
+ for number, line in enumerate(_lines(text), start=1):
1031
+ if line.lstrip().startswith(("//", "/*", "*")):
1032
+ continue
1033
+ if "import" not in line and "require" not in line and "from" not in line:
1034
+ continue
1035
+ for m in _JS_IMPORT.finditer(line):
1036
+ spec = m.group("s1") or m.group("s2")
1037
+ if not spec or "${" in spec:
1038
+ continue
1039
+ if spec in (".", "..") or spec.startswith(("./", "../")):
1040
+ resolved = _resolve_relative(base, spec)
1041
+ if resolved is not None:
1042
+ records.append((number, "path", resolved))
1043
+ elif not spec.startswith("/"):
1044
+ records.append((number, "module", spec))
1045
+ return records
1046
+
1047
+
1048
+ def _py_names(text: str) -> tuple[list[str], bool]:
1049
+ """Names of a `from m import ...` list, and whether its `)` was seen."""
1050
+ closed = True
1051
+ if text.startswith("("):
1052
+ text = text[1:]
1053
+ closed = False
1054
+ if not closed and ")" in text:
1055
+ text = text.split(")", 1)[0]
1056
+ closed = True
1057
+ names = []
1058
+ for item in text.split(","):
1059
+ item = item.strip().split(" as ", 1)[0].strip()
1060
+ if item and item != "*" and re.fullmatch(r"[A-Za-z_]\w*", item):
1061
+ names.append(item)
1062
+ return names, closed
1063
+
1064
+
1065
+ def _py_from_records(number: int, mod: str, names: list[str], base: str) -> list[Record]:
1066
+ if not mod.startswith("."):
1067
+ return [(number, "module", mod)] + [(number, "module", f"{mod}.{n}") for n in names]
1068
+ level = len(mod) - len(mod.lstrip("."))
1069
+ rest = mod[level:]
1070
+ target = base
1071
+ for _ in range(level - 1):
1072
+ if not target:
1073
+ return []
1074
+ target = posixpath.dirname(target)
1075
+ if rest:
1076
+ sub = rest.replace(".", "/")
1077
+ target = posixpath.join(target, sub) if target else sub
1078
+ records: list[Record] = [(number, "path", target)]
1079
+ for n in names:
1080
+ records.append((number, "path", posixpath.join(target, n) if target else n))
1081
+ return records
1082
+
1083
+
1084
+ def _py_statements(tree: ast.Module):
1085
+ """Every statement of a module, the ones in nested blocks included."""
1086
+ queue: list[ast.AST] = list(tree.body)
1087
+ for node in queue:
1088
+ yield node
1089
+ for field in ("body", "orelse", "finalbody", "handlers", "cases"):
1090
+ queue.extend(getattr(node, field, ()))
1091
+
1092
+
1093
+ def _python_records(text: str, rel: str) -> list[Record]:
1094
+ base = posixpath.dirname(rel)
1095
+ if "import" not in text:
1096
+ return []
1097
+ if len(text) > PYTHON_AST_MAX_CHARS:
1098
+ return _python_line_records(text, base)
1099
+ try:
1100
+ with warnings.catch_warnings():
1101
+ # an invalid escape sequence warns; the file is only read
1102
+ warnings.simplefilter("ignore")
1103
+ tree = ast.parse(text)
1104
+ except (SyntaxError, ValueError, RecursionError, MemoryError):
1105
+ # Python 2, a syntax error, syntax newer than this interpreter, or
1106
+ # nesting too deep to parse
1107
+ return _python_line_records(text, base)
1108
+ # an import statement sits in a block; a dynamic import is a call in any expression
1109
+ dynamic = "import_module" in text or "__import__" in text
1110
+ records: list[Record] = []
1111
+ for node in ast.walk(tree) if dynamic else _py_statements(tree):
1112
+ if isinstance(node, ast.Import):
1113
+ records.extend((node.lineno, "module", alias.name) for alias in node.names)
1114
+ elif isinstance(node, ast.ImportFrom):
1115
+ mod = "." * node.level + (node.module or "")
1116
+ names = [alias.name for alias in node.names if alias.name != "*"]
1117
+ records.extend(_py_from_records(node.lineno, mod, names, base))
1118
+ elif isinstance(node, ast.Call) and node.args:
1119
+ func, arg = node.func, node.args[0]
1120
+ called = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None)
1121
+ if (
1122
+ called in ("import_module", "__import__")
1123
+ and isinstance(arg, ast.Constant)
1124
+ and isinstance(arg.value, str)
1125
+ and _PY_DOTTED.fullmatch(arg.value)
1126
+ ):
1127
+ records.append((node.lineno, "module", arg.value))
1128
+ return sorted(records, key=lambda record: record[0])
1129
+
1130
+
1131
+ def _py_code(line: str, quote: str | None) -> tuple[str, str | None]:
1132
+ """The code of one line, and the triple quote still open at its end.
1133
+
1134
+ `quote` is the triple quote open when the line starts. Triple-quoted text
1135
+ and the comment are left out; a `#` inside a string literal is kept.
1136
+ """
1137
+ code: list[str] = []
1138
+ i, end = 0, len(line)
1139
+ while i < end:
1140
+ if quote:
1141
+ if line.startswith(quote, i):
1142
+ quote = None
1143
+ i += 3
1144
+ else:
1145
+ i += 2 if line[i] == "\\" else 1
1146
+ continue
1147
+ char = line[i]
1148
+ if char == "#":
1149
+ break
1150
+ if line.startswith(('"""', "'''"), i):
1151
+ quote = line[i : i + 3]
1152
+ i += 3
1153
+ elif char in "'\"":
1154
+ close = i + 1
1155
+ while close < end and line[close] != char:
1156
+ close += 2 if line[close] == "\\" else 1
1157
+ code.append(line[i : close + 1])
1158
+ i = close + 1
1159
+ else:
1160
+ code.append(char)
1161
+ i += 1
1162
+ return "".join(code), quote
1163
+
1164
+
1165
+ def _python_line_records(text: str, base: str) -> list[Record]:
1166
+ """The imports of a file `ast` cannot parse, read line by line."""
1167
+ records: list[Record] = []
1168
+ quote: str | None = None
1169
+ pending: tuple[int, str] | None = None # (line, module) of an open `from m import (`
1170
+ for number, line in enumerate(_lines(text), start=1):
1171
+ code, quote = _py_code(line, quote)
1172
+ if pending is not None:
1173
+ names, closed = _py_names("(" + code)
1174
+ records.extend(_py_from_records(pending[0], pending[1], names, base)[1:])
1175
+ if closed:
1176
+ pending = None
1177
+ continue
1178
+ for statement in code.split(";"):
1179
+ statement = statement.strip()
1180
+ if statement.startswith("import "):
1181
+ for item in statement[len("import "):].split(","):
1182
+ item = item.strip().split(" as ", 1)[0].strip()
1183
+ if _PY_DOTTED.fullmatch(item):
1184
+ records.append((number, "module", item))
1185
+ elif statement.startswith("from "):
1186
+ m = _PY_FROM.match(statement)
1187
+ if not m:
1188
+ continue
1189
+ names, closed = _py_names(m.group("names").strip())
1190
+ records.extend(_py_from_records(number, m.group("mod"), names, base))
1191
+ if not closed:
1192
+ pending = (number, m.group("mod"))
1193
+ if "import_module" in code or "__import__" in code:
1194
+ for m in _PY_DYNAMIC.finditer(code):
1195
+ records.append((number, "module", m.group("mod")))
1196
+ return records
1197
+
1198
+
1199
+ def _rust_records(text: str, rel: str) -> list[Record]:
1200
+ records: list[Record] = []
1201
+ for number, line in enumerate(_lines(text), start=1):
1202
+ if line.lstrip().startswith(("//", "/*", "*")):
1203
+ continue
1204
+ for regex in (_RS_USE, _RS_EXTERN):
1205
+ m = regex.match(line)
1206
+ if m:
1207
+ records.append((number, "module", m.group("path")))
1208
+ if "::" in line:
1209
+ for m in _RS_PATH.finditer(line):
1210
+ records.append((number, "module", m.group("path")))
1211
+ return records
1212
+
1213
+
1214
+ def _go_records(text: str, rel: str) -> list[Record]:
1215
+ records: list[Record] = []
1216
+ in_block = False
1217
+ for number, line in enumerate(_lines(text), start=1):
1218
+ code = line.split("//", 1)[0]
1219
+ if in_block:
1220
+ if code.strip().startswith(")"):
1221
+ in_block = False
1222
+ continue
1223
+ m = _GO_QUOTED.search(code)
1224
+ if m:
1225
+ records.append((number, "module", m.group("path")))
1226
+ if ")" in code.split('"')[-1]:
1227
+ in_block = False
1228
+ continue
1229
+ if _GO_IMPORT_OPEN.match(code):
1230
+ rest = code.split("(", 1)[1]
1231
+ for m in _GO_QUOTED.finditer(rest):
1232
+ records.append((number, "module", m.group("path")))
1233
+ in_block = ")" not in rest.split('"')[-1]
1234
+ continue
1235
+ m = _GO_IMPORT_ONE.match(code)
1236
+ if m:
1237
+ records.append((number, "module", m.group("path")))
1238
+ return records
1239
+
1240
+
1241
+ def _jvm_records(text: str, rel: str) -> list[Record]:
1242
+ records: list[Record] = []
1243
+ for number, line in enumerate(_lines(text), start=1):
1244
+ m = _JVM_IMPORT.match(line)
1245
+ if m:
1246
+ path = m.group("path")
1247
+ if path.endswith(".*"):
1248
+ path = path[:-2]
1249
+ records.append((number, "module", path))
1250
+ return records
1251
+
1252
+
1253
+ def _ruby_records(text: str, rel: str) -> list[Record]:
1254
+ records: list[Record] = []
1255
+ base = posixpath.dirname(rel)
1256
+ for number, line in enumerate(_lines(text), start=1):
1257
+ m = _RB_REQUIRE.match(line)
1258
+ if not m:
1259
+ continue
1260
+ path = m.group("path")
1261
+ if m.group("kind") == "require_relative" or path.startswith(("./", "../")):
1262
+ resolved = _resolve_relative(base, path)
1263
+ if resolved is not None:
1264
+ records.append((number, "path", resolved))
1265
+ else:
1266
+ records.append((number, "module", path))
1267
+ return records
1268
+
1269
+
1270
+ def _php_records(text: str, rel: str) -> list[Record]:
1271
+ records: list[Record] = []
1272
+ for number, line in enumerate(_lines(text), start=1):
1273
+ m = _PHP_USE.match(line)
1274
+ if not m:
1275
+ continue
1276
+ body = m.group("body").strip()
1277
+ if body.startswith(("function ", "const ")):
1278
+ body = body.split(None, 1)[1] if " " in body else ""
1279
+ if "{" in body:
1280
+ prefix = body.split("{", 1)[0].strip().strip("\\")
1281
+ items = [prefix] if prefix else []
1282
+ else:
1283
+ items = [
1284
+ item.strip().split(" as ", 1)[0].strip().strip("\\") for item in body.split(",")
1285
+ ]
1286
+ for item in items:
1287
+ if re.fullmatch(r"[A-Za-z_]\w*(?:\\[A-Za-z_]\w*)*", item):
1288
+ records.append((number, "module", item))
1289
+ return records
1290
+
1291
+
1292
+ def _swift_records(text: str, rel: str) -> list[Record]:
1293
+ records: list[Record] = []
1294
+ for number, line in enumerate(_lines(text), start=1):
1295
+ m = _SWIFT_IMPORT.match(line)
1296
+ if m:
1297
+ records.append((number, "module", m.group("mod")))
1298
+ return records
1299
+
1300
+
1301
+ EXTRACTORS = {
1302
+ "js": _js_records,
1303
+ "python": _python_records,
1304
+ "rust": _rust_records,
1305
+ "go": _go_records,
1306
+ "jvm": _jvm_records,
1307
+ "ruby": _ruby_records,
1308
+ "php": _php_records,
1309
+ "swift": _swift_records,
1310
+ }
1311
+
1312
+
1313
+ def extract_records(language: str, text: str, rel: str) -> list[Record]:
1314
+ """Import records of one source file, in line order."""
1315
+ return EXTRACTORS[language](text.lstrip("\ufeff"), rel)
1316
+
1317
+
1318
+ def _key(language: str, name: str) -> str:
1319
+ return name.lower() if language == "php" else name
1320
+
1321
+
1322
+ def candidates(language: str, module: str) -> list[str]:
1323
+ """Every prefix of `module` at the language's separator, longest first."""
1324
+ module = _key(language, module)
1325
+ sep = SEPARATORS[language]
1326
+ if sep is None:
1327
+ return [module]
1328
+ parts = module.split(sep)
1329
+ return [sep.join(parts[:i]) for i in range(len(parts), 0, -1)]
1330
+
1331
+
1332
+ # --------------------------------------------------------------------------
1333
+ # Counting
1334
+ # --------------------------------------------------------------------------
1335
+
1336
+
1337
+ def _walk(root: Path, excludes: Excludes, extensions: dict[str, str], start: str = ""):
1338
+ """Yield (rel, path, language) for every source file left after exclusions,
1339
+ only those under the `start` folder (relative to `root`) when one is given."""
1340
+ for dirpath, dirnames, filenames in os.walk(root / start if start else root):
1341
+ rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/")
1342
+ rel_dir = "" if rel_dir == "." else rel_dir
1343
+ kept = []
1344
+ for name in sorted(dirnames):
1345
+ rel = f"{rel_dir}/{name}" if rel_dir else name
1346
+ if os.path.islink(os.path.join(dirpath, name)) or excludes.skip_dir(name, rel):
1347
+ continue
1348
+ kept.append(name)
1349
+ dirnames[:] = kept
1350
+ for name in sorted(filenames):
1351
+ language = extensions.get(os.path.splitext(name)[1])
1352
+ if language is None:
1353
+ continue
1354
+ rel = f"{rel_dir}/{name}" if rel_dir else name
1355
+ path = os.path.join(dirpath, name)
1356
+ if os.path.islink(path) or excludes.skip_file(name, rel):
1357
+ continue
1358
+ yield rel, path, language
1359
+
1360
+
1361
+ def _files(first_lines: dict[str, int]) -> list[dict]:
1362
+ return [{"path": p, "line": first_lines[p]} for p in sorted(first_lines)]
1363
+
1364
+
1365
+ def _languages(ecosystem: str | None) -> list[str]:
1366
+ """The languages whose files a target of this ecosystem is matched in."""
1367
+ if ecosystem in ECOSYSTEM_LANGUAGE:
1368
+ return [ECOSYSTEM_LANGUAGE[ecosystem]]
1369
+ return list(LANGUAGE_EXTENSIONS)
1370
+
1371
+
1372
+ _JVM_PACKAGE = re.compile(r"^\s*package\s+(?P<name>[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", re.M)
1373
+
1374
+
1375
+ def _declared_jvm_packages(root: Path, unit_path: str, owns, excludes: Excludes) -> list[str]:
1376
+ """The packages a unit's own .java, .kt, .groovy and .scala files declare;
1377
+ `owns(rel)` says whether a file belongs to the unit, not to one inside it."""
1378
+ extensions = {ext: "jvm" for ext in LANGUAGE_EXTENSIONS["jvm"]}
1379
+ packages: set[str] = set()
1380
+ for rel, path, _ in _walk(root, excludes, extensions, unit_path):
1381
+ if not owns(rel):
1382
+ continue
1383
+ try:
1384
+ with open(path, "rb") as handle:
1385
+ text = handle.read().decode("utf-8", errors="replace")
1386
+ except OSError:
1387
+ continue
1388
+ m = _JVM_PACKAGE.search(text)
1389
+ if m:
1390
+ packages.add(m.group("name"))
1391
+ return sorted(packages)
1392
+
1393
+
1394
+ def _unit_import_names(root: Path, unit: dict, owns, excludes: Excludes) -> list[str]:
1395
+ """A unit's import names: its `modules`, else what it declares (the
1396
+ packages of its JVM files, its composer.json autoload, its Package.swift
1397
+ targets), else its `manifest_name` by the dependency rules."""
1398
+ if unit["modules"] is not None:
1399
+ return list(unit["modules"])
1400
+ ecosystem, base = unit["ecosystem"], root / unit["path"]
1401
+ declared: list[str] = []
1402
+ if ecosystem in ("maven", "gradle"):
1403
+ declared = _declared_jvm_packages(root, unit["path"], owns, excludes)
1404
+ elif ecosystem == "composer":
1405
+ declared = _composer_autoload(base / "composer.json")
1406
+ elif ecosystem == "swift":
1407
+ declared = _swift_targets(base / "Package.swift")
1408
+ if declared:
1409
+ return declared
1410
+ if unit["manifest_name"] and ecosystem in ECOSYSTEM_LANGUAGE:
1411
+ return import_names(unit["manifest_name"], ecosystem, root)[0]
1412
+ return []
1413
+
1414
+
1415
+ def count_imports(
1416
+ root: Path,
1417
+ dependencies: list[dict],
1418
+ units: list[dict] | None = None,
1419
+ threshold: int = DEFAULT_THRESHOLD,
1420
+ extra_excludes: list[str] | None = None,
1421
+ ) -> dict:
1422
+ """Count the files under `root` that import each dependency and unit."""
1423
+ units = units or []
1424
+ targets: list[dict] = []
1425
+ unresolved: list[dict] = []
1426
+ excludes = Excludes(list(DEFAULT_EXCLUDES) + list(extra_excludes or []))
1427
+
1428
+ unit_by_path = {unit["path"]: pos for pos, unit in enumerate(units)}
1429
+
1430
+ def unit_of(rel: str) -> int | None:
1431
+ """The unit with the longest path that contains `rel`."""
1432
+ while True:
1433
+ if rel in unit_by_path:
1434
+ return unit_by_path[rel]
1435
+ if not rel:
1436
+ return None
1437
+ rel = posixpath.dirname(rel)
1438
+
1439
+ unit_names = [
1440
+ _unit_import_names(root, unit, lambda rel, pos=pos: unit_of(rel) == pos, excludes)
1441
+ for pos, unit in enumerate(units)
1442
+ ]
1443
+ # a dependency that has one of these (language, import name) keys is the unit
1444
+ internal = {
1445
+ (language, _key(language, name))
1446
+ for pos, unit in enumerate(units)
1447
+ for language in _languages(unit["ecosystem"])
1448
+ for name in unit_names[pos]
1449
+ }
1450
+
1451
+ for dep in dependencies:
1452
+ ecosystem = dep["ecosystem"]
1453
+ if "modules" in dep:
1454
+ names, resolution = list(dep["modules"]), "explicit"
1455
+ elif ecosystem is not None and ecosystem not in ECOSYSTEM_LANGUAGE:
1456
+ unresolved.append(_unresolved(dep, [], "unsupported-ecosystem"))
1457
+ continue
1458
+ else:
1459
+ names, resolution = import_names(dep["name"], ecosystem, root)
1460
+ if not names:
1461
+ unresolved.append(_unresolved(dep, names, "no-import-name"))
1462
+ continue
1463
+ languages = _languages(ecosystem)
1464
+ if any((lang, _key(lang, n)) in internal for lang in languages for n in names):
1465
+ continue
1466
+ targets.append(
1467
+ {
1468
+ "kind": "dependency",
1469
+ "dep": dep,
1470
+ "names": names,
1471
+ "resolution": resolution,
1472
+ "languages": languages,
1473
+ }
1474
+ )
1475
+
1476
+ unit_index: dict[int, int] = {} # position in `units` -> position in `targets`
1477
+ for pos, unit in enumerate(units):
1478
+ unit_index[pos] = len(targets)
1479
+ targets.append(
1480
+ {
1481
+ "kind": "unit",
1482
+ "unit": pos,
1483
+ "names": unit_names[pos],
1484
+ "languages": _languages(unit["ecosystem"]),
1485
+ }
1486
+ )
1487
+
1488
+ # language -> import-name key -> target positions
1489
+ index: dict[str, dict[str, list[int]]] = {lang: {} for lang in LANGUAGE_EXTENSIONS}
1490
+ for t_pos, target in enumerate(targets):
1491
+ for language in target["languages"]:
1492
+ for name in target["names"]:
1493
+ index[language].setdefault(_key(language, name), []).append(t_pos)
1494
+
1495
+ needed = {lang for target in targets for lang in target["languages"] if target["names"]}
1496
+ if units:
1497
+ # relative imports resolve into unit paths in these languages
1498
+ needed.update(("js", "python", "ruby"))
1499
+ extensions = {ext: lang for lang in needed for ext in LANGUAGE_EXTENSIONS[lang]}
1500
+
1501
+ first_lines: list[dict[str, int]] = [{} for _ in targets]
1502
+ # (importing unit, imported unit) -> importing file -> its first line
1503
+ edge_lines: dict[tuple[str, str], dict[str, int]] = {}
1504
+ external: dict[int, set[str]] = {pos: set() for pos in range(len(units))}
1505
+ warnings: list[str] = []
1506
+ files_scanned = 0
1507
+
1508
+ for rel, path, language in _walk(root, excludes, extensions):
1509
+ try:
1510
+ with open(path, "rb") as handle:
1511
+ text = handle.read().decode("utf-8", errors="replace")
1512
+ except OSError as exc:
1513
+ warnings.append(f"{rel}: unreadable ({exc.strerror or exc})")
1514
+ continue
1515
+ files_scanned += 1
1516
+ records = extract_records(language, text, rel)
1517
+ if not records:
1518
+ continue
1519
+ own_unit = unit_of(rel) if units else None
1520
+ lang_index = index[language]
1521
+ for number, kind, value in records:
1522
+ if kind == "module":
1523
+ hits: list[int] = []
1524
+ for candidate in candidates(language, value):
1525
+ if candidate in lang_index:
1526
+ hits = lang_index[candidate]
1527
+ break
1528
+ else:
1529
+ pos = unit_of(value) if units else None
1530
+ hits = [unit_index[pos]] if pos is not None else []
1531
+ for t_pos in hits:
1532
+ target = targets[t_pos]
1533
+ if target["kind"] == "unit":
1534
+ if target["unit"] == own_unit:
1535
+ continue
1536
+ if own_unit is not None:
1537
+ edge = (units[own_unit]["name"], units[target["unit"]]["name"])
1538
+ lines = edge_lines.setdefault(edge, {})
1539
+ if rel not in lines or number < lines[rel]:
1540
+ lines[rel] = number
1541
+ elif own_unit is not None:
1542
+ external[own_unit].add(target["dep"]["name"])
1543
+ seen = first_lines[t_pos].get(rel)
1544
+ if seen is None or number < seen:
1545
+ first_lines[t_pos][rel] = number
1546
+
1547
+ dependencies_out: list[dict] = []
1548
+ for t_pos, target in enumerate(targets):
1549
+ if target["kind"] != "dependency":
1550
+ continue
1551
+ dep = target["dep"]
1552
+ if target["resolution"] == "guessed" and not first_lines[t_pos]:
1553
+ unresolved.append(_unresolved(dep, target["names"], "guess-unmatched"))
1554
+ continue
1555
+ entry = {
1556
+ "name": dep["name"],
1557
+ "ecosystem": dep["ecosystem"],
1558
+ "import_names": target["names"],
1559
+ "resolution": target["resolution"],
1560
+ }
1561
+ if dep["scope"] == "dev":
1562
+ entry["scope"] = "dev"
1563
+ entry["files"] = _files(first_lines[t_pos])
1564
+ entry["file_count"] = len(first_lines[t_pos])
1565
+ entry["above_threshold"] = entry["file_count"] >= threshold
1566
+ dependencies_out.append(entry)
1567
+ dependencies_out.sort(key=lambda d: (-d["file_count"], d["name"], d["ecosystem"] or ""))
1568
+ unresolved.sort(key=lambda d: (d["name"], d["ecosystem"] or ""))
1569
+
1570
+ result: dict = {
1571
+ "threshold": threshold,
1572
+ "files_scanned": files_scanned,
1573
+ "dependencies": dependencies_out,
1574
+ "unresolved": unresolved,
1575
+ }
1576
+ if units:
1577
+ imported_by: dict[str, set[str]] = {u["name"]: set() for u in units}
1578
+ imports_from: dict[str, set[str]] = {u["name"]: set() for u in units}
1579
+ for src, dst in edge_lines:
1580
+ imports_from[src].add(dst)
1581
+ imported_by[dst].add(src)
1582
+ units_out = []
1583
+ for pos, unit in enumerate(units):
1584
+ lines = first_lines[unit_index[pos]]
1585
+ units_out.append(
1586
+ {
1587
+ "name": unit["name"],
1588
+ "path": unit["path"],
1589
+ "import_names": unit_names[pos],
1590
+ "files": _files(lines),
1591
+ "file_count": len(lines),
1592
+ "imported_by": sorted(imported_by[unit["name"]]),
1593
+ "imports_from": sorted(imports_from[unit["name"]]),
1594
+ "external_deps": sorted(external[pos]),
1595
+ }
1596
+ )
1597
+ result["units"] = sorted(units_out, key=lambda u: u["name"])
1598
+ result["edges"] = [list(edge) for edge in sorted(edge_lines)]
1599
+ result["edge_files"] = [
1600
+ {"from": src, "to": dst, "files": _files(edge_lines[(src, dst)])}
1601
+ for src, dst in sorted(edge_lines)
1602
+ ]
1603
+ if warnings:
1604
+ result["warnings"] = warnings
1605
+ return result
1606
+
1607
+
1608
+ def _unresolved(dep: dict, names: list[str], reason: str) -> dict:
1609
+ entry = {
1610
+ "name": dep["name"],
1611
+ "ecosystem": dep["ecosystem"],
1612
+ "import_names": names,
1613
+ "reason": reason,
1614
+ }
1615
+ if dep["scope"] == "dev":
1616
+ entry["scope"] = "dev"
1617
+ return entry
1618
+
1619
+
1620
+ def prefix_paths(result: dict, prefix: str) -> dict:
1621
+ """Write every `files[].path` of `result` under `prefix`, a forward-slash folder."""
1622
+ if not prefix:
1623
+ return result
1624
+
1625
+ def moved(files: list[dict]) -> list[dict]:
1626
+ return [{**f, "path": f"{prefix}/{f['path']}"} for f in files]
1627
+
1628
+ for entry in result["dependencies"] + result.get("units", []) + result.get("edge_files", []):
1629
+ entry["files"] = moved(entry["files"])
1630
+ return result
1631
+
1632
+
1633
+ def _relative_prefix(root: Path, base: Path) -> str:
1634
+ """The forward-slash path of `root` inside `base` ('' when they are one folder).
1635
+
1636
+ Raises InputError when `base` is not a folder or does not hold `root`.
1637
+ """
1638
+ if not base.is_dir():
1639
+ raise InputError(f"--relative-to not a directory: {base}")
1640
+ try:
1641
+ prefix = root.resolve().relative_to(base.resolve()).as_posix()
1642
+ except ValueError:
1643
+ raise InputError(f"root {root} is not inside --relative-to {base}") from None
1644
+ return "" if prefix == "." else prefix
1645
+
1646
+
1647
+ def as_libraries(result: dict) -> list[dict]:
1648
+ """The skf-pair-intersect.py `--libraries` input: dependencies, then units."""
1649
+ rows = [
1650
+ {"name": d["name"], "files": [f["path"] for f in d["files"]]}
1651
+ for d in result["dependencies"]
1652
+ ]
1653
+ rows += [
1654
+ {"name": u["name"], "files": [f["path"] for f in u["files"]]}
1655
+ for u in result.get("units", [])
1656
+ ]
1657
+ return rows
1658
+
1659
+
1660
+ SUMMARY_MAX_FILES = 5
1661
+ SUMMARY_IMPORTS = "the imports input (a `count --units` envelope)"
1662
+ SUMMARY_MANIFESTS = "the --manifests input (a skf-scan-manifests.py scan)"
1663
+
1664
+
1665
+ def _object_rows(data: object, key: str, what: str) -> list[dict]:
1666
+ """`data[key]` when it is a list of objects, else an InputError naming the input."""
1667
+ rows = data.get(key) if isinstance(data, dict) else None
1668
+ if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows):
1669
+ raise InputError(f"{what} holds no `{key}` list of objects")
1670
+ return rows
1671
+
1672
+
1673
+ def summarize(result: object, manifests: object | None = None, max_files: int = SUMMARY_MAX_FILES) -> dict:
1674
+ """The unit graph of a `count --units` envelope without its file lists, and
1675
+ the fields of a manifest scan the cohesion triggers read (see `summary`)."""
1676
+ units = _object_rows(result, "units", SUMMARY_IMPORTS)
1677
+ edge_files = _object_rows(result, "edge_files", SUMMARY_IMPORTS)
1678
+ try:
1679
+ out: dict = {
1680
+ "units": [
1681
+ {"name": u["name"], "imports_from": u["imports_from"], "imported_by": u["imported_by"],
1682
+ "file_count": u["file_count"], "external_dep_count": len(u["external_deps"])}
1683
+ for u in units
1684
+ ],
1685
+ "edge_files": [
1686
+ {"from": e["from"], "to": e["to"], "file_count": len(e["files"]), "files": e["files"][:max_files]}
1687
+ for e in edge_files
1688
+ ],
1689
+ }
1690
+ except (KeyError, TypeError) as exc:
1691
+ raise InputError(f"{SUMMARY_IMPORTS} has a malformed units or edge_files entry "
1692
+ f"({type(exc).__name__}: {exc})") from None
1693
+ if manifests is not None:
1694
+ rows = _object_rows(manifests, "manifests", SUMMARY_MANIFESTS)
1695
+ umbrellas = manifests.get("umbrella_candidates")
1696
+ out["umbrella_candidates"] = umbrellas if isinstance(umbrellas, list) else []
1697
+ out["manifests"] = [
1698
+ {"path": m.get("path"), "name": m.get("name"), "private": m.get("private"),
1699
+ "internal_deps": m.get("internal_deps") or []}
1700
+ for m in rows
1701
+ ]
1702
+ if result.get("warnings"):
1703
+ out["warnings"] = result["warnings"]
1704
+ return out
1705
+
1706
+
1707
+ # --------------------------------------------------------------------------
1708
+ # CLI
1709
+ # --------------------------------------------------------------------------
1710
+
1711
+
1712
+ def _cmd_count(args: argparse.Namespace) -> int:
1713
+ root = Path(args.root)
1714
+ if not root.is_dir():
1715
+ print(f"error: root not a directory: {root}", file=sys.stderr)
1716
+ return 1
1717
+ if args.deps is None and args.units is None:
1718
+ print("error: pass --deps, --units or both", file=sys.stderr)
1719
+ return 1
1720
+ if args.deps == "-" and args.units == "-":
1721
+ print("error: only one of --deps and --units may read stdin", file=sys.stderr)
1722
+ return 1
1723
+ if args.threshold < 0:
1724
+ print(f"error: --threshold must be >= 0; got {args.threshold}", file=sys.stderr)
1725
+ return 1
1726
+ try:
1727
+ prefix = "" if args.relative_to is None else _relative_prefix(root, Path(args.relative_to))
1728
+ dependencies: list[dict] = []
1729
+ units: list[dict] | None = None
1730
+ if args.deps is not None:
1731
+ dependencies = parse_dependencies(_read_json_source(args.deps, "--deps"))
1732
+ if args.units is not None:
1733
+ units = parse_units(_read_json_source(args.units, "--units"))
1734
+ except InputError as exc:
1735
+ print(f"error: {exc}", file=sys.stderr)
1736
+ return 1
1737
+ try:
1738
+ result = count_imports(
1739
+ root,
1740
+ dependencies,
1741
+ units=units,
1742
+ threshold=args.threshold,
1743
+ extra_excludes=args.exclude,
1744
+ )
1745
+ except OSError as exc:
1746
+ print(f"error: filesystem error during scan: {exc}", file=sys.stderr)
1747
+ return 1
1748
+ result = prefix_paths(result, prefix)
1749
+ output = as_libraries(result) if args.format == "libraries" else result
1750
+ json.dump(output, sys.stdout, indent=2)
1751
+ sys.stdout.write("\n")
1752
+ return 0
1753
+
1754
+
1755
+ def _cmd_summary(args: argparse.Namespace) -> int:
1756
+ if args.imports == "-" and args.manifests == "-":
1757
+ print("error: only one of the imports file and --manifests may read stdin", file=sys.stderr)
1758
+ return 1
1759
+ if args.max_files < 0:
1760
+ print(f"error: --max-files must be >= 0; got {args.max_files}", file=sys.stderr)
1761
+ return 1
1762
+ try:
1763
+ result = _read_json_source(args.imports, "imports")
1764
+ manifests = None if args.manifests is None else _read_json_source(args.manifests, "--manifests")
1765
+ output = summarize(result, manifests, args.max_files)
1766
+ except InputError as exc:
1767
+ print(f"error: {exc}", file=sys.stderr)
1768
+ return 1
1769
+ json.dump(output, sys.stdout)
1770
+ sys.stdout.write("\n")
1771
+ return 0
1772
+
1773
+
1774
+ def _build_parser() -> argparse.ArgumentParser:
1775
+ parser = argparse.ArgumentParser(
1776
+ prog="skf-count-imports",
1777
+ description=(
1778
+ "Count, per dependency and per unit, the source files that import "
1779
+ "it, with boundary-anchored matching and distribution-to-import names."
1780
+ ),
1781
+ )
1782
+ sub = parser.add_subparsers(dest="cmd", required=True)
1783
+
1784
+ p_count = sub.add_parser("count", help="count importing files and emit JSON")
1785
+ p_count.add_argument("root", help="path to the tree to scan")
1786
+ p_count.add_argument(
1787
+ "--deps",
1788
+ default=None,
1789
+ help=(
1790
+ "dependencies JSON file, or '-' for stdin: the skf-scan-manifests.py "
1791
+ 'envelope, {"dependencies": [...]} or a list of names or '
1792
+ '{"name", "ecosystem", "modules", "scope"} objects'
1793
+ ),
1794
+ )
1795
+ p_count.add_argument(
1796
+ "--units",
1797
+ default=None,
1798
+ help=(
1799
+ "units JSON file, or '-' for stdin: a list of "
1800
+ '{"name", "path", "modules", "manifest_name", "ecosystem"} objects'
1801
+ ),
1802
+ )
1803
+ p_count.add_argument(
1804
+ "--threshold",
1805
+ type=int,
1806
+ default=DEFAULT_THRESHOLD,
1807
+ help=f"file count at which a dependency is above_threshold (default {DEFAULT_THRESHOLD})",
1808
+ )
1809
+ p_count.add_argument(
1810
+ "--exclude",
1811
+ action="append",
1812
+ default=[],
1813
+ metavar="GLOB",
1814
+ help=(
1815
+ "another path glob to leave out, added to the defaults (repeatable; "
1816
+ "a leading / or ./ anchors it to the root)"
1817
+ ),
1818
+ )
1819
+ p_count.add_argument(
1820
+ "--format",
1821
+ choices=("full", "libraries"),
1822
+ default="full",
1823
+ help="full envelope (default) or the skf-pair-intersect.py --libraries list",
1824
+ )
1825
+ p_count.add_argument(
1826
+ "--relative-to",
1827
+ default=None,
1828
+ metavar="DIR",
1829
+ help=(
1830
+ "write each files[].path relative to DIR, a folder that holds the "
1831
+ "root, instead of to the root"
1832
+ ),
1833
+ )
1834
+ p_count.set_defaults(func=_cmd_count)
1835
+
1836
+ p_summary = sub.add_parser(
1837
+ "summary", help="print the unit graph of a saved count --units envelope without its file lists"
1838
+ )
1839
+ p_summary.add_argument("imports", help="the count --units envelope, a JSON file or '-' for stdin")
1840
+ p_summary.add_argument(
1841
+ "--manifests",
1842
+ default=None,
1843
+ help=(
1844
+ "the skf-scan-manifests.py scan envelope, a JSON file or '-' for stdin: adds "
1845
+ "umbrella_candidates and each manifest's path, name, private and internal_deps"
1846
+ ),
1847
+ )
1848
+ p_summary.add_argument(
1849
+ "--max-files",
1850
+ type=int,
1851
+ default=SUMMARY_MAX_FILES,
1852
+ metavar="N",
1853
+ help=f"files listed per edge_files entry (default {SUMMARY_MAX_FILES})",
1854
+ )
1855
+ p_summary.set_defaults(func=_cmd_summary)
1856
+
1857
+ return parser
1858
+
1859
+
1860
+ def main(argv: list[str] | None = None) -> int:
1861
+ parser = _build_parser()
1862
+ args = parser.parse_args(argv)
1863
+ return args.func(args)
1864
+
1865
+
1866
+ if __name__ == "__main__":
1867
+ raise SystemExit(main())