bmad-module-skill-forge 2.1.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +51 -49
  3. package/docs/404.md +10 -8
  4. package/docs/_data/pinned.yaml +22 -13
  5. package/docs/_internal/STABILITY.md +58 -35
  6. package/docs/agents.md +39 -53
  7. package/docs/architecture.md +65 -51
  8. package/docs/bmad-synergy.md +53 -51
  9. package/docs/campaign.md +127 -75
  10. package/docs/concepts.md +34 -28
  11. package/docs/examples.md +70 -68
  12. package/docs/forge-auto.md +34 -31
  13. package/docs/getting-started.md +118 -57
  14. package/docs/how-it-works.md +25 -21
  15. package/docs/index.md +15 -15
  16. package/docs/skill-model.md +70 -59
  17. package/docs/troubleshooting.md +135 -18
  18. package/docs/verifying-a-skill.md +83 -57
  19. package/docs/why-skf.md +24 -22
  20. package/docs/workflows.md +274 -110
  21. package/package.json +18 -10
  22. package/src/forger/preferences.yaml +8 -1
  23. package/src/knowledge/agentskills-spec.md +2 -0
  24. package/src/knowledge/ccc-bridge.md +40 -24
  25. package/src/knowledge/confidence-tiers.md +9 -5
  26. package/src/knowledge/overview.md +3 -3
  27. package/src/knowledge/progressive-capability.md +8 -8
  28. package/src/knowledge/provenance-tracking.md +9 -5
  29. package/src/knowledge/qmd-registry.md +2 -2
  30. package/src/knowledge/skf-knowledge-index.csv +2 -2
  31. package/src/knowledge/skill-lifecycle.md +18 -11
  32. package/src/knowledge/split-body-strategy.md +0 -3
  33. package/src/knowledge/tool-resolution.md +4 -2
  34. package/src/knowledge/version-paths.md +101 -15
  35. package/src/module-help.csv +3 -3
  36. package/src/module.yaml +10 -0
  37. package/src/shared/data/ast-grep-recipes.yaml +608 -0
  38. package/src/shared/data/ide-context-files.json +30 -0
  39. package/src/shared/health-check.md +32 -12
  40. package/src/shared/references/description-guard-protocol.md +14 -9
  41. package/src/shared/references/feasibility-report-schema.md +19 -13
  42. package/src/shared/references/headless-gate-convention.md +55 -3
  43. package/src/shared/references/output-contract-schema.md +19 -8
  44. package/src/shared/references/pipeline-contracts.md +94 -29
  45. package/src/shared/references/tessl-review.md +77 -0
  46. package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
  47. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
  48. package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
  49. package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
  50. package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
  51. package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
  52. package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
  53. package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
  54. package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
  55. package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
  56. package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
  57. package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
  58. package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
  59. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
  60. package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
  61. package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
  62. package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
  63. package/src/shared/scripts/skf-atomic-write.py +30 -14
  64. package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
  65. package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
  66. package/src/shared/scripts/skf-check-unit-records.py +184 -0
  67. package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
  68. package/src/shared/scripts/skf-classify-changed-files.py +694 -0
  69. package/src/shared/scripts/skf-comention-pairs.py +1333 -123
  70. package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
  71. package/src/shared/scripts/skf-count-imports.py +1867 -0
  72. package/src/shared/scripts/skf-count-tokens.py +1 -1
  73. package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
  74. package/src/shared/scripts/skf-description-guard.py +117 -9
  75. package/src/shared/scripts/skf-detect-docs.py +892 -4
  76. package/src/shared/scripts/skf-detect-language.py +486 -107
  77. package/src/shared/scripts/skf-detect-registry.py +887 -0
  78. package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
  79. package/src/shared/scripts/skf-detect-tools.py +215 -27
  80. package/src/shared/scripts/skf-detect-workspaces.py +506 -30
  81. package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
  82. package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
  83. package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
  84. package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
  85. package/src/shared/scripts/skf-extract-public-api.py +4084 -85
  86. package/src/shared/scripts/skf-extraction-inventory.py +746 -0
  87. package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
  88. package/src/shared/scripts/skf-fetch-temporal.py +674 -0
  89. package/src/shared/scripts/skf-find-cycles.py +97 -29
  90. package/src/shared/scripts/skf-find-test-report.py +476 -0
  91. package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
  92. package/src/shared/scripts/skf-github-fetch.py +365 -0
  93. package/src/shared/scripts/skf-github-probe.py +661 -0
  94. package/src/shared/scripts/skf-hash-content.py +166 -12
  95. package/src/shared/scripts/skf-language-corpora.py +1 -1
  96. package/src/shared/scripts/skf-load-provenance.py +159 -27
  97. package/src/shared/scripts/skf-manifest-ops.py +1 -1
  98. package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
  99. package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
  100. package/src/shared/scripts/skf-names-present.py +221 -0
  101. package/src/shared/scripts/skf-pair-intersect.py +119 -28
  102. package/src/shared/scripts/skf-parse-gaps.py +1384 -0
  103. package/src/shared/scripts/skf-preapply.py +1 -1
  104. package/src/shared/scripts/skf-preflight.py +161 -27
  105. package/src/shared/scripts/skf-promote-staged.py +430 -0
  106. package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
  107. package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
  108. package/src/shared/scripts/skf-quick-batch.py +774 -0
  109. package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
  110. package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
  111. package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
  112. package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
  113. package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
  114. package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
  115. package/src/shared/scripts/skf-resolve-package.py +523 -106
  116. package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
  117. package/src/shared/scripts/skf-run-lock.py +577 -0
  118. package/src/shared/scripts/skf-scan-manifests.py +998 -57
  119. package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
  120. package/src/shared/scripts/skf-severity-classify.py +712 -108
  121. package/src/shared/scripts/skf-shape-detect.py +298 -204
  122. package/src/shared/scripts/skf-shard-body.py +9 -8
  123. package/src/shared/scripts/skf-skill-inventory.py +2412 -89
  124. package/src/shared/scripts/skf-skills-module.py +472 -0
  125. package/src/shared/scripts/skf-source-tree.py +1796 -0
  126. package/src/shared/scripts/skf-structural-diff.py +670 -119
  127. package/src/shared/scripts/skf-tessl-review.py +706 -0
  128. package/src/shared/scripts/skf-update-active-symlink.py +5 -5
  129. package/src/shared/scripts/skf-update-run-state.py +624 -0
  130. package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
  131. package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
  132. package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
  133. package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
  134. package/src/shared/scripts/skf-validate-output.py +712 -64
  135. package/src/shared/scripts/skf-validate-pins.py +1 -1
  136. package/src/shared/scripts/skf-verify-no-trace.py +116 -10
  137. package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
  138. package/src/shared/scripts/skf-write-skill-brief.py +367 -19
  139. package/src/shared/tool-requirements.yaml +201 -0
  140. package/src/skf-analyze-source/SKILL.md +41 -36
  141. package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
  142. package/src/skf-analyze-source/customize.toml +36 -24
  143. package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
  144. package/src/skf-analyze-source/references/continue.md +2 -13
  145. package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
  146. package/src/skf-analyze-source/references/generate-briefs.md +121 -55
  147. package/src/skf-analyze-source/references/headless-contract.md +42 -15
  148. package/src/skf-analyze-source/references/identify-units.md +96 -110
  149. package/src/skf-analyze-source/references/init.md +73 -71
  150. package/src/skf-analyze-source/references/map-and-detect.md +123 -105
  151. package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
  152. package/src/skf-analyze-source/references/recommend.md +51 -70
  153. package/src/skf-analyze-source/references/scan-project.md +60 -84
  154. package/src/skf-analyze-source/references/scan-root.md +39 -0
  155. package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
  156. package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
  157. package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
  158. package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
  159. package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
  160. package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
  161. package/src/skf-audit-skill/SKILL.md +43 -51
  162. package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
  163. package/src/skf-audit-skill/customize.toml +41 -20
  164. package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
  165. package/src/skf-audit-skill/references/doc-drift.md +141 -0
  166. package/src/skf-audit-skill/references/headless-contract.md +29 -0
  167. package/src/skf-audit-skill/references/init.md +148 -210
  168. package/src/skf-audit-skill/references/re-index.md +96 -108
  169. package/src/skf-audit-skill/references/report.md +96 -33
  170. package/src/skf-audit-skill/references/semantic-diff.md +27 -18
  171. package/src/skf-audit-skill/references/severity-classify.md +79 -65
  172. package/src/skf-audit-skill/references/severity-rules.md +15 -6
  173. package/src/skf-audit-skill/references/structural-diff.md +75 -90
  174. package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
  175. package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
  176. package/src/skf-brief-skill/SKILL.md +36 -25
  177. package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
  178. package/src/skf-brief-skill/assets/scope-templates.md +17 -24
  179. package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
  180. package/src/skf-brief-skill/customize.toml +48 -20
  181. package/src/skf-brief-skill/references/analyze-target.md +142 -125
  182. package/src/skf-brief-skill/references/confirm-brief.md +10 -8
  183. package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
  184. package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
  185. package/src/skf-brief-skill/references/gather-intent.md +87 -182
  186. package/src/skf-brief-skill/references/headless-args.md +84 -16
  187. package/src/skf-brief-skill/references/health-check.md +1 -1
  188. package/src/skf-brief-skill/references/invocation-contract.md +18 -13
  189. package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
  190. package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
  191. package/src/skf-brief-skill/references/scope-definition.md +123 -47
  192. package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
  193. package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
  194. package/src/skf-brief-skill/references/version-resolution.md +3 -2
  195. package/src/skf-brief-skill/references/write-brief.md +66 -57
  196. package/src/skf-campaign/SKILL.md +31 -34
  197. package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
  198. package/src/skf-campaign/customize.toml +47 -16
  199. package/src/skf-campaign/references/campaign-contracts.md +39 -29
  200. package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
  201. package/src/skf-campaign/references/health-check.md +7 -16
  202. package/src/skf-campaign/references/step-01-setup.md +46 -84
  203. package/src/skf-campaign/references/step-02-strategy.md +31 -30
  204. package/src/skf-campaign/references/step-03-pins.md +26 -18
  205. package/src/skf-campaign/references/step-04-provenance.md +24 -23
  206. package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
  207. package/src/skf-campaign/references/step-06-batch.md +41 -34
  208. package/src/skf-campaign/references/step-07-capstone.md +40 -25
  209. package/src/skf-campaign/references/step-08-verify.md +28 -29
  210. package/src/skf-campaign/references/step-09-refine.md +29 -30
  211. package/src/skf-campaign/references/step-10-export.md +48 -36
  212. package/src/skf-campaign/references/step-11-maintenance.md +31 -34
  213. package/src/skf-campaign/references/step-resume.md +37 -68
  214. package/src/skf-campaign/scripts/campaign-deps.py +57 -18
  215. package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
  216. package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
  217. package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
  218. package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
  219. package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
  220. package/src/skf-campaign/scripts/campaign-report.py +204 -37
  221. package/src/skf-campaign/scripts/campaign-state.py +1128 -0
  222. package/src/skf-campaign/scripts/campaign-status.py +10 -74
  223. package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
  224. package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
  225. package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
  226. package/src/skf-campaign/templates/campaign-report-template.md +5 -1
  227. package/src/skf-campaign/templates/kickoff-template.md +0 -14
  228. package/src/skf-create-skill/SKILL.md +27 -20
  229. package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
  230. package/src/skf-create-skill/assets/skill-sections.md +17 -6
  231. package/src/skf-create-skill/customize.toml +20 -8
  232. package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
  233. package/src/skf-create-skill/references/batch-mode.md +81 -0
  234. package/src/skf-create-skill/references/ccc-index-check.md +27 -0
  235. package/src/skf-create-skill/references/compile.md +107 -29
  236. package/src/skf-create-skill/references/component-extraction.md +113 -91
  237. package/src/skf-create-skill/references/enrich.md +41 -23
  238. package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
  239. package/src/skf-create-skill/references/extract.md +200 -161
  240. package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
  241. package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
  242. package/src/skf-create-skill/references/extraction-patterns.md +584 -196
  243. package/src/skf-create-skill/references/generate-artifacts.md +145 -93
  244. package/src/skf-create-skill/references/health-check.md +1 -1
  245. package/src/skf-create-skill/references/load-brief.md +53 -36
  246. package/src/skf-create-skill/references/relabel-rule.md +12 -0
  247. package/src/skf-create-skill/references/report.md +28 -59
  248. package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
  249. package/src/skf-create-skill/references/step-auto-shard.md +20 -93
  250. package/src/skf-create-skill/references/step-doc-rot.md +44 -60
  251. package/src/skf-create-skill/references/step-doc-sources.md +55 -18
  252. package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
  253. package/src/skf-create-skill/references/sub/fetch-docs.md +106 -57
  254. package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
  255. package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
  256. package/src/skf-create-skill/references/validate.md +173 -99
  257. package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
  258. package/src/skf-create-stack-skill/SKILL.md +29 -56
  259. package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
  260. package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
  261. package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
  262. package/src/skf-create-stack-skill/customize.toml +23 -12
  263. package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
  264. package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
  265. package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
  266. package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
  267. package/src/skf-create-stack-skill/references/generate-output.md +138 -128
  268. package/src/skf-create-stack-skill/references/init.md +90 -32
  269. package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
  270. package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
  271. package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
  272. package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
  273. package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
  274. package/src/skf-create-stack-skill/references/report.md +39 -33
  275. package/src/skf-create-stack-skill/references/validate.md +20 -39
  276. package/src/skf-drop-skill/SKILL.md +28 -51
  277. package/src/skf-drop-skill/customize.toml +44 -30
  278. package/src/skf-drop-skill/references/execute.md +79 -146
  279. package/src/skf-drop-skill/references/invocation-contract.md +64 -0
  280. package/src/skf-drop-skill/references/report.md +46 -30
  281. package/src/skf-drop-skill/references/select.md +118 -124
  282. package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
  283. package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
  284. package/src/skf-export-skill/SKILL.md +40 -62
  285. package/src/skf-export-skill/assets/managed-section-format.md +14 -50
  286. package/src/skf-export-skill/assets/snippet-format.md +3 -1
  287. package/src/skf-export-skill/customize.toml +42 -13
  288. package/src/skf-export-skill/references/generate-snippet.md +54 -46
  289. package/src/skf-export-skill/references/health-check.md +2 -0
  290. package/src/skf-export-skill/references/invocation-contract.md +15 -0
  291. package/src/skf-export-skill/references/load-skill.md +88 -62
  292. package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
  293. package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
  294. package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
  295. package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
  296. package/src/skf-export-skill/references/package.md +15 -73
  297. package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
  298. package/src/skf-export-skill/references/result-envelope.md +31 -16
  299. package/src/skf-export-skill/references/summary.md +41 -17
  300. package/src/skf-export-skill/references/token-report.md +4 -12
  301. package/src/skf-export-skill/references/update-context.md +118 -237
  302. package/src/skf-forger/SKILL.md +61 -37
  303. package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
  304. package/src/skf-forger/customize.toml +43 -0
  305. package/src/skf-forger/references/pipeline-mode.md +73 -18
  306. package/src/skf-forger/scripts/forge-status.py +299 -0
  307. package/src/skf-forger/scripts/parse-pipeline.py +322 -54
  308. package/src/skf-forger/scripts/pipeline-gate.py +428 -0
  309. package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
  310. package/src/skf-quick-skill/SKILL.md +54 -53
  311. package/src/skf-quick-skill/assets/skill-template.md +4 -2
  312. package/src/skf-quick-skill/customize.toml +51 -20
  313. package/src/skf-quick-skill/references/batch-mode.md +74 -21
  314. package/src/skf-quick-skill/references/compile.md +30 -31
  315. package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
  316. package/src/skf-quick-skill/references/finalize.md +69 -18
  317. package/src/skf-quick-skill/references/halt-contract.md +41 -29
  318. package/src/skf-quick-skill/references/health-check.md +5 -3
  319. package/src/skf-quick-skill/references/quick-extract.md +114 -45
  320. package/src/skf-quick-skill/references/registry-resolution.md +9 -64
  321. package/src/skf-quick-skill/references/resolve-target.md +116 -66
  322. package/src/skf-quick-skill/references/write-and-validate.md +79 -40
  323. package/src/skf-refine-architecture/SKILL.md +43 -53
  324. package/src/skf-refine-architecture/customize.toml +49 -20
  325. package/src/skf-refine-architecture/references/compile.md +97 -54
  326. package/src/skf-refine-architecture/references/exit-codes.md +30 -0
  327. package/src/skf-refine-architecture/references/finding-storage.md +21 -0
  328. package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
  329. package/src/skf-refine-architecture/references/improvements.md +14 -20
  330. package/src/skf-refine-architecture/references/init.md +98 -57
  331. package/src/skf-refine-architecture/references/issue-detection.md +32 -39
  332. package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
  333. package/src/skf-refine-architecture/references/report.md +65 -54
  334. package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
  335. package/src/skf-rename-skill/SKILL.md +19 -54
  336. package/src/skf-rename-skill/customize.toml +32 -26
  337. package/src/skf-rename-skill/references/execute.md +230 -184
  338. package/src/skf-rename-skill/references/exit-codes.md +7 -7
  339. package/src/skf-rename-skill/references/health-check.md +1 -7
  340. package/src/skf-rename-skill/references/invocation-contract.md +25 -0
  341. package/src/skf-rename-skill/references/report.md +63 -52
  342. package/src/skf-rename-skill/references/select.md +197 -115
  343. package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
  344. package/src/skf-setup/SKILL.md +26 -26
  345. package/src/skf-setup/customize.toml +14 -10
  346. package/src/skf-setup/references/auto-index.md +50 -63
  347. package/src/skf-setup/references/ccc-index.md +29 -89
  348. package/src/skf-setup/references/detect-and-tier.md +58 -66
  349. package/src/skf-setup/references/health-check.md +2 -0
  350. package/src/skf-setup/references/invocation-contract.md +14 -0
  351. package/src/skf-setup/references/report.md +43 -166
  352. package/src/skf-setup/references/tier-rules.md +5 -5
  353. package/src/skf-setup/references/write-config.md +34 -57
  354. package/src/skf-test-skill/SKILL.md +17 -57
  355. package/src/skf-test-skill/assets/output-section-formats.md +33 -34
  356. package/src/skf-test-skill/customize.toml +27 -13
  357. package/src/skf-test-skill/references/coherence-check.md +134 -62
  358. package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
  359. package/src/skf-test-skill/references/coverage-check.md +217 -254
  360. package/src/skf-test-skill/references/detect-mode.md +5 -5
  361. package/src/skf-test-skill/references/external-validators.md +94 -82
  362. package/src/skf-test-skill/references/health-check.md +6 -7
  363. package/src/skf-test-skill/references/init.md +163 -88
  364. package/src/skf-test-skill/references/invocation-contract.md +66 -0
  365. package/src/skf-test-skill/references/migration-section-rules.md +5 -2
  366. package/src/skf-test-skill/references/report.md +162 -142
  367. package/src/skf-test-skill/references/score.md +80 -107
  368. package/src/skf-test-skill/references/scoring-rules.md +41 -155
  369. package/src/skf-test-skill/references/source-access-protocol.md +19 -92
  370. package/src/skf-test-skill/references/step-hard-gate.md +51 -33
  371. package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
  372. package/src/skf-test-skill/scripts/build-result-context.py +287 -0
  373. package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
  374. package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
  375. package/src/skf-test-skill/scripts/compute-score.py +223 -45
  376. package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
  377. package/src/skf-test-skill/scripts/hard-gate.py +203 -0
  378. package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
  379. package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
  380. package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
  381. package/src/skf-test-skill/scripts/score-signatures.py +459 -0
  382. package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
  383. package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
  384. package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
  385. package/src/skf-test-skill/templates/test-report-template.md +21 -15
  386. package/src/skf-update-skill/SKILL.md +38 -25
  387. package/src/skf-update-skill/customize.toml +27 -12
  388. package/src/skf-update-skill/references/detect-changes.md +233 -222
  389. package/src/skf-update-skill/references/gap-driven.md +336 -0
  390. package/src/skf-update-skill/references/health-check.md +31 -5
  391. package/src/skf-update-skill/references/init.md +250 -84
  392. package/src/skf-update-skill/references/invocation-contract.md +12 -0
  393. package/src/skf-update-skill/references/manual-section-rules.md +6 -23
  394. package/src/skf-update-skill/references/merge.md +180 -47
  395. package/src/skf-update-skill/references/re-extract.md +53 -262
  396. package/src/skf-update-skill/references/report.md +100 -35
  397. package/src/skf-update-skill/references/write.md +331 -149
  398. package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
  399. package/src/skf-verify-stack/SKILL.md +23 -42
  400. package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
  401. package/src/skf-verify-stack/customize.toml +55 -22
  402. package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
  403. package/src/skf-verify-stack/references/coverage.md +67 -64
  404. package/src/skf-verify-stack/references/exit-codes.md +21 -8
  405. package/src/skf-verify-stack/references/init.md +107 -62
  406. package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
  407. package/src/skf-verify-stack/references/integrations.md +106 -65
  408. package/src/skf-verify-stack/references/report.md +59 -42
  409. package/src/skf-verify-stack/references/requirements.md +32 -9
  410. package/src/skf-verify-stack/references/synthesize.md +53 -39
  411. package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
  412. package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
  413. package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
  414. package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
  415. package/tools/cli/commands/install.js +11 -4
  416. package/tools/cli/commands/status.js +15 -3
  417. package/tools/cli/commands/update.js +11 -4
  418. package/tools/cli/lib/tool-check.js +320 -0
  419. package/tools/cli/lib/version-check.js +22 -9
  420. package/tools/cli/skf-cli.js +4 -1
  421. package/.gitattributes +0 -16
  422. package/.nvmrc +0 -1
  423. package/docs/_internal/RELEASING.md +0 -645
  424. package/release-audits/v1.0.0-launch-audit.md +0 -728
  425. package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
  426. package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
  427. package/src/skf-campaign/manifest.yaml +0 -15
  428. package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
  429. package/src/skf-create-skill/references/ecosystem-check.md +0 -74
  430. package/src/skf-drop-skill/references/headless-contract.md +0 -18
  431. package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
  432. package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
  433. package/src/skf-update-skill/references/validate.md +0 -124
  434. package/tools/validate-docs-drift.js +0 -233
  435. package/tools/validate-docs-links.js +0 -665
@@ -1,40 +1,155 @@
1
1
  # /// script
2
2
  # requires-python = ">=3.11"
3
- # dependencies = []
3
+ # dependencies = ["pyyaml"]
4
4
  # ///
5
- """SKF Extract Public API — pure parser for manifest + entry-point content.
5
+ """SKF Extract Public API: one package's public surface, by parser or by recipe.
6
6
 
7
- Takes a JSON payload on stdin describing one logical package (one
8
- manifest, one or more entry-point files) and emits a structured
9
- extraction inventory: package name, version, description, public
10
- exports, declared dependencies, and (for Maven/Gradle) discovered
11
- sub-modules. The helper does NO file I/O — the caller fetches files
12
- (via gh api / web browsing / local filesystem / wherever) and pipes
13
- their contents in. Multi-module monorepos are caller-orchestrated:
14
- invoke once per module, aggregate the results.
7
+ Three modes:
15
8
 
16
- Supported languages (--language values; first listed is preferred):
9
+ --mode quick (the default) A pure parser: takes one logical package (one
10
+ manifest, one or more entry-point files) and emits its
11
+ package name, version, description, public exports,
12
+ declared dependencies and (for Maven/Gradle) sub-modules.
13
+ The files come as a JSON payload on stdin, or by path with
14
+ --manifest-file and --entry-file, which the script reads
15
+ from disk (the caller stages them first: skf-github-fetch.py
16
+ raw fetches into a folder, or a local checkout), so a file's
17
+ text never passes through the model or a shell string.
18
+ --follow also reads the modules an `export *` chain passes
19
+ names on from, fetching them first with --repo. Multi-module
20
+ monorepos are caller-orchestrated: invoke once per module,
21
+ aggregate.
22
+ --mode full The ast-grep recipe runner (skf-create-skill and the
23
+ workflows that extract as it does): runs the recipes in
24
+ src/shared/data/ast-grep-recipes.yaml over a source tree
25
+ and emits every export with its exact file, line, name and
26
+ node kind, the export surface the package's entry points
27
+ declare, and the counts compile needs. See "Full mode" below.
28
+ --mode entries The files quick mode reads for one package, found in the
29
+ repository's file listing with full mode's entry-point
30
+ rules, so no caller derives an entry file from a
31
+ distribution name or reads a manifest by eye. See "Entries
32
+ mode" below.
33
+
34
+ Quick mode
35
+ ----------
36
+
37
+ Supported languages (the payload's `language`, or --language; first listed
38
+ is preferred):
17
39
 
18
40
  js, ts, javascript, typescript package.json index.{js,ts}, src/index.{js,ts}
19
- python pyproject.toml __init__.py / setup.py
41
+ python pyproject.toml __init__.py / setup.py / setup.cfg
20
42
  rust Cargo.toml src/lib.rs
21
43
  go go.mod top-level *.go
22
44
  java pom.xml src/main/java/**/*.java
23
45
  kotlin build.gradle* src/main/kotlin/**/*.kt
46
+ swift Package.swift Sources/<target>/*.swift
24
47
 
25
- Mode flag (currently `quick` only; `full` reserved for skf-create-skill):
48
+ Top-level exports only, read from the entry files. Quick mode reads no other
49
+ file (--follow aside, below), so a name an entry file passes on from another
50
+ module without naming it is not an export here: a warning names each such
51
+ statement, and an `unlisted[]` record gives its file, line, statement,
52
+ specifier and `module_file`, so the caller can pass that module as a
53
+ --follow-file, or have --follow read it (a module_file), or read it by eye
54
+ (none). A warning also names
55
+ each statement below whose names the file holds in a form quick mode does
56
+ not read (an anonymous default export, a destructuring declaration, an
57
+ assignment inside a block).
26
58
 
27
- --mode quick Top-level exports only (one entry file per language).
59
+ JavaScript / TypeScript read with comments, strings and regular
60
+ expressions blanked: column-0 `export` declarations (`const`, `let`
61
+ and `var`, every declarator of the statement, `function`,
62
+ `function*`, `async function`, `class`, `abstract class`,
63
+ `interface`, `type`, `enum`, `const enum`, `namespace`, each may be
64
+ `default` or `declare`), typed by keyword; `export { ... }` and
65
+ `export type { ... }` lists (`re-export`); `export * as ns from`
66
+ (`namespace`); `export default <name>` (the type of that local
67
+ declaration, else `re-export`); TypeScript's `export = <value>`, read
68
+ as `module.exports = <value>` is; and CommonJS: each `exports.<name> =`
69
+ and `module.exports.<name> =` (each name of a chain such as tsc's
70
+ `exports.a = exports.b = void 0`), the keys of a `module.exports =
71
+ { ... }` object, and the one name of `module.exports = <name>`
72
+ (`exports = module.exports = <name>` included), a named function or a
73
+ named class (`function`, `class`, or the local declaration's type,
74
+ else `variable`). Warned: `export * from`, `module.exports =
75
+ require(...)`, a spread in the `module.exports` object, a
76
+ `module.exports` or `export =` that is anonymous or an expression, an
77
+ `export default` that is anonymous or an expression, an `export const`
78
+ that destructures, and a `module.exports`, `exports` or
79
+ `exports.<name>` assignment that does not open its line (in an `if`, a
80
+ block or a function, so which one runs the file alone cannot tell).
81
+ Python with `__all__` (literal lists and tuples, however the module
82
+ builds it, read as full mode reads it), every public name it lists,
83
+ typed by what binds it in the file: `def` (async def included),
84
+ `class`, `variable` (an assignment), `module` (`import x`),
85
+ `re-export` (an import, or no binding in the file); a part that is no
86
+ literal (another module's `__all__`) is warned. Without `__all__`, the
87
+ public def, async def and class names, and the names imported from
88
+ the package itself (`re-export`: `from .x import Y`, or, in an
89
+ `__init__.py`, an absolute import that names the package by any
90
+ trailing run of its folders: `storage`, `cloud.storage` or
91
+ `acme.cloud.storage` for acme/cloud/storage/__init__.py); a star
92
+ import from the package is warned. Imports under `if TYPE_CHECKING:`
93
+ do not count. A module this Python cannot parse is read from its
94
+ text, with a warning.
95
+ Rust column-0 unrestricted `pub` items, qualifiers included (`const`,
96
+ `async`, `unsafe`, `extern "C"`: `pub const fn f` is the fn `f`),
97
+ typed by item keyword; each `#[macro_export]` `macro_rules!` macro
98
+ (`macro`); and each name a column-0 `pub use` brings in (`re-export`,
99
+ under its alias when it has one); a glob `pub use x::*` is warned. An
100
+ indented `pub` item is a method or sits in an inline module, not a
101
+ top-level export.
102
+ Go capitalized top-level func, type, var and const names.
103
+ Java public classes, interfaces, enums and records, and the classes a
104
+ Spring or Jakarta annotation marks.
105
+ Kotlin declarations not marked internal or private.
106
+ Swift public and open declarations.
28
107
 
29
- Input JSON shape (stdin):
108
+ Input: a JSON payload on stdin,
30
109
 
31
110
  {
32
111
  "language": "python",
33
112
  "manifest": {"path": "pyproject.toml", "content": "..."},
34
- "entries": [{"path": "src/foo/__init__.py", "content": "..."}, ...],
113
+ "entries": [{"path": "src/foo/__init__.py", "content": "...",
114
+ "followed": false}, ...],
35
115
  "mode": "quick"
36
116
  }
37
117
 
118
+ or the files, read from disk (stdin is then never read):
119
+
120
+ --mode quick --language <language> [--source-root <dir>]
121
+ [--manifest-file <path>] [--entry-file <path>]...
122
+ [--follow-file <path>]... [--tree-file <listing>] [--fetch-list <file>]
123
+ [--follow [--repo <owner/repo> [--ref <ref>]]]
124
+
125
+ An entry with `followed` true (a --follow-file) is a module an `unlisted[]`
126
+ record named, read as an entry is, less a JS/TS `export default`, which
127
+ `export *` does not pass on.
128
+
129
+ --follow (with --tree-file and --source-root) reads those modules itself,
130
+ so no caller loops over the files: while `unlisted[]` names a module_file
131
+ not tried yet, at most 5 rounds, each one is read as one more --follow-file
132
+ and the files are parsed again, so an `export *` chain is read to its end.
133
+ With --repo, each is first fetched into --source-root at --ref (HEAD by
134
+ default) by skf-github-fetch.py, which reads it from
135
+ raw.githubusercontent.com and through gh only for a private repository;
136
+ without it, each is read where --source-root holds it (a local checkout).
137
+ A module that cannot be fetched or read stays in `unlisted[]`, and a
138
+ warning names it and why. The output then also lists `followed`, the
139
+ modules read this way, in the order they were read.
140
+
141
+ Each path is relative to --source-root, the folder the files were staged in
142
+ laid out like the repository (or a local checkout), else to the current
143
+ folder, and is the `path` the output names (`source_file`), written with
144
+ `/`. A path under --source-root may not be absolute or lead outside it. A
145
+ file's text is read as UTF-8 (a byte order mark dropped, an invalid byte
146
+ replaced), so a quote or an apostrophe in it reaches the parser as it is.
147
+ An entry named twice is read once (a file both an --entry-file and a
148
+ --follow-file is an entry). Without --manifest-file the package metadata is
149
+ empty, as for a payload whose manifest `content` is empty. A python
150
+ manifest is read by its name: setup.py and setup.cfg by their own parsers,
151
+ any other as pyproject.toml.
152
+
38
153
  Output JSON shape (stdout):
39
154
 
40
155
  {
@@ -45,26 +160,343 @@ Output JSON shape (stdout):
45
160
  "exports": [{"name": "Bar", "type": "class", "source_file": "..."}, ...],
46
161
  "dependencies": ["requests", "pydantic", ...],
47
162
  "modules": ["server", "client"], (Maven/Gradle only)
163
+ "module_folders": ["server", "client"], (each module's folder in
164
+ the repository)
48
165
  "extra": {"group_id": "com.example"}, (Maven only)
49
- "warnings": ["..."] (parse failures, fallbacks)
166
+ "warnings": ["..."], (parse failures, fallbacks)
167
+ "unlisted": [{"file", "line", "statement", "specifier",
168
+ "module_file"}, ...],
169
+ "followed": ["src/core.ts", ...] (--follow only)
50
170
  }
51
171
 
52
- Exit codes:
172
+ `unlisted` holds one record per statement whose names come from a module
173
+ quick mode did not read: `export * from`, `module.exports = require(...)`
174
+ (and such an assignment in a block), a spread in the `module.exports`
175
+ object, a Python star import from the package and a Rust `pub use x::*`.
176
+ `module_file` is the file of the listing (--tree-file, or a payload's
177
+ `tree`) that module is, found as full mode finds it (resolve_js_module, a
178
+ NodeNext `./x.js` naming x.ts, and _py_module_file); null without a
179
+ listing, and for a package import, a spread, a Rust glob or a module the
180
+ listing does not hold. A statement whose module_file is already an entry
181
+ file is left out, with its warning: its names are read. --fetch-list
182
+ writes each module_file still to read, one per line, so a caller fetches
183
+ them, passes them as more --follow-file and runs again until the list is
184
+ empty, which --follow does itself. `module_folders` gives each of
185
+ `modules` as a folder of the repository: the manifest's folder joined with
186
+ the module (a Gradle `core:api` read as `core/api`).
187
+
188
+ Exit codes (quick):
53
189
 
54
190
  0 success
55
- 1 payload-level error (unknown language, no manifest content)
56
- 2 stdin / argparse / JSON-decode error
191
+ 1 payload-level error (a language quick mode does not parse)
192
+ 2 input error: stdin, argparse or JSON-decode error, --language missing
193
+ or given twice with the file inputs, a --manifest-file, --entry-file
194
+ or --follow-file that cannot be read or leads outside --source-root,
195
+ a --tree-file that cannot be read or a --fetch-list that cannot be
196
+ written, --follow without --tree-file and --source-root, --repo or
197
+ --ref without --follow (--ref without --repo), or a --repo or --ref
198
+ skf-github-fetch.py does not accept. A module --follow could not
199
+ read changes no exit code.
200
+
201
+ Entries mode
202
+ ------------
203
+
204
+ --mode entries --language <language> --tree-file <listing>
205
+ --source-root <dir> [--scope <folder>] [--fetch-list <file>]
206
+
207
+ The entry files of the one package at --scope (the repository root
208
+ without it), read from the listing (skf-github-probe.py tree output, or any
209
+ listing skf-detect-language.py's --tree-file reads). The package's manifest
210
+ is read from --source-root, the folder it was fetched into (a folder not
211
+ made yet holds none):
212
+
213
+ js, ts the files its package.json exports map, else its
214
+ types/module/main/browser fields, name (a build output not
215
+ committed mapped back to its source: dist/index.js ->
216
+ src/index.ts), else index.* or src/index.* (_js_entries)
217
+ python the top __init__.py of each package among its .py files
218
+ (_py_entries), less those under a top folder named test*,
219
+ docs, examples or benchmarks, the one whose folder is the
220
+ manifest name as Python imports it preferred; with no
221
+ package, <name>.py or src/<name>.py (a single-module
222
+ package such as six). The name is the first one the
223
+ manifests give, read in the order pyproject.toml ([project]
224
+ or [tool.poetry]), setup.py, setup.cfg ([metadata]), and
225
+ `manifest` is the one that gave it (else the first listed)
226
+ rust the crate root, `[lib] path` else src/lib.rs (_rust_entries)
227
+ go the .go files directly in the folder, less *_test.go
228
+ java src/main/java/<groupId as a path>/*.java (the pom's groupId,
229
+ else its parent's)
230
+ kotlin src/main/kotlin/**/*.kt
231
+
232
+ Go, Java and Kotlin read the first 5 files in path order, and no language
233
+ more than 25. Output (stdout):
234
+
235
+ {
236
+ "mode": "entries",
237
+ "language": "python",
238
+ "scope": "." | "<folder>",
239
+ "manifest": "<path in the listing>" | null,
240
+ "entry_files": ["src/PIL/__init__.py", ...],
241
+ "unresolved": [{"package", "subpath", "targets"}, ...],
242
+ "warnings": ["..."]
243
+ }
244
+
245
+ `manifest` is the manifest the language reads at the folder (null when the
246
+ listing has none), `unresolved` names what no listed file answered (a
247
+ package.json target, or the paths tried), and --fetch-list writes
248
+ `entry_files` one per line. A language quick mode parses but has no rule
249
+ here (swift) gives no entry file and a warning.
250
+
251
+ Exit codes (entries): 0 (an empty `entry_files` included); 1 for a language
252
+ quick mode does not parse; 2 for an input error (a --tree-file that cannot
253
+ be read, a --source-root that is a file, a --fetch-list that cannot be
254
+ written, a flag of another mode).
255
+
256
+ Full mode
257
+ ---------
258
+
259
+ --mode full --source-root <dir>
260
+ [--brief <skill-brief.yaml>] [--include GLOB]... [--exclude GLOB]...
261
+ [--tier-a-include GLOB]... [--scope-type TYPE] [--language LANG]...
262
+ [--files-from <file>] [--recipe-set standard|component-library]
263
+ [--tier Quick|Forge|Forge+|Deep] [--head-cap N] [--recipes <file>]
264
+ [--timeout <seconds>] [-o <out.json>]
265
+
266
+ Which files. The files under --source-root are those git lists there
267
+ (tracked, and untracked but not ignored), or every file when it is not in
268
+ a git work tree, leaving out hidden entries and symbolic links: the files
269
+ ast-grep itself would walk. With --files-from (paths relative to
270
+ --source-root: a JSON list when the whole file is one, else one path per
271
+ line, so a first path such as `[slug]/page.tsx` is read as a path) only
272
+ the files it names are read, and a named file that is missing, or whose
273
+ language has no recipe, is listed in `file_issues`. A file is in scope when an include glob matches it (every
274
+ file when there is none) and no exclude glob does, and when its language
275
+ is one the recipes read (python, typescript, tsx, javascript, rust, go and
276
+ .vue files) and --language allows it. Globs follow
277
+ skf-resolve-authoritative-files.py's rules: `**` spans any number of path
278
+ segments, none included, and `*` and `?` stay inside one, so `src/**/*.ts`
279
+ matches `src/index.ts` and `**/test_*` matches a top-level `test_x.py`.
280
+
281
+ --brief supplies scope.include, scope.exclude, scope.tier_a_include,
282
+ scope.type and language; --include, --exclude, --tier-a-include,
283
+ --scope-type and --language replace the brief's value when given.
284
+ --language takes a language family (`typescript`, `javascript`, `ts`,
285
+ `js`, `tsx` and `vue` all select .ts, .tsx, .js and .vue files; `python`,
286
+ `rust`, `go`).
287
+
288
+ Which recipes. --recipe-set picks the recipes whose metadata.sets lists it:
289
+ `component-library` when the scope type is component-library, else
290
+ `standard`. Each recipe runs in the languages its metadata.languages
291
+ lists. A match whose name starts with `_` is never an export. --head-cap
292
+ keeps at most N matches of each recipe, in file and line order (0: no
293
+ cap); without it the cap is 200, or at Forge+ and Deep tiers 500 for a
294
+ full-library scope and 300 for a component-library one. `truncated` is
295
+ true when a recipe had more.
296
+
297
+ Merging. The matches of one name in one file become one export, at the
298
+ first line that names it (overload signatures, and a definition in both
299
+ branches of an `if`, repeat a name). When several recipes match that line,
300
+ the export records the one no other of them lists under
301
+ metadata.prefer_over, else the first in the recipe file. Its `source_line`
302
+ is the line of `$NAME` (a decorator or an `export` line can come first),
303
+ `signature_line` the whole source line holding it (the full list line for
304
+ a list item), `signature` the declaration that line opens on one line (the
305
+ lines after it joined while a ( or [ it opens is still open, each without
306
+ its line comment, and a trailing comma before the closing bracket dropped,
307
+ so a parameter list a formatter split one per line reads as its one-line
308
+ form; at most 40 lines; otherwise `signature_line`), `ast_node_type` the
309
+ kind the recipe declares, and
310
+ `export_type` its metadata.export_type (for a list, the declaration
311
+ keyword before the name picks one). A function (export_type `function`)
312
+ also gets `params`, each parameter read from `signature` as {name, type,
313
+ default, optional} (Python's through its own parser; a rest or variadic
314
+ parameter, `...rest`, `*args`, `**kwargs` or Go's `...T`, is optional, and
315
+ a Go name shares the type written after it), and `return_type`, the type
316
+ the declaration states, null when it states none. Both are null for any
317
+ other export, and for a signature the reader does not parse (one cut at
318
+ the line cap, say), which the caller then reads by eye. An item of a local `export { ... }`
319
+ list takes instead the type of the top-level binding it names in its file:
320
+ what the recipes record for that declaration exported in place (a const
321
+ bound to an arrow function or a function expression is a function), or
322
+ `re-export` for an import.
323
+
324
+ Entry points. The export surface each package declares: every subpath of a
325
+ package.json `exports` map (committed .d.ts/.d.mts targets outside src/
326
+ included, a build output mapped back to its source when the target is not
327
+ committed, a pattern subpath's `*` matched against the tree, one entry per
328
+ file, less the subpaths a `null` target takes out), else its
329
+ types/module/main fields, else index.* or src/index.*; a Python package's
330
+ top __init__.py (its __all__ when it has one, else the public names it
331
+ defines or imports from the package, and those its star imports give); a
332
+ crate's lib.rs (its `pub use` items, globs included, and column-0
333
+ unrestricted `pub` items); a Go package's capitalized top-level names
334
+ outside _test.go, package main and internal/ folders. Each entry point is
335
+ `in_scope` when it is one of the files in scope (a warning names each
336
+ JS/TS one that is not). Chains are followed to the end, a cycle cut: an
337
+ `export *`, a star import or a glob passes on what its module exports,
338
+ that module's own chains included; a re-exported name is traced through
339
+ each re-export, import and glob between to the file that defines it (a
340
+ default export to the name it declares). A namespace (`export * as ns`, a
341
+ Python submodule, a Rust `pub mod`) counts once, and what it makes
342
+ reachable is not internal: the names its module passes on, the modules
343
+ among them in turn, and for Rust every recipe match in the crate root and
344
+ in each module file a `pub mod` or `pub use` of a reachable module names.
345
+ A chain the trace cannot follow (a module no file here holds, a `#[path]`
346
+ module) is listed in `unresolved`. The diff against the recipe inventory,
347
+ one language family at a time (JavaScript/TypeScript, Python, Rust, Go):
348
+ `public` (the entry points' names), `internal` (names the recipes found
349
+ that no entry point exports or makes reachable), `extraction_gaps` (names
350
+ an entry point exports that no recipe found, defined in a file in scope or
351
+ in none the trace reached: read them by eye) and `outside_scope` (such
352
+ names defined in a file out of scope: widen scope.include to document
353
+ them). In a family with no entry point, or only empty ones, every name the
354
+ recipes found is public.
355
+
356
+ Counts. `exports_public_api` counts the public names (the package's whole
357
+ public API, `outside_scope` names included) and `exports_internal` the
358
+ internal ones. `effective_denominator` counts the public names whose
359
+ definition file matches scope.tier_a_include (else scope.include) and no
360
+ scope.exclude glob: each name once, so a type counts once whatever its
361
+ methods, and a barrel that only re-exports adds nothing of its own. `arms` gives compile's effective_denominator arms:
362
+ (a) `monorepo` from skf-detect-workspaces.py, (b) `specific_modules`
363
+ (scope type specific-modules on a repository that is not a monorepo) and
364
+ (c) `multi_subpath_exports` (an in-scope package.json whose `exports` map
365
+ has more than one non-root subpath without a `*`, on a repository that is
366
+ not a monorepo).
367
+
368
+ Output JSON (stdout, or -o):
369
+
370
+ {
371
+ "mode": "full",
372
+ "status": "ok" | "incomplete" | "no-ast-grep",
373
+ "source_root": "<as given>",
374
+ "recipes_file": "<path>",
375
+ "recipe_set": "standard" | "component-library",
376
+ "ast_grep": {"path": "<exe>" | null, "version": "<x.y.z>" | null},
377
+ "scope": {"include": [...], "exclude": [...],
378
+ "tier_a_include": [...] | null, "type": "<type>" | null,
379
+ "languages": [...], "files_from": "<file>" | null},
380
+ "files_in_scope": N,
381
+ "files_by_language": {"<language>": N, ...},
382
+ "files_without_recipes": {".java": N, ...}, # in scope, no recipe reads them
383
+ "file_issues": [{"file", "issue": "missing" | "no-recipes" | "not-utf8"
384
+ | "unreadable" | "syntax-errors",
385
+ "count": N, "line": N}, ...], # count, line: syntax-errors
386
+ "head_cap": N | null,
387
+ "truncated": bool,
388
+ "recipes": [{"id", "languages": [...], "matches": N, "truncated": bool}, ...],
389
+ "exports": [{"export_name", "source_file", "source_line", "signature_line",
390
+ "signature", "params": [{"name", "type", "default",
391
+ "optional"}, ...] | null, "return_type": "<type>" | null,
392
+ "citation": "[AST:<file>:L<line>]", "ast_recipe",
393
+ "ast_node_type", "export_type", "language",
394
+ "from": "<module>" | null, "from_file": "<file>" | null,
395
+ "confidence": "T1", "extraction_method": "ast-grep"}, ...],
396
+ "aggregates": {"exports": N, "by_type": {"function": N, ...},
397
+ "t1": N, "t1_low": 0},
398
+ "entry_points": {"status": "barrel" | "empty-barrel" | "no-entry-point"
399
+ | null, # null: no-ast-grep
400
+ "by_language": {"<family>": "<status>", ...},
401
+ "files": [{"language", "file", "package", "subpath",
402
+ "resolution": "file" | "source-guess"
403
+ | "conventional" | "package",
404
+ "in_scope": bool}, ...],
405
+ "exports_maps": [{"package", "subpaths": [...],
406
+ "non_root_subpaths": N,
407
+ "wildcard_subpaths": N}, ...],
408
+ "unresolved": [{"package", "subpath", "targets"}
409
+ | {"entry", "name", "from", "file"}, ...]},
410
+ "entry_point_diff": {
411
+ "public": [{"name", "language", "entry", "via", "from", "local",
412
+ "file", "line"}, ...],
413
+ "internal": [{"name", "language", "source_file", "source_line"}, ...],
414
+ "extraction_gaps": [{"name", "language", "entry", "file", "line"}, ...],
415
+ "outside_scope": [{"name", "language", "entry", "file", "line"}, ...]},
416
+ "reexport_targets": [{"name", "language", "local", "from", "entry",
417
+ "file", "line", "via"}, ...],
418
+ "counts": {"exports_public_api": N, "exports_internal": N,
419
+ "effective_denominator": N,
420
+ "effective_denominator_basis": "tier_a_include" | "scope.include"
421
+ | "all-files",
422
+ "denominator_files": N},
423
+ "arms": {"monorepo": bool, "monorepo_kind": "<kind>" | null,
424
+ "specific_modules": bool, "multi_subpath_exports": bool},
425
+ "errors": [{"reason": "ast-grep-error" | "ast-grep-timeout" | "time-limit",
426
+ "files": N, "first_file", "detail",
427
+ "unread": [...]}, ...], # unread: time-limit only
428
+ "warnings": ["..."]
429
+ }
430
+
431
+ `via` says how an entry point exports a name: `declaration` (defined in the
432
+ entry point), `re-export` (from another file, which `file` names, `local`
433
+ giving the name it has there when it differs), `star` (through `export *`,
434
+ `from .x import *` or `pub use x::*`) or `namespace` (a module: `export *
435
+ as ns`, a Python submodule, a Rust module). `line` is null where no line
436
+ defines the name (a module, a name from another package). An `unresolved`
437
+ item names a subpath whose targets name no file, or a chain met tracing
438
+ `entry` that stops at `file`, the module `from` naming no file here.
439
+ Without ast-grep the JSON still has the entry points' `files`, the
440
+ `exports_maps`, the unresolved subpaths and the `arms`.
441
+ `file_issues` lists the files to read by eye: a file ast-grep skips (not
442
+ UTF-8), and a file where the parser met code it could not read (an ERROR
443
+ node), where a recipe can miss an export. `t1_low` is 0: every export here
444
+ comes from a recipe, and an export the caller then reads by eye (an
445
+ extraction gap, a Known Limitation #11 form) is T1-low.
446
+
447
+ Time limit. The ast-grep runs stop once --timeout seconds (540 by default,
448
+ under the 10-minute limit an agent's shell call often has) have passed
449
+ since the run started: the run going on is stopped, no other starts, and
450
+ an `errors[]` item with the reason `time-limit` lists in `unread` the
451
+ files left unread (each ast-grep call also stops after 600 seconds,
452
+ `ast-grep-timeout`). The JSON is still written, status "incomplete".
453
+
454
+ Exit codes (full):
455
+
456
+ 0 the recipes ran on every file in scope ("ok")
457
+ 1 an ast-grep run failed (exited non-zero) or timed out, or the
458
+ --timeout ran out: its files may be unread and the result is
459
+ incomplete ("incomplete"); `errors` lists them
460
+ 2 input error (no source root, an unreadable brief, file list or
461
+ recipe file, a recipe without its metadata, no PyYAML, an -o file
462
+ that cannot be written, a --timeout not above 0), or a failure the
463
+ runner did not foresee: one line on stderr and no JSON
464
+ 3 no ast-grep the runner can run ("no-ast-grep"): none on PATH, or
465
+ only an npm .cmd/.bat shim (Windows) with no native ast-grep.exe
466
+ beside it (cmd.exe would read a repository path's `&` as syntax);
467
+ the JSON has the scope and no exports; extract by source reading
468
+
469
+ CLI examples:
470
+ uv run skf-extract-public-api.py --mode quick < {payload_json}
471
+ uv run skf-extract-public-api.py --mode entries --language python \\
472
+ --tree-file {tree_json} --source-root {staged_dir} --fetch-list {entries_txt}
473
+ uv run skf-extract-public-api.py --mode quick --language rust \\
474
+ --source-root {staged_dir} --manifest-file Cargo.toml --entry-file src/lib.rs
475
+ uv run skf-extract-public-api.py --mode full --source-root {source_root} \\
476
+ --brief {brief_path} --tier Forge -o {extraction_json}
477
+ uv run skf-extract-public-api.py --mode full --source-root {source_root} \\
478
+ --files-from {scan_list} --head-cap 0
57
479
  """
58
480
 
59
481
  from __future__ import annotations
60
482
 
61
483
  import argparse
484
+ import ast
485
+ import configparser
486
+ import importlib.util
62
487
  import json
488
+ import os
489
+ import posixpath
63
490
  import re
491
+ import subprocess
64
492
  import sys
493
+ import tempfile
494
+ import time
65
495
  import tomllib
66
496
  import xml.etree.ElementTree as ET
67
- from typing import Callable
497
+ from collections import Counter
498
+ from pathlib import Path
499
+ from typing import Callable, Iterator
68
500
 
69
501
  # --------------------------------------------------------------------------
70
502
  # Manifest parsers
@@ -127,6 +559,31 @@ def parse_setup_py(content: str) -> dict:
127
559
  }
128
560
 
129
561
 
562
+ def parse_setup_cfg(content: str) -> dict:
563
+ """setup.cfg's [metadata] (a literal version only) and [options]
564
+ install_requires, read with configparser."""
565
+ parser = configparser.ConfigParser(interpolation=None, strict=False)
566
+ try:
567
+ parser.read_string(content)
568
+ except configparser.Error as e:
569
+ return {"_parse_error": f"setup.cfg parse error: {e}"}
570
+ meta = parser["metadata"] if parser.has_section("metadata") else {}
571
+ version = (meta.get("version") or "").strip()
572
+ deps: list[str] = []
573
+ if parser.has_option("options", "install_requires"):
574
+ for line in parser.get("options", "install_requires").splitlines():
575
+ m = re.match(r"^([A-Za-z0-9_.\-]+)", line.strip())
576
+ if m:
577
+ deps.append(m.group(1))
578
+ return {
579
+ "name": (meta.get("name") or "").strip() or None,
580
+ "version": None if not version or version.startswith(("attr:", "file:")) else version,
581
+ "description": (meta.get("description") or "").strip() or None,
582
+ "dependencies": deps,
583
+ "modules": [],
584
+ }
585
+
586
+
130
587
  def parse_cargo_toml(content: str) -> dict:
131
588
  try:
132
589
  data = tomllib.loads(content)
@@ -288,69 +745,449 @@ def parse_package_swift(content: str) -> dict:
288
745
  # Export scanners
289
746
  # --------------------------------------------------------------------------
290
747
 
291
- _JS_DECL_RE = re.compile(
292
- r"^export\s+(?:default\s+)?(const|function|class|type|interface|enum)\s+(\w+)",
293
- re.MULTILINE,
294
- )
295
- _JS_REEXPORT_RE = re.compile(r"^export\s*\{([^}]+)\}", re.MULTILINE)
748
+ def _note(warnings: list[str] | None, message: str) -> None:
749
+ if warnings is not None:
750
+ warnings.append(message)
296
751
 
297
752
 
298
- def scan_exports_js(content: str, source_file: str) -> list[dict]:
753
+ def _unlisted(source_file: str, line: int, statement: str) -> str:
754
+ """The warning for a statement that exports names quick mode cannot list."""
755
+ return (f"{source_file} line {line}: {statement} passes on names from a module quick mode does not read: "
756
+ "pass that module as an entry too, or read it by eye")
757
+
758
+
759
+ def _note_unlisted(warnings: list[str] | None, unlisted: list[dict] | None, source_file: str, line: int,
760
+ statement: str, specifier: str | None = None, module: tuple | None = None,
761
+ shown: str | None = None) -> None:
762
+ """Note a statement whose names come from a module quick mode does not
763
+ read: its warning (_unlisted, naming `shown`, else the statement in
764
+ backticks) and, when `unlisted` collects them, its record. `module` says
765
+ how extract() finds the module's file: ("js", specifier) or ("python",
766
+ level, module); None when it cannot (a package import, a Rust glob)."""
767
+ message = _unlisted(source_file, line, shown or f"`{statement}`")
768
+ _note(warnings, message)
769
+ if unlisted is not None:
770
+ unlisted.append({"file": source_file, "line": line, "statement": statement, "specifier": specifier,
771
+ "_module": module, "_warning": message})
772
+
773
+
774
+ def _exports(found: list[tuple[int, str, str]], source_file: str) -> list[dict]:
775
+ """The export records of (offset, name, type) found, in file order, each
776
+ name once (its first form wins)."""
299
777
  out: list[dict] = []
300
778
  seen: set[str] = set()
301
- for m in _JS_DECL_RE.finditer(content):
302
- kind, name = m.group(1), m.group(2)
779
+ for _, name, kind in sorted(found, key=lambda item: item[0]):
303
780
  if name not in seen:
304
781
  seen.add(name)
305
782
  out.append({"name": name, "type": kind, "source_file": source_file})
306
- for m in _JS_REEXPORT_RE.finditer(content):
307
- for piece in m.group(1).split(","):
308
- piece = piece.strip()
309
- if not piece:
310
- continue
311
- # Handle `foo as bar` — keep the local name (foo) and the alias (bar);
312
- # we only emit one entry, the exposed name.
313
- if " as " in piece:
314
- _, _, exposed = piece.partition(" as ")
315
- exposed = exposed.strip()
316
- else:
317
- exposed = piece
318
- m2 = re.match(r"\w+$", exposed)
319
- if m2 and exposed not in seen:
320
- seen.add(exposed)
321
- out.append({"name": exposed, "type": "re-export", "source_file": source_file})
322
783
  return out
323
784
 
324
785
 
325
- def scan_exports_python(content: str, source_file: str) -> list[dict]:
326
- all_names: list[str] | None = None
327
- m = re.search(r"^__all__\s*=\s*\[([^\]]+)\]", content, re.MULTILINE | re.DOTALL)
328
- if m:
329
- all_names = re.findall(r"""['"]([^'"]+)['"]""", m.group(1))
786
+ # A column-0 declaration: `export` (with `declare` or `default`) when it is
787
+ # exported, its keyword and its name. A generator's `*` may touch the name.
788
+ _JS_DECL_RE = re.compile(
789
+ r"^(export\s+(?:declare\s+)?(?:default\s+)?)?(?:declare\s+)?(?:abstract\s+)?(?:async\s+)?"
790
+ r"(const\s+enum|const|let|var|function\s*\*|function|class|interface|type|enum|namespace|module)"
791
+ r"(?:(?<=\*)\s*|\s+)([A-Za-z_$][\w$]*)",
792
+ re.MULTILINE,
793
+ )
794
+ _JS_LIST_RE = re.compile(r"^export\s+(?:type\s+)?\{([^}]*)\}", re.MULTILINE)
795
+ _JS_STAR_RE = re.compile(
796
+ r"^export\s+(?:type\s+)?\*\s*(?:as\s+([A-Za-z_$][\w$]*)\s*)?from\s*(['\"])", re.MULTILINE)
797
+ _JS_DEFAULT_RE = re.compile(r"^export\s+default\s+([A-Za-z_$][\w$]*)\s*(?:;|$)", re.MULTILINE)
798
+ _JS_ANY_DEFAULT_RE = re.compile(r"^export\s+default\b\s*", re.MULTILINE)
799
+ # `export const { a, b } = obj` or `export let [x, y] = pair`
800
+ _JS_DESTRUCTURE_RE = re.compile(r"^export\s+(?:declare\s+)?(const|let|var)\s*[{[]", re.MULTILINE)
801
+ # A name after a comma of a `const`/`let`/`var` statement: its next declarator.
802
+ _JS_DECLARATOR_RE = re.compile(r"\s*([A-Za-z_$][\w$]*)\s*(?:[=,;:!]|$)", re.MULTILINE)
803
+ _JS_LEADING_COMMA_RE = re.compile(r"\s*,")
804
+ # TypeScript's `export = <value>`, read as `module.exports = <value>` is.
805
+ _TS_EXPORT_ASSIGN_RE = re.compile(r"^export\s*=(?![=>])\s*", re.MULTILINE)
806
+ _CJS_NAMED_RE = re.compile(r"^(?:module\.)?exports\.([A-Za-z_$][\w$]*)\s*=(?![=>])", re.MULTILINE)
807
+ # A column-0 `module.exports =`, also as Express writes it: `exports =
808
+ # module.exports = <value>` (or `module.exports = exports = <value>`).
809
+ _CJS_MODULE_RE = re.compile(
810
+ r"^(?:exports\s*=\s*)?module\.exports\s*=(?![=>])\s*(?:exports\s*=(?![=>])\s*)?", re.MULTILINE)
811
+ # Any `module.exports`, `exports` or `exports.<name>` assignment; the ones
812
+ # that do not open their line or follow a chain that does (inside an `if`, a
813
+ # block, a function) are warned, since which of them runs the entry file
814
+ # alone cannot tell. TypeScript's CommonJS output opens with a chain:
815
+ # `exports.a = exports.b = void 0;`.
816
+ _CJS_ANY_RE = re.compile(r"(?<![\w$.])(?:module\.exports|exports)(?:\.([A-Za-z_$][\w$]*))?\s*=(?![=>])")
817
+ _CJS_CHAIN_RE = re.compile(r"(?:(?:module\.)?exports(?:\.[A-Za-z_$][\w$]*)?\s*=(?![=>])\s*)*")
818
+ _JS_NAME_RE = re.compile(r"[A-Za-z_$][\w$]*")
819
+ _JS_SPACE_RE = re.compile(r"\s*")
820
+ _JS_REQUIRE_RE = re.compile(r"require\s*\(\s*(['\"])")
821
+ _JS_NAMED_VALUE_RE = re.compile(r"(?:async\s+)?function\s*\*?\s*([A-Za-z_$][\w$]*)|class\s+([A-Za-z_$][\w$]*)")
822
+ _JS_ANONYMOUS_VALUE_RE = re.compile(r"(?:async\s+)?(?:function\b|class\b|\([^()]*\)\s*=>|[A-Za-z_$][\w$]*\s*=>)")
823
+ _JS_FUNCTION_VALUE_RE = re.compile(r"\s*(?:async\s+)?(?:function\b|(?:\([^()]*\)|[A-Za-z_$][\w$]*)\s*=>)")
824
+ _JS_CLASS_VALUE_RE = re.compile(r"\s*class\b")
825
+ _JS_NAME_VALUE_RE = re.compile(r"\s*([A-Za-z_$][\w$]*)\s*(?:[;,})\]]|$)", re.MULTILINE)
826
+ _JS_MEMBER_PREFIX_RE = re.compile(r"(?:(?:get|set|static|async)\s+(?=[\w$'\"*\[]))*\*?\s*")
827
+ # Words the name patterns can meet that name no export: `class extends
828
+ # Base`, `export default true`, and the interop flag compilers set.
829
+ _JS_NOT_NAMES = frozenset({
830
+ "extends", "implements", "true", "false", "null", "undefined", "this", "new", "void", "typeof",
831
+ "function", "class", "async", "await", "__esModule",
832
+ })
330
833
 
331
- out: list[dict] = []
332
- for m in re.finditer(r"^(def|class)\s+(\w+)", content, re.MULTILINE):
333
- kind, name = m.group(1), m.group(2)
334
- if name.startswith("_"):
834
+
835
+ def _js_type(keyword: str) -> str:
836
+ """The export type of a declaration keyword: `function*` and `async
837
+ function` are functions, a `const enum` an enum, a `module` a
838
+ namespace."""
839
+ keyword = " ".join(keyword.split())
840
+ if keyword.startswith("function"):
841
+ return "function"
842
+ return {"const enum": "enum", "module": "namespace"}.get(keyword, keyword)
843
+
844
+
845
+ def _js_string(content: str, code: str, quote: int) -> str:
846
+ """The text of the string literal whose opening quote is at `quote`."""
847
+ close = code.find(code[quote], quote + 1)
848
+ return content[quote + 1: close if close >= 0 else len(content)]
849
+
850
+
851
+ def _js_value_type(code: str, pos: int, declared: dict[str, str]) -> str:
852
+ """The export type of the value assigned at `pos`: a function or class
853
+ expression, the local declaration a name refers to, else a variable."""
854
+ if _JS_FUNCTION_VALUE_RE.match(code, pos):
855
+ return "function"
856
+ if _JS_CLASS_VALUE_RE.match(code, pos):
857
+ return "class"
858
+ name = _JS_NAME_VALUE_RE.match(code, pos)
859
+ return declared.get(name.group(1), "variable") if name else "variable"
860
+
861
+
862
+ def _js_object_members(content: str, code: str, start: int,
863
+ declared: dict[str, str]) -> tuple[list[tuple[int, str, str]], list[str]]:
864
+ """((offset, key, type) of each member of the object literal whose `{`
865
+ is at `start` in `code`, the spread expressions it holds). A method is
866
+ a function, `key: value` takes the value's type, a shorthand key the
867
+ type of its local declaration; a computed key names nothing."""
868
+ spans: list[tuple[int, int]] = []
869
+ depth, begin = 0, start + 1
870
+ for i in range(start, len(code)):
871
+ ch = code[i]
872
+ if ch in "{[(":
873
+ depth += 1
874
+ elif ch in "}])":
875
+ depth -= 1
876
+ if depth == 0:
877
+ spans.append((begin, i))
878
+ break
879
+ elif ch == "," and depth == 1:
880
+ spans.append((begin, i))
881
+ begin = i + 1
882
+ members: list[tuple[int, str, str]] = []
883
+ spreads: list[str] = []
884
+ for b, e in spans:
885
+ pos = b + len(code[b:e]) - len(code[b:e].lstrip())
886
+ if pos >= e:
335
887
  continue
336
- if all_names is not None and name not in all_names:
888
+ if code.startswith("...", pos):
889
+ spreads.append(" ".join(content[pos:e].split()))
337
890
  continue
338
- out.append({"name": name, "type": kind, "source_file": source_file})
339
- return out
891
+ pos = _JS_MEMBER_PREFIX_RE.match(code, pos, e).end()
892
+ if pos >= e:
893
+ continue
894
+ if code[pos] in "'\"":
895
+ close = code.find(code[pos], pos + 1, e)
896
+ if close < 0:
897
+ continue
898
+ key, after = content[pos + 1:close], close + 1
899
+ else:
900
+ ident = _JS_NAME_RE.match(code, pos, e)
901
+ if not ident:
902
+ continue
903
+ key, after = ident.group(0), ident.end()
904
+ rest = code[after:e].lstrip()
905
+ if rest.startswith("("):
906
+ kind = "function"
907
+ elif rest.startswith(":"):
908
+ kind = _js_value_type(code, code.index(":", after, e) + 1, declared)
909
+ else:
910
+ kind = declared.get(key, "variable")
911
+ if key:
912
+ members.append((pos, key, kind))
913
+ return members, spreads
340
914
 
341
915
 
342
- def scan_exports_rust(content: str, source_file: str) -> list[dict]:
343
- out: list[dict] = []
344
- for m in re.finditer(
345
- r"^\s*pub\s+(fn|struct|enum|trait|mod|type|const|static)\s+(\w+)",
346
- content,
347
- re.MULTILINE,
348
- ):
349
- out.append({"name": m.group(2), "type": m.group(1), "source_file": source_file})
350
- return out
916
+ def _js_module_exports(content: str, code: str, match: re.Match, source_file: str, declared: dict[str, str],
917
+ found: list[tuple[int, str, str]], warnings: list[str] | None,
918
+ target: str = "module.exports", unlisted: list[dict] | None = None) -> None:
919
+ """Add the names one `module.exports = <value>` (or TypeScript's
920
+ `export = <value>`, the `target`) exports."""
921
+ pos, line = match.end(), _line_of(code, match.start())
922
+ assign = target if target.endswith("=") else f"{target} ="
923
+ if code.startswith("{", pos):
924
+ members, spreads = _js_object_members(content, code, pos, declared)
925
+ found.extend(members)
926
+ for spread in spreads:
927
+ _note_unlisted(warnings, unlisted, source_file, line, spread,
928
+ shown=f"the `{target}` object's `{spread}`")
929
+ return
930
+ require = _JS_REQUIRE_RE.match(code, pos)
931
+ if require:
932
+ spec = _js_string(content, code, require.end(1) - 1)
933
+ _note_unlisted(warnings, unlisted, source_file, line, f"{assign} require('{spec}')", spec, ("js", spec))
934
+ return
935
+ named = _JS_NAMED_VALUE_RE.match(code, pos)
936
+ if named:
937
+ found.append((pos, named.group(1) or named.group(2), "function" if named.group(1) else "class"))
938
+ return
939
+ if _JS_ANONYMOUS_VALUE_RE.match(code, pos):
940
+ _note(warnings, f"{source_file} line {line}: `{target}` is an anonymous function or class: "
941
+ "the package's one export has no name here")
942
+ return
943
+ name = _JS_NAME_VALUE_RE.match(code, pos)
944
+ if name and name.group(1) not in _JS_NOT_NAMES:
945
+ found.append((pos, name.group(1), declared.get(name.group(1), "variable")))
946
+ return
947
+ _note(warnings, f"{source_file} line {line}: `{target}` is set to an expression quick mode does not "
948
+ "read: read the names it exports by eye")
949
+
950
+
951
+ def _js_more_declarators(code: str, pos: int) -> list[tuple[int, str]]:
952
+ """(offset, name) of each declarator after the first of the `const`,
953
+ `let` or `var` statement whose first name ends at `pos`: `export const
954
+ a = 1, b = 2` declares `b` too. The statement ends at a `;`, a bracket
955
+ it did not open, or a line end that no comma joins to the next line."""
956
+ names: list[tuple[int, str]] = []
957
+ depth, last, i = 0, "", pos
958
+ while i < len(code):
959
+ ch = code[i]
960
+ if ch in "([{":
961
+ depth += 1
962
+ elif ch in ")]}":
963
+ if depth == 0:
964
+ break
965
+ depth -= 1
966
+ elif depth == 0 and ch == ";":
967
+ break
968
+ elif depth == 0 and ch == "\n":
969
+ if last != "," and not _JS_LEADING_COMMA_RE.match(code, i):
970
+ break
971
+ elif depth == 0 and ch == ",":
972
+ declarator = _JS_DECLARATOR_RE.match(code, i + 1)
973
+ if declarator and declarator.group(1) not in _JS_NOT_NAMES:
974
+ names.append((declarator.start(1), declarator.group(1)))
975
+ if not ch.isspace():
976
+ last = ch
977
+ i += 1
978
+ return names
979
+
980
+
981
+ def scan_exports_js(content: str, source_file: str, warnings: list[str] | None = None,
982
+ unlisted: list[dict] | None = None, defaults: bool = True) -> list[dict]:
983
+ """The names a JS/TS entry file exports (see "Quick mode" above), read
984
+ from its text with comments, string contents and regular expressions
985
+ blanked. Without `defaults` (a module reached through `export *`) its
986
+ `export default` is left out, unread and unwarned."""
987
+ code = _blank(content, "js")
988
+ declared: dict[str, str] = {}
989
+ found: list[tuple[int, str, str]] = []
990
+ read_defaults: set[int] = set() # the `export default` statements a name was read from
991
+ for m in _JS_DECL_RE.finditer(code):
992
+ name, kind = m.group(3), _js_type(m.group(2))
993
+ if name in _JS_NOT_NAMES:
994
+ continue
995
+ declared.setdefault(name, kind)
996
+ if m.group(1):
997
+ if "default" in m.group(1).split():
998
+ read_defaults.add(m.start())
999
+ if not defaults:
1000
+ continue
1001
+ found.append((m.start(), name, kind))
1002
+ if kind in ("const", "let", "var"):
1003
+ for offset, more in _js_more_declarators(code, m.end(3)):
1004
+ declared.setdefault(more, kind)
1005
+ if m.group(1):
1006
+ found.append((offset, more, kind))
1007
+ for m in _JS_DESTRUCTURE_RE.finditer(code):
1008
+ _note(warnings, f"{source_file} line {_line_of(code, m.start())}: `export {m.group(1)}` destructures, "
1009
+ "and quick mode does not read the names it binds: read them by eye")
1010
+ for m in _JS_LIST_RE.finditer(code):
1011
+ for piece in m.group(1).split(","):
1012
+ # `a`, `a as b`, `type T`, `"a-b" as c`: the exposed name comes last
1013
+ words = piece.split()
1014
+ if words and _JS_NAME_RE.fullmatch(words[-1]):
1015
+ found.append((m.start(), words[-1], "re-export"))
1016
+ for m in _JS_STAR_RE.finditer(code):
1017
+ if m.group(1):
1018
+ found.append((m.start(), m.group(1), "namespace"))
1019
+ else:
1020
+ spec = _js_string(content, code, m.end(2) - 1)
1021
+ _note_unlisted(warnings, unlisted, source_file, _line_of(code, m.start()), f"export * from '{spec}'",
1022
+ spec, ("js", spec))
1023
+ for m in _JS_DEFAULT_RE.finditer(code):
1024
+ if m.group(1) not in _JS_NOT_NAMES and defaults:
1025
+ found.append((m.start(), m.group(1), declared.get(m.group(1), "re-export")))
1026
+ read_defaults.add(m.start())
1027
+ for m in _JS_ANY_DEFAULT_RE.finditer(code):
1028
+ if m.start() in read_defaults or not defaults:
1029
+ continue
1030
+ line = _line_of(code, m.start())
1031
+ if _JS_ANONYMOUS_VALUE_RE.match(code, m.end()):
1032
+ _note(warnings, f"{source_file} line {line}: `export default` is an anonymous function or class: "
1033
+ "the module's default export has no name here")
1034
+ else:
1035
+ _note(warnings, f"{source_file} line {line}: `export default` is set to an expression quick mode "
1036
+ "does not read: read the names it exports by eye")
1037
+ for m in _TS_EXPORT_ASSIGN_RE.finditer(code):
1038
+ _js_module_exports(content, code, m, source_file, declared, found, warnings, "export =", unlisted)
1039
+ for m in _CJS_NAMED_RE.finditer(code):
1040
+ if m.group(1) not in _JS_NOT_NAMES:
1041
+ found.append((m.start(), m.group(1), _js_value_type(code, m.end(), declared)))
1042
+ for m in _CJS_MODULE_RE.finditer(code):
1043
+ _js_module_exports(content, code, m, source_file, declared, found, warnings, unlisted=unlisted)
1044
+ nested_lines: set[int] = set()
1045
+ for m in _CJS_ANY_RE.finditer(code):
1046
+ line_start = code.rfind("\n", 0, m.start()) + 1
1047
+ if _CJS_CHAIN_RE.fullmatch(code, line_start, m.start()):
1048
+ # A statement that opens its line, its first target read above;
1049
+ # a later `exports.<name>` of its chain is one more export.
1050
+ if m.start() > line_start and m.group(1) and m.group(1) not in _JS_NOT_NAMES:
1051
+ found.append((m.start(), m.group(1), _js_value_type(code, m.end(), declared)))
1052
+ continue
1053
+ line = _line_of(code, m.start())
1054
+ if line in nested_lines:
1055
+ continue
1056
+ nested_lines.add(line)
1057
+ target = " ".join(m.group(0).rstrip("=").split())
1058
+ require = _JS_REQUIRE_RE.match(code, _JS_SPACE_RE.match(code, m.end()).end())
1059
+ if require:
1060
+ spec = _js_string(content, code, require.end(1) - 1)
1061
+ _note_unlisted(warnings, unlisted, source_file, line, f"{target} = require('{spec}')", spec, ("js", spec))
1062
+ else:
1063
+ _note(warnings, f"{source_file} line {line}: `{target}` is assigned inside a block or a condition: "
1064
+ "read the names it exports by eye")
1065
+ return _exports(found, source_file)
1066
+
1067
+
1068
+ def _py_export_type(binding: dict | None) -> str:
1069
+ """The quick-mode type of a name by what binds it in the module."""
1070
+ if binding is None or binding["kind"] == "import":
1071
+ return "re-export"
1072
+ if binding["kind"] == "module-import":
1073
+ return "module"
1074
+ return binding.get("form", "def")
1075
+
1076
+
1077
+ def _py_all_unread(node: ast.AST | None) -> bool:
1078
+ """Whether an `__all__` value holds a part that is no string literal
1079
+ (another module's `__all__`, a list built at run time)."""
1080
+ if isinstance(node, ast.Constant):
1081
+ return not isinstance(node.value, str)
1082
+ if isinstance(node, (ast.List, ast.Tuple)):
1083
+ return any(_py_all_unread(e) for e in node.elts)
1084
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
1085
+ return _py_all_unread(node.left) or _py_all_unread(node.right)
1086
+ return True
1087
+
1088
+
1089
+ def _py_all_unread_line(text: str) -> int | None:
1090
+ """The line of the first statement that builds `__all__` from a part
1091
+ that is no string literal, whose names no reader of this file alone can
1092
+ list; None when there is none or the module does not parse."""
1093
+ try:
1094
+ tree = ast.parse(text)
1095
+ except SyntaxError:
1096
+ return None
1097
+ for node in _module_statements(tree.body):
1098
+ value = None
1099
+ if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
1100
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]
1101
+ if "__all__" in [n for t in targets for n in _target_names(t)]:
1102
+ value = node.value
1103
+ elif (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
1104
+ and isinstance(node.value.func, ast.Attribute)
1105
+ and isinstance(node.value.func.value, ast.Name) and node.value.func.value.id == "__all__"
1106
+ and node.value.func.attr in ("extend", "append") and node.value.args):
1107
+ value = node.value.args[0]
1108
+ if value is not None and _py_all_unread(value):
1109
+ return node.lineno
1110
+ return None
1111
+
1112
+
1113
+ def scan_exports_python(content: str, source_file: str, warnings: list[str] | None = None,
1114
+ unlisted: list[dict] | None = None) -> list[dict]:
1115
+ """The names a Python entry file exports (see "Quick mode" above), read
1116
+ with full mode's module reader: __all__ when the module has one, else
1117
+ its public definitions and the names it imports from its own
1118
+ package."""
1119
+ try:
1120
+ all_names, bound, stars = _py_bindings(content)
1121
+ except SyntaxError as exc:
1122
+ _note(warnings, f"{source_file}: this Python cannot parse it ({exc.msg}, line {exc.lineno}); "
1123
+ "its names are read from its text")
1124
+ all_names, bound, stars = _py_text_bindings(content)
1125
+ # The package an __init__.py opens, whose own modules its absolute imports
1126
+ # can name: by any trailing run of its folders, as a nested or namespace
1127
+ # package is imported (storage, cloud.storage, acme.cloud.storage).
1128
+ packages: list[str] = []
1129
+ if posixpath.basename(source_file) == "__init__.py":
1130
+ folders = [p for p in posixpath.dirname(source_file).split("/") if p and p != "."]
1131
+ packages = [".".join(folders[i:]) for i in range(len(folders))]
351
1132
 
1133
+ def own(module: str | None, level: int) -> bool:
1134
+ return level > 0 or bool(module and any(module == p or module.startswith(p + ".") for p in packages))
352
1135
 
353
- def scan_exports_go(content: str, source_file: str) -> list[dict]:
1136
+ unread = _py_all_unread_line(content)
1137
+ if unread is not None:
1138
+ _note(warnings, f"{source_file} line {unread}: `__all__` takes names quick mode cannot read (another "
1139
+ "module's `__all__`, or a list built at run time): read the names it exports by eye")
1140
+ if all_names is not None:
1141
+ return [{"name": name, "type": _py_export_type(bound.get(name)), "source_file": source_file}
1142
+ for name in dict.fromkeys(all_names) if name.isidentifier() and not name.startswith("_")]
1143
+ found: list[tuple[int, str, str]] = []
1144
+ for name, binding in bound.items():
1145
+ if name.startswith("_"):
1146
+ continue
1147
+ if binding["kind"] == "definition" and binding.get("form", "def") in ("def", "class"):
1148
+ found.append((binding["line"], name, binding.get("form", "def")))
1149
+ elif binding["kind"] == "import" and own(binding["module"], binding["level"]):
1150
+ found.append((binding["line"], name, "re-export"))
1151
+ for star in stars:
1152
+ if own(star["module"], star["level"]):
1153
+ source = "." * star["level"] + (star["module"] or "")
1154
+ _note_unlisted(warnings, unlisted, source_file, star["line"], f"from {source} import *", source,
1155
+ ("python", star["level"], star["module"]))
1156
+ return _exports(found, source_file)
1157
+
1158
+
1159
+ # A `#[macro_export]` macro, exported at the crate root wherever it sits;
1160
+ # other attributes may stand between the two.
1161
+ _RUST_MACRO_EXPORT_RE = re.compile(
1162
+ r"^[ \t]*#\[macro_export\b[^\]]*\](?:\s*#\[[^\]]*\])*\s*macro_rules!\s*((?:r#)?[A-Za-z_]\w*)", re.M)
1163
+
1164
+
1165
+ def scan_exports_rust(content: str, source_file: str, warnings: list[str] | None = None,
1166
+ unlisted: list[dict] | None = None) -> list[dict]:
1167
+ """The names a Rust entry file exports (see "Quick mode" above): its
1168
+ column-0 unrestricted `pub` items, `#[macro_export]` macros and `pub
1169
+ use` names, read with full mode's readers from its text with comments
1170
+ and strings blanked."""
1171
+ code = _blank(content, "rust")
1172
+ found = [(line, name, kind) for name, kind, line in _rust_items(code)]
1173
+ found += [(_line_of(code, m.start(1)), m.group(1), "macro") for m in _RUST_MACRO_EXPORT_RE.finditer(code)]
1174
+ for use in _RUST_USE_RE.finditer(code):
1175
+ line = _line_of(code, use.start())
1176
+ for path, alias, glob in _rust_use_leaves(use.group(1)):
1177
+ if path and path[-1] == "self" and not glob:
1178
+ path = path[:-1]
1179
+ if not path or alias == "_":
1180
+ continue
1181
+ if glob:
1182
+ _note_unlisted(warnings, unlisted, source_file, line, f"pub use {'::'.join(path)}::*",
1183
+ "::".join(path))
1184
+ else:
1185
+ found.append((line, alias or path[-1], "re-export"))
1186
+ return _exports(found, source_file)
1187
+
1188
+
1189
+ def scan_exports_go(content: str, source_file: str, warnings: list[str] | None = None,
1190
+ unlisted: list[dict] | None = None) -> list[dict]:
354
1191
  out: list[dict] = []
355
1192
  for m in re.finditer(r"^(func|type|var|const)\s+([A-Z]\w*)", content, re.MULTILINE):
356
1193
  out.append({"name": m.group(2), "type": m.group(1), "source_file": source_file})
@@ -367,7 +1204,8 @@ _JAVA_ANNOTATION_RE = re.compile(
367
1204
  )
368
1205
 
369
1206
 
370
- def scan_exports_java(content: str, source_file: str) -> list[dict]:
1207
+ def scan_exports_java(content: str, source_file: str, warnings: list[str] | None = None,
1208
+ unlisted: list[dict] | None = None) -> list[dict]:
371
1209
  out: list[dict] = []
372
1210
  seen: set[str] = set()
373
1211
  for m in _JAVA_PUBLIC_RE.finditer(content):
@@ -398,7 +1236,8 @@ _KOTLIN_DECL_RE = re.compile(
398
1236
  )
399
1237
 
400
1238
 
401
- def scan_exports_kotlin(content: str, source_file: str) -> list[dict]:
1239
+ def scan_exports_kotlin(content: str, source_file: str, warnings: list[str] | None = None,
1240
+ unlisted: list[dict] | None = None) -> list[dict]:
402
1241
  """Kotlin defaults to public — omit internal/private declarations."""
403
1242
  out: list[dict] = []
404
1243
  seen: set[str] = set()
@@ -425,7 +1264,8 @@ _SWIFT_DECL_KEYWORDS = {
425
1264
  }
426
1265
 
427
1266
 
428
- def scan_exports_swift(content: str, source_file: str) -> list[dict]:
1267
+ def scan_exports_swift(content: str, source_file: str, warnings: list[str] | None = None,
1268
+ unlisted: list[dict] | None = None) -> list[dict]:
429
1269
  """Swift defaults to internal — emit only public/open declarations."""
430
1270
  out: list[dict] = []
431
1271
  seen: set[str] = set()
@@ -445,7 +1285,9 @@ def scan_exports_swift(content: str, source_file: str) -> list[dict]:
445
1285
  # --------------------------------------------------------------------------
446
1286
 
447
1287
  ManifestParser = Callable[[str], dict]
448
- ExportScanner = Callable[[str, str], list[dict]]
1288
+ # (content, source file, warnings, unlisted): a scanner may note a form it
1289
+ # cannot list, and record each statement whose names another module gives.
1290
+ ExportScanner = Callable[[str, str, list[str], list[dict]], list[dict]]
449
1291
 
450
1292
  LANGUAGE_DISPATCH: dict[str, tuple[ManifestParser, ExportScanner]] = {
451
1293
  "js": (parse_package_json, scan_exports_js),
@@ -462,9 +1304,11 @@ LANGUAGE_DISPATCH: dict[str, tuple[ManifestParser, ExportScanner]] = {
462
1304
 
463
1305
 
464
1306
  def _select_manifest_parser(language: str, manifest_path: str) -> ManifestParser:
465
- """Pick the right manifest parser, special-casing Python's setup.py."""
1307
+ """Pick the right manifest parser, special-casing Python's setup.py and setup.cfg."""
466
1308
  if language == "python" and manifest_path.endswith("setup.py"):
467
1309
  return parse_setup_py
1310
+ if language == "python" and manifest_path.endswith("setup.cfg"):
1311
+ return parse_setup_cfg
468
1312
  return LANGUAGE_DISPATCH[language][0]
469
1313
 
470
1314
 
@@ -520,15 +1364,22 @@ def extract(payload: dict) -> dict:
520
1364
 
521
1365
  _, scanner = LANGUAGE_DISPATCH[language]
522
1366
  exports: list[dict] = []
1367
+ unlisted: list[dict] = []
1368
+ entry_paths: set[str] = set()
523
1369
  for entry in payload.get("entries") or []:
524
1370
  if not isinstance(entry, dict):
525
1371
  continue
526
1372
  content = entry.get("content") or ""
527
1373
  path = entry.get("path") or ""
1374
+ entry_paths.add(path)
528
1375
  if not content:
529
1376
  continue
530
1377
  try:
531
- exports.extend(scanner(content, path))
1378
+ if entry.get("followed") and scanner is scan_exports_js:
1379
+ # `export *` passes on no default export
1380
+ exports.extend(scan_exports_js(content, path, warnings, unlisted, defaults=False))
1381
+ else:
1382
+ exports.extend(scanner(content, path, warnings, unlisted))
532
1383
  except Exception as e: # noqa: BLE001 — best-effort: scanner errors are warnings, not fatal
533
1384
  warnings.append(f"export scan failed for {path}: {e}")
534
1385
 
@@ -540,29 +1391,3177 @@ def extract(payload: dict) -> dict:
540
1391
  "exports": exports,
541
1392
  "dependencies": parsed.get("dependencies", []),
542
1393
  "modules": parsed.get("modules", []),
1394
+ "module_folders": [_module_folder(manifest_path, module) for module in parsed.get("modules", [])],
543
1395
  "warnings": warnings,
1396
+ "unlisted": _resolve_unlisted(unlisted, payload.get("tree"), entry_paths, warnings),
544
1397
  }
545
1398
  if "_extra" in parsed:
546
1399
  result["extra"] = parsed["_extra"]
547
1400
  return result
548
1401
 
549
1402
 
1403
+ def _module_folder(manifest_path: str, module: str) -> str:
1404
+ """The folder of a Maven or Gradle module in the repository: the
1405
+ manifest's folder joined with the module, a Gradle `core:api` read as
1406
+ `core/api`."""
1407
+ joined = posixpath.join(posixpath.dirname(manifest_path), module.strip(":").replace(":", "/"))
1408
+ return posixpath.normpath(joined)
1409
+
1410
+
1411
+ def _resolve_unlisted(unlisted: list[dict], tree: object, entry_paths: set[str],
1412
+ warnings: list[str]) -> list[dict]:
1413
+ """The `unlisted` records, each with its `module_file`: the file in the
1414
+ listing (`tree`) that the statement's module names, by full mode's
1415
+ resolvers (resolve_js_module for `export * from` and `module.exports =
1416
+ require(...)`, _py_module_file for a Python star import), else None. A
1417
+ record whose module is already an entry file is dropped with its
1418
+ warning: its names are read. Without a listing every module_file is
1419
+ None, since only the entry files are known."""
1420
+ listed = set(tree) if isinstance(tree, list) else None
1421
+ known = (listed or set()) | entry_paths
1422
+ records = []
1423
+ for item in unlisted:
1424
+ module, message = item.pop("_module"), item.pop("_warning")
1425
+ module_file = None
1426
+ if module and module[0] == "js":
1427
+ module_file = resolve_js_module(module[1], item["file"], known)
1428
+ elif module and module[0] == "python":
1429
+ module_file = _py_module_file(posixpath.dirname(item["file"]), module[1], module[2], known)
1430
+ if module_file is not None and module_file in entry_paths:
1431
+ if message in warnings:
1432
+ warnings.remove(message)
1433
+ continue
1434
+ item["module_file"] = module_file if listed is not None else None
1435
+ records.append(item)
1436
+ return records
1437
+
1438
+
550
1439
  # --------------------------------------------------------------------------
551
- # CLI
1440
+ # Full mode: the ast-grep recipe runner
552
1441
  # --------------------------------------------------------------------------
553
1442
 
1443
+ RECIPES_FILE = Path(__file__).resolve().parent.parent / "data" / "ast-grep-recipes.yaml"
1444
+ RECIPE_SETS = ("standard", "component-library")
1445
+ RECIPE_LANGUAGES = ("python", "typescript", "tsx", "javascript", "rust", "go")
1446
+ EXPORT_TYPES = ("function", "class", "const", "interface", "type", "enum", "re-export")
1447
+ TIERS = ("Quick", "Forge", "Forge+", "Deep")
554
1448
 
555
- def main(argv: list[str]) -> int:
556
- parser = argparse.ArgumentParser(
557
- description="Extract public-API surface from manifest + entry-point content (pure parser, no I/O).",
558
- )
559
- parser.add_argument(
560
- "--mode",
561
- default="quick",
562
- choices=("quick",),
563
- help="Extraction mode (only 'quick' currently; 'full' reserved for skf-create-skill).",
564
- )
565
- args = parser.parse_args(argv)
1449
+ # A file's extension -> its language here: the ast-grep language its
1450
+ # recipes run in, or `vue`, a file ast-grep reads as HTML through the
1451
+ # scratch sgconfig.yml, where the typescript and tsx recipes run in its
1452
+ # <script> blocks (the Vue note in extraction-patterns-by-hand.md).
1453
+ EXTENSION_LANGUAGES = {
1454
+ ".py": "python",
1455
+ ".pyi": "python",
1456
+ ".ts": "typescript",
1457
+ ".mts": "typescript",
1458
+ ".cts": "typescript",
1459
+ ".tsx": "tsx",
1460
+ ".js": "javascript",
1461
+ ".jsx": "javascript",
1462
+ ".mjs": "javascript",
1463
+ ".cjs": "javascript",
1464
+ ".rs": "rust",
1465
+ ".go": "go",
1466
+ ".vue": "vue",
1467
+ }
1468
+ JS_FAMILY = ("typescript", "tsx", "javascript", "vue")
1469
+ # A file language -> the language family whose entry points declare its
1470
+ # exports: TypeScript, JavaScript and .vue files share package.json barrels.
1471
+ FAMILY_OF = {
1472
+ "python": "python",
1473
+ "typescript": "javascript",
1474
+ "tsx": "javascript",
1475
+ "javascript": "javascript",
1476
+ "vue": "javascript",
1477
+ "rust": "rust",
1478
+ "go": "go",
1479
+ }
1480
+ FAMILIES = ("javascript", "python", "rust", "go")
1481
+ # --language (or a brief's language) -> the file languages it selects.
1482
+ LANGUAGE_FAMILIES = {
1483
+ "python": ("python",),
1484
+ "rust": ("rust",),
1485
+ "go": ("go",),
1486
+ "golang": ("go",),
1487
+ **{name: JS_FAMILY for name in ("typescript", "ts", "tsx", "javascript", "js", "jsx", "vue")},
1488
+ }
1489
+ # Source files no recipe reads, counted in `files_without_recipes`.
1490
+ NO_RECIPE_EXTENSIONS = frozenset({
1491
+ ".java", ".kt", ".kts", ".swift", ".rb", ".php", ".cs", ".scala", ".c", ".h",
1492
+ ".cc", ".cpp", ".hpp", ".m", ".mm", ".dart", ".ex", ".exs", ".lua", ".zig",
1493
+ })
1494
+ DEFAULT_HEAD_CAP = 200
1495
+ # (tier, scope type) -> head cap, as extraction-patterns.md's head cap
1496
+ # selection sets it; DEFAULT_HEAD_CAP otherwise.
1497
+ HEAD_CAPS = {
1498
+ ("Forge+", "full-library"): 500,
1499
+ ("Deep", "full-library"): 500,
1500
+ ("Forge+", "component-library"): 300,
1501
+ ("Deep", "component-library"): 300,
1502
+ }
1503
+ SCAN_TIMEOUT_SEC = 600 # per ast-grep call
1504
+ # The whole run's limit (--timeout), below the 10-minute cap many agents put
1505
+ # on one shell call, so the JSON is written before the shell stops the run.
1506
+ DEFAULT_TIMEOUT_SEC = 540
1507
+ # The most source lines one export's `signature` spans (a formatter splits a
1508
+ # long parameter list one parameter per line).
1509
+ SIGNATURE_MAX_LINES = 40
1510
+ # Characters of file paths per ast-grep call: Windows caps a whole command
1511
+ # line at 32,767 characters.
1512
+ ARGV_BUDGET = 24000
1513
+ SGCONFIG = 'ruleDirs: []\nlanguageGlobs:\n html: ["*.vue"]\n'
1514
+ # Rules a scan runs beside the recipes; a match of one is never an export.
1515
+ SYNTAX_ERROR_RULE = "skf-syntax-error"
1516
+ STAR_RULE = "skf-star-reexport"
1517
+ IMPORT_NAMED_RULE = "skf-import-named"
1518
+ IMPORT_DEFAULT_RULE = "skf-import-default"
1519
+ IMPORT_NAMESPACE_RULE = "skf-import-namespace"
1520
+ IMPORT_RULES = (IMPORT_NAMED_RULE, IMPORT_DEFAULT_RULE, IMPORT_NAMESPACE_RULE)
1521
+ BINDING_RULE = "skf-binding" # skf-binding-<form>: a top-level binding a local list names
1522
+ # The build folders a package.json target can name when the build output is
1523
+ # not committed (`dist/index.js`): the target maps back to its source.
1524
+ BUILD_DIRS = frozenset({"dist", "build", "lib", "out", "esm", "cjs", "es", "module", "umd", "types", "typings"})
1525
+ JS_SOURCE_EXTENSIONS = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs")
1526
+ JS_RESOLVE_EXTENSIONS = (
1527
+ ".ts", ".tsx", ".d.ts", ".mts", ".d.mts", ".cts", ".d.cts", ".js", ".jsx", ".mjs", ".cjs",
1528
+ )
1529
+ # A `.js` import specifier names its TypeScript source (`./x.js` -> x.ts).
1530
+ JS_SOURCE_OF = {
1531
+ ".js": (".ts", ".tsx", ".d.ts"),
1532
+ ".jsx": (".tsx",),
1533
+ ".mjs": (".mts", ".d.mts"),
1534
+ ".cjs": (".cts", ".d.cts"),
1535
+ }
1536
+ JS_TARGET_SUFFIXES = (".d.mts", ".d.cts", ".d.ts", ".mjs", ".cjs", ".jsx", ".js", ".mts", ".cts", ".tsx", ".ts")
1537
+ PACKAGE_ENTRY_FIELDS = ("types", "typings", "module", "main", "browser")
1538
+ WORKSPACE_MANIFESTS = ("package.json", "pnpm-workspace.yaml", "lerna.json", "Cargo.toml", "rush.json", "nx.json")
1539
+
1540
+
1541
+ class RunnerError(Exception):
1542
+ """An input full mode cannot use (exit 2)."""
1543
+
1544
+
1545
+ class _Clock:
1546
+ """The --timeout of one full run: the seconds left before it runs out."""
1547
+
1548
+ def __init__(self, seconds: float):
1549
+ self.seconds = seconds
1550
+ self.end = time.monotonic() + seconds
1551
+
1552
+ def left(self) -> float:
1553
+ return self.end - time.monotonic()
1554
+
1555
+
1556
+ _SIBLINGS: dict[str, object] = {}
1557
+
1558
+
1559
+ def _sibling(filename: str):
1560
+ """A helper installed beside this script, loaded once as a module:
1561
+ skf-source-tree.py (git without the caller's hooks and git variables,
1562
+ and a tool lookup that never takes a shim planted in the current
1563
+ folder), skf-resolve-authoritative-files.py (the brief's glob rules) and
1564
+ skf-detect-workspaces.py (monorepo detection). Raises RunnerError."""
1565
+ module = _SIBLINGS.get(filename)
1566
+ if module is None:
1567
+ path = Path(__file__).resolve().parent / filename
1568
+ name = "skf_" + filename.removeprefix("skf-").removesuffix(".py").replace("-", "_")
1569
+ spec = importlib.util.spec_from_file_location(name, path)
1570
+ if spec is None or spec.loader is None:
1571
+ raise RunnerError(f"cannot load {filename} beside {Path(__file__).name}")
1572
+ module = importlib.util.module_from_spec(spec)
1573
+ try:
1574
+ spec.loader.exec_module(module)
1575
+ except (OSError, ImportError, SyntaxError) as exc:
1576
+ raise RunnerError(f"cannot load {filename} beside {Path(__file__).name}: {exc}") from exc
1577
+ _SIBLINGS[filename] = module
1578
+ return module
1579
+
1580
+
1581
+ def _yaml():
1582
+ try:
1583
+ import yaml # full mode only; `uv run` installs it from the header
1584
+ except ImportError as exc:
1585
+ raise RunnerError("full mode reads YAML with PyYAML: run the helper with `uv run`") from exc
1586
+ return yaml
1587
+
1588
+
1589
+ def _norm(path: object) -> str:
1590
+ """A path or glob as the runner compares them: forward slashes, no
1591
+ leading `./` or `/`."""
1592
+ text = str(path or "").strip().replace("\\", "/")
1593
+ while text.startswith("./"):
1594
+ text = text[2:]
1595
+ return text.lstrip("/")
1596
+
1597
+
1598
+ def _line_of(text: str, offset: int) -> int:
1599
+ return text.count("\n", 0, offset) + 1
1600
+
1601
+
1602
+ # --------------------------------------------------------------------------
1603
+ # Recipes
1604
+ # --------------------------------------------------------------------------
1605
+
1606
+
1607
+ def _recipe_problem(doc: object) -> str | None:
1608
+ """Why `doc` is not a recipe the runner can run, or None."""
1609
+ if not isinstance(doc, dict) or not isinstance(doc.get("id"), str) or not doc["id"]:
1610
+ return "a document that is not a recipe with an id"
1611
+ rid, rule, meta = doc["id"], doc.get("rule"), doc.get("metadata")
1612
+ if doc.get("language") not in RECIPE_LANGUAGES or not isinstance(rule, dict) or not isinstance(rule.get("kind"), str):
1613
+ return f"{rid}: a language the runner does not know, or a rule that names no kind"
1614
+ if not isinstance(meta, dict):
1615
+ return f"{rid}: no metadata"
1616
+ languages, sets = meta.get("languages"), meta.get("sets")
1617
+ if (not isinstance(languages, list) or not languages or languages[0] != doc["language"]
1618
+ or not all(language in RECIPE_LANGUAGES for language in languages)):
1619
+ return f"{rid}: metadata.languages must list its own language first, then other recipe languages"
1620
+ if not isinstance(sets, list) or not sets or not all(s in RECIPE_SETS for s in sets):
1621
+ return f"{rid}: metadata.sets must list {' or '.join(RECIPE_SETS)}"
1622
+ types = meta.get("export_type")
1623
+ types = types if isinstance(types, list) else [types]
1624
+ if not types or not all(t in EXPORT_TYPES for t in types):
1625
+ return f"{rid}: metadata.export_type must be one of {', '.join(EXPORT_TYPES)}, or a list of them"
1626
+ prefer = meta.get("prefer_over", [])
1627
+ if not isinstance(prefer, list) or not all(isinstance(p, str) for p in prefer):
1628
+ return f"{rid}: metadata.prefer_over must be a list of recipe ids"
1629
+ return None
1630
+
1631
+
1632
+ def load_recipe_file(path: Path) -> list[dict]:
1633
+ """The recipes in `path`, in file order, each checked for the metadata
1634
+ the runner reads: the `recipes` list of a YAML mapping (the module's
1635
+ data file), a list, or one recipe per YAML document, as kind-at reads
1636
+ them. Raises RunnerError."""
1637
+ yaml = _yaml()
1638
+ try:
1639
+ loaded = list(yaml.safe_load_all(path.read_text(encoding="utf-8")))
1640
+ except (OSError, UnicodeDecodeError) as exc:
1641
+ raise RunnerError(f"cannot read recipes {path}: {exc}") from exc
1642
+ except yaml.YAMLError as exc:
1643
+ raise RunnerError(f"malformed YAML in recipes {path}: {exc}") from exc
1644
+ docs: list[object] = []
1645
+ for doc in loaded:
1646
+ if isinstance(doc, dict) and isinstance(doc.get("recipes"), list):
1647
+ docs.extend(doc["recipes"])
1648
+ elif isinstance(doc, list):
1649
+ docs.extend(doc)
1650
+ elif doc is not None:
1651
+ docs.append(doc)
1652
+ if not docs:
1653
+ raise RunnerError(f"no recipe in {path}")
1654
+ seen: set[tuple[str, str]] = set()
1655
+ for doc in docs:
1656
+ problem = _recipe_problem(doc)
1657
+ for language in [] if problem else doc["metadata"]["languages"]:
1658
+ if (doc["id"], language) in seen:
1659
+ problem = f"{doc['id']} runs in {language} twice"
1660
+ seen.add((doc["id"], language))
1661
+ if problem is not None:
1662
+ raise RunnerError(f"recipes {path}: {problem}")
1663
+ return docs
1664
+
1665
+
1666
+ def _forms(recipes: list[dict]) -> dict[tuple[str, str], dict]:
1667
+ """(recipe id, language) -> the recipe form that runs in that language."""
1668
+ return {(r["id"], language): r for r in recipes for language in r["metadata"]["languages"]}
1669
+
1670
+
1671
+ def _recipe_rules(recipes: list[dict], languages: set[str]) -> list[dict]:
1672
+ """One ast-grep rule per recipe form and language it runs in, among
1673
+ `languages`, with `language` rewritten and the metadata left out."""
1674
+ rules = []
1675
+ for recipe in recipes:
1676
+ for language in recipe["metadata"]["languages"]:
1677
+ if language in languages:
1678
+ rule = {k: v for k, v in recipe.items() if k != "metadata"}
1679
+ rule["language"] = language
1680
+ rules.append(rule)
1681
+ return rules
1682
+
1683
+
1684
+ def _syntax_error_rule(language: str) -> dict:
1685
+ """An outermost ERROR node: code the parser could not read."""
1686
+ return {"id": SYNTAX_ERROR_RULE, "language": language,
1687
+ "rule": {"kind": "ERROR", "not": {"inside": {"kind": "ERROR", "stopBy": "end"}}}}
1688
+
1689
+
1690
+ def _trace_rules(language: str) -> list[dict]:
1691
+ """A JS/TS file's `export * from` statements and its imported names,
1692
+ which one-level re-export tracing follows."""
1693
+ source = {"field": "source", "has": {"kind": "string_fragment", "pattern": "$SOURCE"}}
1694
+ clause = {"kind": "import_clause",
1695
+ "inside": {"kind": "import_statement", "inside": {"kind": "program"}, "has": source}}
1696
+ named = {"kind": "import_specifier", "inside": {"kind": "named_imports", "inside": clause},
1697
+ "any": [{"has": {"field": "alias", "pattern": "$NAME"}},
1698
+ {"all": [{"not": {"has": {"field": "alias", "regex": "."}}},
1699
+ {"has": {"field": "name", "pattern": "$NAME"}}]}]}
1700
+ star = {"kind": "export_statement", "inside": {"kind": "program"}, "has": source,
1701
+ "not": {"any": [{"has": {"kind": "export_clause"}}, {"has": {"kind": "namespace_export"}}]}}
1702
+ return [
1703
+ {"id": STAR_RULE, "language": language, "rule": star},
1704
+ {"id": IMPORT_NAMED_RULE, "language": language, "rule": named},
1705
+ {"id": IMPORT_DEFAULT_RULE, "language": language,
1706
+ "rule": {"kind": "identifier", "pattern": "$NAME", "inside": clause}},
1707
+ {"id": IMPORT_NAMESPACE_RULE, "language": language,
1708
+ "rule": {"kind": "identifier", "pattern": "$NAME",
1709
+ "inside": {"kind": "namespace_import", "inside": clause}}},
1710
+ ]
1711
+
1712
+
1713
+ # A binding's form -> the export type the recipes record for it exported in
1714
+ # place, most telling first (a const bound to an arrow function or a
1715
+ # function expression is a function; an import is a re-export).
1716
+ BINDING_TYPES = {"function": "function", "class": "class", "enum": "enum", "arrow": "function",
1717
+ "const": "const", "interface": "interface", "type": "type", "import": "re-export"}
1718
+
1719
+
1720
+ def _binding_rules(language: str) -> list[dict]:
1721
+ """A JS/TS file's top-level bindings (outside every block and class
1722
+ body), one rule per form, whose id ends in it: the types a local
1723
+ `export { ... }` list's items take. TypeScript only kinds stay out of
1724
+ a javascript rule, which ast-grep would refuse."""
1725
+ top = {"not": {"inside": {"any": [{"kind": "statement_block"}, {"kind": "class_body"}], "stopBy": "end"}}}
1726
+ named = {"has": {"field": "name", "pattern": "$NAME"}}
1727
+ ts = language != "javascript"
1728
+ kinds = {
1729
+ "function": ["function_declaration", "generator_function_declaration"] + (["function_signature"] if ts else []),
1730
+ "class": ["class_declaration"] + (["abstract_class_declaration"] if ts else []),
1731
+ "interface": ["interface_declaration"] if ts else [],
1732
+ "type": ["type_alias_declaration"] if ts else [],
1733
+ "enum": ["enum_declaration"] if ts else [],
1734
+ }
1735
+ rules = [{"id": f"{BINDING_RULE}-{form}", "language": language,
1736
+ "rule": {"any": [{"kind": kind} for kind in found], **named, **top}}
1737
+ for form, found in kinds.items() if found]
1738
+ declarator = {"kind": "variable_declarator",
1739
+ "has": {"field": "name", "kind": "identifier", "pattern": "$NAME"}, **top}
1740
+ value = {"has": {"field": "value", "any": [{"kind": "arrow_function"}, {"kind": "function_expression"},
1741
+ {"kind": "generator_function"}]}}
1742
+ rules.append({"id": f"{BINDING_RULE}-arrow", "language": language, "rule": dict(declarator, all=[value])})
1743
+ rules.append({"id": f"{BINDING_RULE}-const", "language": language, "rule": declarator})
1744
+ return rules + [r for r in _trace_rules(language) if r["id"] in IMPORT_RULES]
1745
+
1746
+
1747
+ def _binding_types(matches: list[dict]) -> dict[tuple[str, str], str]:
1748
+ """(file, name) -> the export type of the top-level binding `name` has
1749
+ in that file, from the matches of _binding_rules."""
1750
+ rank = list(BINDING_TYPES)
1751
+ best: dict[tuple[str, str], str] = {}
1752
+ for m in matches:
1753
+ form = "import" if m["rule"] in IMPORT_RULES else m["rule"].removeprefix(f"{BINDING_RULE}-")
1754
+ if form in BINDING_TYPES and m["name"]:
1755
+ key = (m["file"], m["name"])
1756
+ if key not in best or rank.index(form) < rank.index(best[key]):
1757
+ best[key] = form
1758
+ return {key: BINDING_TYPES[form] for key, form in best.items()}
1759
+
1760
+
1761
+ # --------------------------------------------------------------------------
1762
+ # Files in scope
1763
+ # --------------------------------------------------------------------------
1764
+
1765
+
1766
+ def _hidden(path: str) -> bool:
1767
+ return any(part.startswith(".") for part in path.split("/"))
1768
+
1769
+
1770
+ def _git_files(root: Path) -> list[str] | None:
1771
+ """The files git lists under `root` (tracked, and untracked but not
1772
+ ignored), or None when git is missing or `root` is no work tree."""
1773
+ try:
1774
+ res = _sibling("skf-source-tree.py")._git(
1775
+ root, "ls-files", "-z", "--cached", "--others", "--exclude-standard")
1776
+ except RunnerError:
1777
+ return None
1778
+ if res is None or res[0] != 0:
1779
+ return None
1780
+ files = []
1781
+ for raw in res[1].split(b"\0"):
1782
+ rel = os.fsdecode(raw).replace("\\", "/") if raw else ""
1783
+ if rel and (root / rel).is_file() and not (root / rel).is_symlink():
1784
+ files.append(rel)
1785
+ return files
1786
+
1787
+
1788
+ def _walked_files(root: Path) -> list[str]:
1789
+ files = []
1790
+ for dirpath, dirnames, filenames in os.walk(root):
1791
+ rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/")
1792
+ rel_dir = "" if rel_dir == "." else rel_dir
1793
+ dirnames[:] = sorted(d for d in dirnames
1794
+ if not d.startswith(".") and not os.path.islink(os.path.join(dirpath, d)))
1795
+ for name in filenames:
1796
+ if not name.startswith(".") and not os.path.islink(os.path.join(dirpath, name)):
1797
+ files.append(f"{rel_dir}/{name}" if rel_dir else name)
1798
+ return files
1799
+
1800
+
1801
+ def list_tree(root: Path) -> list[str]:
1802
+ """The files under `root` a scan may read, as sorted POSIX paths
1803
+ relative to it: those git lists when `root` is in a git work tree (so
1804
+ .gitignore holds, as in ast-grep's own walk), else every file, as when
1805
+ git lists none (a folder the work tree ignores). Hidden entries and
1806
+ symbolic links are left out either way."""
1807
+ listed = _git_files(root) or _walked_files(root)
1808
+ return sorted(p for p in listed if not _hidden(p))
1809
+
1810
+
1811
+ def _file_language(path: str) -> str | None:
1812
+ return EXTENSION_LANGUAGES.get(posixpath.splitext(path)[1].lower())
1813
+
1814
+
1815
+ def _rule_languages(file_language: str) -> tuple[str, ...]:
1816
+ """The recipe languages that run in a file of `file_language`."""
1817
+ return ("typescript", "tsx") if file_language == "vue" else (file_language,)
1818
+
1819
+
1820
+ def _wanted_languages(values: list[str]) -> tuple[set[str], list[str]]:
1821
+ """The file languages the --language values select, and the values no
1822
+ recipe language family knows. No value selects every language."""
1823
+ if not values:
1824
+ return set(EXTENSION_LANGUAGES.values()), []
1825
+ wanted: set[str] = set()
1826
+ unknown = []
1827
+ for value in values:
1828
+ family = LANGUAGE_FAMILIES.get(str(value).strip().lower())
1829
+ if family is None:
1830
+ unknown.append(str(value))
1831
+ else:
1832
+ wanted.update(family)
1833
+ return wanted, unknown
1834
+
1835
+
1836
+ def _read_file_list(path: Path) -> list[str]:
1837
+ """--files-from: a JSON list of paths when the whole text is one, else
1838
+ one path per line, so a first path such as `[slug]/page.tsx` is a path."""
1839
+ try:
1840
+ text = path.read_text(encoding="utf-8")
1841
+ except (OSError, UnicodeDecodeError) as exc:
1842
+ raise RunnerError(f"cannot read file list {path}: {exc}") from exc
1843
+ items = None
1844
+ if text.lstrip().startswith("["):
1845
+ try:
1846
+ value = json.loads(text)
1847
+ except ValueError:
1848
+ value = None
1849
+ if isinstance(value, list) and all(isinstance(item, str) for item in value):
1850
+ items = value
1851
+ if items is None:
1852
+ items = text.splitlines()
1853
+ return list(dict.fromkeys(rel for rel in (_norm(item) for item in items) if rel))
1854
+
1855
+
1856
+ def _read_brief(path: Path) -> dict:
1857
+ """The scope fields and language of a skill brief."""
1858
+ yaml = _yaml()
1859
+ try:
1860
+ data = yaml.safe_load(path.read_text(encoding="utf-8"))
1861
+ except (OSError, UnicodeDecodeError) as exc:
1862
+ raise RunnerError(f"cannot read brief {path}: {exc}") from exc
1863
+ except yaml.YAMLError as exc:
1864
+ raise RunnerError(f"brief {path} is not valid YAML: {exc}") from exc
1865
+ if not isinstance(data, dict):
1866
+ raise RunnerError(f"brief {path} is not a YAML mapping")
1867
+ scope = data.get("scope") if isinstance(data.get("scope"), dict) else {}
1868
+
1869
+ def strings(value: object) -> list[str]:
1870
+ return [v for v in value if isinstance(v, str)] if isinstance(value, list) else []
1871
+
1872
+ return {
1873
+ "language": data.get("language") if isinstance(data.get("language"), str) else None,
1874
+ "type": scope.get("type") if isinstance(scope.get("type"), str) else None,
1875
+ "include": strings(scope.get("include")),
1876
+ "exclude": strings(scope.get("exclude")),
1877
+ "tier_a_include": strings(scope.get("tier_a_include")),
1878
+ }
1879
+
1880
+
1881
+ # --------------------------------------------------------------------------
1882
+ # Running ast-grep
1883
+ # --------------------------------------------------------------------------
1884
+
1885
+
1886
+ def _batches(files: list[str]) -> Iterator[list[str]]:
1887
+ batch: list[str] = []
1888
+ size = 0
1889
+ for rel in files:
1890
+ if batch and size + len(rel) + 3 > ARGV_BUDGET:
1891
+ yield batch
1892
+ batch, size = [], 0
1893
+ batch.append(rel)
1894
+ size += len(rel) + 3
1895
+ if batch:
1896
+ yield batch
1897
+
1898
+
1899
+ def _position(node: object) -> int | None:
1900
+ """The 1-based line a JSON match (or metavariable) starts on."""
1901
+ if not isinstance(node, dict):
1902
+ return None
1903
+ line = ((node.get("range") or {}).get("start") or {}).get("line")
1904
+ return line + 1 if isinstance(line, int) and not isinstance(line, bool) else None
1905
+
1906
+
1907
+ _SIGNATURE_QUOTES = {"python": "'\"", "rust": '"'} # a Rust `'a` is a lifetime
1908
+
1909
+
1910
+ def _code_end(text: str, language: str, depth: int) -> tuple[int, int]:
1911
+ """(the bracket depth after `text`, one source line, where its line
1912
+ comment starts, else its length): each ( or [ outside a string opens
1913
+ one, each ) or ] closes one."""
1914
+ quotes = _SIGNATURE_QUOTES.get(language, "'\"`")
1915
+ comment = "#" if language == "python" else "//"
1916
+ quote = None
1917
+ i = 0
1918
+ while i < len(text):
1919
+ ch = text[i]
1920
+ if quote:
1921
+ if ch == "\\":
1922
+ i += 1
1923
+ elif ch == quote:
1924
+ quote = None
1925
+ elif ch in quotes:
1926
+ quote = ch
1927
+ elif text.startswith(comment, i):
1928
+ return depth, i
1929
+ elif ch in "([":
1930
+ depth += 1
1931
+ elif ch in ")]":
1932
+ depth -= 1
1933
+ i += 1
1934
+ return depth, len(text)
1935
+
1936
+
1937
+ def _declaration_signature(rows: list[str], start: int, language: str) -> str:
1938
+ """The declaration a match names, on one line: the source line holding
1939
+ `$NAME`, and, while a ( or [ it opens is still open at a line's end (or
1940
+ a generic's < that ends the line, outside Python), the lines after it
1941
+ (at most SIGNATURE_MAX_LINES), each without its line comment, as is a
1942
+ one-line declaration. A parameter list a formatter split one parameter
1943
+ per line, its trailing comma included, reads as its one-line form does:
1944
+ `def f(a: int, b: str) -> R:`, and so does a destructured parameter
1945
+ (`({ a, b }: Props) => {`) or a split generic (`f<T, U>(a: T)`). A {
1946
+ adds no depth: it opens a body, an interface or an object as often as a
1947
+ parameter, and a ( or [ around it already holds the line open."""
1948
+ out, depth, angle = "", 0, 0
1949
+ for row in rows[start:start + SIGNATURE_MAX_LINES]:
1950
+ text = row.strip()
1951
+ if out and angle and text.startswith(">"):
1952
+ angle -= 1
1953
+ depth, end = _code_end(text, language, depth)
1954
+ piece = text[:end].rstrip()
1955
+ if language != "python" and piece.endswith("<"):
1956
+ angle += 1
1957
+ if not out:
1958
+ out = piece
1959
+ elif piece:
1960
+ if piece[0] in ")]}>" and out.endswith(","):
1961
+ out = out[:-1]
1962
+ out += piece if out[-1] in "([<" or piece[0] in ")]>" else " " + piece
1963
+ if depth <= 0 and angle <= 0:
1964
+ break
1965
+ return out
1966
+
1967
+
1968
+ def _parse_match(raw: object) -> dict | None:
1969
+ """One `--json=stream` match: its rule, language, file, `$NAME`'s line
1970
+ (the match's own first line when it binds no `$NAME`), `$NAME`,
1971
+ `$SOURCE`, the source line holding `$NAME` (from `lines`, the whole
1972
+ lines the match spans, so a list item gives its full line), the
1973
+ declaration signature that line opens (_declaration_signature) and the
1974
+ match text."""
1975
+ first = _position(raw)
1976
+ if first is None:
1977
+ return None
1978
+ single = (raw.get("metaVariables") or {}).get("single") or {}
1979
+ name_var, source_var = single.get("NAME"), single.get("SOURCE")
1980
+ name, line = None, first
1981
+ if isinstance(name_var, dict):
1982
+ name = str(name_var.get("text") or "")
1983
+ line = _position(name_var) or first
1984
+ text = str(raw.get("text") or "")
1985
+ rows = str(raw.get("lines") or text).split("\n")
1986
+ row = min(max(line - first, 0), len(rows) - 1)
1987
+ language = str(raw.get("language") or "").lower()
1988
+ return {
1989
+ "rule": str(raw.get("ruleId") or ""),
1990
+ "language": language,
1991
+ "file": _norm(raw.get("file")),
1992
+ "line": line,
1993
+ "start_line": first,
1994
+ "name": name,
1995
+ "source": (str(source_var.get("text") or "") or None) if isinstance(source_var, dict) else None,
1996
+ "signature_line": rows[row].strip(),
1997
+ "signature": _declaration_signature(rows, row, language),
1998
+ "text": text,
1999
+ }
2000
+
2001
+
2002
+ def _scan(exe: str, root: Path, folder: Path, rules: list[dict], files: list[str],
2003
+ errors: list[dict], clock: _Clock | None = None) -> list[dict]:
2004
+ """ast-grep's matches of `rules` in `files` (relative to `root`), in
2005
+ batches. The rules go in a file as JSON documents (YAML reads JSON),
2006
+ never on the command line. A call that times out, exits non-zero (its
2007
+ rules carry no severity, so a scan that read every file exits 0, matches
2008
+ or not) or prints other than JSON lines adds to `errors`: its batch's
2009
+ files may be unread. The matches it printed are kept. When `clock` (the
2010
+ run's --timeout) runs out, the call running is stopped and no other
2011
+ starts: one `time-limit` item names every file left unread."""
2012
+ if not rules or not files:
2013
+ return []
2014
+ rule_file = folder / "rules.yml"
2015
+ rule_file.write_text("\n---\n".join(json.dumps(r, ensure_ascii=False) for r in rules) + "\n",
2016
+ encoding="utf-8")
2017
+ config = folder / "sgconfig.yml"
2018
+ config.write_text(SGCONFIG, encoding="utf-8")
2019
+ found = []
2020
+ batches = list(_batches(files))
2021
+ for index, batch in enumerate(batches):
2022
+ left = None if clock is None else clock.left()
2023
+ if left is not None and left <= 0:
2024
+ unread = [rel for rest in batches[index:] for rel in rest]
2025
+ errors.append({"reason": "time-limit", "files": len(unread), "first_file": unread[0],
2026
+ "detail": f"the --timeout of {clock.seconds:g}s ran out before ast-grep read them",
2027
+ "unread": unread})
2028
+ break
2029
+ limit = SCAN_TIMEOUT_SEC if left is None else min(SCAN_TIMEOUT_SEC, left)
2030
+ cmd = [exe, "scan", "--config", str(config), "-r", str(rule_file), "--json=stream", "--", *batch]
2031
+ failure = None
2032
+ try:
2033
+ res = subprocess.run(cmd, cwd=root, capture_output=True, timeout=limit, check=False)
2034
+ except subprocess.TimeoutExpired:
2035
+ if clock is not None and clock.left() <= 0:
2036
+ failure = ("time-limit", f"the --timeout of {clock.seconds:g}s ran out while ast-grep read them")
2037
+ else:
2038
+ failure = ("ast-grep-timeout", f"no answer in {limit:g}s")
2039
+ except (OSError, ValueError) as exc:
2040
+ failure = ("ast-grep-error", f"{type(exc).__name__}: {exc}")
2041
+ else:
2042
+ stdout = res.stdout.decode("utf-8", errors="replace")
2043
+ if res.returncode != 0:
2044
+ stderr = res.stderr.decode("utf-8", errors="replace")
2045
+ detail = next((ln.strip() for ln in stderr.splitlines() if ln.strip()), "")
2046
+ failure = ("ast-grep-error", f"exit {res.returncode}: {detail}")
2047
+ for raw in stdout.splitlines():
2048
+ if not raw.strip():
2049
+ continue
2050
+ try:
2051
+ match = _parse_match(json.loads(raw))
2052
+ except ValueError:
2053
+ failure = failure or ("ast-grep-error", "output is not JSON lines")
2054
+ break
2055
+ if match is not None:
2056
+ found.append(match)
2057
+ if failure is not None:
2058
+ error = {"reason": failure[0], "files": len(batch), "first_file": batch[0], "detail": failure[1]}
2059
+ if failure[0] == "time-limit":
2060
+ error["unread"] = list(batch)
2061
+ errors.append(error)
2062
+ return found
2063
+
2064
+
2065
+ def _native_ast_grep(exe: str) -> str | None:
2066
+ """The executable a scan runs for `exe`, the ast-grep PATH gives: `exe`
2067
+ itself, or for an npm `.cmd` or `.bat` shim (Windows) the native
2068
+ ast-grep.exe it wraps, which @ast-grep/cli's postinstall puts in its
2069
+ package folder (node_modules/@ast-grep/cli beside a global install's
2070
+ shim, ../@ast-grep/cli from node_modules/.bin), else its platform
2071
+ package's. None when neither holds one: a shim runs through cmd.exe,
2072
+ which would read the `&`, `^` and `%` of the repository paths a scan
2073
+ passes as its own syntax."""
2074
+ if not exe.lower().endswith((".cmd", ".bat")):
2075
+ return exe
2076
+ shim_dir = Path(exe).parent
2077
+ for scope in (shim_dir / "node_modules" / "@ast-grep", shim_dir.parent / "@ast-grep"):
2078
+ for package in [scope / "cli", *sorted(scope.glob("cli-win32-*"))]:
2079
+ native = package / "ast-grep.exe"
2080
+ if native.is_file():
2081
+ return str(native)
2082
+ return None
2083
+
2084
+
2085
+ def _ast_grep() -> tuple[str | None, str | None, str | None]:
2086
+ """(the ast-grep executable a scan runs, its version, the npm shim PATH
2087
+ gives when no native binary backs it), each None when unknown."""
2088
+ exe = _sibling("skf-source-tree.py")._resolve_outside_cwd("ast-grep")
2089
+ if exe is None:
2090
+ return None, None, None
2091
+ native = _native_ast_grep(exe)
2092
+ if native is None:
2093
+ return None, None, exe
2094
+ try:
2095
+ res = subprocess.run([native, "--version"], capture_output=True, text=True, timeout=30, check=False)
2096
+ except (OSError, subprocess.SubprocessError):
2097
+ return native, None, None
2098
+ words = res.stdout.split()
2099
+ return native, (words[-1] if res.returncode == 0 and words else None), None
2100
+
2101
+
2102
+ # --------------------------------------------------------------------------
2103
+ # Merging the matches into exports
2104
+ # --------------------------------------------------------------------------
2105
+
2106
+ _WORD_RE = re.compile(r"[A-Za-z_$][\w$]*")
2107
+
2108
+
2109
+ def _keyword_type(text: str, name: str, choices: list[str]) -> str:
2110
+ """The last of `choices` that stands as a word before `name` in the
2111
+ match text (`export declare const enum E` -> enum)."""
2112
+ picked = choices[0]
2113
+ for word in _WORD_RE.findall(text):
2114
+ if word == name:
2115
+ break
2116
+ if word in choices:
2117
+ picked = word
2118
+ return picked
2119
+
2120
+
2121
+ def _export_record(match: dict, form: dict) -> dict:
2122
+ export_type = form["metadata"]["export_type"]
2123
+ if isinstance(export_type, list):
2124
+ export_type = _keyword_type(match["text"], match["name"], export_type)
2125
+ params, return_type = (signature_parts(match["signature"], match["language"], match["name"])
2126
+ if export_type == "function" else (None, None))
2127
+ return {
2128
+ "export_name": match["name"],
2129
+ "source_file": match["file"],
2130
+ "source_line": match["line"],
2131
+ "signature_line": match["signature_line"],
2132
+ "signature": match["signature"],
2133
+ "params": params,
2134
+ "return_type": return_type,
2135
+ "citation": f"[AST:{match['file']}:L{match['line']}]",
2136
+ "ast_recipe": match["rule"],
2137
+ "ast_node_type": form["rule"]["kind"],
2138
+ "export_type": export_type,
2139
+ "language": match["language"],
2140
+ "from": match["source"],
2141
+ "from_file": None,
2142
+ "confidence": "T1",
2143
+ "extraction_method": "ast-grep",
2144
+ }
2145
+
2146
+
2147
+ # --------------------------------------------------------------------------
2148
+ # A function's parameters and return type, read from its one-line signature
2149
+ # --------------------------------------------------------------------------
2150
+
2151
+ _OPENERS = {"(": ")", "[": "]", "{": "}", "<": ">"}
2152
+
2153
+
2154
+ def _split_top(text: str, sep: str, angle: bool = True, quotes: str = "'\"`") -> list[str]:
2155
+ """`text` split at each `sep` outside brackets and strings (`<` counts
2156
+ as a bracket when `angle`, and the `>` of `=>` or `->` closes none).
2157
+ `quotes` are the characters that open a string: only `"` in Rust, whose
2158
+ `'a` is a lifetime."""
2159
+ parts, depth, quote, start, i = [], 0, None, 0, 0
2160
+ while i < len(text):
2161
+ ch = text[i]
2162
+ if quote:
2163
+ if ch == "\\":
2164
+ i += 1
2165
+ elif ch == quote:
2166
+ quote = None
2167
+ elif ch in quotes:
2168
+ quote = ch
2169
+ elif ch in "([{" or (angle and ch == "<"):
2170
+ depth += 1
2171
+ elif ch in ")]}" or (angle and ch == ">" and text[i - 1:i] not in ("=", "-")):
2172
+ depth -= 1
2173
+ elif depth == 0 and text.startswith(sep, i) and not (sep == "=" and text[i + 1:i + 2] in ("=", ">")):
2174
+ parts.append(text[start:i])
2175
+ start = i + len(sep)
2176
+ i = start
2177
+ continue
2178
+ i += 1
2179
+ parts.append(text[start:])
2180
+ return parts
2181
+
2182
+
2183
+ def _close(text: str, start: int, angle: bool = True, quotes: str = "'\"`") -> int | None:
2184
+ """The index of the bracket that closes the one at `start`, else None
2185
+ (strings as _split_top reads them)."""
2186
+ depth, quote, i = 0, None, start
2187
+ while i < len(text):
2188
+ ch = text[i]
2189
+ if quote:
2190
+ if ch == "\\":
2191
+ i += 1
2192
+ elif ch == quote:
2193
+ quote = None
2194
+ elif ch in quotes:
2195
+ quote = ch
2196
+ elif ch in "([{" or (angle and ch == "<"):
2197
+ depth += 1
2198
+ elif ch in ")]}" or (angle and ch == ">" and text[i - 1:i] not in ("=", "-")):
2199
+ depth -= 1
2200
+ if depth == 0:
2201
+ return i
2202
+ i += 1
2203
+ return None
2204
+
2205
+
2206
+ def _param(name: str | None, type_: str | None, default: str | None, optional: bool) -> dict:
2207
+ return {"name": name or None, "type": type_ or None, "default": default or None, "optional": optional}
2208
+
2209
+
2210
+ def _python_parts(signature: str) -> tuple[list[dict] | None, str | None]:
2211
+ """A Python def's parameters and return annotation, through its own
2212
+ parser, each as the source writes it."""
2213
+ head = signature.strip()
2214
+ if head.endswith(":"):
2215
+ head = head[:-1]
2216
+ source = head + ": pass"
2217
+ try:
2218
+ tree = ast.parse(source)
2219
+ except SyntaxError:
2220
+ return None, None
2221
+ node = tree.body[0] if tree.body else None
2222
+ if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
2223
+ return None, None
2224
+ a = node.args
2225
+ positional = a.posonlyargs + a.args
2226
+ defaults = [None] * (len(positional) - len(a.defaults)) + list(a.defaults)
2227
+ params = []
2228
+
2229
+ def text(value):
2230
+ # the source's own text (`"x"`, `0x10`), never ast.unparse's normal form (`'x'`, `16`)
2231
+ return (ast.get_source_segment(source, value) or ast.unparse(value)) if value is not None else None
2232
+
2233
+ for arg, default in zip(positional, defaults):
2234
+ params.append(_param(arg.arg, text(arg.annotation), text(default), default is not None))
2235
+ if a.vararg is not None:
2236
+ params.append(_param("*" + a.vararg.arg, text(a.vararg.annotation), None, True))
2237
+ for arg, default in zip(a.kwonlyargs, a.kw_defaults):
2238
+ params.append(_param(arg.arg, text(arg.annotation), text(default), default is not None))
2239
+ if a.kwarg is not None:
2240
+ params.append(_param("**" + a.kwarg.arg, text(a.kwarg.annotation), None, True))
2241
+ return params, text(node.returns)
2242
+
2243
+
2244
+ def _js_param(part: str) -> dict | None:
2245
+ """One JS/TS parameter: `name`, `name?: T`, `name: T = d`, `...rest: T[]`
2246
+ or a destructuring pattern; None for TypeScript's `this` parameter."""
2247
+ part = part.strip()
2248
+ default = None
2249
+ pieces = _split_top(part, "=")
2250
+ if len(pieces) > 1:
2251
+ part, default = pieces[0].strip(), "=".join(pieces[1:]).strip()
2252
+ typed = _split_top(part, ":")
2253
+ name, type_ = typed[0].strip(), (":".join(typed[1:]).strip() if len(typed) > 1 else None)
2254
+ optional = default is not None or name.startswith("...")
2255
+ if name.endswith("?"):
2256
+ name, optional = name[:-1].rstrip(), True
2257
+ if name == "this":
2258
+ return None
2259
+ return _param(name, type_, default, optional)
2260
+
2261
+
2262
+ def _js_parts(signature: str, name: str) -> tuple[list[dict] | None, str | None]:
2263
+ """An exported JS/TS function's parameters and return type: a function
2264
+ declaration (`function name<T>(...): R {`), or a const bound to an arrow
2265
+ function (`const name = async <T>(...): R => ...`, `const name = x => ...`)."""
2266
+ # `$` is an identifier character in JS, so `\b` would never bound `$fetch`
2267
+ at = re.search(r"(?<![\w$])" + re.escape(name) + r"(?![\w$])", signature)
2268
+ if at is None:
2269
+ return None, None
2270
+ rest = signature[at.end():]
2271
+ arrow = re.match(r"\s*(?::[^=]*?)?=\s*(?:async\s+)?", rest)
2272
+ if arrow and not re.match(r"\s*\(", rest):
2273
+ rest = rest[arrow.end():]
2274
+ bare = re.match(r"([A-Za-z_$][\w$]*)\s*=>", rest)
2275
+ if bare:
2276
+ return [_param(bare.group(1), None, None, False)], None
2277
+ else:
2278
+ arrow = None
2279
+ rest = rest.lstrip()
2280
+ if rest.startswith("<"):
2281
+ end = _close(rest, 0)
2282
+ if end is None:
2283
+ return None, None
2284
+ rest = rest[end + 1:].lstrip()
2285
+ if not rest.startswith("("):
2286
+ return None, None
2287
+ end = _close(rest, 0)
2288
+ if end is None:
2289
+ return None, None
2290
+ inner = rest[1:end].strip()
2291
+ params = [p for p in (_js_param(part) for part in _split_top(inner, ",") if part.strip()) if p is not None]
2292
+ tail = rest[end + 1:].strip()
2293
+ if not tail.startswith(":"):
2294
+ return params, None
2295
+ tail = tail[1:].strip()
2296
+ if arrow is not None:
2297
+ pieces = _split_top(tail, "=>")
2298
+ if len(pieces) < 2:
2299
+ return None, None
2300
+ returns = pieces[0].strip()
2301
+ else:
2302
+ returns = tail[:-1].rstrip() if tail.endswith(("{", ";")) else tail
2303
+ return (params, returns) if returns else (None, None)
2304
+
2305
+
2306
+ def _rust_parts(signature: str, name: str) -> tuple[list[dict] | None, str | None]:
2307
+ """A Rust fn's parameters (`self` forms included) and its `->` type."""
2308
+ at = re.search(r"\bfn\s+" + re.escape(name) + r"\b", signature)
2309
+ if at is None:
2310
+ return None, None
2311
+ rest = signature[at.end():].lstrip()
2312
+ if rest.startswith("<"):
2313
+ end = _close(rest, 0, quotes='"')
2314
+ if end is None:
2315
+ return None, None
2316
+ rest = rest[end + 1:].lstrip()
2317
+ if not rest.startswith("("):
2318
+ return None, None
2319
+ end = _close(rest, 0, angle=False, quotes='"')
2320
+ if end is None:
2321
+ return None, None
2322
+ params = []
2323
+ for part in _split_top(rest[1:end], ",", quotes='"'):
2324
+ part = part.strip()
2325
+ if not part:
2326
+ continue
2327
+ typed = _split_top(part, ":", quotes='"')
2328
+ if len(typed) == 1:
2329
+ params.append(_param(part, None, None, False)) # self, &self, &mut self
2330
+ else:
2331
+ params.append(_param(typed[0].strip(), ":".join(typed[1:]).strip(), None, False))
2332
+ tail = rest[end + 1:].strip()
2333
+ if not tail.startswith("->"):
2334
+ return params, None
2335
+ returns = re.split(r"\s+where\s+", tail[2:].strip(), maxsplit=1)[0].strip()
2336
+ returns = returns[:-1].rstrip() if returns.endswith(("{", ";")) else returns
2337
+ return (params, returns) if returns else (None, None)
2338
+
2339
+
2340
+ _GO_TYPE_WORDS = frozenset({"chan", "func", "map", "struct", "interface"})
2341
+
2342
+
2343
+ def _go_parts(signature: str, name: str) -> tuple[list[dict] | None, str | None]:
2344
+ """A Go func's parameters (a type shared by the names before it, as in
2345
+ `a, b int`, given to each) and its result list or type."""
2346
+ at = re.search(r"\bfunc\s+" + re.escape(name) + r"\b", signature)
2347
+ if at is None:
2348
+ return None, None
2349
+ rest = signature[at.end():].lstrip()
2350
+ if rest.startswith("["):
2351
+ end = _close(rest, 0, angle=False, quotes='"`')
2352
+ if end is None:
2353
+ return None, None
2354
+ rest = rest[end + 1:].lstrip()
2355
+ if not rest.startswith("("):
2356
+ return None, None
2357
+ end = _close(rest, 0, angle=False, quotes='"`')
2358
+ if end is None:
2359
+ return None, None
2360
+ parts = [p.strip() for p in _split_top(rest[1:end], ",", angle=False, quotes='"`') if p.strip()]
2361
+ # `a, b int` names its parameters; `chan int` alone is a type
2362
+ named = any(re.match(r"[A-Za-z_]\w*\s+\S", p) and p.split(None, 1)[0] not in _GO_TYPE_WORDS for p in parts)
2363
+ params: list[dict] = []
2364
+ pending: list[str] = []
2365
+ for part in parts:
2366
+ if not named:
2367
+ params.append(_param(None, part, None, part.startswith("...")))
2368
+ continue
2369
+ words = part.split(None, 1)
2370
+ if len(words) == 1:
2371
+ pending.append(words[0])
2372
+ continue
2373
+ type_ = words[1].strip()
2374
+ for waiting in pending + [words[0]]:
2375
+ params.append(_param(waiting, type_, None, type_.startswith("...")))
2376
+ pending = []
2377
+ if pending:
2378
+ return None, None
2379
+ tail = rest[end + 1:].strip()
2380
+ tail = tail[:-1].rstrip() if tail.endswith("{") else tail
2381
+ return params, (tail or None)
2382
+
2383
+
2384
+ def signature_parts(signature: str, language: str, name: str) -> tuple[list[dict] | None, str | None]:
2385
+ """(params, return_type) of an exported function, read from the
2386
+ one-line `signature` its recipe matched: each parameter as {name, type,
2387
+ default, optional}, and the declared return type, null when the
2388
+ declaration states none. (None, None) when the signature is a form this
2389
+ reader does not parse, such as a declaration cut at its line cap."""
2390
+ if language == "python":
2391
+ return _python_parts(signature)
2392
+ if language in ("typescript", "tsx", "javascript"):
2393
+ return _js_parts(signature, name)
2394
+ if language == "rust":
2395
+ return _rust_parts(signature, name)
2396
+ if language == "go":
2397
+ return _go_parts(signature, name)
2398
+ return None, None
2399
+
2400
+
2401
+ def _preferred(at_line: list[dict], forms: dict, order: dict[str, int]) -> dict:
2402
+ """The match an export records among several recipes' matches of one
2403
+ name on one line: one that no other lists under prefer_over, then the
2404
+ first in the recipe file."""
2405
+ ids = {m["rule"] for m in at_line}
2406
+ beaten = {loser for m in at_line
2407
+ for loser in forms[(m["rule"], m["language"])]["metadata"].get("prefer_over", [])
2408
+ if loser in ids and loser != m["rule"]}
2409
+ left = [m for m in at_line if m["rule"] not in beaten] or at_line
2410
+ return min(left, key=lambda m: (order[m["rule"]], m["language"]))
2411
+
2412
+
2413
+ def merge_exports(matches: list[dict], recipes: list[dict], recipe_set: str,
2414
+ head_cap: int | None) -> tuple[list[dict], list[dict]]:
2415
+ """(exports, per-recipe stats) from the matches of one recipe set: at
2416
+ most `head_cap` matches of each recipe, in file and line order, merged
2417
+ by (file, name) at the first line that names it."""
2418
+ forms = _forms(recipes)
2419
+ order: dict[str, int] = {}
2420
+ languages: dict[str, list[str]] = {}
2421
+ for recipe in recipes:
2422
+ if recipe_set in recipe["metadata"]["sets"]:
2423
+ order.setdefault(recipe["id"], len(order))
2424
+ langs = languages.setdefault(recipe["id"], [])
2425
+ langs.extend(lang for lang in recipe["metadata"]["languages"] if lang not in langs)
2426
+ per_recipe: dict[str, list[dict]] = {rid: [] for rid in order}
2427
+ for m in matches:
2428
+ if m["rule"] in per_recipe and m["name"] and not m["name"].startswith("_") and (m["rule"], m["language"]) in forms:
2429
+ per_recipe[m["rule"]].append(m)
2430
+ stats, kept = [], []
2431
+ for rid, found in per_recipe.items():
2432
+ found.sort(key=lambda m: (m["file"], m["line"], m["name"]))
2433
+ cut = head_cap is not None and len(found) > head_cap
2434
+ stats.append({"id": rid, "languages": languages[rid], "matches": len(found), "truncated": cut})
2435
+ kept.extend(found[:head_cap] if cut else found)
2436
+ groups: dict[tuple[str, str], list[dict]] = {}
2437
+ for m in kept:
2438
+ groups.setdefault((m["file"], m["name"]), []).append(m)
2439
+ exports = []
2440
+ for group in groups.values():
2441
+ first_line = min(m["line"] for m in group)
2442
+ chosen = _preferred([m for m in group if m["line"] == first_line], forms, order)
2443
+ exports.append(_export_record(chosen, forms[(chosen["rule"], chosen["language"])]))
2444
+ exports.sort(key=lambda e: (e["source_file"], e["source_line"], e["export_name"]))
2445
+ return exports, stats
2446
+
2447
+
2448
+ def _syntax_issues(matches: list[dict]) -> list[dict]:
2449
+ """One `syntax-errors` issue per file with an outermost ERROR node."""
2450
+ per_file: dict[str, tuple[int, int]] = {}
2451
+ for m in matches:
2452
+ if m["rule"] == SYNTAX_ERROR_RULE:
2453
+ count, line = per_file.get(m["file"], (0, m["start_line"]))
2454
+ per_file[m["file"]] = (count + 1, min(line, m["start_line"]))
2455
+ return [{"file": f, "issue": "syntax-errors", "count": c, "line": ln}
2456
+ for f, (c, ln) in sorted(per_file.items())]
2457
+
2458
+
2459
+ def _type_list_items(exe: str, root: Path, folder: Path, matches: list[dict], exports: list[dict],
2460
+ errors: list[dict], clock: _Clock | None = None) -> None:
2461
+ """Give each export a local `export { ... }` list names (an
2462
+ export_specifier with no `from`) the type of the top-level binding it
2463
+ names in its file, as BINDING_TYPES has it; the recipe's own
2464
+ export_type stays when no such binding has the name (a destructured
2465
+ one)."""
2466
+ items = [e for e in exports if e["ast_node_type"] == "export_specifier" and e["from"] is None]
2467
+ if not items:
2468
+ return
2469
+ local: dict[tuple[str, str, str], str] = {}
2470
+ for m in matches:
2471
+ if m["name"] and m["source"] is None:
2472
+ local.setdefault((m["file"], m["rule"], m["name"]), _specifier_local(m["text"]))
2473
+ files = sorted({e["source_file"] for e in items})
2474
+ languages = sorted({lang for f in files for lang in _rule_languages(_file_language(f) or "")}
2475
+ & set(RECIPE_LANGUAGES))
2476
+ rules = [rule for language in languages for rule in _binding_rules(language)]
2477
+ types = _binding_types(_scan(exe, root, folder, rules, files, errors, clock))
2478
+ for e in items:
2479
+ name = local.get((e["source_file"], e["ast_recipe"], e["export_name"]), e["export_name"])
2480
+ e["export_type"] = types.get((e["source_file"], name), e["export_type"])
2481
+
2482
+
2483
+ # --------------------------------------------------------------------------
2484
+ # Comments and strings, blanked for the text readers below
2485
+ # --------------------------------------------------------------------------
2486
+
2487
+ _RUST_RAW_OPEN_RE = re.compile(r'r(#*)"')
2488
+ _RUST_CHAR_RE = re.compile(r"'(?:\\(?:x[0-9A-Fa-f]{2}|u\{[0-9A-Fa-f]{1,6}\}|.)|[^\\'\n])'")
2489
+ # What a JavaScript `/` follows when it opens a regular expression literal
2490
+ # rather than dividing: punctuation no operand ends with, or a keyword.
2491
+ _JS_REGEX_AFTER = frozenset("(,=:[!&|?{};")
2492
+ _JS_REGEX_AFTER_WORDS = frozenset({
2493
+ "return", "typeof", "case", "do", "else", "in", "of", "instanceof", "new", "delete", "void", "throw",
2494
+ "yield", "await",
2495
+ })
2496
+
2497
+
2498
+ def _js_regex_end(text: str, done: list[str], i: int) -> int | None:
2499
+ """The index of the `/` that closes the regular expression literal the
2500
+ `/` at `i` opens, or None when that `/` divides. `done` is the text up
2501
+ to `i` with its comments and strings blanked; a literal opens a line or
2502
+ follows _JS_REGEX_AFTER or a word of _JS_REGEX_AFTER_WORDS (a `}` before
2503
+ a JSX `/>` aside), and ends on its line."""
2504
+ k = i - 1
2505
+ while k >= 0 and done[k] in " \t\r":
2506
+ k -= 1
2507
+ before = done[k] if k >= 0 else "\n"
2508
+ if before == "}" and text.startswith(">", i + 1):
2509
+ return None
2510
+ if before != "\n" and before not in _JS_REGEX_AFTER:
2511
+ end = k + 1
2512
+ while k >= 0 and (done[k].isalnum() or done[k] in "_$"):
2513
+ k -= 1
2514
+ if "".join(done[k + 1:end]) not in _JS_REGEX_AFTER_WORDS or (k >= 0 and done[k] == "."):
2515
+ return None
2516
+ in_class, j = False, i + 1
2517
+ while j < len(text) and text[j] != "\n":
2518
+ ch = text[j]
2519
+ if ch == "\\":
2520
+ j += 2
2521
+ continue
2522
+ if in_class:
2523
+ in_class = ch != "]"
2524
+ elif ch == "[":
2525
+ in_class = True
2526
+ elif ch == "/":
2527
+ return j
2528
+ j += 1
2529
+ return None
2530
+
2531
+
2532
+ def _blank(text: str, language: str) -> str:
2533
+ """`text` with its comments, and the insides of its strings, turned into
2534
+ spaces, its lines kept: a pattern run over it finds code only.
2535
+ JavaScript/TypeScript (`js`): `//`, `/* */` and '...', "..." and `...`
2536
+ strings, a string followed within its line (a template literal spanning
2537
+ lines is not tracked), and the insides of a regular expression literal
2538
+ (`/\\/*$/` opens no comment). Go: `//`, `/* */`, "..." and '...' within a
2539
+ line and `...` raw strings across lines. Rust: `//`, nested `/* */`,
2540
+ "..." and r#"..."# strings across lines, and char literals."""
2541
+ out = list(text)
2542
+ n = len(text)
2543
+
2544
+ def blank(start: int, end: int) -> None:
2545
+ for k in range(start, min(end, n)):
2546
+ if out[k] != "\n":
2547
+ out[k] = " "
2548
+
2549
+ quotes = "\"'`" if language in ("js", "go") else '"'
2550
+ i = 0
2551
+ while i < n:
2552
+ if text.startswith("//", i):
2553
+ end = text.find("\n", i)
2554
+ end = n if end < 0 else end
2555
+ blank(i, end)
2556
+ i = end
2557
+ continue
2558
+ if text.startswith("/*", i):
2559
+ depth, j = 1, i + 2
2560
+ while j < n and depth:
2561
+ if language == "rust" and text.startswith("/*", j):
2562
+ depth, j = depth + 1, j + 2
2563
+ elif text.startswith("*/", j):
2564
+ depth, j = depth - 1, j + 2
2565
+ else:
2566
+ j += 1
2567
+ blank(i, j)
2568
+ i = j
2569
+ continue
2570
+ ch = text[i]
2571
+ if language == "js" and ch == "/":
2572
+ end = _js_regex_end(text, out, i)
2573
+ if end is not None:
2574
+ blank(i + 1, end)
2575
+ i = end + 1
2576
+ continue
2577
+ if language == "rust":
2578
+ raw = _RUST_RAW_OPEN_RE.match(text, i) if ch == "r" else None
2579
+ if raw and not (i and (text[i - 1].isalnum() or text[i - 1] == "_")):
2580
+ close = '"' + raw.group(1)
2581
+ end = text.find(close, raw.end())
2582
+ end = n if end < 0 else end
2583
+ blank(raw.end(), end)
2584
+ i = end + len(close)
2585
+ continue
2586
+ char = _RUST_CHAR_RE.match(text, i) if ch == "'" else None
2587
+ if char:
2588
+ i = char.end()
2589
+ continue
2590
+ if ch in quotes:
2591
+ across = language == "rust" or (language == "go" and ch == "`")
2592
+ escapes = not (language == "go" and ch == "`")
2593
+ j = i + 1
2594
+ while j < n and text[j] != ch and (across or text[j] != "\n"):
2595
+ j += 2 if escapes and text[j] == "\\" else 1
2596
+ blank(i + 1, j)
2597
+ i = j + 1 if j < n and text[j] == ch else j
2598
+ continue
2599
+ i += 1
2600
+ return "".join(out)
2601
+
2602
+
2603
+ def _read_text(root: Path, rel: str) -> str | None:
2604
+ try:
2605
+ return (root / rel).read_text(encoding="utf-8")
2606
+ except (OSError, UnicodeDecodeError):
2607
+ return None
2608
+
2609
+
2610
+ # --------------------------------------------------------------------------
2611
+ # Entry points: JavaScript and TypeScript
2612
+ # --------------------------------------------------------------------------
2613
+
2614
+ _JS_DECLARATION_RE = re.compile(
2615
+ r"^export\s+(?:declare\s+)?(default\s+)?(?:abstract\s+)?(?:async\s+)?"
2616
+ r"(?:const\s+enum|const|let|var|function\s*\*?|class|interface|type|enum|namespace|module)"
2617
+ r"\s+([A-Za-z_$][\w$]*)",
2618
+ re.M,
2619
+ )
2620
+ _JS_IMPORT_EQUALS_RE = re.compile(r"^export\s+import()\s+([A-Za-z_$][\w$]*)\s*=", re.M)
2621
+ _JS_DEFAULT_NAME_RE = re.compile(r"^export\s+default\s+([A-Za-z_$][\w$]*)\s*;?\s*$", re.M)
2622
+ _JS_DEFAULT_EXPORT_RE = re.compile(r"export\s+default\b")
2623
+
2624
+
2625
+ def _js_text_names(text: str) -> tuple[list[tuple[str, int, bool]], list[tuple[str, int]]]:
2626
+ """The names a JS/TS file's column-0 `export` declarations bind, read
2627
+ from its text for the forms no recipe matches (`export let`, `export
2628
+ declare const`, `export namespace`), as (name, line, is the default
2629
+ export), and its `export default <name>;` statements, as (name, line)."""
2630
+ code = _blank(text, "js")
2631
+ declared = [(m.group(2), _line_of(code, m.start(2)), bool(m.group(1)))
2632
+ for rx in (_JS_DECLARATION_RE, _JS_IMPORT_EQUALS_RE) for m in rx.finditer(code)]
2633
+ defaults = [(m.group(1), _line_of(code, m.start(1))) for m in _JS_DEFAULT_NAME_RE.finditer(code)]
2634
+ return declared, defaults
2635
+
2636
+
2637
+ def resolve_js_module(spec: str | None, from_file: str, tree: set[str]) -> str | None:
2638
+ """The file a relative import or re-export names: the path, then with
2639
+ each JS/TS extension, then its index file, a `.js` specifier naming its
2640
+ TypeScript source first. None for a package name or a path outside the
2641
+ root."""
2642
+ if not spec or not spec.startswith("."):
2643
+ return None
2644
+ base = posixpath.normpath(posixpath.join(posixpath.dirname(from_file), spec))
2645
+ if base == ".." or base.startswith("../"):
2646
+ return None
2647
+ base = "" if base == "." else base
2648
+ stem, ext = posixpath.splitext(base)
2649
+ candidates = [base] if base else []
2650
+ candidates += [stem + e for e in JS_SOURCE_OF.get(ext, ())]
2651
+ candidates += [base + e for e in JS_RESOLVE_EXTENSIONS]
2652
+ candidates += [posixpath.join(base, "index" + e) for e in JS_RESOLVE_EXTENSIONS]
2653
+ return next((c for c in candidates if c in tree), None)
2654
+
2655
+
2656
+ def _targets(value: object) -> list[str]:
2657
+ """Every target path of a package.json `exports` value, conditions and
2658
+ fallbacks included."""
2659
+ if isinstance(value, str):
2660
+ return [value]
2661
+ if isinstance(value, list):
2662
+ return [t for item in value for t in _targets(item)]
2663
+ if isinstance(value, dict):
2664
+ return [t for item in value.values() for t in _targets(item)]
2665
+ return []
2666
+
2667
+
2668
+ def exports_subpaths(exports: object) -> dict[str, list[str]]:
2669
+ """A package.json `exports` value as {subpath: its target paths}."""
2670
+ if isinstance(exports, dict):
2671
+ keys = [k for k in exports if isinstance(k, str)]
2672
+ if keys and all(k.startswith(".") for k in keys):
2673
+ return {k: _targets(exports[k]) for k in keys}
2674
+ targets = _targets(exports)
2675
+ return {".": targets} if targets else {}
2676
+
2677
+
2678
+ def _source_stems(rel: str) -> list[str]:
2679
+ """The stems a build output's source can have, in the order tried: the
2680
+ path without its extension, then with one more of its leading build
2681
+ folders swapped for `src/` or dropped at each try (`dist/esm/index.js`
2682
+ -> dist/esm/index, src/esm/index, esm/index, src/index, index)."""
2683
+ stem = next((rel[: -len(s)] for s in JS_TARGET_SUFFIXES if rel.endswith(s)), rel)
2684
+ parts = stem.split("/")
2685
+ stems = [stem]
2686
+ for depth in range(1, len(parts)):
2687
+ if parts[depth - 1] not in BUILD_DIRS:
2688
+ break
2689
+ rest = "/".join(parts[depth:])
2690
+ stems += [f"src/{rest}", rest]
2691
+ return stems
2692
+
2693
+
2694
+ def _package_target(pkg: str, target: str, tree: set[str]) -> tuple[str | None, str]:
2695
+ """(the file a package.json target names, `file`), or, for a build
2696
+ output that is not committed, (its source, `source-guess`): each of its
2697
+ source stems with each source extension (`dist/esm/index.js` ->
2698
+ src/index.ts, `build/lib/util.js` -> lib/util.ts)."""
2699
+
2700
+ def join(rel: str) -> str:
2701
+ return f"{pkg}/{rel}" if pkg else rel
2702
+
2703
+ rel = posixpath.normpath(_norm(target))
2704
+ if rel in (".", "..") or rel.startswith("../"):
2705
+ return None, "unresolved"
2706
+ if join(rel) in tree:
2707
+ return join(rel), "file"
2708
+ for stem in _source_stems(rel):
2709
+ for ext in JS_SOURCE_EXTENSIONS:
2710
+ if join(stem + ext) in tree:
2711
+ return join(stem + ext), "source-guess"
2712
+ return None, "unresolved"
2713
+
2714
+
2715
+ def _pattern_files(pkg: str, pattern: str, tree: set[str]) -> dict[str, str]:
2716
+ """{what the `*` stands for: file} of the files a path pattern with one
2717
+ `*`, which spans folders as in Node, names in package folder `pkg`."""
2718
+ head, _, tail = (f"{pkg}/{pattern}" if pkg else pattern).partition("*")
2719
+ return {path[len(head): len(path) - len(tail)]: path for path in sorted(tree)
2720
+ if path.startswith(head) and path.endswith(tail) and len(path) > len(head) + len(tail)}
2721
+
2722
+
2723
+ def _package_pattern(pkg: str, target: str, tree: set[str]) -> tuple[dict[str, str], str]:
2724
+ """A package.json target with one `*` (a subpath pattern's): ({what the
2725
+ `*` stands for: file} of the files it names, `file`), else, for a
2726
+ build output that is not committed, those of the first source stem
2727
+ that names any (`source-guess`)."""
2728
+ rel = posixpath.normpath(_norm(target))
2729
+ if rel.count("*") != 1 or rel.startswith("../"):
2730
+ return {}, "unresolved"
2731
+ found = _pattern_files(pkg, rel, tree)
2732
+ if found:
2733
+ return found, "file"
2734
+ for stem in _source_stems(rel):
2735
+ for ext in JS_SOURCE_EXTENSIONS:
2736
+ for key, file in _pattern_files(pkg, stem + ext, tree).items():
2737
+ found.setdefault(key, file)
2738
+ if found:
2739
+ return found, "source-guess"
2740
+ return {}, "unresolved"
2741
+
2742
+
2743
+ def _subpath_excluded(subpath: str, excluded: list[str]) -> bool:
2744
+ """Whether a subpath (or subpath pattern) the exports map maps to `null`
2745
+ takes `subpath` out of the package's surface."""
2746
+ for pattern in excluded:
2747
+ head, star, tail = pattern.partition("*")
2748
+ if subpath == pattern or (star and subpath.startswith(head) and subpath.endswith(tail)
2749
+ and len(subpath) >= len(head) + len(tail)):
2750
+ return True
2751
+ return False
2752
+
2753
+
2754
+ def _owners(files: list[str], manifest: str, tree: list[str]) -> list[str]:
2755
+ """The folders ('' for the root) of the nearest `manifest` above each
2756
+ file."""
2757
+ dirs = {posixpath.dirname(p) for p in tree if posixpath.basename(p) == manifest}
2758
+ owners = set()
2759
+ for rel in files:
2760
+ folder = posixpath.dirname(rel)
2761
+ while True:
2762
+ if folder in dirs:
2763
+ owners.add(folder)
2764
+ break
2765
+ if not folder:
2766
+ break
2767
+ folder = posixpath.dirname(folder)
2768
+ return sorted(owners)
2769
+
2770
+
2771
+ def _js_entries(root: Path, files: list[str], tree: list[str], tree_set: set[str],
2772
+ warnings: list[str]) -> tuple[list[dict], list[dict], list[dict]]:
2773
+ """(entry points, exports-map facts, unresolved subpaths) of the
2774
+ packages that own the in-scope JS/TS files: the files each subpath of
2775
+ an `exports` map names (a pattern subpath, one with `*`, one entry per
2776
+ file its target matches, less those a `null` target takes out), else
2777
+ the package's types/module/main/browser fields, else its index file."""
2778
+ js_files = [f for f in files if _file_language(f) in JS_FAMILY]
2779
+ packages = _owners(js_files, "package.json", tree) or ([""] if js_files else [])
2780
+ entries: list[dict] = []
2781
+ maps: list[dict] = []
2782
+ unresolved: list[dict] = []
2783
+ for pkg in packages:
2784
+ found, facts, missing = _js_package_entries(root, pkg, tree_set, warnings)
2785
+ entries += found
2786
+ maps += facts
2787
+ unresolved += missing
2788
+ return entries, maps, unresolved
2789
+
2790
+
2791
+ def _js_package_entries(root: Path, pkg: str, tree_set: set[str],
2792
+ warnings: list[str]) -> tuple[list[dict], list[dict], list[dict]]:
2793
+ """(entry points, exports-map facts, unresolved subpaths) of the one
2794
+ package in folder `pkg` ("" for the root), by _js_entries's rules, its
2795
+ package.json read from `root`."""
2796
+ label = pkg or "."
2797
+ entries: list[dict] = []
2798
+ maps: list[dict] = []
2799
+ unresolved: list[dict] = []
2800
+
2801
+ def add(file: str, how: str, subpath: str) -> None:
2802
+ if not any(e["file"] == file and e["subpath"] == subpath for e in entries):
2803
+ entries.append({"language": _file_language(file) or "javascript", "file": file,
2804
+ "package": label, "subpath": subpath, "resolution": how})
2805
+
2806
+ manifest: dict = {}
2807
+ manifest_path = f"{pkg}/package.json" if pkg else "package.json"
2808
+ if manifest_path in tree_set:
2809
+ try:
2810
+ loaded = json.loads((root / manifest_path).read_text(encoding="utf-8"))
2811
+ manifest = loaded if isinstance(loaded, dict) else {}
2812
+ except (OSError, UnicodeDecodeError, ValueError) as exc:
2813
+ warnings.append(f"{manifest_path}: cannot read ({exc}); its entry points are the index files")
2814
+ subpaths = exports_subpaths(manifest.get("exports")) if "exports" in manifest else {}
2815
+ if subpaths:
2816
+ code = [s for s in subpaths if s != "." and "*" not in s
2817
+ and any(t.endswith(JS_TARGET_SUFFIXES) for t in subpaths[s])]
2818
+ maps.append({"package": label, "subpaths": sorted(subpaths),
2819
+ "non_root_subpaths": len(code),
2820
+ "wildcard_subpaths": sum(1 for s in subpaths if "*" in s)})
2821
+ else:
2822
+ fields = [manifest[f] for f in PACKAGE_ENTRY_FIELDS if isinstance(manifest.get(f), str)]
2823
+ subpaths = {".": fields} if fields else {}
2824
+ excluded = [s for s, targets in subpaths.items() if not targets]
2825
+ for subpath, targets in subpaths.items():
2826
+ code_targets = [t for t in targets if t.endswith(JS_TARGET_SUFFIXES)]
2827
+ found = 0
2828
+ for target in code_targets:
2829
+ if "*" in target:
2830
+ matched, how = _package_pattern(pkg, target, tree_set)
2831
+ found += len(matched)
2832
+ for key, file in matched.items():
2833
+ named = subpath.replace("*", key, 1)
2834
+ if not _subpath_excluded(named, excluded):
2835
+ add(file, how, named)
2836
+ continue
2837
+ file, how = _package_target(pkg, target, tree_set)
2838
+ if file:
2839
+ add(file, how, subpath)
2840
+ found += 1
2841
+ if code_targets and not found:
2842
+ unresolved.append({"package": label, "subpath": subpath, "targets": code_targets})
2843
+ if "." in subpaths or not subpaths:
2844
+ if not any(e["subpath"] == "." for e in entries):
2845
+ folder = f"{pkg}/" if pkg else ""
2846
+ conventional = next((f"{folder}{d}index{e}" for d in ("", "src/") for e in JS_SOURCE_EXTENSIONS
2847
+ if f"{folder}{d}index{e}" in tree_set), None)
2848
+ if conventional:
2849
+ entries.append({"language": _file_language(conventional), "file": conventional,
2850
+ "package": label, "subpath": ".", "resolution": "conventional"})
2851
+ unresolved = [u for u in unresolved if u["subpath"] != "."]
2852
+ return entries, maps, unresolved
2853
+
2854
+
2855
+ def _specifier_local(text: str) -> str:
2856
+ """The local (or imported) side of an export or import specifier's text:
2857
+ `a as b` -> a, `type T` -> T, `"a-b" as c` -> a-b."""
2858
+ item = text.strip()
2859
+ if item.startswith("type ") or item.startswith("typeof "):
2860
+ item = item.split(None, 1)[1]
2861
+ return re.split(r"\s+as\s+", item, maxsplit=1)[0].strip().strip("'\"")
2862
+
2863
+
2864
+ class _Surface:
2865
+ """The public names one language family's entry points declare:
2866
+ {name: info}, the first entry point naming it wins; the names, and the
2867
+ files, a namespace export (a module) makes reachable, which are not
2868
+ internal; and the re-exports the trace could not follow."""
2869
+
2870
+ def __init__(self) -> None:
2871
+ self.names: dict[str, dict] = {}
2872
+ self.reachable: set[str] = set()
2873
+ self.reachable_files: set[str] = set()
2874
+ self.unresolved: list[dict] = []
2875
+
2876
+ def add(self, name: str, entry: str, via: str, file: str | None, line: int | None,
2877
+ source: str | None = None, local: str | None = None) -> dict | None:
2878
+ if not name or name.startswith("_"):
2879
+ return None
2880
+ if name not in self.names:
2881
+ self.names[name] = {"name": name, "entry": entry, "via": via, "from": source,
2882
+ "local": local if local and local != name else None,
2883
+ "file": file, "line": line}
2884
+ return self.names[name]
2885
+
2886
+ def unresolve(self, entry: str, name: str, source: str | None, file: str) -> None:
2887
+ """A re-export in `file`, met tracing `entry`, that names no file here."""
2888
+ item = {"entry": entry, "name": name, "from": source, "file": file}
2889
+ if item not in self.unresolved:
2890
+ self.unresolved.append(item)
2891
+
2892
+
2893
+ def _js_facts(matches: list[dict], text: str, forms: dict, barrel_ids: set[str]) -> dict:
2894
+ """What one JS/TS module exports, read from its scan matches (the
2895
+ standard export recipes' and the trace rules') and, for the forms no
2896
+ recipe matches, its text: {"names": {exposed name: fact}, "stars":
2897
+ [(source, line)], "imports": {local name: (imported, source)}}, the
2898
+ imported name being `default` for a default import and `*` for a
2899
+ namespace import. A fact is {"how", "line", "local", "source"}:
2900
+ `declaration` (the module's own binding `local`), `re-export` (name
2901
+ `local` of module `source`: a named re-export, or a list item or the
2902
+ default export of an imported binding) or `namespace` (module `source`
2903
+ as a whole). The default export is the name `default`, `local` naming
2904
+ what it declares. Of two facts for one name, a declaration beats a list
2905
+ item, which beats a name read from the text, then the first line wins."""
2906
+ imports: dict[str, tuple[str, str | None]] = {}
2907
+ stars: list[tuple[str | None, int]] = []
2908
+ ordered = sorted(matches, key=lambda m: (m["line"], m["name"] or ""))
2909
+ for m in ordered:
2910
+ if m["rule"] in IMPORT_RULES and m["name"]:
2911
+ imported = {IMPORT_DEFAULT_RULE: "default", IMPORT_NAMESPACE_RULE: "*"}.get(m["rule"])
2912
+ imports.setdefault(m["name"], (imported or _specifier_local(m["text"]), m["source"]))
2913
+ elif m["rule"] == STAR_RULE:
2914
+ stars.append((m["source"], m["line"]))
2915
+ ranked: dict[str, tuple[tuple[int, int], dict]] = {}
2916
+
2917
+ def put(name: str, rank: int, how: str, line: int | None, local: str | None, source: str | None) -> None:
2918
+ key = (rank, line or 0)
2919
+ if name and (name not in ranked or key < ranked[name][0]):
2920
+ ranked[name] = (key, {"how": how, "line": line, "local": local, "source": source})
2921
+
2922
+ def bound(name: str, local: str, line: int | None, rank: int) -> None:
2923
+ """`name` exported for the module's binding `local`, which an import may bind."""
2924
+ imported, source = imports.get(local, (None, None))
2925
+ if imported == "*":
2926
+ put(name, rank, "namespace", line, None, source)
2927
+ elif imported is not None:
2928
+ put(name, rank, "re-export", line, imported, source)
2929
+ else:
2930
+ put(name, rank, "declaration", line, local, None)
2931
+
2932
+ for m in ordered:
2933
+ if m["rule"] not in barrel_ids or not m["name"]:
2934
+ continue
2935
+ if forms[(m["rule"], m["language"])]["rule"]["kind"] == "export_specifier":
2936
+ if m["source"]:
2937
+ put(m["name"], 1, "re-export", m["line"], _specifier_local(m["text"]), m["source"])
2938
+ else:
2939
+ bound(m["name"], _specifier_local(m["text"]), m["line"], 1)
2940
+ elif m["source"]:
2941
+ put(m["name"], 0, "namespace", m["line"], None, m["source"])
2942
+ elif _JS_DEFAULT_EXPORT_RE.match(m["text"]):
2943
+ put("default", 0, "declaration", m["line"], m["name"], None)
2944
+ else:
2945
+ put(m["name"], 0, "declaration", m["line"], m["name"], None)
2946
+ declared, defaults = _js_text_names(text)
2947
+ for name, line, is_default in declared:
2948
+ put("default" if is_default else name, 2, "declaration", line, name, None)
2949
+ for local, line in defaults:
2950
+ bound("default", local, line, 2)
2951
+ return {"names": {name: fact for name, (_, fact) in ranked.items()}, "stars": stars, "imports": imports}
2952
+
2953
+
2954
+ def _js_surface(root: Path, entries: list[dict], tree_set: set[str], forms: dict,
2955
+ barrel_ids: set[str], scan: Callable[[list[str]], dict[str, list[dict]]],
2956
+ surface: _Surface) -> None:
2957
+ """Add the names the JS/TS entry points export: those each declares,
2958
+ re-exports or exports as a namespace, and those its `export *`
2959
+ statements pass on through every module they chain to (a default
2960
+ export aside). A re-exported name is traced to the module that declares
2961
+ it, through each re-export, imported binding and `export *` between (a
2962
+ cycle ends the trace); a namespace export makes reachable the names its
2963
+ module passes on, traced the same way, and those of the namespaces
2964
+ among them. The modules the exports lead to are scanned a level at a
2965
+ time."""
2966
+ entry_files = sorted({e["file"] for e in entries})
2967
+ by_file = dict(scan(entry_files))
2968
+ cache: dict[str, dict] = {}
2969
+
2970
+ def facts(file: str) -> dict:
2971
+ if file not in cache:
2972
+ cache[file] = _js_facts(by_file.get(file, []), _read_text(root, file) or "", forms, barrel_ids)
2973
+ return cache[file]
2974
+
2975
+ def target(source: str | None, file: str) -> str | None:
2976
+ return resolve_js_module(source, file, tree_set)
2977
+
2978
+ seen, level = set(entry_files), entry_files
2979
+ while level:
2980
+ found: set[str] = set()
2981
+ for file in level:
2982
+ module = facts(file)
2983
+ sources = [s for s, _ in module["stars"]] + [fact["source"] for fact in module["names"].values()]
2984
+ found.update(t for t in (target(s, file) for s in sources) if t is not None and t not in seen)
2985
+ seen |= found
2986
+ level = sorted(found)
2987
+ by_file.update(scan(level))
2988
+
2989
+ closures: dict[str, list[str]] = {}
2990
+ passed: dict[str, dict[str, str]] = {}
2991
+
2992
+ def closure(file: str) -> list[str]:
2993
+ """`file`, then each module its `export *` statements chain to."""
2994
+ if file not in closures:
2995
+ order: list[str] = []
2996
+ stack, visited = [file], set()
2997
+ while stack:
2998
+ module = stack.pop()
2999
+ if module in visited:
3000
+ continue
3001
+ visited.add(module)
3002
+ order.append(module)
3003
+ stack += reversed([t for t in (target(s, module) for s, _ in facts(module)["stars"]) if t])
3004
+ closures[file] = order
3005
+ return closures[file]
3006
+
3007
+ def passed_on(file: str) -> dict[str, str]:
3008
+ """{name: the module that exports it} of what `export * from` the
3009
+ module `file` passes on: its own names, then its chain's."""
3010
+ if file not in passed:
3011
+ out: dict[str, str] = {}
3012
+ for module in closure(file):
3013
+ for name in facts(module)["names"]:
3014
+ if name != "default":
3015
+ out.setdefault(name, module)
3016
+ passed[file] = out
3017
+ return passed[file]
3018
+
3019
+ def resolve(file: str, name: str, chain: set[str]) -> tuple[str, str, int | None, dict | None]:
3020
+ """(module, its name there, line, fact) of what `name`, as module
3021
+ `file` exports it, comes from: a declaration or a namespace export,
3022
+ or a re-export from a module no file here holds, where the trace
3023
+ stops. The fact and the line are None where no module exports it;
3024
+ `chain` collects each name met."""
3025
+ visited: set[tuple[str, str]] = set()
3026
+ while (file, name) not in visited:
3027
+ visited.add((file, name))
3028
+ chain.add(name)
3029
+ fact = facts(file)["names"].get(name)
3030
+ if fact is None:
3031
+ owner = passed_on(file).get(name) if name != "default" else None
3032
+ if owner is None or owner == file:
3033
+ return file, name, None, None
3034
+ file = owner
3035
+ continue
3036
+ follow = target(fact["source"], file) if fact["how"] == "re-export" else None
3037
+ if follow is None:
3038
+ return file, fact["local"] or name, fact["line"], fact
3039
+ file, name = follow, fact["local"]
3040
+ return file, name, None, None
3041
+
3042
+ reached: set[str] = set()
3043
+
3044
+ def reach(file: str | None) -> None:
3045
+ """Make reachable what the namespace module `file` passes on."""
3046
+ if file is None or file in reached:
3047
+ return
3048
+ reached.add(file)
3049
+ for name, owner in passed_on(file).items():
3050
+ chain: set[str] = set()
3051
+ where, local, _, fact = resolve(owner, name, chain)
3052
+ surface.reachable.update(chain | {local})
3053
+ if fact is not None and fact["how"] == "namespace":
3054
+ reach(target(fact["source"], where))
3055
+
3056
+ def publish(entry: str, name: str, via: str, module: str, source: str | None) -> None:
3057
+ """Add public `name`, which `module` exports, traced to what it is."""
3058
+ chain: set[str] = set()
3059
+ where, local, line, fact = resolve(module, name, chain)
3060
+ surface.reachable.update(chain)
3061
+ how = fact["how"] if fact is not None else None
3062
+ if how == "namespace":
3063
+ namespace = target(fact["source"], where)
3064
+ surface.add(name, entry, "namespace", namespace, None, source)
3065
+ if namespace is None:
3066
+ surface.unresolve(entry, name, fact["source"], where)
3067
+ reach(namespace)
3068
+ elif how == "re-export":
3069
+ surface.add(name, entry, via, None, None, source, local)
3070
+ surface.unresolve(entry, name, fact["source"], where)
3071
+ else:
3072
+ surface.add(name, entry, via, where, line, source, local)
3073
+
3074
+ for entry in entry_files:
3075
+ for name, fact in facts(entry)["names"].items():
3076
+ if fact["how"] == "declaration":
3077
+ surface.add(name, entry, "declaration", entry, fact["line"], local=fact["local"])
3078
+ else:
3079
+ publish(entry, name, "re-export", entry, fact["source"])
3080
+ for entry in entry_files:
3081
+ for source, _ in facts(entry)["stars"]:
3082
+ start = target(source, entry)
3083
+ if start is None:
3084
+ surface.unresolve(entry, "*", source, entry)
3085
+ continue
3086
+ for name, owner in passed_on(start).items():
3087
+ publish(entry, name, "star", owner, source)
3088
+ for module in closure(start):
3089
+ for inner, _ in facts(module)["stars"]:
3090
+ if target(inner, module) is None:
3091
+ surface.unresolve(entry, "*", inner, module)
3092
+
3093
+
3094
+ # --------------------------------------------------------------------------
3095
+ # Entry points: Python
3096
+ # --------------------------------------------------------------------------
3097
+
3098
+
3099
+ def _type_checking(node: ast.stmt) -> bool:
3100
+ """An `if TYPE_CHECKING:` (or `typing.TYPE_CHECKING`) block: its body
3101
+ never runs, so what it imports is no attribute of the module."""
3102
+ test = node.test if isinstance(node, ast.If) else None
3103
+ return (isinstance(test, ast.Name) and test.id == "TYPE_CHECKING") or (
3104
+ isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING"
3105
+ and isinstance(test.value, ast.Name) and test.value.id in ("typing", "typing_extensions"))
3106
+
3107
+
3108
+ def _module_statements(body: list) -> Iterator[ast.stmt]:
3109
+ """A module's statements, those inside its module-level if, try, with,
3110
+ for, while and match blocks included, never a function's or a class's,
3111
+ nor those of an `if TYPE_CHECKING:` body (its `else` runs)."""
3112
+ for node in body:
3113
+ yield node
3114
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
3115
+ continue
3116
+ for field in ("orelse",) if _type_checking(node) else ("body", "orelse", "finalbody"):
3117
+ inner = getattr(node, field, None)
3118
+ if isinstance(inner, list):
3119
+ yield from _module_statements(inner)
3120
+ for handler in getattr(node, "handlers", None) or []:
3121
+ yield from _module_statements(handler.body)
3122
+ for case in getattr(node, "cases", None) or []:
3123
+ yield from _module_statements(case.body)
3124
+
3125
+
3126
+ def _target_names(node: ast.AST) -> list[str]:
3127
+ if isinstance(node, ast.Name):
3128
+ return [node.id]
3129
+ if isinstance(node, (ast.Tuple, ast.List)):
3130
+ return [n for elt in node.elts for n in _target_names(elt)]
3131
+ if isinstance(node, ast.Starred):
3132
+ return _target_names(node.value)
3133
+ return []
3134
+
3135
+
3136
+ def _literal_names(node: ast.AST | None) -> list[str] | None:
3137
+ """The strings of a literal `__all__` value (a list, a tuple, a sum of
3138
+ them or one string); None when it holds no literal part."""
3139
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
3140
+ return [node.value]
3141
+ if isinstance(node, (ast.List, ast.Tuple)):
3142
+ return [e.value for e in node.elts if isinstance(e, ast.Constant) and isinstance(e.value, str)]
3143
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
3144
+ left, right = _literal_names(node.left), _literal_names(node.right)
3145
+ if left is None and right is None:
3146
+ return None
3147
+ return (left or []) + (right or [])
3148
+ return None
3149
+
3150
+
3151
+ def _py_bindings(text: str) -> tuple[list[str] | None, dict[str, dict], list[dict]]:
3152
+ """A Python module's (__all__ or None, the names it binds at module
3153
+ level, its star imports). A binding: {"line", "kind": "definition" |
3154
+ "import" | "module-import"}; a definition's "form" (`def` for a def or
3155
+ async def, `class`, `variable` for an assignment); an import's
3156
+ "module", "level" and "imported" name. Raises SyntaxError."""
3157
+ tree = ast.parse(text)
3158
+ all_names: list[str] | None = None
3159
+ bound: dict[str, dict] = {}
3160
+ stars: list[dict] = []
3161
+ for node in _module_statements(tree.body):
3162
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
3163
+ form = "class" if isinstance(node, ast.ClassDef) else "def"
3164
+ bound.setdefault(node.name, {"line": node.lineno, "kind": "definition", "form": form})
3165
+ elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
3166
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]
3167
+ names = [n for t in targets for n in _target_names(t)]
3168
+ if "__all__" in names:
3169
+ listed = _literal_names(node.value)
3170
+ if isinstance(node, ast.AugAssign):
3171
+ all_names = (all_names or []) + (listed or [])
3172
+ elif listed is not None:
3173
+ all_names = listed
3174
+ if not isinstance(node, ast.AugAssign):
3175
+ for name in names:
3176
+ bound.setdefault(name, {"line": node.lineno, "kind": "definition", "form": "variable"})
3177
+ elif isinstance(node, ast.ImportFrom):
3178
+ for alias in node.names:
3179
+ if alias.name == "*":
3180
+ stars.append({"module": node.module, "level": node.level, "line": node.lineno})
3181
+ else:
3182
+ bound.setdefault(alias.asname or alias.name, {
3183
+ "line": node.lineno, "kind": "import", "module": node.module,
3184
+ "level": node.level, "imported": alias.name})
3185
+ elif isinstance(node, ast.Import):
3186
+ for alias in node.names:
3187
+ bound.setdefault(alias.asname or alias.name.split(".")[0],
3188
+ {"line": node.lineno, "kind": "module-import"})
3189
+ elif (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)
3190
+ and isinstance(node.value.func, ast.Attribute)
3191
+ and isinstance(node.value.func.value, ast.Name) and node.value.func.value.id == "__all__"
3192
+ and node.value.func.attr in ("extend", "append") and node.value.args):
3193
+ all_names = (all_names or []) + (_literal_names(node.value.args[0]) or [])
3194
+ return all_names, bound, stars
3195
+
3196
+
3197
+ _PY_ALL_RE = re.compile(r"^__all__\s*\+?=\s*[\[(]([^\])]*)[\])]", re.M)
3198
+ _PY_DEF_RE = re.compile(r"^(?:async\s+)?(def|class)\s+(\w+)", re.M)
3199
+ _PY_FROM_RE = re.compile(r"^from\s+(\.*)([\w.]*)\s+import\s+(\([^)]*\)|[^\n]+)", re.M)
3200
+
3201
+
3202
+ def _py_text_bindings(text: str) -> tuple[list[str] | None, dict[str, dict], list[dict]]:
3203
+ """_py_bindings read from text, for a module this Python cannot parse
3204
+ (newer syntax): __all__, column-0 def and class, from-imports."""
3205
+ all_names = None
3206
+ for m in _PY_ALL_RE.finditer(text):
3207
+ all_names = (all_names or []) + re.findall(r"""['"]([^'"]+)['"]""", m.group(1))
3208
+ bound: dict[str, dict] = {}
3209
+ stars: list[dict] = []
3210
+ for m in _PY_DEF_RE.finditer(text):
3211
+ bound.setdefault(m.group(2), {"line": _line_of(text, m.start(2)), "kind": "definition", "form": m.group(1)})
3212
+ for m in _PY_FROM_RE.finditer(text):
3213
+ line, level, module = _line_of(text, m.start()), len(m.group(1)), m.group(2) or None
3214
+ for item in m.group(3).strip("() \t").split(","):
3215
+ parts = item.split()
3216
+ if parts == ["*"]:
3217
+ stars.append({"module": module, "level": level, "line": line})
3218
+ elif parts and re.fullmatch(r"\w+", parts[0]):
3219
+ name = parts[2] if len(parts) == 3 and parts[1] == "as" else parts[0]
3220
+ bound.setdefault(name, {"line": line, "kind": "import", "module": module,
3221
+ "level": level, "imported": parts[0]})
3222
+ return all_names, bound, stars
3223
+
3224
+
3225
+ def _py_module_file(pkg_dir: str, level: int, module: str | None, tree: set[str]) -> str | None:
3226
+ """The file of the module `from <level dots><module> import` names, from
3227
+ a module of the package in `pkg_dir`: relative imports, or an absolute
3228
+ one within that package."""
3229
+ if level > 0:
3230
+ base = pkg_dir
3231
+ for _ in range(level - 1):
3232
+ if not base:
3233
+ return None
3234
+ base = posixpath.dirname(base)
3235
+ else:
3236
+ top = posixpath.basename(pkg_dir)
3237
+ if not top or not module or module.split(".")[0] != top:
3238
+ return None
3239
+ base = posixpath.dirname(pkg_dir)
3240
+ path = "/".join(p for p in (base, *(module.split(".") if module else [])) if p)
3241
+ candidates = ([f"{path}.py"] if module else []) + [f"{path}/__init__.py" if path else "__init__.py"]
3242
+ return next((c for c in candidates if c in tree), None)
3243
+
3244
+
3245
+ def _py_submodule(folder: str, name: str, tree: set[str]) -> str | None:
3246
+ base = f"{folder}/{name}" if folder else name
3247
+ return next((c for c in (f"{base}.py", f"{base}/__init__.py") if c in tree), None)
3248
+
3249
+
3250
+ def _py_entries(files: list[str], tree_set: set[str]) -> list[dict]:
3251
+ """The top __init__.py of each package that holds an in-scope file."""
3252
+ tops = set()
3253
+ for rel in files:
3254
+ if _file_language(rel) != "python":
3255
+ continue
3256
+ folder = posixpath.dirname(rel)
3257
+ if (f"{folder}/__init__.py" if folder else "__init__.py") not in tree_set:
3258
+ continue
3259
+ while folder and (f"{posixpath.dirname(folder)}/__init__.py" if posixpath.dirname(folder)
3260
+ else "__init__.py") in tree_set:
3261
+ folder = posixpath.dirname(folder)
3262
+ tops.add(f"{folder}/__init__.py" if folder else "__init__.py")
3263
+ return [{"language": "python", "file": f, "package": posixpath.dirname(f) or ".", "subpath": None,
3264
+ "resolution": "file"} for f in sorted(tops)]
3265
+
3266
+
3267
+ def _py_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _Surface,
3268
+ warnings: list[str]) -> None:
3269
+ """Add the names each package __init__.py exports: its __all__ when it
3270
+ has one, else the public names it defines or imports from its package
3271
+ and those its star imports give, through every module they chain to.
3272
+ Each name is traced to the module that defines it, through each import
3273
+ and star import between (a cycle ends the trace); a submodule is a
3274
+ namespace, which makes reachable the names it exports, traced the same
3275
+ way, and those of the submodules among them."""
3276
+ parsed: dict[str, tuple] = {}
3277
+
3278
+ def bindings(rel: str) -> tuple[list[str] | None, dict[str, dict], list[dict]]:
3279
+ if rel not in parsed:
3280
+ text = _read_text(root, rel) or ""
3281
+ try:
3282
+ parsed[rel] = _py_bindings(text)
3283
+ except SyntaxError as exc:
3284
+ warnings.append(f"{rel}: this Python cannot parse it ({exc.msg}, line {exc.lineno}); "
3285
+ "its names are read from its text")
3286
+ parsed[rel] = _py_text_bindings(text)
3287
+ return parsed[rel]
3288
+
3289
+ def module_file(rel: str, level: int, module: str | None, top: str) -> str | None:
3290
+ """The module `from <level dots><module> import` names in module
3291
+ `rel`: relative to its package, or absolute within the package in
3292
+ folder `top`."""
3293
+ return _py_module_file(posixpath.dirname(rel) if level else top, level, module, tree_set)
3294
+
3295
+ def packaged(rel: str, binding: dict, top: str) -> bool:
3296
+ """A binding module `rel` defines, or imports from the package in
3297
+ folder `top`: an import from another package is not its own."""
3298
+ if binding["kind"] == "definition":
3299
+ return True
3300
+ return binding["kind"] == "import" and (
3301
+ binding["level"] > 0 or module_file(rel, 0, binding["module"], top) is not None)
3302
+
3303
+ star_cache: dict[tuple[str, str], dict[str, str]] = {}
3304
+
3305
+ def star_names(rel: str, top: str, missing: list[tuple[str, str]] | None = None) -> dict[str, str]:
3306
+ """{name: the module that binds it} of what `from <rel> import *`
3307
+ gives: its __all__, else the public names it defines or imports
3308
+ (from its own package) and those its own star imports give.
3309
+ `missing` collects the (module, source) of each relative star
3310
+ import that names no file."""
3311
+ if missing is None and (rel, top) in star_cache:
3312
+ return star_cache[(rel, top)]
3313
+ out: dict[str, str] = {}
3314
+ stack, visited = [rel], set()
3315
+ while stack:
3316
+ module = stack.pop()
3317
+ if module in visited:
3318
+ continue
3319
+ visited.add(module)
3320
+ listed, bound, stars = bindings(module)
3321
+ if listed is not None:
3322
+ for name in listed:
3323
+ out.setdefault(name, module)
3324
+ continue
3325
+ for name, binding in bound.items():
3326
+ if packaged(module, binding, top) and not name.startswith("_"):
3327
+ out.setdefault(name, module)
3328
+ chained = []
3329
+ for star in stars:
3330
+ found = module_file(module, star["level"], star["module"], top)
3331
+ if found is not None:
3332
+ chained.append(found)
3333
+ elif star["level"] and missing is not None:
3334
+ missing.append((module, "." * star["level"] + (star["module"] or "")))
3335
+ stack += reversed(chained)
3336
+ star_cache[(rel, top)] = out
3337
+ return out
3338
+
3339
+ def resolve(rel: str, name: str, top: str, chain: set[str]) -> tuple[str | None, str, int | None, str]:
3340
+ """(module, its name there, line, what it is) that `name`, bound in
3341
+ module `rel`, comes from, `what` being `definition`, `submodule`
3342
+ (the module is it), `module` (an `import x` binding), `import` (from
3343
+ outside the package: no module) or `missing` (bound nowhere the
3344
+ trace reaches). `chain` collects each name met."""
3345
+ visited: set[tuple[str, str]] = set()
3346
+ while (rel, name) not in visited:
3347
+ visited.add((rel, name))
3348
+ chain.add(name)
3349
+ _, bound, stars = bindings(rel)
3350
+ binding = bound.get(name)
3351
+ package = posixpath.dirname(rel) if rel.endswith("__init__.py") else None
3352
+ if binding is None or binding["kind"] == "module-import":
3353
+ sub = _py_submodule(package, name, tree_set) if package is not None else None
3354
+ if sub:
3355
+ return sub, name, None, "submodule"
3356
+ if binding is not None:
3357
+ return rel, name, binding["line"], "module"
3358
+ owner = None
3359
+ for star in stars:
3360
+ found = module_file(rel, star["level"], star["module"], top)
3361
+ owner = star_names(found, top).get(name) if found is not None else None
3362
+ if owner is not None:
3363
+ break
3364
+ if owner is None:
3365
+ return rel, name, None, "missing"
3366
+ rel = owner
3367
+ continue
3368
+ if binding["kind"] == "definition":
3369
+ return rel, name, binding["line"], "definition"
3370
+ found = module_file(rel, binding["level"], binding["module"], top)
3371
+ if found is None:
3372
+ return None, binding["imported"], None, "import"
3373
+ if found.endswith("__init__.py"):
3374
+ sub = _py_submodule(posixpath.dirname(found), binding["imported"], tree_set)
3375
+ if sub and sub != rel:
3376
+ return sub, binding["imported"], None, "submodule"
3377
+ rel, name = found, binding["imported"]
3378
+ return rel, name, None, "missing"
3379
+
3380
+ reached: set[str] = set()
3381
+
3382
+ def reach(rel: str, top: str) -> None:
3383
+ """Make reachable what the submodule `rel` exports."""
3384
+ if rel in reached:
3385
+ return
3386
+ reached.add(rel)
3387
+ for name, owner in star_names(rel, top).items():
3388
+ chain: set[str] = set()
3389
+ where, local, _, what = resolve(owner, name, top, chain)
3390
+ surface.reachable.update(chain | {local})
3391
+ if what == "submodule" and where is not None:
3392
+ reach(where, top)
3393
+
3394
+ for entry in [e["file"] for e in entries]:
3395
+ top = posixpath.dirname(entry)
3396
+ all_names, bound, stars = bindings(entry)
3397
+ names: dict[str, str | None] # name -> the star import that gives it, if one does
3398
+ if all_names is not None:
3399
+ names = dict.fromkeys(all_names)
3400
+ else:
3401
+ names = {name: None for name, binding in bound.items() if packaged(entry, binding, top)}
3402
+ missing: list[tuple[str, str]] = []
3403
+ for star in stars:
3404
+ source = "." * star["level"] + (star["module"] or "")
3405
+ found = module_file(entry, star["level"], star["module"], top)
3406
+ if found is None:
3407
+ if star["level"] > 0:
3408
+ missing.append((entry, source))
3409
+ continue
3410
+ for name in star_names(found, top, missing):
3411
+ names.setdefault(name, source)
3412
+ for module, source in missing:
3413
+ surface.unresolve(entry, "*", source, module)
3414
+ for name, star_source in names.items():
3415
+ binding = bound.get(name)
3416
+ source = star_source
3417
+ if binding is not None and binding["kind"] == "import":
3418
+ source = "." * binding["level"] + (binding["module"] or "")
3419
+ chain: set[str] = set()
3420
+ where, local, line, what = resolve(entry, name, top, chain)
3421
+ surface.reachable.update(chain)
3422
+ if what == "submodule":
3423
+ surface.add(name, entry, "namespace", where, None, source, local)
3424
+ reach(where, top)
3425
+ elif what == "import":
3426
+ surface.add(name, entry, "re-export", None, None, source, local)
3427
+ surface.unresolve(entry, name, source, entry)
3428
+ elif binding is not None and binding["kind"] == "definition":
3429
+ surface.add(name, entry, "declaration", entry, binding["line"])
3430
+ elif binding is not None and binding["kind"] == "import":
3431
+ surface.add(name, entry, "re-export", where, line, source, local)
3432
+ elif star_source is not None or (what in ("definition", "module") and where != entry):
3433
+ surface.add(name, entry, "star", where, line, source, local)
3434
+ else:
3435
+ # __all__ lists it but nothing binds it, or an `import x`
3436
+ surface.add(name, entry, "declaration", None, None)
3437
+
3438
+
3439
+ # --------------------------------------------------------------------------
3440
+ # Entry points: Rust
3441
+ # --------------------------------------------------------------------------
3442
+
3443
+ _RUST_ITEM_RE = re.compile(
3444
+ r'^pub\s+((?:(?:async|const|unsafe|default|extern(?:\s+"[^"]*")?)\s+)*)'
3445
+ r"(fn|struct|enum|trait|type|const|static|union|mod)\s+(?:mut\s+)?((?:r#)?[A-Za-z_]\w*)",
3446
+ re.M,
3447
+ )
3448
+ _RUST_USE_RE = re.compile(r"^pub\s+use\s+([^;]*);", re.M)
3449
+ _RUST_MOD_RE = re.compile(r"^(?:pub(?:\([^)]*\))?\s+)?mod\s+((?:r#)?[A-Za-z_]\w*)\s*([;{])", re.M)
3450
+ _RUST_TOKEN_RE = re.compile(r"r#\w+|\w+|::|[{},*]")
3451
+
3452
+
3453
+ def _rust_items(code: str) -> list[tuple[str, str, int]]:
3454
+ """(name, kind, line) of each unrestricted column-0 `pub` item."""
3455
+ return [(m.group(3), m.group(2), _line_of(code, m.start(3))) for m in _RUST_ITEM_RE.finditer(code)]
3456
+
3457
+
3458
+ def _rust_use_leaves(tree_text: str) -> list[tuple[list[str], str | None, bool]]:
3459
+ """Each (path, alias, glob) a `pub use` tree brings in:
3460
+ `a::{b, c::D as E, f::*}` -> (a, b), (a, c, D; E), (a, f; glob)."""
3461
+ tokens = _RUST_TOKEN_RE.findall(tree_text)
3462
+ leaves: list[tuple[list[str], str | None, bool]] = []
3463
+
3464
+ def parse(i: int, prefix: list[str]) -> int:
3465
+ segments: list[str] = []
3466
+ while i < len(tokens):
3467
+ token = tokens[i]
3468
+ if token == "::":
3469
+ i += 1
3470
+ elif token == "{":
3471
+ i += 1
3472
+ while i < len(tokens) and tokens[i] != "}":
3473
+ i = parse(i, prefix + segments)
3474
+ if i < len(tokens) and tokens[i] == ",":
3475
+ i += 1
3476
+ return i + 1
3477
+ elif token == "*":
3478
+ leaves.append((prefix + segments, None, True))
3479
+ return i + 1
3480
+ elif token in (",", "}"):
3481
+ break
3482
+ elif token == "as" and segments and i + 1 < len(tokens):
3483
+ leaves.append((prefix + segments, tokens[i + 1], False))
3484
+ return i + 2
3485
+ else:
3486
+ segments.append(token)
3487
+ i += 1
3488
+ if segments:
3489
+ leaves.append((prefix + segments, None, False))
3490
+ return i
3491
+
3492
+ parse(0, [])
3493
+ return leaves
3494
+
3495
+
3496
+ def _rust_file(lib: str, path: list[str], tree: set[str]) -> str | None:
3497
+ """The file of the crate module at `path` (its names from the crate
3498
+ root, whose file is `lib`): <name>.rs or <name>/mod.rs at each level;
3499
+ None for a module no such file holds (an inline one, a `#[path]` one)."""
3500
+ base, current = posixpath.dirname(lib), lib
3501
+ for seg in path:
3502
+ stem = f"{base}/{seg}" if base else seg
3503
+ current = next((c for c in (f"{stem}.rs", f"{stem}/mod.rs") if c in tree), None)
3504
+ if current is None:
3505
+ return None
3506
+ base = stem
3507
+ return current
3508
+
3509
+
3510
+ def _rust_use_path(segments: list[str], here: list[str], mods: set[str]) -> list[str] | None:
3511
+ """A `use` path written in the module at `here` as a path from the
3512
+ crate root: `crate::` starts at the root, `self::` at `here`, each
3513
+ `super::` one module up, and a module `here` declares at `here`; None
3514
+ for another crate."""
3515
+ segs = [s.removeprefix("r#") for s in segments]
3516
+ if not segs:
3517
+ return None
3518
+ if segs[0] == "crate":
3519
+ return segs[1:]
3520
+ base = list(here)
3521
+ if segs[0] == "self":
3522
+ return base + segs[1:]
3523
+ if segs[0] == "super":
3524
+ while segs and segs[0] == "super":
3525
+ if not base:
3526
+ return None
3527
+ base, segs = base[:-1], segs[1:]
3528
+ return base + segs
3529
+ return base + segs if segs[0] in mods else None
3530
+
3531
+
3532
+ def _rust_entries(root: Path, files: list[str], tree: list[str], tree_set: set[str]) -> list[dict]:
3533
+ """The library root (`[lib] path`, else src/lib.rs) of each crate that
3534
+ holds an in-scope file."""
3535
+ rust_files = [f for f in files if _file_language(f) == "rust"]
3536
+ entries = []
3537
+ for crate in _owners(rust_files, "Cargo.toml", tree):
3538
+ lib = "src/lib.rs"
3539
+ try:
3540
+ manifest = tomllib.loads((root / (f"{crate}/Cargo.toml" if crate else "Cargo.toml")).read_text(encoding="utf-8"))
3541
+ path = (manifest.get("lib") or {}).get("path")
3542
+ lib = _norm(path) if isinstance(path, str) and path.strip() else lib
3543
+ except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError, AttributeError):
3544
+ pass
3545
+ file = posixpath.normpath(f"{crate}/{lib}" if crate else lib)
3546
+ if file in tree_set:
3547
+ entries.append({"language": "rust", "file": file, "package": crate or ".", "subpath": None,
3548
+ "resolution": "file"})
3549
+ return entries
3550
+
3551
+
3552
+ def _rust_surface(root: Path, entries: list[dict], tree_set: set[str], surface: _Surface) -> None:
3553
+ """Add the names each crate root exports: its unrestricted column-0
3554
+ `pub` items (a `pub mod` is a namespace) and its `pub use` items, each
3555
+ traced to the module file that declares it through the `pub use` items
3556
+ between (a cycle ends the trace); a glob `pub use m::*` adds what `m`
3557
+ exports, its globs included. The crate root, and each module a `pub
3558
+ mod` or a `pub use` of a reachable module names, in turn, are reachable
3559
+ files: every recipe match in them is reachable (the recipe keeps to the
3560
+ `pub mod` blocks within a file), and so is each name they export."""
3561
+ parsed: dict[str, dict] = {}
3562
+
3563
+ def module(rel: str) -> dict:
3564
+ """A module file's unrestricted column-0 `pub` items, the modules it
3565
+ declares ({name: declared with `;`, its body in a file of its own})
3566
+ and its `pub use` leaves, with their lines."""
3567
+ if rel not in parsed:
3568
+ code = _blank(_read_text(root, rel) or "", "rust")
3569
+ uses = []
3570
+ for use in _RUST_USE_RE.finditer(code):
3571
+ for path, alias, glob in _rust_use_leaves(use.group(1)):
3572
+ if path and path[-1] == "self" and not glob:
3573
+ path = path[:-1]
3574
+ if path and alias != "_":
3575
+ uses.append((path, alias, glob, _line_of(code, use.start())))
3576
+ parsed[rel] = {"items": _rust_items(code), "uses": uses,
3577
+ "mods": {m.group(1).removeprefix("r#"): m.group(2) == ";"
3578
+ for m in _RUST_MOD_RE.finditer(code)}}
3579
+ return parsed[rel]
3580
+
3581
+ def exported(lib: str, here: list[str]) -> list[str]:
3582
+ """The names the module at `here` exports: its `pub` items, its
3583
+ `pub use` names and what its globs pass on, in that order."""
3584
+ out: dict[str, None] = {}
3585
+ stack, visited = [list(here)], set()
3586
+ while stack:
3587
+ path = stack.pop()
3588
+ file = _rust_file(lib, path, tree_set)
3589
+ if tuple(path) in visited or file is None:
3590
+ continue
3591
+ visited.add(tuple(path))
3592
+ info = module(file)
3593
+ out.update(dict.fromkeys(name for name, _, _ in info["items"]))
3594
+ globs = []
3595
+ for use, alias, glob, _ in info["uses"]:
3596
+ if not glob:
3597
+ out.setdefault(alias or use[-1])
3598
+ elif (full := _rust_use_path(use, path, set(info["mods"]))) is not None:
3599
+ globs.append(full)
3600
+ stack += reversed(globs)
3601
+ return list(out)
3602
+
3603
+ def resolve(lib: str, here: list[str], name: str, chain: set[str]) -> tuple:
3604
+ """(file, its name there, line, module path, stop) of what `name`,
3605
+ as the module at `here` exports it, is: an item of that file, or a
3606
+ module (its path, and its file when one holds it). `stop` is the
3607
+ (`use` path, file) where the trace ends in another crate, the file
3608
+ None with it; the line is None where the trace ends at a module
3609
+ that does not export the name, or at a cycle. `chain` collects each
3610
+ name met."""
3611
+ visited: set[tuple[tuple[str, ...], str]] = set()
3612
+ while (tuple(here), name) not in visited:
3613
+ visited.add((tuple(here), name))
3614
+ chain.add(name)
3615
+ file = _rust_file(lib, here, tree_set)
3616
+ if file is None:
3617
+ return None, name, None, None, None
3618
+ info = module(file)
3619
+ item = next(((kind, line) for n, kind, line in info["items"] if n == name), None)
3620
+ if item is not None:
3621
+ if item[0] != "mod":
3622
+ return file, name, item[1], None, None
3623
+ sub = [*here, name.removeprefix("r#")]
3624
+ sub_file = _rust_file(lib, sub, tree_set)
3625
+ return sub_file or file, name, None if sub_file else item[1], sub, None
3626
+ mods = set(info["mods"])
3627
+ use = next(((path, line) for path, alias, glob, line in info["uses"]
3628
+ if not glob and (alias or path[-1]) == name), None)
3629
+ if use is not None:
3630
+ path = use[0]
3631
+ full = _rust_use_path(path, here, mods)
3632
+ if full is None:
3633
+ return None, path[-1], None, None, ("::".join(path), file)
3634
+ if not path[-1][:1].isupper() and _rust_file(lib, full, tree_set) is not None:
3635
+ return _rust_file(lib, full, tree_set), path[-1], None, full, None
3636
+ here, name = full[:-1], path[-1]
3637
+ continue
3638
+ owner = next((full for path, _, glob, _ in info["uses"] if glob
3639
+ for full in [_rust_use_path(path, here, mods)]
3640
+ if full is not None and name in exported(lib, full)), None)
3641
+ if owner is None:
3642
+ return file, name, None, None, None
3643
+ here = owner
3644
+ return _rust_file(lib, here, tree_set), name, None, None, None
3645
+
3646
+ def reach(lib: str, here: list[str], reached: set[tuple[str, ...]]) -> None:
3647
+ """Make reachable the module at `here`: its file and the names it
3648
+ exports, then the modules its `pub mod` and `pub use` items name."""
3649
+ if tuple(here) in reached:
3650
+ return
3651
+ reached.add(tuple(here))
3652
+ file = _rust_file(lib, here, tree_set)
3653
+ if file is None:
3654
+ parent = _rust_file(lib, here[:-1], tree_set)
3655
+ if here and parent is not None and module(parent)["mods"].get(here[-1]):
3656
+ # a `pub mod name;` whose file is not name.rs or name/mod.rs
3657
+ surface.unresolve(lib, here[-1], "::".join(["crate", *here]), parent)
3658
+ return
3659
+ surface.reachable_files.add(file)
3660
+ info = module(file)
3661
+ for name, kind, _ in info["items"]:
3662
+ surface.reachable.add(name)
3663
+ if kind == "mod":
3664
+ reach(lib, [*here, name.removeprefix("r#")], reached)
3665
+ for path, alias, glob, _ in info["uses"]:
3666
+ if glob:
3667
+ full = _rust_use_path(path, here, set(info["mods"]))
3668
+ if full is not None:
3669
+ reach(lib, full, reached)
3670
+ continue
3671
+ chain: set[str] = set()
3672
+ sub = resolve(lib, here, alias or path[-1], chain)[3]
3673
+ surface.reachable.update(chain)
3674
+ if sub is not None:
3675
+ reach(lib, sub, reached)
3676
+
3677
+ def publish(lib: str, here: list[str], name: str, via: str, source: str | None,
3678
+ reached: set[tuple[str, ...]]) -> None:
3679
+ """Add public `name`, which the module at `here` exports, traced to
3680
+ what it is."""
3681
+ chain: set[str] = set()
3682
+ file, local, line, sub, stop = resolve(lib, here, name, chain)
3683
+ surface.reachable.update(chain)
3684
+ if stop is not None:
3685
+ surface.add(name, lib, via, None, None, source, local)
3686
+ surface.unresolve(lib, name, *stop)
3687
+ elif sub is not None:
3688
+ surface.add(name, lib, "namespace", file, line, "::".join(sub))
3689
+ reach(lib, sub, reached)
3690
+ else:
3691
+ surface.add(name, lib, via, file, line, source, local)
3692
+
3693
+ for lib in [e["file"] for e in entries]:
3694
+ surface.reachable_files.add(lib)
3695
+ reached: set[tuple[str, ...]] = set()
3696
+ info = module(lib)
3697
+ for name, kind, line in info["items"]:
3698
+ if kind != "mod":
3699
+ surface.add(name, lib, "declaration", lib, line)
3700
+ continue
3701
+ sub = [name.removeprefix("r#")]
3702
+ sub_file = _rust_file(lib, sub, tree_set)
3703
+ surface.add(name, lib, "namespace", sub_file or lib, None if sub_file else line)
3704
+ reach(lib, sub, reached)
3705
+ for path, alias, glob, _ in info["uses"]:
3706
+ if not glob:
3707
+ publish(lib, [], alias or path[-1], "re-export", "::".join(path[:-1]) or None, reached)
3708
+ continue
3709
+ full = _rust_use_path(path, [], set(info["mods"]))
3710
+ if full is None or _rust_file(lib, full, tree_set) is None:
3711
+ # another crate's module, or one no file of its own holds
3712
+ surface.unresolve(lib, "*", "::".join(path), lib)
3713
+ continue
3714
+ for name in exported(lib, full):
3715
+ publish(lib, full, name, "star", "::".join(path), reached)
3716
+
3717
+
3718
+ # --------------------------------------------------------------------------
3719
+ # Entry points: Go
3720
+ # --------------------------------------------------------------------------
3721
+
3722
+ _GO_FUNC_RE = re.compile(r"^func\s+([^\W\d]\w*)", re.M)
3723
+ _GO_DECL_RE = re.compile(r"^(?:type|var|const)\s+([^\W\d]\w*)", re.M)
3724
+ _GO_GROUP_RE = re.compile(r"^(?:type|var|const)\s*\(\s*$")
3725
+ _GO_SPEC_RE = re.compile(r"^\t([^\W\d]\w*(?:[ \t]*,[ \t]*[^\W\d]\w*)*)")
3726
+ _GO_MAIN_RE = re.compile(r"^package\s+main\b", re.M)
3727
+
3728
+
3729
+ def _go_names(code: str) -> list[tuple[str, int]]:
3730
+ """(name, line) of each capitalized top-level func (not a method),
3731
+ type, var and const, grouped declarations included."""
3732
+ found = [(m.group(1), _line_of(code, m.start(1))) for rx in (_GO_FUNC_RE, _GO_DECL_RE)
3733
+ for m in rx.finditer(code)]
3734
+ lines = code.split("\n")
3735
+ i = 0
3736
+ while i < len(lines):
3737
+ if _GO_GROUP_RE.match(lines[i]):
3738
+ i += 1
3739
+ while i < len(lines) and not lines[i].startswith(")"):
3740
+ spec = _GO_SPEC_RE.match(lines[i])
3741
+ if spec:
3742
+ found += [(n, i + 1) for n in re.split(r"[ \t]*,[ \t]*", spec.group(1))]
3743
+ i += 1
3744
+ i += 1
3745
+ return sorted((n, ln) for n, ln in found if n[0].isupper())
3746
+
3747
+
3748
+ def _go_surface(root: Path, files: list[str], surface: _Surface, entries: list[dict]) -> None:
3749
+ """Add the capitalized top-level names of the in-scope Go files outside
3750
+ _test.go files, package main and internal/ folders."""
3751
+ packages: set[str] = set()
3752
+ for rel in files:
3753
+ if (_file_language(rel) != "go" or rel.endswith("_test.go")
3754
+ or "internal" in rel.split("/")[:-1]):
3755
+ continue
3756
+ code = _blank(_read_text(root, rel) or "", "go")
3757
+ if _GO_MAIN_RE.search(code):
3758
+ continue
3759
+ packages.add(posixpath.dirname(rel) or ".")
3760
+ for name, line in _go_names(code):
3761
+ surface.add(name, rel, "declaration", rel, line)
3762
+ entries.extend({"language": "go", "file": p, "package": p, "subpath": None, "resolution": "package"}
3763
+ for p in sorted(packages))
3764
+
3765
+
3766
+ # --------------------------------------------------------------------------
3767
+ # The run
3768
+ # --------------------------------------------------------------------------
3769
+
3770
+
3771
+ def entry_point_diff(exports: list[dict], surfaces: dict[str, _Surface], entries: list[dict],
3772
+ in_scope: set[str]) -> tuple[dict[str, str], dict[tuple[str, str], dict], list[dict],
3773
+ list[tuple[str, dict]], list[tuple[str, dict]]]:
3774
+ """The entry-point diff, one language family at a time: (status by
3775
+ family, public names {(family, name): info}, internal export records,
3776
+ and (family, info) of the extraction gaps and of the names outside
3777
+ scope). A family whose entry points declare names diffs them with its
3778
+ exports: an export no entry point names, nor traces back to, nor makes
3779
+ reachable (a name, or any export of a reachable file), is internal. A
3780
+ declared name no recipe found is a gap when a file in scope defines it
3781
+ (or the trace found no file), and is outside scope when a file out of
3782
+ scope does; a module is no recipe's target, so neither. In a family
3783
+ with no entry point, or only empty ones, every export is public."""
3784
+ statuses: dict[str, str] = {}
3785
+ public: dict[tuple[str, str], dict] = {}
3786
+ internal: list[dict] = []
3787
+ gaps: list[tuple[str, dict]] = []
3788
+ outside: list[tuple[str, dict]] = []
3789
+ entry_families = {FAMILY_OF[e["language"]] for e in entries}
3790
+ families = {FAMILY_OF[e["language"]] for e in exports} | entry_families
3791
+ for family in sorted(families | {f for f, s in surfaces.items() if s.names}):
3792
+ records = [e for e in exports if FAMILY_OF[e["language"]] == family]
3793
+ declared = surfaces[family].names
3794
+ if not declared:
3795
+ statuses[family] = "empty-barrel" if family in entry_families else "no-entry-point"
3796
+ for e in records:
3797
+ public.setdefault((family, e["export_name"]), {
3798
+ "name": e["export_name"], "entry": None, "via": "declaration", "from": None,
3799
+ "local": None, "file": e["source_file"], "line": e["source_line"]})
3800
+ continue
3801
+ statuses[family] = "barrel"
3802
+ surface = surfaces[family]
3803
+ names = {e["export_name"] for e in records}
3804
+ traced = {info["local"] for info in declared.values() if info["local"]} | surface.reachable
3805
+ public.update(((family, name), info) for name, info in declared.items())
3806
+ seen: set[str] = set()
3807
+ for e in records:
3808
+ name = e["export_name"]
3809
+ if (name not in declared and name not in traced and name not in seen
3810
+ and e["source_file"] not in surface.reachable_files):
3811
+ seen.add(name)
3812
+ internal.append(e)
3813
+ for name, info in declared.items():
3814
+ if name in names or (info["local"] or name) in names or info["via"] == "namespace":
3815
+ continue
3816
+ (gaps if info["file"] is None or info["file"] in in_scope else outside).append((family, info))
3817
+ return statuses, public, internal, gaps, outside
3818
+
3819
+
3820
+ def _head_cap(value: int | None, tier: str | None, scope_type: str | None) -> int | None:
3821
+ if value is not None:
3822
+ return value or None
3823
+ return HEAD_CAPS.get((tier, scope_type), DEFAULT_HEAD_CAP)
3824
+
3825
+
3826
+ def _monorepo(root: Path, tree: list[str], tree_set: set[str]) -> tuple[bool, str | None]:
3827
+ """skf-detect-workspaces.py's verdict on the source root."""
3828
+ manifests = {}
3829
+ for name in WORKSPACE_MANIFESTS:
3830
+ if name in tree_set:
3831
+ text = _read_text(root, name)
3832
+ if text is not None:
3833
+ manifests[name] = text
3834
+ result = _sibling("skf-detect-workspaces.py").detect({"tree": tree, "manifests": manifests})
3835
+ return bool(result.get("is_monorepo")), result.get("manifest_kind")
3836
+
3837
+
3838
+ def _select_files(root: Path, candidates: list[str], listed: bool, includes: list[str], excludes: list[str],
3839
+ wanted: set[str], glob_match: Callable[[str, str], bool], issues: list[dict],
3840
+ warnings: list[str]) -> tuple[list[str], dict[str, int]]:
3841
+ """(the files in scope, by extension the in-scope source files no recipe
3842
+ reads). A file a --files-from list names that is missing, or that no
3843
+ recipe reads, is an issue; an include glob no candidate matches, a
3844
+ warning."""
3845
+ in_scope: list[str] = []
3846
+ without_recipes: Counter = Counter()
3847
+ used = dict.fromkeys(includes, False)
3848
+ for rel in candidates:
3849
+ hits = [g for g in includes if glob_match(rel, g)]
3850
+ used.update(dict.fromkeys(hits, True))
3851
+ if (includes and not hits) or any(glob_match(rel, g) for g in excludes):
3852
+ continue
3853
+ if listed and not (root / rel).is_file():
3854
+ issues.append({"file": rel, "issue": "missing"})
3855
+ continue
3856
+ language = _file_language(rel)
3857
+ if language is None or language not in wanted:
3858
+ extension = posixpath.splitext(rel)[1].lower()
3859
+ if listed:
3860
+ issues.append({"file": rel, "issue": "no-recipes"})
3861
+ elif extension in NO_RECIPE_EXTENSIONS:
3862
+ without_recipes[extension] += 1
3863
+ continue
3864
+ in_scope.append(rel)
3865
+ warnings += [f"scope.include pattern {g!r} matches no file" for g, hit in used.items() if not hit]
3866
+ return in_scope, dict(sorted(without_recipes.items()))
3867
+
3868
+
3869
+ def run_full(args: argparse.Namespace) -> tuple[dict, int]:
3870
+ """Run the recipes over the source tree; (result, exit code)."""
3871
+ clock = _Clock(DEFAULT_TIMEOUT_SEC if args.timeout is None else args.timeout)
3872
+ root = Path(args.source_root)
3873
+ if not root.is_dir():
3874
+ raise RunnerError(f"source root not found: {args.source_root}")
3875
+ recipes_path = Path(args.recipes) if args.recipes else RECIPES_FILE
3876
+ recipes = load_recipe_file(recipes_path)
3877
+ brief = _read_brief(Path(args.brief)) if args.brief else {}
3878
+ includes = [g for g in (_norm(v) for v in (args.include or brief.get("include") or [])) if g]
3879
+ excludes = [g for g in (_norm(v) for v in (args.exclude or brief.get("exclude") or [])) if g]
3880
+ tier_a = [g for g in (_norm(v) for v in (args.tier_a_include or brief.get("tier_a_include") or [])) if g]
3881
+ scope_type = args.scope_type or brief.get("type")
3882
+ language_values = args.language or ([brief["language"]] if brief.get("language") else [])
3883
+ wanted, unknown_languages = _wanted_languages(language_values)
3884
+ recipe_set = args.recipe_set or ("component-library" if scope_type == "component-library" else "standard")
3885
+ head_cap = _head_cap(args.head_cap, args.tier, scope_type)
3886
+ glob_match = _sibling("skf-resolve-authoritative-files.py").glob_match
3887
+
3888
+ warnings: list[str] = [f"no recipe reads {value} files" for value in unknown_languages]
3889
+ errors: list[dict] = []
3890
+ issues: list[dict] = []
3891
+ tree = list_tree(root)
3892
+ tree_set = set(tree)
3893
+ listed = _read_file_list(Path(args.files_from)) if args.files_from else None
3894
+ in_scope, without_recipes = _select_files(root, tree if listed is None else listed, listed is not None,
3895
+ includes, excludes, wanted, glob_match, issues, warnings)
3896
+ readable = []
3897
+ for rel in in_scope:
3898
+ try:
3899
+ (root / rel).read_bytes().decode("utf-8")
3900
+ except OSError:
3901
+ issues.append({"file": rel, "issue": "unreadable"})
3902
+ continue
3903
+ except UnicodeDecodeError:
3904
+ issues.append({"file": rel, "issue": "not-utf8"})
3905
+ continue
3906
+ readable.append(rel)
3907
+
3908
+ # The entry points and the arms need no ast-grep.
3909
+ in_scope_set = set(in_scope)
3910
+ surfaces = {family: _Surface() for family in FAMILIES}
3911
+ js_entries, maps, unresolved_subpaths = _js_entries(root, in_scope, tree, tree_set, warnings)
3912
+ py_entries = _py_entries(in_scope, tree_set)
3913
+ rust_entries = _rust_entries(root, in_scope, tree, tree_set)
3914
+ go_entries: list[dict] = []
3915
+ _go_surface(root, in_scope, surfaces["go"], go_entries)
3916
+ entries = js_entries + py_entries + rust_entries + go_entries
3917
+ for entry in entries:
3918
+ # a Go entry is a package folder, which the files in scope give
3919
+ entry["in_scope"] = entry["language"] == "go" or entry["file"] in in_scope_set
3920
+ if not entry["in_scope"] and entry["subpath"] is not None:
3921
+ warnings.append(f"public entry point {entry['file']} (subpath '{entry['subpath']}' of package "
3922
+ f"{entry['package']}) is outside the files in scope: widen scope.include to document "
3923
+ "the names it exports (entry_point_diff.outside_scope), which effective_denominator "
3924
+ "leaves out")
3925
+ monorepo, monorepo_kind = _monorepo(root, tree, tree_set)
3926
+ arms = {
3927
+ "monorepo": monorepo,
3928
+ "monorepo_kind": monorepo_kind,
3929
+ "specific_modules": scope_type == "specific-modules" and not monorepo,
3930
+ "multi_subpath_exports": not monorepo and any(m["non_root_subpaths"] > 1 for m in maps),
3931
+ }
3932
+
3933
+ exe, version, shim = _ast_grep()
3934
+ result: dict = {
3935
+ "mode": "full",
3936
+ "status": "ok",
3937
+ "source_root": args.source_root,
3938
+ "recipes_file": recipes_path.as_posix(),
3939
+ "recipe_set": recipe_set,
3940
+ "ast_grep": {"path": exe, "version": version},
3941
+ "scope": {"include": includes, "exclude": excludes, "tier_a_include": tier_a or None,
3942
+ "type": scope_type, "languages": sorted(wanted),
3943
+ "files_from": args.files_from},
3944
+ "files_in_scope": len(in_scope),
3945
+ "files_by_language": dict(sorted(Counter(_file_language(f) for f in in_scope).items())),
3946
+ "files_without_recipes": without_recipes,
3947
+ "file_issues": issues,
3948
+ "head_cap": head_cap,
3949
+ "truncated": False,
3950
+ "recipes": [],
3951
+ "exports": [],
3952
+ "aggregates": {"exports": 0, "by_type": {}, "t1": 0, "t1_low": 0},
3953
+ "entry_points": {"status": None, "by_language": {}, "files": entries, "exports_maps": maps,
3954
+ "unresolved": unresolved_subpaths},
3955
+ "entry_point_diff": None,
3956
+ "reexport_targets": [],
3957
+ "counts": None,
3958
+ "arms": arms,
3959
+ "errors": errors,
3960
+ "warnings": warnings,
3961
+ }
3962
+ if exe is None:
3963
+ result["status"] = "no-ast-grep"
3964
+ result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
3965
+ if shim is None:
3966
+ warnings.append("ast-grep is not on PATH: no recipe ran; extract by source reading")
3967
+ else:
3968
+ warnings.append(f"ast-grep on PATH is the npm shim {shim}, with no native ast-grep.exe beside it: "
3969
+ "a shim runs through cmd.exe, which would read the file paths a scan passes as "
3970
+ "commands, so no recipe ran; extract by source reading, or install a native "
3971
+ "ast-grep (pip install ast-grep-cli)")
3972
+ return result, 3
3973
+
3974
+ forms = _forms(recipes)
3975
+ set_recipes = [r for r in recipes if recipe_set in r["metadata"]["sets"]]
3976
+ barrel_recipes = [r for r in recipes if "standard" in r["metadata"]["sets"]
3977
+ and r["rule"]["kind"] in ("export_statement", "export_specifier")
3978
+ and set(r["metadata"]["languages"]) & set(JS_FAMILY)]
3979
+ barrel_ids = {r["id"] for r in barrel_recipes}
3980
+ languages = {lang for f in readable for lang in _rule_languages(_file_language(f))}
3981
+ with tempfile.TemporaryDirectory(prefix="skf-extract-", ignore_cleanup_errors=True) as tmp:
3982
+ folder = Path(tmp)
3983
+ rules = _recipe_rules(set_recipes, languages) + [_syntax_error_rule(lang) for lang in sorted(languages)]
3984
+ matches = _scan(exe, root, folder, rules, readable, errors, clock)
3985
+ issues += _syntax_issues(matches)
3986
+ exports, stats = merge_exports(matches, recipes, recipe_set, head_cap)
3987
+ _type_list_items(exe, root, folder, matches, exports, errors, clock)
3988
+
3989
+ scanned: dict[str, list[dict]] = {}
3990
+
3991
+ def scan_barrels(files: list[str]) -> dict[str, list[dict]]:
3992
+ todo = [f for f in files if f not in scanned and _read_text(root, f) is not None]
3993
+ langs = {lang for f in todo for lang in _rule_languages(_file_language(f) or "")}
3994
+ barrel_rules = _recipe_rules(barrel_recipes, langs)
3995
+ barrel_rules += [r for lang in sorted(langs & set(RECIPE_LANGUAGES)) for r in _trace_rules(lang)]
3996
+ for f in todo:
3997
+ scanned[f] = []
3998
+ for m in _scan(exe, root, folder, barrel_rules, todo, errors, clock):
3999
+ scanned.setdefault(m["file"], []).append(m)
4000
+ return {f: scanned.get(f, []) for f in files}
4001
+
4002
+ if js_entries:
4003
+ _js_surface(root, js_entries, tree_set, forms, barrel_ids, scan_barrels, surfaces["javascript"])
4004
+ _py_surface(root, py_entries, tree_set, surfaces["python"], warnings)
4005
+ _rust_surface(root, rust_entries, tree_set, surfaces["rust"])
4006
+
4007
+ for record in exports:
4008
+ if record["from"] and record["language"] in JS_FAMILY:
4009
+ record["from_file"] = resolve_js_module(record["from"], record["source_file"], tree_set)
4010
+ statuses, public, internal, gaps, outside = entry_point_diff(exports, surfaces, entries, in_scope_set)
4011
+ if "barrel" in statuses.values():
4012
+ status = "barrel"
4013
+ else:
4014
+ status = "empty-barrel" if "empty-barrel" in statuses.values() else "no-entry-point"
4015
+
4016
+ basis_globs = tier_a or includes
4017
+ basis = "tier_a_include" if tier_a else ("scope.include" if includes else "all-files")
4018
+ denominator_files = {f for f in tree if (not basis_globs or any(glob_match(f, g) for g in basis_globs))
4019
+ and not any(glob_match(f, g) for g in excludes)}
4020
+
4021
+ by_type: Counter = Counter(e["export_type"] for e in exports)
4022
+ ordered = sorted(public.items(), key=lambda item: (item[0][1], item[0][0]))
4023
+ result.update({
4024
+ "truncated": any(s["truncated"] for s in stats),
4025
+ "recipes": stats,
4026
+ "exports": exports,
4027
+ "aggregates": {"exports": len(exports), "by_type": dict(sorted(by_type.items())),
4028
+ "t1": len(exports), "t1_low": 0},
4029
+ "entry_points": {"status": status, "by_language": statuses, "files": entries, "exports_maps": maps,
4030
+ "unresolved": unresolved_subpaths + [u for s in surfaces.values() for u in s.unresolved]},
4031
+ "entry_point_diff": {
4032
+ "public": [dict(info, language=family) for (family, _), info in ordered],
4033
+ "internal": [{"name": e["export_name"], "language": FAMILY_OF[e["language"]],
4034
+ "source_file": e["source_file"], "source_line": e["source_line"]} for e in internal],
4035
+ "extraction_gaps": [{"name": info["name"], "language": family, "entry": info["entry"],
4036
+ "file": info["file"], "line": info["line"]}
4037
+ for family, info in sorted(gaps, key=lambda g: (g[1]["name"], g[0]))],
4038
+ "outside_scope": [{"name": info["name"], "language": family, "entry": info["entry"],
4039
+ "file": info["file"], "line": info["line"]}
4040
+ for family, info in sorted(outside, key=lambda g: (g[1]["name"], g[0]))],
4041
+ },
4042
+ "reexport_targets": [
4043
+ {"name": info["name"], "language": family, "local": info["local"], "from": info["from"],
4044
+ "entry": info["entry"], "file": info["file"], "line": info["line"], "via": info["via"]}
4045
+ for (family, _), info in ordered if info["via"] in ("re-export", "star", "namespace")],
4046
+ "counts": {
4047
+ "exports_public_api": len(public),
4048
+ "exports_internal": len(internal),
4049
+ "effective_denominator": sum(1 for info in public.values() if info["file"] in denominator_files),
4050
+ "effective_denominator_basis": basis,
4051
+ "denominator_files": len(denominator_files),
4052
+ },
4053
+ })
4054
+ result["file_issues"] = sorted(issues, key=lambda i: (i["file"], i["issue"]))
4055
+ if errors:
4056
+ result["status"] = "incomplete"
4057
+ return result, 1
4058
+ return result, 0
4059
+
4060
+
4061
+ def _main_full(args: argparse.Namespace) -> int:
4062
+ if not args.source_root:
4063
+ sys.stderr.write("error: --mode full needs --source-root\n")
4064
+ return 2
4065
+ if args.head_cap is not None and args.head_cap < 0:
4066
+ sys.stderr.write("error: --head-cap must be 0 (no cap) or more\n")
4067
+ return 2
4068
+ if args.timeout is not None and not args.timeout > 0:
4069
+ sys.stderr.write("error: --timeout must be more than 0 seconds\n")
4070
+ return 2
4071
+ try:
4072
+ result, code = run_full(args)
4073
+ except RunnerError as exc:
4074
+ sys.stderr.write(f"error: {exc}\n")
4075
+ return 2
4076
+ except Exception as exc: # noqa: BLE001 - one stderr line and exit 2, never a traceback
4077
+ detail = " ".join(str(exc).split())
4078
+ sys.stderr.write(f"error: the run failed: {type(exc).__name__}: {detail}\n")
4079
+ return 2
4080
+ text = json.dumps(result, indent=2) + "\n"
4081
+ if args.output:
4082
+ try:
4083
+ Path(args.output).write_text(text, encoding="utf-8")
4084
+ except OSError as exc:
4085
+ sys.stderr.write(f"error: cannot write {args.output}: {exc}\n")
4086
+ return 2
4087
+ else:
4088
+ sys.stdout.write(text)
4089
+ return code
4090
+
4091
+
4092
+ # --------------------------------------------------------------------------
4093
+ # Entries mode: the files quick mode reads, found in the listing
4094
+ # --------------------------------------------------------------------------
4095
+
4096
+ # The quick languages entries mode finds entry files for, by family.
4097
+ ENTRY_LANGUAGES = {"js": "javascript", "ts": "javascript", "javascript": "javascript",
4098
+ "typescript": "javascript", "python": "python", "rust": "rust", "go": "go",
4099
+ "java": "java", "kotlin": "kotlin"}
4100
+ # The manifest of each family at the package folder, the first one listed read
4101
+ # (Python: the first that names the package, read in this order).
4102
+ ENTRY_MANIFESTS = {"javascript": ("package.json",), "python": ("pyproject.toml", "setup.py", "setup.cfg"),
4103
+ "rust": ("Cargo.toml",), "go": ("go.mod",), "java": ("pom.xml",),
4104
+ "kotlin": ("build.gradle.kts", "build.gradle")}
4105
+ # A glob-picked family (Go, Java, Kotlin) reads at most this many files, as
4106
+ # the fetch's `--limit 5` did; no family reads more than ENTRY_FILE_LIMIT.
4107
+ ENTRY_GLOB_LIMIT = 5
4108
+ ENTRY_FILE_LIMIT = 25
4109
+ # Top folders of a Python tree that hold no package code (test* folders too).
4110
+ PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
4111
+
4112
+
4113
+ def _scope_folder(scope: str | None) -> str:
4114
+ """The package folder a --scope names, "" for the repository root."""
4115
+ folder = _norm(scope or "").strip("/")
4116
+ return "" if folder == "." else folder
4117
+
4118
+
4119
+ def _import_name(name: object) -> str:
4120
+ """A distribution name the way Python imports it: lower case, each run
4121
+ of `-` and `.` read as `_` (`zope.interface` -> zope_interface)."""
4122
+ return re.sub(r"[-.]+", "_", name.strip().lower()) if isinstance(name, str) else ""
4123
+
4124
+
4125
+ def _py_quick_entries(folder: str, tree: list[str], tree_set: set[str],
4126
+ name: str | None) -> tuple[list[str], list[str]]:
4127
+ """(entry files, the paths tried when there is none) of the Python
4128
+ package in `folder`: the top __init__.py of each package full mode finds
4129
+ (_py_entries) among its .py files, less those under a top folder named
4130
+ test*, docs, examples or benchmarks, the one whose folder is the
4131
+ manifest's import name preferred (Pillow's src/PIL/ is kept, as no
4132
+ folder is named pillow); with no package at all, the single module
4133
+ `<name>.py` or `src/<name>.py` (six.py)."""
4134
+ prefix = f"{folder}/" if folder else ""
4135
+ files = []
4136
+ for path in tree:
4137
+ if not path.startswith(prefix) or not path.endswith(".py"):
4138
+ continue
4139
+ rest = path[len(prefix):]
4140
+ top = rest.split("/", 1)[0].lower() if "/" in rest else ""
4141
+ if top.startswith(("test", ".")) or top in PY_NON_CORE_TOPS:
4142
+ continue
4143
+ files.append(path)
4144
+ tops = [entry["file"] for entry in _py_entries(files, tree_set)]
4145
+ wanted = _import_name(name)
4146
+ if tops:
4147
+ named = [top for top in tops if posixpath.basename(posixpath.dirname(top)).lower() == wanted]
4148
+ return named or tops, []
4149
+ tried = [f"{prefix}{wanted}.py", f"{prefix}src/{wanted}.py"] if wanted else []
4150
+ return [path for path in tried if path in tree_set][:1], tried
4151
+
4152
+
4153
+ def _py_manifest_name(manifest: str, content: str) -> str | None:
4154
+ """The distribution name a Python manifest gives: pyproject.toml's
4155
+ [project] or [tool.poetry] `name`, setup.py's `name=`, setup.cfg's
4156
+ [metadata] `name`."""
4157
+ if manifest.endswith("setup.py"):
4158
+ name = parse_setup_py(content).get("name")
4159
+ elif manifest.endswith("setup.cfg"):
4160
+ name = parse_setup_cfg(content).get("name")
4161
+ else:
4162
+ try:
4163
+ data = tomllib.loads(content)
4164
+ except tomllib.TOMLDecodeError:
4165
+ return None
4166
+ project = data.get("project") if isinstance(data.get("project"), dict) else {}
4167
+ tool = data.get("tool") if isinstance(data.get("tool"), dict) else {}
4168
+ poetry = tool.get("poetry") if isinstance(tool.get("poetry"), dict) else {}
4169
+ name = project.get("name") or poetry.get("name")
4170
+ return name.strip() if isinstance(name, str) and name.strip() else None
4171
+
4172
+
4173
+ def _pom_group(content: str) -> str | None:
4174
+ """The groupId of a pom.xml (parse_pom_xml's), else the one its <parent> gives."""
4175
+ group = (parse_pom_xml(content).get("_extra") or {}).get("group_id")
4176
+ if group:
4177
+ return group
4178
+ try:
4179
+ project = ET.fromstring(content)
4180
+ except ET.ParseError:
4181
+ return None
4182
+ for child in project:
4183
+ if _strip_ns(child.tag) == "parent":
4184
+ for item in child:
4185
+ if _strip_ns(item.tag) == "groupId" and (item.text or "").strip():
4186
+ return item.text.strip()
4187
+ return None
4188
+
4189
+
4190
+ def _rust_lib_path(content: str | None) -> str:
4191
+ """The crate root a Cargo.toml names (`[lib] path`), else src/lib.rs."""
4192
+ try:
4193
+ path = (tomllib.loads(content or "").get("lib") or {}).get("path")
4194
+ except (tomllib.TOMLDecodeError, AttributeError):
4195
+ path = None
4196
+ return _norm(path) if isinstance(path, str) and path.strip() else "src/lib.rs"
4197
+
4198
+
4199
+ def find_entries(language: str, tree: list[str], root: Path, scope: str | None = None) -> dict:
4200
+ """The entry files quick mode reads for the one package at `scope` (the
4201
+ repository root by default), found in the listing `tree` with full
4202
+ mode's rules, its manifest read from `root` (the folder it was fetched
4203
+ into). See "Entries mode" above."""
4204
+ lowered = (language or "").lower()
4205
+ if lowered not in LANGUAGE_DISPATCH:
4206
+ return {"_error": f"unknown language: {language!r}; expected one of {sorted(LANGUAGE_DISPATCH)}"}
4207
+ family = ENTRY_LANGUAGES.get(lowered)
4208
+ folder = _scope_folder(scope)
4209
+ prefix = f"{folder}/" if folder else ""
4210
+ label = folder or "."
4211
+ tree_set = set(tree)
4212
+ warnings: list[str] = []
4213
+ out = {"mode": "entries", "language": lowered, "scope": label, "manifest": None,
4214
+ "entry_files": [], "unresolved": [], "warnings": warnings}
4215
+ if family is None:
4216
+ warnings.append(f"no entry-point rule for {lowered}: read its source by eye")
4217
+ return out
4218
+ listed = [prefix + name for name in ENTRY_MANIFESTS[family] if prefix + name in tree_set]
4219
+ contents: dict[str, str] = {}
4220
+ for path in listed if family == "python" else listed[:1]:
4221
+ try:
4222
+ contents[path] = (root / path).read_bytes().decode("utf-8-sig", errors="replace")
4223
+ except OSError:
4224
+ warnings.append(f"{path} is in the listing but not under --source-root: fetch it first")
4225
+ manifest = listed[0] if listed else None
4226
+ py_name = None
4227
+ if family == "python":
4228
+ named = ((path, _py_manifest_name(path, text)) for path, text in contents.items())
4229
+ manifest, py_name = next(((path, name) for path, name in named if name), (manifest, None))
4230
+ out["manifest"] = manifest
4231
+ content = contents.get(manifest) if manifest else None
4232
+ files: list[str] = []
4233
+ tried: list[str] = []
4234
+ if family == "javascript":
4235
+ found, _, out["unresolved"] = _js_package_entries(root, folder, tree_set, warnings)
4236
+ files = [entry["file"] for entry in found]
4237
+ tried = [f"{prefix}index.*", f"{prefix}src/index.*"]
4238
+ elif family == "python":
4239
+ files, tried = _py_quick_entries(folder, tree, tree_set, py_name)
4240
+ elif family == "rust":
4241
+ crate = [path for path in tree if path.startswith(prefix) and path.endswith(".rs")]
4242
+ files = [entry["file"] for entry in _rust_entries(root, crate, tree, tree_set) if entry["package"] == label]
4243
+ tried = [posixpath.normpath(prefix + _rust_lib_path(content))]
4244
+ elif family == "go":
4245
+ files = sorted(path for path in tree if posixpath.dirname(path) == folder and path.endswith(".go")
4246
+ and not path.endswith("_test.go"))[:ENTRY_GLOB_LIMIT]
4247
+ tried = [f"{prefix}*.go"]
4248
+ elif family == "java":
4249
+ group = _pom_group(content) if content else None
4250
+ base = f"{prefix}src/main/java/{group.replace('.', '/')}" if group else None
4251
+ if base:
4252
+ files = sorted(path for path in tree
4253
+ if posixpath.dirname(path) == base and path.endswith(".java"))[:ENTRY_GLOB_LIMIT]
4254
+ tried = [f"{base}/*.java" if base else f"{prefix}src/main/java/<groupId>/*.java"]
4255
+ else:
4256
+ base = f"{prefix}src/main/kotlin/"
4257
+ files = sorted(path for path in tree if path.startswith(base) and path.endswith(".kt"))[:ENTRY_GLOB_LIMIT]
4258
+ tried = [f"{base}**/*.kt"]
4259
+ if len(files) > ENTRY_FILE_LIMIT:
4260
+ warnings.append(f"{len(files)} entry files: quick mode reads the first {ENTRY_FILE_LIMIT}")
4261
+ files = files[:ENTRY_FILE_LIMIT]
4262
+ if not files and not out["unresolved"]:
4263
+ out["unresolved"] = [{"package": label, "subpath": None, "targets": tried}]
4264
+ out["entry_files"] = files
4265
+ return out
4266
+
4267
+
4268
+ # --------------------------------------------------------------------------
4269
+ # CLI
4270
+ # --------------------------------------------------------------------------
4271
+
4272
+
4273
+ def _build_parser() -> argparse.ArgumentParser:
4274
+ parser = argparse.ArgumentParser(
4275
+ description=(
4276
+ "Extract a package's public-API surface: from manifest and entry-point content on stdin or "
4277
+ "in files (quick, a pure parser), or by running the ast-grep recipes over a source tree (full); "
4278
+ "or find the entry files quick mode reads in a repository listing (entries)."
4279
+ ),
4280
+ )
4281
+ parser.add_argument(
4282
+ "--mode",
4283
+ default="quick",
4284
+ choices=("quick", "full", "entries"),
4285
+ help="quick (default): parse the stdin payload, or the --manifest-file and --entry-file files; "
4286
+ "full: run the recipes over --source-root; entries: list one package's entry files from "
4287
+ "--tree-file.",
4288
+ )
4289
+ quick = parser.add_argument_group("quick mode, files read from disk")
4290
+ quick.add_argument("--manifest-file", metavar="PATH",
4291
+ help="the package manifest (relative to --source-root when given); needs --language")
4292
+ quick.add_argument("--entry-file", action="append", metavar="PATH",
4293
+ help="an entry-point file (relative to --source-root when given), repeatable; needs --language")
4294
+ quick.add_argument("--follow-file", action="append", metavar="PATH",
4295
+ help="a module an unlisted[] record named (a --fetch-list path), read as an entry less "
4296
+ "its JS/TS default export, which `export *` does not pass on; repeatable")
4297
+ quick.add_argument("--follow", action="store_true",
4298
+ help=f"read each module an unlisted[] record names as one more --follow-file, round "
4299
+ f"after round (at most {FOLLOW_ROUNDS}), until none is left; needs --tree-file and "
4300
+ f"--source-root")
4301
+ quick.add_argument("--repo", metavar="OWNER/REPO",
4302
+ help="with --follow: fetch each module from this GitHub repository into --source-root "
4303
+ "first (skf-github-fetch.py)")
4304
+ quick.add_argument("--ref", metavar="REF",
4305
+ help="with --repo: the branch, tag or commit to fetch at (default: the default branch)")
4306
+ listing = parser.add_argument_group("entries mode, and the listing quick mode follows chains in")
4307
+ listing.add_argument("--tree-file", metavar="PATH",
4308
+ help="the repository's file listing (skf-github-probe.py tree output, JSON, or one "
4309
+ "path per line): entries mode finds the entry files in it, and quick mode "
4310
+ "resolves each unlisted[] statement's module_file against it")
4311
+ listing.add_argument("--scope", metavar="FOLDER",
4312
+ help="entries mode: the package folder to read (default: the repository root)")
4313
+ listing.add_argument("--fetch-list", metavar="FILE",
4314
+ help="write one path per line for skf-github-fetch.py --patterns-file: the entry "
4315
+ "files (entries mode), or each unlisted[] module_file still to read (quick mode)")
4316
+ full = parser.add_argument_group("full mode")
4317
+ full.add_argument("--source-root",
4318
+ help="the source tree to extract from (quick mode: the folder the file paths are relative to)")
4319
+ full.add_argument("--brief", help="a skill-brief.yaml: its scope and language")
4320
+ full.add_argument("--include", action="append", metavar="GLOB",
4321
+ help="an include glob, repeatable (replaces the brief's scope.include)")
4322
+ full.add_argument("--exclude", action="append", metavar="GLOB",
4323
+ help="an exclude glob, repeatable (replaces the brief's scope.exclude)")
4324
+ full.add_argument("--tier-a-include", action="append", metavar="GLOB",
4325
+ help="a tier-A glob for effective_denominator, repeatable (replaces scope.tier_a_include)")
4326
+ full.add_argument("--scope-type", help="the scope type (replaces the brief's scope.type)")
4327
+ full.add_argument("--language", action="append",
4328
+ help="a language family to extract, repeatable (replaces the brief's language); quick "
4329
+ "mode: the files' language, once")
4330
+ full.add_argument("--files-from", metavar="FILE",
4331
+ help="read only these files: one path per line, or a JSON list")
4332
+ full.add_argument("--recipe-set", choices=RECIPE_SETS,
4333
+ help="the recipe set to run (default: from the scope type)")
4334
+ full.add_argument("--recipes", metavar="FILE", help="the recipe file (default: the module's data file)")
4335
+ full.add_argument("--tier", choices=TIERS, help="the forge tier, for the default head cap")
4336
+ full.add_argument("--head-cap", type=int, metavar="N",
4337
+ help="the most matches kept per recipe (0: no cap; default: by tier and scope type)")
4338
+ full.add_argument("--timeout", type=float, metavar="SECONDS",
4339
+ help=f"stop the ast-grep runs after this many seconds and still write the JSON, "
4340
+ f"status incomplete (default {DEFAULT_TIMEOUT_SEC})")
4341
+ full.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
4342
+ return parser
4343
+
4344
+
4345
+ FULL_MODE_FLAGS = ("source_root", "brief", "include", "exclude", "tier_a_include", "scope_type", "language",
4346
+ "files_from", "recipe_set", "recipes", "tier", "head_cap", "timeout", "output")
4347
+ # The full mode flags quick mode takes with its file inputs, and entries mode always.
4348
+ QUICK_FILE_FLAGS = ("source_root", "language")
4349
+ # The flags of entries mode; quick mode takes --tree-file and --fetch-list with its file inputs.
4350
+ LISTING_FLAGS = ("tree_file", "scope", "fetch_list")
4351
+ # Quick mode's own flags with its file inputs: the chains it follows itself.
4352
+ FOLLOW_FLAGS = ("follow", "repo", "ref")
4353
+ FOLLOW_ROUNDS = 5
4354
+
4355
+
4356
+ class QuickInputError(Exception):
4357
+ """A --manifest-file or --entry-file quick mode cannot read (exit 2)."""
4358
+
4359
+
4360
+ def _read_quick_file(root: Path | None, value: str, flag: str) -> dict:
4361
+ """{"path", "content"} of a --manifest-file or --entry-file: the path as
4362
+ given, written with `/`, and the file's text. Raises QuickInputError."""
4363
+ given = Path(value)
4364
+ path = given
4365
+ if root is not None:
4366
+ if given.is_absolute():
4367
+ raise QuickInputError(f"{flag} must be relative to --source-root: {value}")
4368
+ path = root / given
4369
+ if not path.resolve().is_relative_to(root.resolve()):
4370
+ raise QuickInputError(f"{flag} leads outside --source-root: {value}")
4371
+ try:
4372
+ data = path.read_bytes()
4373
+ except OSError as exc:
4374
+ raise QuickInputError(f"cannot read {flag} {value}: {exc.strerror or exc}") from exc
4375
+ return {"path": given.as_posix(), "content": data.decode("utf-8-sig", errors="replace")}
4376
+
4377
+
4378
+ def _read_listing(value: str) -> tuple[list[str], bool]:
4379
+ """(paths, whether the listing was cut short) of a --tree-file, read by
4380
+ skf-detect-language.py's read_tree_file(). Raises QuickInputError."""
4381
+ try:
4382
+ return _sibling("skf-detect-language.py").read_tree_file(value)
4383
+ except (RunnerError, ValueError) as exc: # ValueError: the sibling's TreeListingError
4384
+ raise QuickInputError(str(exc)) from exc
4385
+
4386
+
4387
+ def _write_fetch_list(path: str, files: list[str]) -> None:
4388
+ """Write `files` one per line (LF on every platform) for --patterns-file. Raises QuickInputError."""
4389
+ try:
4390
+ Path(path).write_text("".join(f"{f}\n" for f in files), encoding="utf-8", newline="\n")
4391
+ except OSError as exc:
4392
+ raise QuickInputError(f"cannot write --fetch-list {path}: {exc.strerror or exc}") from exc
4393
+
4394
+
4395
+ def _quick_files_payload(args: argparse.Namespace) -> dict:
4396
+ """The quick payload of --manifest-file and --entry-file, read from
4397
+ disk, with the --tree-file listing as its `tree`. Raises
4398
+ QuickInputError."""
4399
+ root = None
4400
+ if args.source_root is not None:
4401
+ root = Path(args.source_root)
4402
+ if not root.is_dir():
4403
+ raise QuickInputError(f"--source-root is not a folder: {args.source_root}")
4404
+ manifest = {"path": "", "content": ""}
4405
+ if args.manifest_file is not None:
4406
+ manifest = _read_quick_file(root, args.manifest_file, "--manifest-file")
4407
+ entries: list[dict] = []
4408
+ for value in args.entry_file or []:
4409
+ entry = _read_quick_file(root, value, "--entry-file")
4410
+ if all(e["path"] != entry["path"] for e in entries):
4411
+ entries.append(entry)
4412
+ for value in getattr(args, "follow_file", None) or []:
4413
+ entry = _read_quick_file(root, value, "--follow-file")
4414
+ if all(e["path"] != entry["path"] for e in entries):
4415
+ entries.append({**entry, "followed": True})
4416
+ payload = {"language": args.language[0], "manifest": manifest, "entries": entries, "mode": "quick"}
4417
+ if getattr(args, "tree_file", None) is not None:
4418
+ payload["tree"] = _read_listing(args.tree_file)[0]
4419
+ return payload
4420
+
4421
+
4422
+ def _follow_reader(args: argparse.Namespace):
4423
+ """--follow's reader: paths -> (those now under --source-root, {path:
4424
+ why not} for the rest), each fetched there first with --repo. Raises
4425
+ QuickInputError."""
4426
+ root = Path(args.source_root)
4427
+ if args.repo is None:
4428
+ def read_local(paths: list[str]) -> tuple[list[str], dict[str, str]]:
4429
+ got = [p for p in paths if (root / p).is_file() and (root / p).resolve().is_relative_to(root.resolve())]
4430
+ return got, {p: "it is not under --source-root" for p in paths if p not in got}
4431
+ return read_local
4432
+ try:
4433
+ fetch = _sibling("skf-github-fetch.py")
4434
+ except RunnerError as exc:
4435
+ raise QuickInputError(str(exc)) from exc
4436
+ try:
4437
+ # No pattern: the call checks --repo, --ref and the listing, and fetches nothing.
4438
+ checked = fetch.run(args.repo, args.ref, args.tree_file, str(root), [])
4439
+ except fetch.UsageError as exc:
4440
+ raise QuickInputError(str(exc)) from exc
4441
+ owner, repo = checked["repo"].split("/", 1)
4442
+ fetch._start_clock(fetch.DEFAULT_TIMEOUT_SEC)
4443
+
4444
+ def read_remote(paths: list[str]) -> tuple[list[str], dict[str, str]]:
4445
+ out = fetch.fetch(owner, repo, checked["ref"], paths, root.absolute())
4446
+ return out["fetched"], {f["path"]: f["detail"] for f in out["failed"]}
4447
+ return read_remote
4448
+
4449
+
4450
+ def _follow(args: argparse.Namespace, result: dict) -> dict:
4451
+ """--follow: read each module_file `unlisted[]` names as one more
4452
+ --follow-file and parse the files again, round after round, until no
4453
+ module is left untried or FOLLOW_ROUNDS have run. Raises
4454
+ QuickInputError."""
4455
+ read = _follow_reader(args)
4456
+ tried = set(args.follow_file or [])
4457
+ followed: list[str] = []
4458
+ missed: dict[str, str] = {}
4459
+ for _ in range(FOLLOW_ROUNDS):
4460
+ named = [path for path in dict.fromkeys(item["module_file"] for item in result["unlisted"])
4461
+ if path and path not in tried]
4462
+ if not named:
4463
+ break
4464
+ tried.update(named)
4465
+ got, failed = read(named)
4466
+ missed.update(failed)
4467
+ if got:
4468
+ followed += got
4469
+ files = argparse.Namespace(**{**vars(args), "follow_file": [*(args.follow_file or []), *followed]})
4470
+ result = extract(_quick_files_payload(files))
4471
+ left = [path for path in dict.fromkeys(item["module_file"] for item in result["unlisted"])
4472
+ if path and path not in tried]
4473
+ result["warnings"] += [f"--follow could not read {path} ({why}), so the names it passes on are missing"
4474
+ for path, why in missed.items()]
4475
+ if left:
4476
+ result["warnings"].append(f"--follow stopped after {FOLLOW_ROUNDS} rounds with modules left to read: "
4477
+ f"{', '.join(left)}")
4478
+ result["followed"] = followed
4479
+ return result
4480
+
4481
+
4482
+ def _main_entries(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
4483
+ """`--mode entries`: the entry files of one package, as JSON on stdout."""
4484
+ if args.manifest_file is not None or args.entry_file or args.follow_file:
4485
+ parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
4486
+ given = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
4487
+ if getattr(args, flag) is not None and flag not in QUICK_FILE_FLAGS]
4488
+ if given:
4489
+ parser.error(f"{', '.join(given)}: full mode only (--mode full)")
4490
+ if not args.language or len(args.language) != 1:
4491
+ parser.error("--mode entries takes one --language")
4492
+ if args.tree_file is None or args.source_root is None:
4493
+ parser.error("--mode entries needs --tree-file and --source-root")
4494
+ root = Path(args.source_root)
4495
+ try:
4496
+ if root.exists() and not root.is_dir():
4497
+ raise QuickInputError(f"--source-root is not a folder: {args.source_root}")
4498
+ tree, truncated = _read_listing(args.tree_file)
4499
+ result = find_entries(args.language[0], tree, root, args.scope)
4500
+ if "_error" in result:
4501
+ print(json.dumps(result, indent=2))
4502
+ return 1
4503
+ if truncated:
4504
+ result["warnings"].append("the listing was cut short (a very large tree): an entry file may be "
4505
+ "missing from it")
4506
+ if args.fetch_list is not None:
4507
+ _write_fetch_list(args.fetch_list, result["entry_files"])
4508
+ except QuickInputError as exc:
4509
+ sys.stderr.write(f"error: {exc}\n")
4510
+ return 2
4511
+ print(json.dumps(result, indent=2))
4512
+ return 0
4513
+
4514
+
4515
+ def main(argv: list[str]) -> int:
4516
+ parser = _build_parser()
4517
+ args = parser.parse_args(argv)
4518
+ files = args.manifest_file is not None or bool(args.entry_file) or bool(args.follow_file)
4519
+ listing = [f"--{flag.replace('_', '-')}" for flag in LISTING_FLAGS if getattr(args, flag) is not None]
4520
+ following = [f"--{flag}" for flag in FOLLOW_FLAGS if getattr(args, flag) not in (None, False)]
4521
+ if following and (args.mode != "quick" or not files):
4522
+ parser.error(f"{', '.join(following)}: quick mode only, with --manifest-file or --entry-file")
4523
+ if args.mode == "full":
4524
+ if files:
4525
+ parser.error("--manifest-file, --entry-file, --follow-file: quick mode only (--mode quick)")
4526
+ if listing:
4527
+ parser.error(f"{', '.join(listing)}: entries or quick mode only")
4528
+ return _main_full(args)
4529
+ if args.mode == "entries":
4530
+ return _main_entries(args, parser)
4531
+ if args.scope is not None:
4532
+ parser.error("--scope: entries mode only (--mode entries)")
4533
+ if listing and not files:
4534
+ parser.error(f"{', '.join(listing)}: quick mode takes them only with --manifest-file or --entry-file")
4535
+ given = [f"--{flag.replace('_', '-')}" for flag in FULL_MODE_FLAGS
4536
+ if getattr(args, flag) is not None and not (files and flag in QUICK_FILE_FLAGS)]
4537
+ if given:
4538
+ hint = ""
4539
+ if any(flag in given for flag in ("--source-root", "--language")):
4540
+ hint = " (quick mode takes --source-root and --language only with --manifest-file or --entry-file)"
4541
+ parser.error(f"{', '.join(given)}: full mode only (--mode full){hint}")
4542
+
4543
+ if files:
4544
+ if not args.language or len(args.language) != 1:
4545
+ parser.error("--manifest-file and --entry-file take one --language")
4546
+ if (args.repo is not None or args.ref is not None) and not args.follow:
4547
+ parser.error("--repo and --ref fetch the modules --follow reads, so pass --follow")
4548
+ if args.ref is not None and args.repo is None:
4549
+ parser.error("--ref is the ref of --repo, so pass --repo")
4550
+ if args.follow and (args.tree_file is None or args.source_root is None):
4551
+ parser.error("--follow finds each module in --tree-file and reads it under --source-root: pass both")
4552
+ try:
4553
+ payload = _quick_files_payload(args)
4554
+ result = extract(payload)
4555
+ if args.follow and "_error" not in result:
4556
+ result = _follow(args, result)
4557
+ if args.fetch_list is not None and "_error" not in result:
4558
+ follow = [item["module_file"] for item in result["unlisted"] if item["module_file"]]
4559
+ _write_fetch_list(args.fetch_list, list(dict.fromkeys(follow)))
4560
+ except QuickInputError as exc:
4561
+ sys.stderr.write(f"error: {exc}\n")
4562
+ return 2
4563
+ print(json.dumps(result, indent=2))
4564
+ return 1 if "_error" in result else 0
566
4565
 
567
4566
  raw = sys.stdin.read()
568
4567
  if not raw.strip():