bmad-module-skill-forge 2.2.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (437) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +49 -49
  3. package/docs/404.md +10 -8
  4. package/docs/_data/pinned.yaml +22 -13
  5. package/docs/_internal/STABILITY.md +58 -35
  6. package/docs/agents.md +39 -53
  7. package/docs/architecture.md +65 -51
  8. package/docs/bmad-synergy.md +53 -51
  9. package/docs/campaign.md +127 -75
  10. package/docs/concepts.md +34 -28
  11. package/docs/examples.md +70 -68
  12. package/docs/forge-auto.md +34 -31
  13. package/docs/getting-started.md +118 -57
  14. package/docs/how-it-works.md +25 -21
  15. package/docs/index.md +15 -15
  16. package/docs/skill-model.md +70 -59
  17. package/docs/troubleshooting.md +143 -18
  18. package/docs/verifying-a-skill.md +87 -57
  19. package/docs/why-skf.md +24 -22
  20. package/docs/workflows.md +274 -110
  21. package/package.json +18 -10
  22. package/src/forger/preferences.yaml +8 -1
  23. package/src/knowledge/agentskills-spec.md +2 -0
  24. package/src/knowledge/ccc-bridge.md +40 -24
  25. package/src/knowledge/confidence-tiers.md +9 -5
  26. package/src/knowledge/overview.md +3 -3
  27. package/src/knowledge/progressive-capability.md +8 -8
  28. package/src/knowledge/provenance-tracking.md +9 -5
  29. package/src/knowledge/qmd-registry.md +3 -3
  30. package/src/knowledge/skf-knowledge-index.csv +2 -2
  31. package/src/knowledge/skill-lifecycle.md +18 -11
  32. package/src/knowledge/split-body-strategy.md +0 -3
  33. package/src/knowledge/tool-resolution.md +4 -2
  34. package/src/knowledge/version-paths.md +101 -15
  35. package/src/module-help.csv +3 -3
  36. package/src/module.yaml +10 -0
  37. package/src/shared/data/ast-grep-recipes.yaml +608 -0
  38. package/src/shared/data/ide-context-files.json +30 -0
  39. package/src/shared/health-check.md +31 -59
  40. package/src/shared/references/description-guard-protocol.md +12 -9
  41. package/src/shared/references/feasibility-report-schema.md +19 -13
  42. package/src/shared/references/headless-gate-convention.md +55 -3
  43. package/src/shared/references/output-contract-schema.md +19 -8
  44. package/src/shared/references/pipeline-contracts.md +94 -29
  45. package/src/shared/references/tessl-review.md +77 -0
  46. package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
  47. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
  48. package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
  49. package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
  50. package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
  51. package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
  52. package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
  53. package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
  54. package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
  55. package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
  56. package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
  57. package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
  58. package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
  59. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
  60. package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
  61. package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
  62. package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
  63. package/src/shared/scripts/skf-atomic-write.py +233 -64
  64. package/src/shared/scripts/skf-build-change-manifest.py +2753 -28
  65. package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
  66. package/src/shared/scripts/skf-check-unit-records.py +184 -0
  67. package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
  68. package/src/shared/scripts/skf-classify-changed-files.py +772 -0
  69. package/src/shared/scripts/skf-comention-pairs.py +1333 -123
  70. package/src/shared/scripts/skf-compare-file-hashes.py +267 -105
  71. package/src/shared/scripts/skf-count-imports.py +1867 -0
  72. package/src/shared/scripts/skf-count-tokens.py +1 -1
  73. package/src/shared/scripts/skf-derive-assembly-shape.py +21 -3
  74. package/src/shared/scripts/skf-description-guard.py +99 -11
  75. package/src/shared/scripts/skf-detect-docs.py +694 -2
  76. package/src/shared/scripts/skf-detect-language.py +486 -107
  77. package/src/shared/scripts/skf-detect-registry.py +887 -0
  78. package/src/shared/scripts/skf-detect-scripts-assets.py +197 -55
  79. package/src/shared/scripts/skf-detect-tools.py +209 -27
  80. package/src/shared/scripts/skf-detect-workspaces.py +506 -30
  81. package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
  82. package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
  83. package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
  84. package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
  85. package/src/shared/scripts/skf-extract-public-api.py +5247 -96
  86. package/src/shared/scripts/skf-extraction-inventory.py +1280 -0
  87. package/src/shared/scripts/skf-extraction-snapshot.py +664 -0
  88. package/src/shared/scripts/skf-fetch-temporal.py +674 -0
  89. package/src/shared/scripts/skf-find-cycles.py +97 -29
  90. package/src/shared/scripts/skf-find-test-report.py +476 -0
  91. package/src/shared/scripts/skf-forge-tier-rw.py +469 -205
  92. package/src/shared/scripts/skf-github-fetch.py +365 -0
  93. package/src/shared/scripts/skf-github-probe.py +700 -0
  94. package/src/shared/scripts/skf-hash-content.py +180 -20
  95. package/src/shared/scripts/skf-language-corpora.py +1 -1
  96. package/src/shared/scripts/skf-load-provenance.py +161 -27
  97. package/src/shared/scripts/skf-manifest-ops.py +1 -1
  98. package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
  99. package/src/shared/scripts/skf-merge-doc-urls.py +143 -20
  100. package/src/shared/scripts/skf-names-present.py +221 -0
  101. package/src/shared/scripts/skf-pair-intersect.py +119 -28
  102. package/src/shared/scripts/skf-parse-gaps.py +1384 -0
  103. package/src/shared/scripts/skf-preapply.py +1 -1
  104. package/src/shared/scripts/skf-preflight.py +176 -28
  105. package/src/shared/scripts/skf-promote-staged.py +430 -0
  106. package/src/shared/scripts/skf-provenance-gap-dispatch.py +64 -8
  107. package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
  108. package/src/shared/scripts/skf-quick-batch.py +774 -0
  109. package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
  110. package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
  111. package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
  112. package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
  113. package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
  114. package/src/shared/scripts/skf-resolve-authoritative-files.py +155 -14
  115. package/src/shared/scripts/skf-resolve-package.py +523 -106
  116. package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
  117. package/src/shared/scripts/skf-run-lock.py +577 -0
  118. package/src/shared/scripts/skf-scan-manifests.py +998 -57
  119. package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
  120. package/src/shared/scripts/skf-severity-classify.py +712 -108
  121. package/src/shared/scripts/skf-shape-detect.py +298 -204
  122. package/src/shared/scripts/skf-shard-body.py +9 -8
  123. package/src/shared/scripts/skf-skill-inventory.py +2412 -89
  124. package/src/shared/scripts/skf-skills-module.py +472 -0
  125. package/src/shared/scripts/skf-source-tree.py +1796 -0
  126. package/src/shared/scripts/skf-structural-diff.py +787 -121
  127. package/src/shared/scripts/skf-tessl-review.py +706 -0
  128. package/src/shared/scripts/skf-update-active-symlink.py +120 -67
  129. package/src/shared/scripts/skf-update-run-state.py +624 -0
  130. package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
  131. package/src/shared/scripts/skf-validate-brief-schema.py +23 -2
  132. package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
  133. package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
  134. package/src/shared/scripts/skf-validate-output.py +712 -64
  135. package/src/shared/scripts/skf-validate-pins.py +1 -1
  136. package/src/shared/scripts/skf-verify-no-trace.py +116 -10
  137. package/src/shared/scripts/skf-verify-provenance-completeness.py +3945 -152
  138. package/src/shared/scripts/skf-write-skill-brief.py +406 -25
  139. package/src/shared/tool-requirements.yaml +201 -0
  140. package/src/skf-analyze-source/SKILL.md +41 -36
  141. package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
  142. package/src/skf-analyze-source/customize.toml +36 -24
  143. package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
  144. package/src/skf-analyze-source/references/continue.md +2 -13
  145. package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
  146. package/src/skf-analyze-source/references/generate-briefs.md +121 -55
  147. package/src/skf-analyze-source/references/headless-contract.md +42 -15
  148. package/src/skf-analyze-source/references/identify-units.md +96 -110
  149. package/src/skf-analyze-source/references/init.md +73 -71
  150. package/src/skf-analyze-source/references/map-and-detect.md +123 -105
  151. package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
  152. package/src/skf-analyze-source/references/recommend.md +51 -70
  153. package/src/skf-analyze-source/references/scan-project.md +60 -84
  154. package/src/skf-analyze-source/references/scan-root.md +39 -0
  155. package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
  156. package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
  157. package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
  158. package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
  159. package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
  160. package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
  161. package/src/skf-audit-skill/SKILL.md +43 -51
  162. package/src/skf-audit-skill/assets/drift-report-template.md +28 -3
  163. package/src/skf-audit-skill/customize.toml +41 -20
  164. package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
  165. package/src/skf-audit-skill/references/doc-drift.md +141 -0
  166. package/src/skf-audit-skill/references/headless-contract.md +29 -0
  167. package/src/skf-audit-skill/references/health-check.md +2 -0
  168. package/src/skf-audit-skill/references/init.md +149 -210
  169. package/src/skf-audit-skill/references/re-index.md +103 -110
  170. package/src/skf-audit-skill/references/report.md +96 -33
  171. package/src/skf-audit-skill/references/semantic-diff.md +34 -25
  172. package/src/skf-audit-skill/references/severity-classify.md +79 -65
  173. package/src/skf-audit-skill/references/severity-rules.md +15 -6
  174. package/src/skf-audit-skill/references/structural-diff.md +80 -90
  175. package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
  176. package/src/skf-audit-skill/scripts/render-drift-tables.py +712 -0
  177. package/src/skf-brief-skill/SKILL.md +37 -26
  178. package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
  179. package/src/skf-brief-skill/assets/scope-templates.md +18 -25
  180. package/src/skf-brief-skill/assets/skill-brief-schema.md +30 -23
  181. package/src/skf-brief-skill/customize.toml +48 -20
  182. package/src/skf-brief-skill/references/analyze-target.md +144 -127
  183. package/src/skf-brief-skill/references/confirm-brief.md +11 -9
  184. package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
  185. package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
  186. package/src/skf-brief-skill/references/gather-intent.md +87 -182
  187. package/src/skf-brief-skill/references/headless-args.md +84 -16
  188. package/src/skf-brief-skill/references/health-check.md +1 -1
  189. package/src/skf-brief-skill/references/invocation-contract.md +18 -13
  190. package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
  191. package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
  192. package/src/skf-brief-skill/references/scope-definition.md +123 -47
  193. package/src/skf-brief-skill/references/step-auto-brief.md +26 -100
  194. package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
  195. package/src/skf-brief-skill/references/version-resolution.md +7 -10
  196. package/src/skf-brief-skill/references/write-brief.md +66 -57
  197. package/src/skf-campaign/SKILL.md +31 -34
  198. package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
  199. package/src/skf-campaign/customize.toml +47 -16
  200. package/src/skf-campaign/references/campaign-contracts.md +39 -29
  201. package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
  202. package/src/skf-campaign/references/health-check.md +7 -16
  203. package/src/skf-campaign/references/step-01-setup.md +46 -84
  204. package/src/skf-campaign/references/step-02-strategy.md +31 -30
  205. package/src/skf-campaign/references/step-03-pins.md +26 -18
  206. package/src/skf-campaign/references/step-04-provenance.md +24 -23
  207. package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
  208. package/src/skf-campaign/references/step-06-batch.md +41 -34
  209. package/src/skf-campaign/references/step-07-capstone.md +40 -25
  210. package/src/skf-campaign/references/step-08-verify.md +28 -29
  211. package/src/skf-campaign/references/step-09-refine.md +29 -30
  212. package/src/skf-campaign/references/step-10-export.md +48 -36
  213. package/src/skf-campaign/references/step-11-maintenance.md +31 -34
  214. package/src/skf-campaign/references/step-resume.md +37 -68
  215. package/src/skf-campaign/scripts/campaign-deps.py +57 -18
  216. package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
  217. package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
  218. package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
  219. package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
  220. package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
  221. package/src/skf-campaign/scripts/campaign-report.py +204 -37
  222. package/src/skf-campaign/scripts/campaign-state.py +1128 -0
  223. package/src/skf-campaign/scripts/campaign-status.py +10 -74
  224. package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
  225. package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
  226. package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
  227. package/src/skf-campaign/templates/campaign-report-template.md +5 -1
  228. package/src/skf-campaign/templates/kickoff-template.md +0 -14
  229. package/src/skf-create-skill/SKILL.md +27 -20
  230. package/src/skf-create-skill/assets/compile-assembly-rules.md +11 -8
  231. package/src/skf-create-skill/assets/skill-sections.md +21 -11
  232. package/src/skf-create-skill/customize.toml +20 -8
  233. package/src/skf-create-skill/references/authoritative-files-protocol.md +36 -58
  234. package/src/skf-create-skill/references/batch-mode.md +81 -0
  235. package/src/skf-create-skill/references/ccc-index-check.md +27 -0
  236. package/src/skf-create-skill/references/compile.md +119 -70
  237. package/src/skf-create-skill/references/component-extraction.md +113 -91
  238. package/src/skf-create-skill/references/enrich.md +41 -23
  239. package/src/skf-create-skill/references/entry-points-by-hand.md +37 -0
  240. package/src/skf-create-skill/references/extract.md +204 -162
  241. package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
  242. package/src/skf-create-skill/references/extraction-patterns-tracing.md +11 -11
  243. package/src/skf-create-skill/references/extraction-patterns.md +588 -201
  244. package/src/skf-create-skill/references/generate-artifacts.md +145 -93
  245. package/src/skf-create-skill/references/health-check.md +1 -1
  246. package/src/skf-create-skill/references/load-brief.md +53 -36
  247. package/src/skf-create-skill/references/relabel-rule.md +12 -0
  248. package/src/skf-create-skill/references/report.md +37 -57
  249. package/src/skf-create-skill/references/source-resolution-protocols.md +58 -169
  250. package/src/skf-create-skill/references/step-auto-shard.md +20 -93
  251. package/src/skf-create-skill/references/step-doc-rot.md +44 -60
  252. package/src/skf-create-skill/references/step-doc-sources.md +28 -19
  253. package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
  254. package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
  255. package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
  256. package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
  257. package/src/skf-create-skill/references/validate.md +173 -99
  258. package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
  259. package/src/skf-create-stack-skill/SKILL.md +29 -56
  260. package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
  261. package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
  262. package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
  263. package/src/skf-create-stack-skill/customize.toml +23 -12
  264. package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
  265. package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
  266. package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
  267. package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
  268. package/src/skf-create-stack-skill/references/generate-output.md +138 -128
  269. package/src/skf-create-stack-skill/references/init.md +90 -32
  270. package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
  271. package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
  272. package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
  273. package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
  274. package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
  275. package/src/skf-create-stack-skill/references/report.md +39 -33
  276. package/src/skf-create-stack-skill/references/validate.md +20 -39
  277. package/src/skf-drop-skill/SKILL.md +28 -51
  278. package/src/skf-drop-skill/customize.toml +44 -30
  279. package/src/skf-drop-skill/references/execute.md +83 -146
  280. package/src/skf-drop-skill/references/health-check.md +2 -0
  281. package/src/skf-drop-skill/references/invocation-contract.md +64 -0
  282. package/src/skf-drop-skill/references/report.md +46 -30
  283. package/src/skf-drop-skill/references/select.md +122 -124
  284. package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
  285. package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
  286. package/src/skf-export-skill/SKILL.md +40 -62
  287. package/src/skf-export-skill/assets/managed-section-format.md +14 -50
  288. package/src/skf-export-skill/assets/snippet-format.md +3 -1
  289. package/src/skf-export-skill/customize.toml +42 -13
  290. package/src/skf-export-skill/references/generate-snippet.md +54 -46
  291. package/src/skf-export-skill/references/health-check.md +2 -0
  292. package/src/skf-export-skill/references/invocation-contract.md +15 -0
  293. package/src/skf-export-skill/references/load-skill.md +88 -62
  294. package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
  295. package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
  296. package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
  297. package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
  298. package/src/skf-export-skill/references/package.md +15 -73
  299. package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
  300. package/src/skf-export-skill/references/result-envelope.md +31 -16
  301. package/src/skf-export-skill/references/summary.md +41 -17
  302. package/src/skf-export-skill/references/token-report.md +4 -12
  303. package/src/skf-export-skill/references/update-context.md +118 -237
  304. package/src/skf-forger/SKILL.md +61 -37
  305. package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
  306. package/src/skf-forger/customize.toml +43 -0
  307. package/src/skf-forger/references/pipeline-mode.md +73 -18
  308. package/src/skf-forger/scripts/forge-status.py +299 -0
  309. package/src/skf-forger/scripts/parse-pipeline.py +322 -54
  310. package/src/skf-forger/scripts/pipeline-gate.py +428 -0
  311. package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
  312. package/src/skf-quick-skill/SKILL.md +54 -53
  313. package/src/skf-quick-skill/assets/skill-template.md +4 -2
  314. package/src/skf-quick-skill/customize.toml +51 -20
  315. package/src/skf-quick-skill/references/batch-mode.md +74 -21
  316. package/src/skf-quick-skill/references/compile.md +30 -31
  317. package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
  318. package/src/skf-quick-skill/references/finalize.md +69 -18
  319. package/src/skf-quick-skill/references/halt-contract.md +41 -29
  320. package/src/skf-quick-skill/references/health-check.md +5 -3
  321. package/src/skf-quick-skill/references/quick-extract.md +114 -45
  322. package/src/skf-quick-skill/references/registry-resolution.md +9 -64
  323. package/src/skf-quick-skill/references/resolve-target.md +116 -66
  324. package/src/skf-quick-skill/references/write-and-validate.md +79 -40
  325. package/src/skf-refine-architecture/SKILL.md +43 -53
  326. package/src/skf-refine-architecture/customize.toml +49 -20
  327. package/src/skf-refine-architecture/references/compile.md +97 -54
  328. package/src/skf-refine-architecture/references/exit-codes.md +30 -0
  329. package/src/skf-refine-architecture/references/finding-storage.md +21 -0
  330. package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
  331. package/src/skf-refine-architecture/references/improvements.md +14 -20
  332. package/src/skf-refine-architecture/references/init.md +98 -57
  333. package/src/skf-refine-architecture/references/issue-detection.md +32 -39
  334. package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
  335. package/src/skf-refine-architecture/references/report.md +65 -54
  336. package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
  337. package/src/skf-rename-skill/SKILL.md +19 -54
  338. package/src/skf-rename-skill/customize.toml +32 -26
  339. package/src/skf-rename-skill/references/execute.md +230 -184
  340. package/src/skf-rename-skill/references/exit-codes.md +7 -7
  341. package/src/skf-rename-skill/references/health-check.md +1 -7
  342. package/src/skf-rename-skill/references/invocation-contract.md +25 -0
  343. package/src/skf-rename-skill/references/report.md +63 -52
  344. package/src/skf-rename-skill/references/select.md +197 -115
  345. package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
  346. package/src/skf-setup/SKILL.md +26 -26
  347. package/src/skf-setup/customize.toml +14 -10
  348. package/src/skf-setup/references/auto-index.md +50 -63
  349. package/src/skf-setup/references/ccc-index.md +29 -90
  350. package/src/skf-setup/references/detect-and-tier.md +58 -67
  351. package/src/skf-setup/references/health-check.md +2 -0
  352. package/src/skf-setup/references/invocation-contract.md +14 -0
  353. package/src/skf-setup/references/report.md +43 -166
  354. package/src/skf-setup/references/tier-rules.md +5 -5
  355. package/src/skf-setup/references/write-config.md +34 -57
  356. package/src/skf-test-skill/SKILL.md +17 -57
  357. package/src/skf-test-skill/assets/output-section-formats.md +33 -34
  358. package/src/skf-test-skill/customize.toml +27 -13
  359. package/src/skf-test-skill/references/coherence-check.md +134 -62
  360. package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
  361. package/src/skf-test-skill/references/coverage-check.md +209 -264
  362. package/src/skf-test-skill/references/detect-mode.md +5 -5
  363. package/src/skf-test-skill/references/external-validators.md +94 -82
  364. package/src/skf-test-skill/references/health-check.md +6 -7
  365. package/src/skf-test-skill/references/init.md +164 -91
  366. package/src/skf-test-skill/references/invocation-contract.md +66 -0
  367. package/src/skf-test-skill/references/migration-section-rules.md +5 -2
  368. package/src/skf-test-skill/references/report.md +164 -142
  369. package/src/skf-test-skill/references/score.md +80 -107
  370. package/src/skf-test-skill/references/scoring-rules.md +42 -155
  371. package/src/skf-test-skill/references/source-access-protocol.md +18 -94
  372. package/src/skf-test-skill/references/step-hard-gate.md +51 -33
  373. package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
  374. package/src/skf-test-skill/scripts/build-result-context.py +380 -0
  375. package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
  376. package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
  377. package/src/skf-test-skill/scripts/compute-score.py +223 -45
  378. package/src/skf-test-skill/scripts/gap-ledger.py +1666 -0
  379. package/src/skf-test-skill/scripts/hard-gate.py +203 -0
  380. package/src/skf-test-skill/scripts/load-coverage-inputs.py +1198 -0
  381. package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
  382. package/src/skf-test-skill/scripts/reconcile-coverage.py +547 -79
  383. package/src/skf-test-skill/scripts/score-signatures.py +479 -0
  384. package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
  385. package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
  386. package/src/skf-test-skill/scripts/verify-declared-numerator.py +133 -56
  387. package/src/skf-test-skill/templates/test-report-template.md +21 -15
  388. package/src/skf-update-skill/SKILL.md +38 -25
  389. package/src/skf-update-skill/customize.toml +27 -12
  390. package/src/skf-update-skill/references/detect-changes.md +253 -223
  391. package/src/skf-update-skill/references/gap-driven.md +336 -0
  392. package/src/skf-update-skill/references/health-check.md +31 -5
  393. package/src/skf-update-skill/references/init.md +254 -84
  394. package/src/skf-update-skill/references/invocation-contract.md +12 -0
  395. package/src/skf-update-skill/references/manual-section-rules.md +6 -23
  396. package/src/skf-update-skill/references/merge.md +182 -49
  397. package/src/skf-update-skill/references/re-extract.md +53 -262
  398. package/src/skf-update-skill/references/report.md +102 -35
  399. package/src/skf-update-skill/references/write.md +357 -155
  400. package/src/skf-update-skill/scripts/skf-new-file-diff.py +183 -46
  401. package/src/skf-verify-stack/SKILL.md +23 -42
  402. package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
  403. package/src/skf-verify-stack/customize.toml +55 -22
  404. package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
  405. package/src/skf-verify-stack/references/coverage.md +67 -64
  406. package/src/skf-verify-stack/references/exit-codes.md +21 -8
  407. package/src/skf-verify-stack/references/init.md +107 -62
  408. package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
  409. package/src/skf-verify-stack/references/integrations.md +106 -65
  410. package/src/skf-verify-stack/references/report.md +59 -42
  411. package/src/skf-verify-stack/references/requirements.md +32 -9
  412. package/src/skf-verify-stack/references/synthesize.md +53 -39
  413. package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
  414. package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
  415. package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
  416. package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
  417. package/tools/cli/commands/install.js +11 -4
  418. package/tools/cli/commands/status.js +15 -3
  419. package/tools/cli/commands/update.js +11 -4
  420. package/tools/cli/lib/tool-check.js +320 -0
  421. package/tools/cli/lib/version-check.js +22 -9
  422. package/tools/cli/skf-cli.js +4 -1
  423. package/.gitattributes +0 -16
  424. package/.nvmrc +0 -1
  425. package/docs/_internal/RELEASING.md +0 -679
  426. package/release-audits/v1.0.0-launch-audit.md +0 -728
  427. package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
  428. package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
  429. package/src/skf-campaign/manifest.yaml +0 -15
  430. package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
  431. package/src/skf-create-skill/references/ecosystem-check.md +0 -74
  432. package/src/skf-drop-skill/references/headless-contract.md +0 -18
  433. package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
  434. package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
  435. package/src/skf-update-skill/references/validate.md +0 -124
  436. package/tools/validate-docs-drift.js +0 -233
  437. package/tools/validate-docs-links.js +0 -665
@@ -1,11 +1,11 @@
1
1
  #!/usr/bin/env python3
2
2
  # /// script
3
- # requires-python = ">=3.10"
4
- # dependencies = []
3
+ # requires-python = ">=3.11"
4
+ # dependencies = ["pyyaml"]
5
5
  # ///
6
6
  """SKF Verify Provenance Completeness — deterministic Check D for update-skill.
7
7
 
8
- `skf-update-skill/references/validate.md §2 Check D (Provenance Completeness)`
8
+ update-skill's Check D (Provenance Completeness, now in write.md §6)
9
9
  previously asked the model to perform three deterministic set/citation
10
10
  operations by eye, against in-context data:
11
11
 
@@ -14,7 +14,28 @@ operations by eye, against in-context data:
14
14
  2. **Orphans** — flag provenance entries whose export was removed
15
15
  (provenance `entries[].export_name` \\ metadata `exports[]`).
16
16
  3. **Stale citations** — flag entries whose `source_file:source_line`
17
- no longer resolves (file gone, or line past end of file).
17
+ no longer resolves (file gone, or line past end of file), or whose
18
+ line is not a line that defines the export (`line-not-definition`).
19
+ 4. **Citation prefixes** (only with `--skill-dir`): flag each `[AST:]` /
20
+ `[SRC:]` citation in the skill's markdown whose prefix disagrees with
21
+ the provenance entry it cites, or an `[AST:]` citation in a skill whose
22
+ map holds no ast-grep entry at all.
23
+ 5. **Node kinds** (only with `--check-node-kinds`): flag each ast-grep
24
+ entry whose `ast_node_type` is not a node kind ast-grep knows for the
25
+ entry's language.
26
+
27
+ Reference apps: when `--metadata` has `scope_type: reference-app`, 1 and 2
28
+ do not run. A reference app's `exports[]` is empty by design while its
29
+ entries are per-citation, so the two sets measure different surfaces:
30
+ `missing` and `orphaned` stay empty and `summary.set_diff` is
31
+ `not-applicable` (`checked` otherwise).
32
+
33
+ Four more subcommands act on the same rules: `fix` applies the citation
34
+ prefix and source-line fixes a `verify` result has one answer for,
35
+ `definition-lines` gives the lines that define one export,
36
+ `classify-stale` decides which stale names are fabricated signatures and
37
+ where the source declares the others, and `kind-at` gives the node kind
38
+ of the recipe that matches a source line.
18
39
 
19
40
  An LLM set-diff can silently pass a dropped or orphaned entry, and eyeballing
20
41
  whether a cited line still exists is not something the model can do reliably.
@@ -25,9 +46,12 @@ export-name strings) and `provenance-map.json` (`entries[]` with
25
46
  `write.md` §2 and §3. This script diffs the two sets and re-checks the
26
47
  citations exactly, so Check D consumes JSON rather than computing it.
27
48
 
28
- Timing: this runs **post-write** (`write.md` §6), after `metadata.json`
29
- (§2) and `provenance-map.json` (§3) are on disk. `validate.md` Check D
30
- defers here — the provenance map does not exist yet at validate time.
49
+ Timing: update-skill runs it **post-write**, in write.md §6, after
50
+ `metadata.json` (§2) and `provenance-map.json` (§3) are on disk.
51
+ create-skill runs it too, at validate §7a against the staged package (with
52
+ `--skill-dir` and `--check-node-kinds`, as update-skill does), and
53
+ test-skill at coverage-check §4c (line check only); test-skill's
54
+ coverage-check §2c runs `classify-stale`.
31
55
 
32
56
  Determinism:
33
57
  - Set operations are pure over the two JSON inputs.
@@ -44,9 +68,157 @@ internal name of a barrel-renamed export (e.g. `_Impl` re-exported as
44
68
  `skf-load-provenance.extract_reexport_map`, so an internal-named entry does
45
69
  not read as a missing public export nor as an orphan.
46
70
 
47
- Subcommand:
71
+ Definition lines: after a citation resolves (the file exists and the line
72
+ is in range, so the older reasons win), the line must define the entry's
73
+ export. NAME is the raw `export_name` (not canonicalized); for a dotted
74
+ name (`App.update`) it is the last segment. The rules follow the
75
+ `source_file` extension:
76
+
77
+ - Python (`.py`, `.pyi`): the `def` / `async def` / `class NAME` line
78
+ itself (a decorator line above it does not count); an assignment
79
+ binding NAME (`NAME =`, `NAME: T =`, annotation-only `NAME: T`, a
80
+ target list `A, NAME = ...`, a chain `A = NAME = ...`, PEP 695
81
+ `type NAME =`); an import that binds NAME (`import NAME`,
82
+ `import X as NAME`, `from X import NAME`, `from X import Y as NAME`),
83
+ including NAME on its own line inside a parenthesized or
84
+ backslash-continued import; a `def` / `async def` / `class` line
85
+ ending in a backslash counts through the next line when that line
86
+ starts with NAME, and the name's line is the definition line (the one
87
+ the recipes cite). These count only on a line where a
88
+ statement begins (the stdlib tokenizer says so), so a docstring line,
89
+ a keyword argument or a parameter never matches; a file that does not
90
+ tokenize is matched line by line. A `from X import *` line counts only
91
+ when no other line of the file defines NAME (a re-export the
92
+ extraction could not trace is cited at its star import).
93
+ - TS/JS (`.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`): a
94
+ `function` (or `function*`), `class`, `interface`, `type`, `enum`,
95
+ `namespace`, `const`, `let` or `var` declaration of NAME, with any of
96
+ `export`, `default`, `async`, `declare`, `abstract` and decorators
97
+ (`@name`, `@a.b`, `@name(...)`, before `export` or between `export`
98
+ and `class`; a name inside a decorator's arguments is not declared);
99
+ a later declarator of a `const`, `let` or `var` list
100
+ (`export const a = 1, NAME = 2;`, or `export const a = 1,` then
101
+ ` NAME = 2;` as Prettier writes it: see Declarator lists below); NAME
102
+ on the line right after a line that ends in its declaration keyword
103
+ (`export function`, `export interface`, `export const` and so on),
104
+ which is the line cited;
105
+ `export import NAME =`; `export * as NAME from` (NAME may be quoted);
106
+ `export default NAME`;
107
+ an `export { ... }` list whose exposed name is NAME, an identifier or
108
+ a quoted string on either side of `as`; CommonJS
109
+ `exports.NAME =`, `module.exports.NAME =`, or NAME as a key or
110
+ shorthand inside `module.exports = { ... }`. A list or object spanning
111
+ several lines counts on the line where NAME appears. For a dotted
112
+ name, an indented class member declaring NAME (method, property or
113
+ accessor, with any of `public`, `private`, `protected`, `static`,
114
+ `readonly`, `async`, `abstract`, `override`, `declare`, `get`, `set`)
115
+ also counts. An indented `NAME(...)` with no modifier, followed by `;`
116
+ or another expression rather than a body `{` or a return type `:`, is
117
+ a call, not a member; an indented `NAME: ...` or `NAME = ...` ending
118
+ with `,` is an object-literal entry or a destructuring default.
119
+ Comments never match: `//` to the end of the line and `/* ... */`
120
+ across lines are removed first, outside '...', "..." and `...` strings,
121
+ and brackets inside a string do not count toward a `module.exports`
122
+ object's depth. Strings are followed within one line only, so a
123
+ template literal spanning lines can hide or expose a comment marker.
124
+
125
+ Declarator lists: a `const`, `let` or `var` list is split at its
126
+ top-level commas, outside strings, regular expression literals and
127
+ brackets, a type-argument list included (`Map<K, V>`: a `<` right after
128
+ a name), up to a top-level `;`, and a destructuring pattern declares
129
+ nothing. The list goes on to the next line when a line ends inside a
130
+ bracket, after a top-level comma or after an operator waiting for its
131
+ operand, or when the next line starts with an operator (a member chain or
132
+ a ternary broken over lines), so a declarator on a line of its own
133
+ counts. A `<` where an expression begins (JSX, a type assertion, a generic
134
+ arrow function's type parameters) is not followed: no later declarator of
135
+ the list counts after it.
136
+
137
+ Indentation: for a dotted name every match counts. For an undotted name,
138
+ matches at any indentation count, but when the file has a column-0 match
139
+ the indented ones are dropped, so a local shadow never outranks the
140
+ module-level definition. A line inside a multi-line import, export list
141
+ or `module.exports` object takes the column of the line its statement
142
+ starts on, and so do the name line of a declaration split after its
143
+ keyword and a later declarator on a line of its own. When the recorded
144
+ line is itself an indented match that this rule dropped (a method
145
+ recorded under an undotted name beside a module-level function of the
146
+ same name), the finding lists the recorded line too, so it carries
147
+ several lines and no caller auto-moves it.
148
+
149
+ Entries in other files, and entries whose `export_type` is `module` or
150
+ `package` (case-insensitive; neither has a definition line), are skipped
151
+ and counted in `summary.line_check_skipped`. A `line-not-definition` item
152
+ lists every defining line in the file in `definition_lines`. An empty list
153
+ means the rules found no definition line, which may be a shape they do not
154
+ cover: the entry is unverified, not gone, and no caller auto-fixes it.
155
+
156
+ Citation check (`--skill-dir DIR`, which must hold a `SKILL.md`): scans
157
+ `DIR/SKILL.md` and `DIR/references/**/*.md` for `[AST:path:Ln]` /
158
+ `[SRC:path:Ln]`, ranges `Ln-m` / `Ln-Lm` included (the first line counts);
159
+ a citation with no line is ignored. A citation matches an entry when its
160
+ normalized path equals `source_file` and its line equals `source_line`.
161
+ `summary.skill_citations_matched` counts the citations that matched an
162
+ entry: 0 with citations scanned means the skill cites paths under another
163
+ root. Reasons, one per citation:
164
+
165
+ - `prefix-mismatch`: the matched entry's `extraction_method` implies the
166
+ other prefix (`ast-grep` / `ast_bridge` give AST, `source-read` /
167
+ `source_reading` give SRC). It wins over the reason below.
168
+ - `ast-without-ast-grep`: an `[AST:]` citation in a skill whose map has
169
+ no `ast-grep` or `ast_bridge` entry, raised only when every entry with
170
+ a source line carries a known `extraction_method` (a missing or
171
+ unknown method may hide an ast-grep entry).
172
+
173
+ When several entries share the cited line, the citation is a
174
+ `prefix-mismatch` only when none of them implies its prefix. An
175
+ `extraction_method` that is not a string reads as unknown (it implies no
176
+ prefix). Unmatched citations are not findings unless the second reason
177
+ applies.
178
+
179
+ Node-kind check (`--check-node-kinds`): ast-grep's JSON output never
180
+ reports the kind of the node a rule matched, so an entry's `ast_node_type`
181
+ is copied from the recipe that matched it, and nothing else proves it is a
182
+ real kind. For each entry whose `extraction_method` is `ast-grep` or
183
+ `ast_bridge` and whose `ast_node_type` is a non-blank string, the
184
+ `source_file` extension picks an ast-grep language (`.py` / `.pyi` python,
185
+ `.ts` / `.mts` / `.cts` typescript, `.tsx` tsx, `.js` / `.jsx` / `.mjs` /
186
+ `.cjs` javascript, `.rs` rust, `.go` go, `.java` java, `.kt` / `.kts`
187
+ kotlin, `.cs` csharp, `.rb` ruby, `.swift` swift, `.php` php); an entry in
188
+ any other file is skipped and counted in `summary.node_kind_check_skipped`.
189
+ Reasons, one per entry:
190
+
191
+ - `error-kind`: the kind is `ERROR`, the node tree-sitter makes of code
192
+ it cannot parse. ast-grep accepts it in a rule, so it is not asked.
193
+ - `invalid-kind`: the kind is not shaped like a kind
194
+ (`[A-Za-z_][A-Za-z0-9_]*`, so no whitespace or shell characters reach
195
+ ast-grep), judged without ast-grep; or ast-grep rejected it (its error
196
+ names an invalid kind).
197
+
198
+ Those two are judged whatever the ast-grep lookup finds. Every other kind is
199
+ asked once per distinct (language, kind): `ast-grep scan --config
200
+ <sgconfig.yml> --inline-rules <rule> --stdin` with a rule matching only that
201
+ kind, over an empty stdin, from a temporary folder holding a minimal
202
+ sgconfig.yml that `--config` names (so no other sgconfig.yml is read).
203
+ ast-grep parses the rule before reading input and rejects a kind the
204
+ language's grammar lacks. Before that, ast-grep is asked about a kind no
205
+ grammar has; unless it rejects it, its answers cannot be read and the check
206
+ is `skipped-unrecognized-ast-grep`. When no `ast-grep` executable resolves
207
+ (the CWD-shim guard applies), the check is `skipped-no-ast-grep`. With no
208
+ kind left to ask, ast-grep is not looked up and the check is `checked`.
209
+
210
+ An ast-grep failure that does not name the kind proves nothing about it: the
211
+ entry is listed in `node_kinds_unchecked[]`, not as a finding, with reason
212
+ `ast-grep-error`, `ast-grep-timeout` (after the first timeout the remaining
213
+ kinds are not asked) or `temp-folder-error` (the temporary folder could not
214
+ be made or written). Without the flag the output keeps its shape: there are
215
+ no `node_kinds` / `node_kinds_unchecked` keys, and the one key added is
216
+ `summary.node_kind_check: "not-requested"`.
217
+
218
+ Subcommands:
48
219
  verify --metadata <metadata.json> --provenance <provenance-map.json>
49
- [--source-root <path>] [-o <out.json>]
220
+ [--source-root <path>] [--skill-dir <dir>] [--check-node-kinds]
221
+ [-o <out.json>]
50
222
 
51
223
  Emit JSON:
52
224
  {
@@ -55,9 +227,40 @@ Subcommand:
55
227
  "orphaned": ["<provenance entry with no documented export>", ...],
56
228
  "stale": [
57
229
  {"export_name": "<raw entry name>",
230
+ "entry_index": <int>, # position in entries[]
58
231
  "source_file": "<rel path>",
59
232
  "source_line": <int|null>,
60
- "reason": "file-missing" | "line-out-of-bounds" | "line-invalid"},
233
+ "reason": "file-missing" | "line-out-of-bounds" | "line-invalid"
234
+ | "line-not-definition",
235
+ "definition_lines": [<int>, ...]}, # line-not-definition only
236
+ ...
237
+ ],
238
+ "citations": [
239
+ {"file": "<markdown path relative to --skill-dir>",
240
+ "line": <int>, # line of the markdown file
241
+ "citation": "[AST:<path>:L<n>]",
242
+ "prefix": "AST" | "SRC",
243
+ "cited_file": "<path as written>",
244
+ "cited_line": <int>,
245
+ "reason": "prefix-mismatch" | "ast-without-ast-grep",
246
+ "expected_prefix": "AST" | "SRC",
247
+ "export_name": "<matched entry name>" | null},
248
+ ...
249
+ ],
250
+ "node_kinds": [ # --check-node-kinds only
251
+ {"export_name": "<raw entry name>" | null,
252
+ "entry_index": <int>, # position in entries[]
253
+ "source_file": "<rel path>",
254
+ "source_line": <as recorded>,
255
+ "ast_node_type": "<recorded kind>",
256
+ "language": "<ast-grep language>",
257
+ "reason": "invalid-kind" | "error-kind"},
258
+ ...
259
+ ],
260
+ "node_kinds_unchecked": [ # --check-node-kinds only
261
+ {... the same keys ...,
262
+ "reason": "ast-grep-error" | "ast-grep-timeout"
263
+ | "temp-folder-error"},
61
264
  ...
62
265
  ],
63
266
  "summary": {
@@ -65,39 +268,462 @@ Subcommand:
65
268
  "entries_checked": <int>, # provenance entries with export_name
66
269
  "missing_count": <int>,
67
270
  "orphaned_count": <int>,
271
+ "set_diff": "checked" | "not-applicable", # reference-app
68
272
  "stale_count": <int>,
69
273
  "citations_checked": <int>, # entries whose citation was resolved
70
- "stale_check": "checked" | "skipped-no-source-root"
274
+ "line_check_skipped": <int>, # resolved entries with no line rule
275
+ "stale_check": "checked" | "skipped-no-source-root",
276
+ "skill_citations_scanned": <int>, # [AST:]/[SRC:] with a line
277
+ "skill_citations_matched": <int>, # of those, citing an entry
278
+ "citation_findings_count": <int>,
279
+ "citation_check": "checked" | "skipped-no-skill-dir",
280
+ "node_kind_check": "checked" | "skipped-no-ast-grep"
281
+ | "skipped-unrecognized-ast-grep"
282
+ | "not-requested",
283
+ # the four below appear with --check-node-kinds only
284
+ "node_kinds_checked": <int>, # entries judged, ERROR included
285
+ "node_kind_check_skipped": <int>, # entries in an unmapped file
286
+ "node_kind_check_errors": <int>, # len(node_kinds_unchecked)
287
+ "node_kind_findings_count": <int>
71
288
  }
72
289
  }
73
290
 
74
291
  `missing` / `orphaned` are sorted, deduplicated public names.
75
292
  `stale` is sorted by (export_name, source_file).
293
+ `citations` is sorted by (file, line, column).
294
+ `node_kinds` / `node_kinds_unchecked` are sorted by (export_name,
295
+ source_file, entry_index).
296
+
297
+ fix --verify <verify.json> --provenance <provenance-map.json>
298
+ --skill-dir <dir> [--manual-inventory <inventory.json>]
299
+ [--no-line-moves] [--dry-run] [-o <out.json>]
300
+
301
+ Applies the fixes a `verify` result (the JSON a `verify` run wrote, over
302
+ the same provenance map and skill folder) has one answer for:
303
+
304
+ 1. Citation prefixes: each `citations[]` item's citation, found by its
305
+ `file`, `line` and `citation` text, takes `expected_prefix`; its
306
+ path and line stay.
307
+ 2. Source lines: each `line-not-definition` item with exactly one
308
+ `definition_lines` value moves its entry's `source_line` there (the
309
+ entry at its `entry_index`, or the entries with its name, file and
310
+ line). Every move is planned before any is made. The citations each
311
+ move takes are those of either prefix, in `SKILL.md` and
312
+ `references/**/*.md` as they stand after step 1, whose path,
313
+ normalized as the citation check normalizes it, and first line are
314
+ the entry's. Every planned move is applied in one pass, so a
315
+ citation moves at most once (`LIMIT` moving from 6 to 7 and
316
+ `DEFAULT` from 7 to 8 move a citation of line 6 to 7, never on to
317
+ 8), and a range moves both ends (`L10-12` to `L13-15`). A citation
318
+ of a line another entry also records in that file stays: it may
319
+ cite that other export (`shared-line`).
320
+
321
+ A citation inside a `<!-- [MANUAL:name] -->` block never changes
322
+ (`inside-manual-block`); the blocks are paired as skf-hash-content.py
323
+ pairs them. Each changed file (markdown first, the provenance map last,
324
+ as JSON indented by two spaces, non-ASCII kept, with a final newline;
325
+ bytes that are not UTF-8 survive in markdown) is written through
326
+ skf-atomic-write.py beside this script. `--manual-inventory` (the JSON
327
+ `skf-hash-content.py manual-inventory` wrote for SKILL.md) is checked
328
+ again after the writes. `--no-line-moves` leaves every source-line
329
+ finding for a person (the drift override). `--dry-run` writes nothing.
330
+
331
+ Emit JSON:
332
+ {
333
+ "applied": [
334
+ {"kind": "citation-prefix", "file": "<rel md>", "line": <int>,
335
+ "citation": "<as it was>", "fixed": "<as written>"},
336
+ {"kind": "source-line", "entry_index": <int>,
337
+ "export_name": "<name>", "source_file": "<rel path>",
338
+ "from": <int>, "to": <int>},
339
+ {"kind": "citation-line", "file": "<rel md>", "line": <int>,
340
+ "citation": "<as it was>", "fixed": "<as written>",
341
+ "export_names": ["<name>", ...]},
342
+ ...
343
+ ],
344
+ "left_as_warn": [
345
+ {... the verify item ..., "kind": "citation-prefix",
346
+ "why": "citation-not-found" | "inside-manual-block"},
347
+ {... the verify item ..., "kind": "source-line",
348
+ "why": "no-definition-line" # unverified, never moved
349
+ | "several-definition-lines" | "needs-decision"
350
+ | "line-moves-skipped" | "entry-not-found"},
351
+ {"kind": "citation-line", "file", "line", "citation",
352
+ "export_names": [...],
353
+ "why": "shared-line" | "inside-manual-block"},
354
+ ...
355
+ ],
356
+ "files_changed": ["<path>", ...], # as the arguments name them
357
+ "files_written": ["<path>", ...], # [] with --dry-run
358
+ "dry_run": <bool>,
359
+ "manual_verify": null | {"preserved", "modified", "missing",
360
+ "moved", "ok"},
361
+ "summary": {"citation_prefixes_fixed": <int>,
362
+ "source_lines_moved": <int>,
363
+ "citations_moved": <int>,
364
+ "left_as_warn_count": <int>}
365
+ }
366
+
367
+ `fix` handles no `missing`, `orphaned` or `node_kinds` finding: read
368
+ those from the verify result. Run `verify` again after it.
369
+
370
+ definition-lines --file <source_file> --name <export_name>
371
+ [--source-root <path>] [--line <n>]
372
+ [--export-type <type>] [-o <out.json>]
373
+
374
+ The definition-line rules above for one export (update-skill's
375
+ spot-check). `--file` is relative to `--source-root` (default: the
376
+ current folder).
377
+
378
+ Emit JSON:
379
+ {
380
+ "source_file": "<as given>",
381
+ "export_name": "<as given>",
382
+ "line_check": "checked" | "file-missing" | "skipped-export-type"
383
+ | "skipped-language",
384
+ "definition_lines": [<int>, ...] | null, # checked only
385
+ "source_line": <int> | null, # --line
386
+ "line_is_definition": <bool> | null # --line, checked only
387
+ }
388
+
389
+ With `--line`, a recorded line that is an indented match the column-0
390
+ rule dropped joins `definition_lines`, as in `verify`.
391
+
392
+ classify-stale --names <coverage.json> --provenance <provenance-map.json>
393
+ --source-root <path> [--inventory <inventory.json>]
394
+ [-o <out.json>]
395
+
396
+ test-skill's fabricated-signature test, for each stale name (a
397
+ documented name the enumerated source surface lacks): the
398
+ definition-lines rules above, run for every provenance entry whose raw
399
+ `export_name` is the name, at its `source_file` and `source_line`, with
400
+ its `export_type`. `--names` holds reconcile-coverage.py's result (its
401
+ `stale` names) or a JSON array of names. `--inventory` holds the
402
+ documented inventory, validate-inventory.py's `--output` file or its
403
+ whole result: the `kind` it gives each name decides the module rule
404
+ below.
405
+
406
+ Emit JSON, one item per name, in the order given (a repeated name once):
407
+ [
408
+ {"name": "<as given>",
409
+ "fabricated": <bool>,
410
+ "source": "<source_file>:<source_line>" | "<source_file>" | null,
411
+ "defined_at": "<path>:<line>" | null,
412
+ "declared_in": ["<path>:<line>", ...] | null,
413
+ "reason": "not-defined" | "defined" | "unchecked" | "no-entry",
414
+ "entries": [{"source_file": "<as recorded>",
415
+ "source_line": <as recorded>,
416
+ "line_check": "checked" | "file-missing"
417
+ | "skipped-export-type"
418
+ | "skipped-language" | "unreadable"
419
+ | "no-file",
420
+ "definition_lines": [<int>, ...] | null}, ...]},
421
+ ...
422
+ ]
423
+
424
+ `fabricated` is true, with reason `not-defined`, when the name has an
425
+ entry and every entry's `line_check` is `file-missing`, or `checked`
426
+ with an empty `definition_lines`: the export is in neither the surface
427
+ nor the cited file. Otherwise the reason is `defined` (a cited file
428
+ defines the name), `unchecked` (no rule could check a cited file, and
429
+ none defines it) or `no-entry` (the map has no entry for the name).
430
+ An entry with no `source_file` (update-skill writes one for an
431
+ `unknown` NEW_EXPORT) cites no file to check: its `line_check` is
432
+ `no-file`, which, like a skipped or unreadable check, neither proves
433
+ the name fabricated nor shows it defined, so the name is `unchecked`
434
+ unless another entry defines it. `source` is the citation of the first
435
+ entry with a `source_file` (the file alone when its line is not a
436
+ number), null when no entry has one.
437
+
438
+ `defined_at` is where the source declares a name that is not
439
+ fabricated, so test-skill can tell a documented extra (a name outside
440
+ the surface that the source still declares) from stale documentation;
441
+ a fabricated name is not looked up (null). A declaration is a line at
442
+ column 0 that the definition-line rules match with imports left out:
443
+ a `def`, `class`, assignment, annotation, type alias, TS/JS
444
+ declaration, `exports.NAME =`, or an alias that renames
445
+ (`import X as NAME`, `from X import Y as NAME`,
446
+ `export { Y as NAME }`, `export * as NAME from`), or a
447
+ `module.exports` key with a value or a method. A plain or star import,
448
+ an `as` that keeps the imported name (`from m import NAME as NAME`,
449
+ `export { NAME as NAME }`), `export import NAME =`,
450
+ `export default NAME`, an export-list item without `as`, a
451
+ `module.exports` shorthand key, a binding whose value is an import
452
+ (`require(...)` or `(await) import(...)`, on its line or the next:
453
+ `const NAME = require(...)`, `exports.NAME = require(...)`,
454
+ `{ NAME: require(...) }`), an indented line and any line of a Python
455
+ file that does not tokenize never count. A module or package named
456
+ NAME (`NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
457
+ `NAME/index.<ext>`) declares NAME too: at its line 1 when no line of it
458
+ declares NAME, unless `--inventory` gives NAME a `function`, `hook`,
459
+ `class`, `interface`, `enum` or `type` kind (any of its kinds, when it
460
+ gives several; a `method` kind is not read), no entry of NAME is a
461
+ `module` or `package` one, and the file is not a TS/JS one that is
462
+ itself the export (a column-0 `export default` or `module.exports =`),
463
+ so that a removed function named after its file is no declaration
464
+ (the inventory has no `module` kind). `declared_line` gives this
465
+ answer for one file, the module rule optional, to a script that loads
466
+ this one, and
467
+ `baseline_gaps` applies it to the map's entries an extraction left
468
+ out (skf-extraction-snapshot.py's baseline gaps and
469
+ skf-build-change-manifest.py's `baseline-gaps`, which apply the module
470
+ rule only to a `module` or `package` entry).
471
+
472
+ The files: when a file the name's entries cite declares it,
473
+ `defined_at` is that file's first declaration (the files in entry
474
+ order), and `declared_in` is null. Otherwise `declared_in` lists, as
475
+ sorted `<path>:<line>`, every file of the walk that declares the name,
476
+ at its first declaration (a type stub, `NAME.pyi` or `NAME.d.ts`,
477
+ beside its implementation is one file with it: the implementation),
478
+ and `defined_at` is the one item when the list has exactly one and no
479
+ walked Python file that does not tokenize holds the name (a homonym,
480
+ two files or more, gives null: test-skill decides it by the files the
481
+ skill cites). The walk reads, sorted by path, every file below the
482
+ walk root: the deepest folder that the files of the map's entries
483
+ share, widened to its package root (Python: the nearest folder at or
484
+ above it that holds `__init__.py`, then up while the parent holds
485
+ one, never the source root itself; TS/JS: the nearest folder at or
486
+ above it that holds `package.json`), leaving out the entries whose
487
+ file lies outside every package; the source root when none lies in
488
+ one. Only the rule families (Python, TS/JS) the map's entries cite are
489
+ read (both when no entry cites a file). The walk skips symlinks, hidden folders, test
490
+ files and test configuration files (`conftest.py`, `jest.config.*`),
491
+ `*.min.js`, and the folders generated, built, installed, vendored or
492
+ test output lives in (`node_modules`, `dist`, `build`, `vendor`,
493
+ `_vendor`, `tests`, `testing`, `fixtures`, `__mocks__`,
494
+ `site-packages`, `env`, `bower_components`, `esm`, `cjs`, `umd` and
495
+ the like), `docs`, `examples` and `benchmarks`, each judged on the
496
+ path below the walk root, the end-to-end test folders `e2e` and
497
+ `__fixtures__` wherever they lie, and, as the first folder below the
498
+ walk root or below a package root inside it (a folder that holds
499
+ `package.json`, the top of an `__init__.py` chain), `example`, `demo`,
500
+ `demos`, `sample`, `samples`, `scripts`, `playground`, `bench`,
501
+ `benchmark` and `migrations` unless it holds a file an entry cites
502
+ (deeper in a package such a folder can be package code). A cited file
503
+ is read only inside the source root (a path that is absolute or climbs
504
+ out of it is not read), with no symlink on its path and under the same
505
+ skips, those walk-only ones aside. The path is relative to the source
506
+ root. A dotted name, or a map whose entries cite files
507
+ but no Python or TS/JS one, is not looked up (`defined_at` and
508
+ `declared_in` null, as for a fabricated name), and a filesystem error
509
+ during the lookup gives every name null in both (the exit code stays
510
+ 0).
511
+
512
+ kind-at --file <path> --line <n> --recipes <recipes>
513
+ [--language <lang>] [--name <export_name>]
514
+ [--source-root <path>] [-o <out.json>]
515
+
516
+ The `kind` the matching recipe declares for a source line, the value an
517
+ ast-grep entry records as `ast_node_type`. `--recipes` is a markdown
518
+ file whose ```yaml fences hold the recipes (extraction-patterns.md) or
519
+ a YAML file: one recipe per document, a list of recipes, or a mapping
520
+ whose `recipes` key holds that list. A recipe is a mapping with `id`,
521
+ `language` and a `rule` naming its `kind`. The file's language
522
+ (`--language`, else its extension, mapped as for `--check-node-kinds`)
523
+ picks the recipes: each id's form for that language, else its
524
+ typescript or tsx form (for javascript, a form whose kinds the grammar
525
+ lacks is skipped), with `language` rewritten. Each runs once through
526
+ `ast-grep scan --stdin` on the file's bytes, from a temporary folder
527
+ with a minimal sgconfig.yml. A recipe matches when its `$NAME` is on
528
+ `--line` (and, with `--name`, is that name's last segment).
529
+
530
+ Emit JSON:
531
+ {
532
+ "file": "<as given>",
533
+ "language": "<ast-grep language>" | null,
534
+ "line": <int>,
535
+ "name": "<as given>" | null,
536
+ "status": "found" | "ambiguous" | "no-match" | "incomplete"
537
+ | "skipped-no-language" | "skipped-no-recipes"
538
+ | "skipped-no-ast-grep",
539
+ "kind": "<the one kind the matches declare>" | null, # found only
540
+ "matches": [{"recipe": "<id>", "kind": "<kind>",
541
+ "name": "<$NAME>"}, ...],
542
+ "errors": [{"recipe": "<id>" | null, "language": "<language>",
543
+ "reason": "ast-grep-error" | "ast-grep-timeout"
544
+ | "temp-folder-error",
545
+ "detail": "<text>"}, ...]
546
+ }
547
+
548
+ `incomplete`: a recipe run failed, so no kind is given even when others
549
+ matched. Never invent a kind: anything but `found` leaves the entry as
550
+ it is.
76
551
 
77
552
  CLI examples:
78
553
  uv run skf-verify-provenance-completeness.py verify \\
79
554
  --metadata {skill_package}/metadata.json \\
80
555
  --provenance {forge_version}/provenance-map.json \\
81
- --source-root {source_root}
556
+ --source-root {source_root} \\
557
+ --skill-dir {skill_package} \\
558
+ --check-node-kinds -o {verify_json}
559
+ uv run skf-verify-provenance-completeness.py fix --verify {verify_json} \\
560
+ --provenance {forge_version}/provenance-map.json \\
561
+ --skill-dir {skill_package}
562
+ uv run skf-verify-provenance-completeness.py definition-lines \\
563
+ --source-root {source_root} --file src/api.py --name search --line 26
564
+ uv run skf-verify-provenance-completeness.py classify-stale \\
565
+ --names {run_dir}/coverage.json \\
566
+ --provenance {forge_version}/provenance-map.json \\
567
+ --source-root {source_root} --inventory {run_dir}/inventory.json \\
568
+ -o {run_dir}/stale.json
569
+ uv run skf-verify-provenance-completeness.py kind-at \\
570
+ --source-root {source_root} --file src/api.ts --line 12 --name Store \\
571
+ --recipes {extractionPatternsData}
82
572
 
83
573
  Exit codes:
84
- 0 — verification ran and found nothing (status "pass")
85
- 1 — verification ran and found missing / orphaned / stale entries
86
- (status "findings"); advisory — the caller decides how to surface it
87
- 2 — error (input file not found, malformed JSON, invalid structure)
574
+ verify
575
+ 0 verification ran and found nothing (status "pass")
576
+ 1 verification ran and found missing / orphaned / stale entries,
577
+ citation findings or node-kind findings (status "findings");
578
+ advisory: the caller decides how to surface it
579
+ fix
580
+ 0 nothing left as a WARN (and manual_verify ok when it ran)
581
+ 1 something left as a WARN, or manual_verify not ok
582
+ definition-lines
583
+ 0 the rules answered (whatever `line_check` says)
584
+ classify-stale
585
+ 0 every name was classified (whatever `fabricated` says)
586
+ kind-at
587
+ 0 status "found"
588
+ 1 any other status
589
+ 2 error, for every subcommand (an input file or folder not found,
590
+ --skill-dir without a SKILL.md, malformed JSON or YAML, a verify
591
+ result without stale[] and citations[], --names without a list of
592
+ names, --inventory without an `exports` list, no recipe in
593
+ --recipes, no PyYAML for kind-at, an atomic write that failed, an -o
594
+ file that cannot be written, or any unexpected failure, such as an
595
+ unreadable file; once fix has started writing, stderr names the files
596
+ already written); one line on stderr and no JSON
88
597
  """
89
598
 
90
599
  from __future__ import annotations
91
600
 
92
601
  import argparse
602
+ import copy
603
+ import importlib.util
604
+ import io
93
605
  import json
606
+ import os
607
+ import posixpath
608
+ import re
609
+ import shutil
610
+ import subprocess
94
611
  import sys
612
+ import tempfile
613
+ import tokenize
95
614
  from pathlib import Path
615
+ from typing import Callable
96
616
 
97
617
 
98
618
  STALE_FILE_MISSING = "file-missing"
99
619
  STALE_LINE_OOB = "line-out-of-bounds"
100
620
  STALE_LINE_INVALID = "line-invalid"
621
+ STALE_LINE_NOT_DEFINITION = "line-not-definition"
622
+
623
+ CITATION_PREFIX_MISMATCH = "prefix-mismatch"
624
+ CITATION_AST_WITHOUT_AST_GREP = "ast-without-ast-grep"
625
+
626
+ PYTHON_EXTENSIONS = frozenset({".py", ".pyi"})
627
+ TSJS_EXTENSIONS = frozenset(
628
+ {".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"}
629
+ )
630
+ # export_type values that name a whole module or package: no definition line.
631
+ NO_DEFINITION_EXPORT_TYPES = frozenset({"module", "package"})
632
+ # The scope_type whose exports[] is empty by design (entries are per-citation).
633
+ REFERENCE_APP = "reference-app"
634
+
635
+ # extraction_method -> the citation prefix it implies
636
+ METHOD_PREFIX = {
637
+ "ast-grep": "AST",
638
+ "ast_bridge": "AST",
639
+ "source-read": "SRC",
640
+ "source_reading": "SRC",
641
+ }
642
+ AST_METHODS = frozenset({"ast-grep", "ast_bridge"})
643
+ # Every extraction_method SKF writes (the same list skf-render-metadata-stats
644
+ # checks labels against).
645
+ KNOWN_METHODS = frozenset(
646
+ {
647
+ "ast-grep",
648
+ "source-read",
649
+ "ast_bridge",
650
+ "source_reading",
651
+ "qmd_bridge",
652
+ "compose-from-skill",
653
+ }
654
+ )
655
+
656
+ NODE_KIND_INVALID = "invalid-kind"
657
+ NODE_KIND_ERROR = "error-kind"
658
+ # Why an entry's kind went unjudged (node_kinds_unchecked[].reason).
659
+ UNCHECKED_AST_GREP_ERROR = "ast-grep-error"
660
+ UNCHECKED_TIMEOUT = "ast-grep-timeout"
661
+ UNCHECKED_TEMP_FOLDER = "temp-folder-error"
662
+ # The node tree-sitter makes of code it cannot parse. ast-grep accepts it as
663
+ # a kind, but no recipe declares it, so an entry recording it guessed.
664
+ ERROR_NODE_KIND = "ERROR"
665
+ # Every named node kind of a grammar ast-grep ships has this shape. Anything
666
+ # else is judged without ast-grep: ast-grep trims surrounding whitespace, and
667
+ # `& | % ^` would reach cmd.exe through an npm `.cmd` shim on Windows.
668
+ NODE_KIND_SHAPE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
669
+ # A kind no grammar has. ast-grep must reject it before its answers are read.
670
+ BOGUS_NODE_KIND = "skf_no_such_node_kind"
671
+ # An explicit, valid sgconfig.yml, so no sgconfig.yml in the temporary
672
+ # folder's ancestors is read.
673
+ MINIMAL_SGCONFIG = "ruleDirs: []\n"
674
+ # source_file extension -> the ast-grep language whose grammar names its kinds
675
+ AST_GREP_LANGUAGES = {
676
+ ".py": "python",
677
+ ".pyi": "python",
678
+ ".ts": "typescript",
679
+ ".mts": "typescript",
680
+ ".cts": "typescript",
681
+ ".tsx": "tsx",
682
+ ".js": "javascript",
683
+ ".jsx": "javascript",
684
+ ".mjs": "javascript",
685
+ ".cjs": "javascript",
686
+ ".rs": "rust",
687
+ ".go": "go",
688
+ ".java": "java",
689
+ ".kt": "kotlin",
690
+ ".kts": "kotlin",
691
+ ".cs": "csharp",
692
+ ".rb": "ruby",
693
+ ".swift": "swift",
694
+ ".php": "php",
695
+ }
696
+ NODE_KIND_TIMEOUT_SEC = 20 # per ast-grep call; each parses one tiny rule
697
+
698
+ # kind-at: a ```yaml fence in a markdown recipe file (extraction-patterns.md).
699
+ _MD_YAML_FENCE_RE = re.compile(r"^```yaml[ \t]*\n(.*?)^```", re.M | re.S)
700
+ # file language -> the recipe languages whose form of a recipe it runs, with
701
+ # `language:` rewritten, when the recipe has no form of its own for it (the
702
+ # language selection note in extraction-patterns-by-hand.md): typescript and tsx
703
+ # share their recipes, and javascript runs a TypeScript recipe with no
704
+ # javascript form (one whose kinds the grammar lacks is skipped).
705
+ RECIPE_LANGUAGE_FALLBACKS = {
706
+ "typescript": ("tsx",),
707
+ "tsx": ("typescript",),
708
+ "javascript": ("typescript", "tsx"),
709
+ }
710
+ KIND_AT_TIMEOUT_SEC = 30 # per ast-grep call; each scans one file
711
+
712
+ # fix: what each applied or left item fixes.
713
+ FIX_CITATION_PREFIX = "citation-prefix"
714
+ FIX_SOURCE_LINE = "source-line"
715
+ FIX_CITATION_LINE = "citation-line"
716
+ # The [MANUAL] markers skf-hash-content.py pairs (its _OPEN_RE / _CLOSE_RE,
717
+ # on text instead of bytes; the test pins the two copies together). ASCII,
718
+ # so `\s` is the whitespace a bytes pattern knows: a marker spaced with
719
+ # U+00A0 is no marker to either.
720
+ _MANUAL_OPEN_RE = re.compile(r"<!--\s*\[MANUAL:([^\]]+)\]\s*-->", re.ASCII)
721
+ _MANUAL_CLOSE_RE = re.compile(r"<!--\s*\[/MANUAL:([^\]]+)\]\s*-->", re.ASCII)
722
+ # One line with its ending, split where `check_citation` counts a line.
723
+ _LINE_WITH_END_RE = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+$")
724
+ ATOMIC_WRITE_HELPER = "skf-atomic-write.py"
725
+ HASH_CONTENT_HELPER = "skf-hash-content.py"
726
+ ATOMIC_WRITE_TIMEOUT_SEC = 60
101
727
 
102
728
 
103
729
  # --------------------------------------------------------------------------
@@ -174,6 +800,12 @@ def extract_reexport_map(prov: dict) -> dict[str, str]:
174
800
  return out
175
801
 
176
802
 
803
+ def is_reference_app(metadata: dict) -> bool:
804
+ """True when metadata.json's `scope_type` is `reference-app`."""
805
+ scope_type = metadata.get("scope_type")
806
+ return isinstance(scope_type, str) and scope_type.strip() == REFERENCE_APP
807
+
808
+
177
809
  def canon(name: str, reexport_map: dict[str, str]) -> str:
178
810
  """Canonicalize an export name to its public form via reexport_map.
179
811
 
@@ -261,171 +893,3332 @@ def check_citation(
261
893
 
262
894
 
263
895
  # --------------------------------------------------------------------------
264
- # Verification
896
+ # Definition lines
265
897
  # --------------------------------------------------------------------------
266
898
 
899
+ _PY_FROM_IMPORT_RE = re.compile(r"^\s*from\s+[\w.]+\s+import\s+(.*)$")
900
+ _PY_IMPORT_RE = re.compile(r"^\s*import\s+(.*)$")
901
+ _PY_IMPORT_ITEM_RE = re.compile(r"^([\w.]+)(?:\s+as\s+(\w+))?$")
902
+ _PY_IDENT_RE = re.compile(r"[A-Za-z_]\w*")
903
+ # `A = value`, `A = B = value`: every name left of an `=` that is not `==`.
904
+ _PY_ASSIGN_CHAIN_RE = re.compile(r"^\s*((?:[A-Za-z_]\w*\s*=(?!=)\s*)+)")
905
+ # `A, B = value`, `(A, *B) = value`, `A, = value`.
906
+ _PY_TARGET_LIST_RE = re.compile(
907
+ r"^\s*[(\[]?\s*"
908
+ r"(\*?[A-Za-z_]\w*\s*,(?:\s*\*?[A-Za-z_]\w*\s*,)*(?:\s*\*?[A-Za-z_]\w*)?)"
909
+ r"\s*[)\]]?\s*=(?!=)"
910
+ )
911
+ # `def \` or `class \`: the name is on the next line.
912
+ _PY_SPLIT_DECL_RE = re.compile(r"^\s*(?:async\s+def|def|class)\s*\\$")
913
+ # A line that can matter to a name it does not hold: an import (which may go
914
+ # on over the next lines) or a `def \` / `class \` whose name is on the next.
915
+ _PY_NAMELESS_MATCH_RE = re.compile(r"^\s*(?:from|import|async|def|class)\b")
916
+ # Tokens that never begin a statement.
917
+ _PY_LAYOUT_TOKENS = frozenset(
918
+ {
919
+ tokenize.NL,
920
+ tokenize.COMMENT,
921
+ tokenize.INDENT,
922
+ tokenize.DEDENT,
923
+ tokenize.ENDMARKER,
924
+ tokenize.ENCODING,
925
+ }
926
+ )
267
927
 
268
- def resolve_source_root(prov: dict, override: str | None) -> Path | None:
269
- """Pick the source root for citation resolution.
928
+ # `@name`, `@a.b`, `@name(...)` (one level of nested brackets in the call).
929
+ _JS_DECORATOR = r"@[\w$]+(?:\.[\w$]+)*(?:\((?:[^()]|\([^()]*\))*\))?"
930
+ _JS_MODIFIERS = (
931
+ rf"(?:(?:export|default|async|declare|abstract)\s+|{_JS_DECORATOR}\s*)*"
932
+ )
933
+ _JS_KINDS = (
934
+ r"(?:function(?:\s*\*\s*|\s+)"
935
+ r"|(?:class|interface|type|enum|namespace|let|var)\s+"
936
+ r"|const\s+(?:enum\s+)?)"
937
+ )
938
+ _JS_MEMBER_MODIFIERS = (
939
+ r"(?:(?:public|private|protected|static|readonly|async|abstract"
940
+ r"|override|declare|get|set)\s+)*"
941
+ )
942
+ # An identifier, or a string name (`"a-b"`) in an export list.
943
+ _JS_NAME_OR_STRING = r"(?:[\w$]+|\"[^\"]*\"|'[^']*')"
944
+ _JS_EXPORT_LIST_RE = re.compile(r"^\s*export\s+(?:type\s+)?\{(.*)$")
945
+ _JS_EXPORT_ITEM_RE = re.compile(
946
+ rf"^(?:type\s+)?({_JS_NAME_OR_STRING})(?:\s+as\s+({_JS_NAME_OR_STRING}))?$"
947
+ )
948
+ _JS_MODULE_EXPORTS_OBJECT_RE = re.compile(r"^\s*module\.exports\s*=\s*\{(.*)$")
949
+ # A bound value that is an import: `require(...)`, `import(...)`, awaited, in
950
+ # parentheses (`(await import("./m")).X`) or in one helper call
951
+ # (`__importDefault(require("./m"))`).
952
+ _JS_IMPORT_VALUE_RE = re.compile(
953
+ r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?(?:await\s+)?(?:require|import)\s*\("
954
+ )
955
+ # The start of such a value that goes on on the next line: nothing yet, a
956
+ # parenthesis, or a helper call's name and parenthesis.
957
+ _JS_IMPORT_VALUE_OPEN_RE = re.compile(r"^[\s(]*(?:await\s+)?(?:[\w$.]+\s*\(\s*)?$")
958
+ # A `const`, `let` or `var` statement: its declarator list.
959
+ _JS_VAR_DECL_RE = re.compile(
960
+ rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(?!enum(?![\w$]))(.*)$"
961
+ )
962
+ # A declaration whose name is on the next line: the line ends in its keyword.
963
+ _JS_SPLIT_DECL_RE = re.compile(
964
+ rf"^\s*{_JS_MODIFIERS}(?P<kind>function(?:\s*\*)?|class|interface|type"
965
+ r"|enum|namespace|let|var|const(?:\s+enum)?)$"
966
+ )
967
+ # The keywords every line that opens a statement going on below holds (a
968
+ # `const` / `let` / `var` list, a split declaration, `export {`,
969
+ # `module.exports = {`): a line with none of them, and not the name, can
970
+ # match no rule while no statement is open.
971
+ _JS_OPENING_WORDS = ("const", "let", "var", "function", "class", "interface", "type", "enum", "namespace",
972
+ "export", "module")
270
973
 
271
- `--source-root` override wins; otherwise the provenance map's top-level
272
- `source_root`. Returns None when neither resolves to an on-disk dir.
974
+
975
+ def _js_quiet_lines(code_lines: list[str]) -> list[bool]:
976
+ """For each comment-free line, True when it holds none of
977
+ `_JS_OPENING_WORDS`."""
978
+ return [not any(word in code for word in _JS_OPENING_WORDS) for code in code_lines]
979
+ _JS_VAR_KEYWORDS = frozenset({"const", "let", "var"})
980
+ # How a `const` / `let` / `var` list that did not end on its line goes on
981
+ # (`_js_declarators`): the next line begins a declarator (this one ended
982
+ # with a top-level comma); it finishes the current one (this one ended
983
+ # inside a bracket, or after an operator waiting for its operand); or it
984
+ # goes on only when it starts with an operator (this one ended with a
985
+ # value, as a member chain or a ternary broken over lines does).
986
+ _DECL_NEXT = "next"
987
+ _DECL_OPEN = "open"
988
+ _DECL_MAYBE = "maybe"
989
+ # What `_js_declarators` gives: None, or (brackets left open, how) the list
990
+ # goes on with, and the pieces that begin a declarator.
991
+ _DeclSplit = tuple[tuple[str, str] | None, list[str]]
992
+ # After one of these, `/` begins a regular expression and `<` begins JSX, a
993
+ # type assertion or a generic arrow function's type parameters.
994
+ _JS_EXPRESSION_START = frozenset("([{,;=?:&|^%+-*/<>!~")
995
+ # A line that ends in one of these leaves an operator waiting for its operand.
996
+ _JS_OPERAND_PENDING = frozenset("=?:&|^%+-*/<>.")
997
+ # A line that begins with one of these goes on with the expression above it.
998
+ _JS_CONTINUATION_START = frozenset(".?:,&|^%+-*/=<>")
999
+
1000
+
1001
+ def definition_name(export_name: str) -> str:
1002
+ """The name a definition line must carry: the last segment of a dotted
1003
+ export name (`App.update` -> `update`), else the name itself."""
1004
+ return export_name.rsplit(".", 1)[-1]
1005
+
1006
+
1007
+ def _read_lines(path: Path) -> list[str] | None:
1008
+ """Read a text file as lines numbered like `check_citation` counts them
1009
+ (universal newlines; a UTF-8 BOM is dropped so it never hides line 1).
1010
+ None when the file cannot be read."""
1011
+ try:
1012
+ with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
1013
+ return [ln.rstrip("\n") for ln in fh]
1014
+ except OSError:
1015
+ return None
1016
+
1017
+
1018
+ def _read_source(path: Path) -> str:
1019
+ """A text file's content, decoded as `_read_lines` decodes it. Raises
1020
+ OSError when the file cannot be read."""
1021
+ with path.open("r", encoding="utf-8-sig", errors="replace") as fh:
1022
+ return fh.read()
1023
+
1024
+
1025
+ def _text_lines(text: str) -> list[str]:
1026
+ """`_read_source`'s text as the lines `_read_lines` gives for the file."""
1027
+ lines = text.split("\n")
1028
+ if lines[-1] == "":
1029
+ lines.pop()
1030
+ return lines
1031
+
1032
+
1033
+ def _is_column_0(text: str) -> bool:
1034
+ """True when a line starts at column 0 (no leading whitespace)."""
1035
+ return not text[:1].isspace()
1036
+
1037
+
1038
+ def _py_statement_starts(lines: list[str]) -> set[int] | None:
1039
+ """Line numbers on which a Python statement begins.
1040
+
1041
+ Uses the stdlib tokenizer, so a line inside a string or inside brackets
1042
+ (a docstring, a keyword argument, a dict entry, a parameter on its own
1043
+ line) never reads as a definition. None when the file does not tokenize:
1044
+ every line is then a candidate.
273
1045
  """
274
- candidate: str | None = None
275
- if override:
276
- candidate = override
277
- else:
278
- sr = prov.get("source_root")
279
- if isinstance(sr, str) and sr:
280
- candidate = sr
281
- if not candidate:
1046
+ starts: set[int] = set()
1047
+ at_start = True
1048
+ readline = io.StringIO("\n".join(lines) + "\n").readline
1049
+ try:
1050
+ for tok in tokenize.generate_tokens(readline):
1051
+ if tok.type == tokenize.NEWLINE:
1052
+ at_start = True
1053
+ elif tok.type in _PY_LAYOUT_TOKENS:
1054
+ continue
1055
+ elif at_start:
1056
+ starts.add(tok.start[0])
1057
+ at_start = False
1058
+ except (tokenize.TokenError, SyntaxError, ValueError):
282
1059
  return None
283
- root = Path(candidate)
284
- return root if root.is_dir() else None
1060
+ return starts
285
1061
 
286
1062
 
287
- def verify(metadata: dict, prov: dict, source_root: Path | None) -> dict:
288
- """Compute the completeness / orphan / stale report over the two maps."""
289
- reexport_map = extract_reexport_map(prov)
1063
+ def _js_code_lines(lines: list[str]) -> list[str]:
1064
+ """Each TS/JS line with its comments removed: `//` to the end of the
1065
+ line, and `/* ... */` across lines, both only outside a '...', "..." or
1066
+ `...` string. A string is followed within its line only: a template
1067
+ literal that spans lines is not tracked, so each line starts outside
1068
+ any string. A backslash outside a string escapes the next character, so
1069
+ a regex literal such as `/\\/*/` opens no comment."""
1070
+ out: list[str] = []
1071
+ in_block = False
1072
+ for text in lines:
1073
+ buf: list[str] = []
1074
+ quote: str | None = None
1075
+ i, n = 0, len(text)
1076
+ while i < n:
1077
+ if in_block:
1078
+ end = text.find("*/", i)
1079
+ if end < 0:
1080
+ break
1081
+ in_block = False
1082
+ buf.append(" ")
1083
+ i = end + 2
1084
+ continue
1085
+ ch = text[i]
1086
+ if ch == "\\" and i + 1 < n:
1087
+ buf.append(text[i : i + 2])
1088
+ i += 2
1089
+ continue
1090
+ if quote is not None:
1091
+ if ch == quote:
1092
+ quote = None
1093
+ elif ch in "'\"`":
1094
+ quote = ch
1095
+ elif text.startswith("//", i):
1096
+ break
1097
+ elif text.startswith("/*", i):
1098
+ in_block = True
1099
+ i += 2
1100
+ continue
1101
+ buf.append(ch)
1102
+ i += 1
1103
+ out.append("".join(buf).rstrip())
1104
+ return out
290
1105
 
291
- documented = extract_export_names(metadata)
292
- documented_canon = {canon(n, reexport_map) for n in documented}
293
1106
 
294
- raw_entry_names = provenance_entry_names(prov)
295
- entry_canon = {canon(n, reexport_map) for n in raw_entry_names}
1107
+ class _SourceCache:
1108
+ """Source lines, Python statement starts and TS/JS comment-free lines,
1109
+ read once per file, and TS/JS declarator lists, split once per line
1110
+ (the split does not depend on the export looked up)."""
296
1111
 
297
- missing = sorted(documented_canon - entry_canon)
298
- orphaned = sorted(entry_canon - documented_canon)
1112
+ def __init__(self) -> None:
1113
+ self._lines: dict[Path, list[str] | None] = {}
1114
+ self._starts: dict[Path, set[int] | None] = {}
1115
+ self._js_code: dict[Path, list[str]] = {}
1116
+ self._js_quiet: dict[Path, list[bool]] = {}
1117
+ self._js_decl: dict[tuple[str, str, bool], _DeclSplit] = {}
299
1118
 
300
- stale: list[dict] = []
301
- citations_checked = 0
302
- if source_root is not None:
303
- entries = prov.get("entries")
304
- if isinstance(entries, list):
305
- for entry in entries:
306
- if not isinstance(entry, dict):
307
- continue
308
- name = entry.get("export_name")
309
- if not isinstance(name, str) or not name:
310
- continue
311
- sf = entry.get("source_file")
312
- sl = entry.get("source_line")
313
- # Only count a resolvable (non-null) citation as "checked".
314
- if isinstance(sf, str) and sf.strip() and _coerce_line(sl) is not None:
315
- citations_checked += 1
316
- reason = check_citation(sf, sl, source_root)
317
- if reason is not None:
318
- stale.append(
319
- {
320
- "export_name": name,
321
- "source_file": sf if isinstance(sf, str) else None,
322
- "source_line": sl if isinstance(sl, int)
323
- and not isinstance(sl, bool) else sl,
324
- "reason": reason,
325
- }
326
- )
327
- stale.sort(key=lambda s: (s["export_name"], s.get("source_file") or ""))
1119
+ def lines(self, path: Path) -> list[str] | None:
1120
+ if path not in self._lines:
1121
+ self._lines[path] = _read_lines(path)
1122
+ return self._lines[path]
328
1123
 
329
- status = "pass" if not (missing or orphaned or stale) else "findings"
330
- return {
331
- "status": status,
332
- "missing": missing,
333
- "orphaned": orphaned,
334
- "stale": stale,
335
- "summary": {
336
- "exports_checked": len(documented_canon),
337
- "entries_checked": len(raw_entry_names),
338
- "missing_count": len(missing),
339
- "orphaned_count": len(orphaned),
340
- "stale_count": len(stale),
341
- "citations_checked": citations_checked,
342
- "stale_check": "checked"
343
- if source_root is not None
344
- else "skipped-no-source-root",
345
- },
346
- }
1124
+ def py_statement_starts(self, path: Path, lines: list[str]) -> set[int] | None:
1125
+ if path not in self._starts:
1126
+ self._starts[path] = _py_statement_starts(lines)
1127
+ return self._starts[path]
347
1128
 
1129
+ def js_code_lines(self, path: Path, lines: list[str]) -> list[str]:
1130
+ if path not in self._js_code:
1131
+ self._js_code[path] = _js_code_lines(lines)
1132
+ return self._js_code[path]
348
1133
 
349
- # --------------------------------------------------------------------------
350
- # CLI
351
- # --------------------------------------------------------------------------
1134
+ def js_quiet_lines(self, path: Path, code_lines: list[str]) -> list[bool]:
1135
+ if path not in self._js_quiet:
1136
+ self._js_quiet[path] = _js_quiet_lines(code_lines)
1137
+ return self._js_quiet[path]
352
1138
 
1139
+ def js_declarators(
1140
+ self, text: str, opened: str = "", fresh: bool = True
1141
+ ) -> _DeclSplit:
1142
+ key = (text, opened, fresh)
1143
+ if key not in self._js_decl:
1144
+ self._js_decl[key] = _js_declarators(text, opened, fresh)
1145
+ return self._js_decl[key]
353
1146
 
354
- def _cmd_verify(args: argparse.Namespace) -> int:
355
- meta_path = Path(args.metadata)
356
- prov_path = Path(args.provenance)
357
- if not meta_path.is_file():
358
- print(f"error: metadata not found: {meta_path}", file=sys.stderr)
359
- return 2
360
- if not prov_path.is_file():
361
- print(f"error: provenance map not found: {prov_path}", file=sys.stderr)
362
- return 2
363
- try:
364
- metadata = load_json_object(meta_path, "metadata")
365
- prov = load_json_object(prov_path, "provenance map")
366
- except ValueError as exc:
367
- print(f"error: {exc}", file=sys.stderr)
368
- return 2
369
1147
 
370
- source_root = resolve_source_root(prov, args.source_root)
371
- if args.verbose:
372
- if source_root is None:
373
- print(
374
- "verbose: no source root resolved — skipping stale-citation "
375
- "check (completeness + orphan diffs only)",
376
- file=sys.stderr,
377
- )
1148
+ def _py_import_binds(
1149
+ items: str, name: str, from_import: bool, alias_only: bool = False
1150
+ ) -> bool:
1151
+ """True when an import item list binds `name` (with `alias_only`, only
1152
+ through an explicit `as name` that renames: `from m import name as
1153
+ name` and `import a.name as name` keep the imported name)."""
1154
+ items = items.split("#", 1)[0]
1155
+ for raw in items.replace("(", " ").replace(")", " ").replace("\\", " ").split(","):
1156
+ m = _PY_IMPORT_ITEM_RE.match(raw.strip())
1157
+ if not m:
1158
+ continue
1159
+ target, alias = m.group(1), m.group(2)
1160
+ if alias is not None:
1161
+ if alias_only and alias == target.rsplit(".", 1)[-1]:
1162
+ continue
1163
+ bound = alias
1164
+ elif alias_only:
1165
+ continue
1166
+ elif from_import:
1167
+ bound = target
378
1168
  else:
379
- print(f"verbose: resolving citations under {source_root}", file=sys.stderr)
1169
+ bound = target.split(".", 1)[0] # `import a.b` binds `a`
1170
+ if bound == name:
1171
+ return True
1172
+ return False
380
1173
 
381
- result = verify(metadata, prov, source_root)
382
1174
 
383
- out_text = json.dumps(result, indent=2) + "\n"
384
- if args.output and args.output != "-":
385
- Path(args.output).write_text(out_text, encoding="utf-8")
386
- else:
387
- sys.stdout.write(out_text)
1175
+ def _py_assigned_names(code: str) -> set[str]:
1176
+ """Names an assignment statement binds (`A = B = v`, `A, *B = v`)."""
1177
+ names: set[str] = set()
1178
+ m = _PY_ASSIGN_CHAIN_RE.match(code)
1179
+ if m:
1180
+ names.update(_PY_IDENT_RE.findall(m.group(1)))
1181
+ m = _PY_TARGET_LIST_RE.match(code)
1182
+ if m:
1183
+ names.update(_PY_IDENT_RE.findall(m.group(1)))
1184
+ return names
388
1185
 
389
- return 0 if result["status"] == "pass" else 1
390
1186
 
1187
+ def _python_definition_lines(
1188
+ lines: list[str],
1189
+ name: str,
1190
+ starts: set[int] | None,
1191
+ declarations_only: bool = False,
1192
+ ) -> list[tuple[int, bool]]:
1193
+ """(line, at column 0) for every line that defines `name`.
391
1194
 
392
- def _build_parser() -> argparse.ArgumentParser:
393
- parser = argparse.ArgumentParser(
394
- prog="skf-verify-provenance-completeness",
395
- description=(
396
- "Cross-reference documented exports (metadata.json) against "
397
- "provenance-map entries — completeness, orphans, and stale "
398
- "file:line citations — emitting findings as JSON."
399
- ),
400
- )
401
- sub = parser.add_subparsers(dest="cmd", required=True)
402
- p = sub.add_parser(
403
- "verify",
404
- help="emit provenance completeness / orphan / stale-citation findings",
405
- )
406
- p.add_argument("--metadata", required=True, help="path to metadata.json")
407
- p.add_argument(
408
- "--provenance", required=True, help="path to provenance-map.json"
1195
+ A line inside a parenthesized or backslash-continued import takes the
1196
+ column of the line the import statement starts on, and so does the name
1197
+ line of a `def \\` or `class \\` whose name is on the next line. A
1198
+ `from X import *` line counts only when no other line defines `name` (a
1199
+ re-export the extraction could not trace further is cited at its star
1200
+ import). With `declarations_only`, an import counts only when it binds
1201
+ `name` through an explicit `as name` that renames (`_py_import_binds`),
1202
+ and a star import never does.
1203
+ """
1204
+ esc = re.escape(name)
1205
+ decl = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{esc}(?!\w)")
1206
+ leading_name = re.compile(rf"^\s*{esc}(?!\w)")
1207
+ annotated = re.compile(rf"^\s*{esc}\s*:(?!=)")
1208
+ type_alias = re.compile(rf"^\s*type\s+{esc}\s*(?:\[[^\]]*\]\s*)?=(?!=)")
1209
+ found: list[tuple[int, bool]] = []
1210
+ star_imports: list[tuple[int, bool]] = []
1211
+ # An import statement that continues on the next line:
1212
+ # (closing mark "paren" or "backslash", column 0, from-import).
1213
+ cont: tuple[str, bool, bool] | None = None
1214
+ for number, text in enumerate(lines, start=1):
1215
+ code = text.split("#", 1)[0]
1216
+ if cont is not None:
1217
+ mark, top, from_import = cont
1218
+ if mark == "paren":
1219
+ if ")" in code:
1220
+ code = code.split(")", 1)[0]
1221
+ cont = None
1222
+ else:
1223
+ stripped = code.rstrip()
1224
+ if stripped.endswith("\\"):
1225
+ code = stripped[:-1]
1226
+ else:
1227
+ cont = None
1228
+ if _py_import_binds(code, name, from_import, declarations_only):
1229
+ found.append((number, top))
1230
+ continue
1231
+ if starts is not None and number not in starts:
1232
+ continue
1233
+ if name not in code and not _PY_NAMELESS_MATCH_RE.match(code):
1234
+ continue # no rule below can match it: spares a scan per name
1235
+ top = _is_column_0(text)
1236
+ if (
1237
+ decl.match(code)
1238
+ or annotated.match(code)
1239
+ or type_alias.match(code)
1240
+ or name in _py_assigned_names(code)
1241
+ ):
1242
+ found.append((number, top))
1243
+ continue
1244
+ if _PY_SPLIT_DECL_RE.match(code.rstrip()):
1245
+ # `def \` then the name: cite the name's line, as the recipes do.
1246
+ if number < len(lines) and leading_name.match(
1247
+ lines[number].split("#", 1)[0]
1248
+ ):
1249
+ found.append((number + 1, top))
1250
+ continue
1251
+ m = _PY_FROM_IMPORT_RE.match(code)
1252
+ from_import = m is not None
1253
+ if m is None:
1254
+ m = _PY_IMPORT_RE.match(code)
1255
+ if m is None:
1256
+ continue
1257
+ rest = m.group(1)
1258
+ if from_import and rest.strip() == "*":
1259
+ if not declarations_only:
1260
+ star_imports.append((number, top))
1261
+ continue
1262
+ if from_import and rest.lstrip().startswith("(") and ")" not in rest:
1263
+ cont = ("paren", top, True)
1264
+ elif rest.rstrip().endswith("\\"):
1265
+ cont = ("backslash", top, from_import)
1266
+ rest = rest.rstrip()[:-1]
1267
+ if _py_import_binds(rest, name, from_import, declarations_only):
1268
+ found.append((number, top))
1269
+ return found or star_imports
1270
+
1271
+
1272
+ def _js_export_item_binds(item: str, name: str, alias_only: bool = False) -> bool:
1273
+ """True when one `export { ... }` item exposes `name`: the name after
1274
+ `as`, else (unless `alias_only`) the item's own name, an identifier or
1275
+ a string. With `alias_only`, an `as` that keeps the item's own name
1276
+ (`NAME as NAME`) does not count either."""
1277
+ m = _JS_EXPORT_ITEM_RE.match(item.strip())
1278
+ if not m or (alias_only and m.group(2) is None):
1279
+ return False
1280
+ own, exposed = (
1281
+ s[1:-1] if s[:1] in ("'", '"') else s
1282
+ for s in (m.group(1), m.group(2) or m.group(1))
409
1283
  )
410
- p.add_argument(
411
- "--source-root",
412
- default=None,
413
- help=(
414
- "source tree root for citation resolution; overrides the "
415
- "provenance map's source_root. Stale check is skipped when no "
416
- "root resolves on disk."
1284
+ if alias_only and exposed == own:
1285
+ return False
1286
+ return exposed == name
1287
+
1288
+
1289
+ def _js_regex_end(text: str, start: int) -> int | None:
1290
+ """The end of the regular expression literal whose opening `/` is at
1291
+ `start`, flags included. None when it does not close on the line, or
1292
+ when the `/` begins a comment."""
1293
+ if text.startswith(("//", "/*"), start):
1294
+ return None
1295
+ in_class = False
1296
+ i, n = start + 1, len(text)
1297
+ while i < n:
1298
+ ch = text[i]
1299
+ if ch == "\\":
1300
+ i += 2
1301
+ continue
1302
+ if in_class:
1303
+ in_class = ch != "]"
1304
+ elif ch == "[":
1305
+ in_class = True
1306
+ elif ch == "/":
1307
+ i += 1
1308
+ while i < n and (text[i].isalnum() or text[i] in "_$"):
1309
+ i += 1
1310
+ return i
1311
+ i += 1
1312
+ return None
1313
+
1314
+
1315
+ def _js_declarators(text: str, opened: str = "", fresh: bool = True) -> _DeclSplit:
1316
+ """The declarators of a `const` / `let` / `var` list that begin in
1317
+ `text`: the pieces between its top-level commas.
1318
+
1319
+ `text` is the list's first line after the keyword, or a later line with
1320
+ the brackets the line above left open (`opened`, innermost last) and
1321
+ whether it begins a declarator (`fresh`) or finishes one, whose piece is
1322
+ then left out. Brackets, '...', "..." and `...` strings, a regular
1323
+ expression literal where an expression begins and a type-argument list
1324
+ (`Map<K, V>`: a `<` right after a name, closed by a `>` that is not the
1325
+ one of `=>`) are skipped. The list ends at a top-level `;`, at a bracket
1326
+ that closes one opened before it, and at a `<` where an expression
1327
+ begins: JSX, a type assertion or a generic arrow function's type
1328
+ parameters, which are not followed.
1329
+
1330
+ Returns None when the list ends on this line, else the (brackets left
1331
+ open, `_DECL_NEXT` | `_DECL_OPEN` | `_DECL_MAYBE`) it goes on with, and
1332
+ the pieces.
1333
+ """
1334
+ pieces: list[str] = []
1335
+ buf: list[str] = []
1336
+ stack = list(opened)
1337
+ keep = fresh # the current piece begins a declarator
1338
+ last = "," if fresh else "=" # the last character outside a string
1339
+ quote: str | None = None
1340
+ escaped = False
1341
+ ended = False
1342
+ i, n = 0, len(text)
1343
+ while i < n:
1344
+ ch = text[i]
1345
+ i += 1
1346
+ if quote is not None:
1347
+ buf.append(ch)
1348
+ if escaped:
1349
+ escaped = False
1350
+ elif ch == "\\":
1351
+ escaped = True
1352
+ elif ch == quote:
1353
+ quote = None
1354
+ continue
1355
+ prev = text[i - 2] if i > 1 else ""
1356
+ if ch in "/<" and last in _JS_EXPRESSION_START and prev + ch != "<<":
1357
+ end = _js_regex_end(text, i - 1) if ch == "/" else None
1358
+ if end is None:
1359
+ ended = True
1360
+ break
1361
+ buf.append(text[i - 1 : end])
1362
+ i = end
1363
+ last = "a" # the literal is a value, as a name is
1364
+ continue
1365
+ if ch in "'\"`":
1366
+ quote = ch
1367
+ elif ch == "<" and (prev.isalnum() or prev in ("_", "$")) and (
1368
+ text[i : i + 1] not in ("<", "=")
1369
+ ):
1370
+ stack.append(ch) # `Map<K, V>`, where `a < b` compares
1371
+ elif ch == ">" and stack[-1:] == ["<"] and prev != "=":
1372
+ stack.pop()
1373
+ buf.append(ch)
1374
+ last = "a" # the type is a value's, as a name is
1375
+ continue
1376
+ elif ch in "([{":
1377
+ stack.append(ch)
1378
+ elif ch in ")]}":
1379
+ while stack[-1:] == ["<"]:
1380
+ stack.pop() # a `<` that compared, written without spaces
1381
+ if not stack:
1382
+ ended = True
1383
+ break
1384
+ stack.pop()
1385
+ elif ch == ";" and all(b == "<" for b in stack):
1386
+ ended = True
1387
+ break
1388
+ elif ch == "," and not stack:
1389
+ if keep:
1390
+ pieces.append("".join(buf))
1391
+ buf, keep, last = [], True, ch
1392
+ continue
1393
+ if not ch.isspace():
1394
+ last = ch
1395
+ buf.append(ch)
1396
+ if keep:
1397
+ pieces.append("".join(buf))
1398
+ if ended or quote is not None:
1399
+ return None, pieces
1400
+ if stack:
1401
+ return ("".join(stack), _DECL_OPEN), pieces
1402
+ if last == ",":
1403
+ return ("", _DECL_NEXT), pieces
1404
+ return ("", _DECL_OPEN if last in _JS_OPERAND_PENDING else _DECL_MAYBE), pieces
1405
+
1406
+
1407
+ def _js_depth1_pieces(text: str, depth: int) -> tuple[int, list[str]]:
1408
+ """Split the text of a `module.exports = { ... }` object or an
1409
+ `export { ... }` list into its top-level pieces (the text between commas
1410
+ at depth 1).
1411
+
1412
+ Returns the bracket depth after `text` (0 once the object closed) and
1413
+ the pieces. An opening bracket at depth 1 stays in its piece, so a
1414
+ method shorthand `name(` and a key `name: {` keep their marker. Brackets
1415
+ and commas inside a string on the line are not counted.
1416
+ """
1417
+ pieces: list[str] = []
1418
+ buf: list[str] = []
1419
+ quote: str | None = None
1420
+ escaped = False
1421
+ for ch in text:
1422
+ if quote is not None:
1423
+ # text inside a string: kept in its piece, never counted
1424
+ if depth == 1:
1425
+ buf.append(ch)
1426
+ if escaped:
1427
+ escaped = False
1428
+ elif ch == "\\":
1429
+ escaped = True
1430
+ elif ch == quote:
1431
+ quote = None
1432
+ continue
1433
+ if ch in "'\"`":
1434
+ quote = ch
1435
+ if depth == 1:
1436
+ buf.append(ch)
1437
+ elif ch in "([{":
1438
+ if depth == 1:
1439
+ buf.append(ch)
1440
+ depth += 1
1441
+ elif ch in ")]}":
1442
+ depth -= 1
1443
+ if depth == 0:
1444
+ break
1445
+ elif depth == 1:
1446
+ if ch == ",":
1447
+ pieces.append("".join(buf))
1448
+ buf = []
1449
+ else:
1450
+ buf.append(ch)
1451
+ pieces.append("".join(buf))
1452
+ return depth, pieces
1453
+
1454
+
1455
+ def _js_paren_tail_is_call(after_open: str) -> bool:
1456
+ """For `name(` on an indented line: True when the text after the
1457
+ parameters says a call, not a method (`name(x);`). A method's
1458
+ parameters are followed by its body `{` or a return type `:`, or they
1459
+ continue on the next line."""
1460
+ depth = 1
1461
+ for i, ch in enumerate(after_open):
1462
+ if ch in "([{":
1463
+ depth += 1
1464
+ elif ch in ")]}":
1465
+ depth -= 1
1466
+ if depth == 0:
1467
+ rest = after_open[i + 1 :].strip()
1468
+ return not (rest == "" or rest[0] in "{:")
1469
+ return False
1470
+
1471
+
1472
+ def _tsjs_definition_lines(
1473
+ lines: list[str],
1474
+ code_lines: list[str],
1475
+ name: str,
1476
+ dotted: bool,
1477
+ declarators: Callable[..., _DeclSplit] = _js_declarators,
1478
+ declarations_only: bool = False,
1479
+ quiet: list[bool] | None = None,
1480
+ ) -> list[tuple[int, bool]]:
1481
+ """(line, at column 0) for every line that defines `name`.
1482
+
1483
+ `quiet` is `_js_quiet_lines(code_lines)` when the caller has it (it does
1484
+ not depend on the name): a quiet line without the name is passed over
1485
+ while no statement is open. `code_lines` are `lines` with comments removed (`_js_code_lines`), so a
1486
+ JSDoc line or a commented-out declaration never matches. A line inside
1487
+ a multi-line `export { ... }` list or `module.exports = { ... }` object
1488
+ takes the column of the line the statement starts on, and so do the
1489
+ name line of a declaration whose keyword ends the line above it and a
1490
+ later declarator on a line of its own. Class members count only for a
1491
+ dotted export name. `declarators` splits a declarator list
1492
+ (`_js_declarators`, or `_SourceCache.js_declarators`, which splits each
1493
+ line once for every export looked up). With `declarations_only`,
1494
+ `export import NAME =`, `export default NAME`, an export-list item
1495
+ without an `as` that renames, a `module.exports` shorthand key and an
1496
+ import-valued binding (a declarator, `exports.NAME =` or a
1497
+ `module.exports` key whose value starts with `require(` or an
1498
+ `(await) import(`, on its line or the next one) never count, nor does
1499
+ a class member.
1500
+ """
1501
+ esc = re.escape(name)
1502
+ end = r"(?![\w$])"
1503
+ statements = [
1504
+ re.compile(rf"^\s*{_JS_MODIFIERS}{_JS_KINDS}{esc}{end}"),
1505
+ re.compile(
1506
+ rf"^\s*export\s+(?:type\s+)?\*\s+as\s+"
1507
+ rf"(?:{esc}\s+|([\"']){esc}\1\s*)from{end}"
417
1508
  ),
1509
+ re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])"),
1510
+ ]
1511
+ if not declarations_only:
1512
+ statements += [
1513
+ re.compile(rf"^\s*export\s+import\s+(?:type\s+)?{esc}\s*=(?![=>])"),
1514
+ re.compile(rf"^\s*export\s+default\s+{esc}\s*;?\s*$"),
1515
+ ]
1516
+ # One declarator of a `const` / `let` / `var` list: `NAME = v`,
1517
+ # `NAME: T = v`, `NAME!: T`, or a bare `NAME`. A destructuring pattern
1518
+ # starts with `{` or `[` and never matches.
1519
+ declarator = re.compile(rf"^\s*{esc}{end}\s*(?:!?\s*:|=|$)")
1520
+ # With declarations_only, the value bound to NAME (the rest of a
1521
+ # declarator, of `exports.NAME =` or of a `module.exports` key), to
1522
+ # test for an import.
1523
+ # (A type annotation holds no `=` but in `=>`: `NAME: () => T = v`.)
1524
+ declarator_value = re.compile(
1525
+ rf"^\s*{esc}{end}\s*(?:!?\s*:(?:[^=]|=>)*)?=(?![=>])(.*)$"
418
1526
  )
419
- p.add_argument(
420
- "-o",
421
- "--output",
422
- default=None,
423
- help="write JSON to this file instead of stdout ('-' for stdout)",
1527
+ var_statement = re.compile(rf"^\s*{_JS_MODIFIERS}(?:const|let|var)\s+(.*)$")
1528
+ exports_value = re.compile(rf"^\s*(?:module\.)?exports\.{esc}\s*=(?![=>])(.*)$")
1529
+ key_value = re.compile(rf"^\s*(?:(['\"]){esc}\1|{esc}{end})\s*:(.*)$")
1530
+
1531
+ def imported(value: str, number: int) -> bool:
1532
+ """True when a bound value is an import: `require(...)`, or an
1533
+ `import(...)` awaited or not, alone or in one helper call. A value
1534
+ that goes on on the next line (`NAME =`, `NAME = (` or
1535
+ `NAME = helper(` ends line `number`) is read there too."""
1536
+ if _JS_IMPORT_VALUE_OPEN_RE.match(value):
1537
+ value += next((c for c in code_lines[number:] if c.strip()), "")
1538
+ return _JS_IMPORT_VALUE_RE.match(value) is not None
1539
+
1540
+ def declares(piece: str, number: int) -> bool:
1541
+ if declarator.match(piece) is None:
1542
+ return False
1543
+ if not declarations_only:
1544
+ return True
1545
+ m = declarator_value.match(piece)
1546
+ return m is None or not imported(m.group(1), number)
1547
+
1548
+ def is_statement(code: str, number: int) -> bool:
1549
+ if not any(p.match(code) for p in statements):
1550
+ return False
1551
+ if not declarations_only:
1552
+ return True
1553
+ m = var_statement.match(code)
1554
+ if m is not None:
1555
+ m = declarator_value.match(m.group(1))
1556
+ else:
1557
+ m = exports_value.match(code)
1558
+ return m is None or not imported(m.group(1), number)
1559
+
1560
+ leading_name = re.compile(rf"^\s*{esc}{end}")
1561
+ # A key with a value, a method, or (unless declarations_only) a shorthand.
1562
+ key_end = r"[:(]" if declarations_only else r"[:(]|$"
1563
+ object_key = re.compile(
1564
+ rf"^\s*(?:(?:async|get|set)\s+|\*\s*)*(?:(['\"]){esc}\1|{esc}{end})"
1565
+ rf"\s*(?:{key_end})"
424
1566
  )
425
- p.add_argument(
426
- "--verbose", action="store_true", help="diagnostics to stderr"
1567
+
1568
+ def binds_key(piece: str, number: int) -> bool:
1569
+ if object_key.match(piece) is None:
1570
+ return False
1571
+ if not declarations_only:
1572
+ return True
1573
+ m = key_value.match(piece)
1574
+ return m is None or not imported(m.group(2), number)
1575
+
1576
+ dotted = dotted and not declarations_only
1577
+ member = re.compile(
1578
+ rf"^\s+(?P<mods>{_JS_MEMBER_MODIFIERS})(?:\*\s*)?{esc}{end}\s*[?!]?\s*"
1579
+ rf"(?:<[^>]*>\s*)?(?P<tail>[(:=;]|$)"
427
1580
  )
428
- p.set_defaults(func=_cmd_verify)
1581
+ found: list[tuple[int, bool]] = []
1582
+ # A statement that continues on the next line:
1583
+ # ("list" or "object", column 0, bracket depth).
1584
+ cont: tuple[str, bool, int] | None = None
1585
+ # A declaration whose keyword ends the line: (keyword, column 0).
1586
+ split: tuple[str, bool] | None = None
1587
+ # The `const` / `let` / `var` lists that go on on the next line, one
1588
+ # nested in another's initializer included: (column 0, brackets left
1589
+ # open, how, as `_js_declarators` gives them).
1590
+ decls: list[tuple[bool, str, str]] = []
1591
+ if quiet is None:
1592
+ quiet = _js_quiet_lines(code_lines)
1593
+ for number, (text, code) in enumerate(zip(lines, code_lines), start=1):
1594
+ if quiet[number - 1] and not (decls or split or cont) and name not in code:
1595
+ continue # no rule below can match it: spares a scan per name
1596
+ # A line of a list carried from above still goes through the rules
1597
+ # below: it may hold a class member or a local of an arrow function.
1598
+ if decls and code.strip():
1599
+ going_on = code.lstrip()[0] in _JS_CONTINUATION_START
1600
+ carried = []
1601
+ for decl_top, opened, how in decls:
1602
+ if how == _DECL_MAYBE and not going_on:
1603
+ continue # the statement ended with the line above
1604
+ carry, pieces = declarators(code, opened, how == _DECL_NEXT)
1605
+ if carry is not None:
1606
+ carried.append((decl_top, *carry))
1607
+ if any(declares(p, number) for p in pieces):
1608
+ # a later declarator on a line of its own, as Prettier
1609
+ # writes it: `export const a = 1,` then ` NAME = 2;`
1610
+ found.append((number, decl_top))
1611
+ decls = carried
1612
+ if split is not None:
1613
+ keyword, top = split
1614
+ split = None
1615
+ if keyword in _JS_VAR_KEYWORDS:
1616
+ carry, pieces = declarators(code)
1617
+ if carry is not None:
1618
+ decls.append((top, *carry))
1619
+ hit = any(declares(p, number) for p in pieces)
1620
+ else:
1621
+ hit = leading_name.match(code) is not None
1622
+ if hit:
1623
+ # the name's line, as the recipes cite it
1624
+ found.append((number, top))
1625
+ continue
1626
+ if cont is not None:
1627
+ kind, top, depth = cont
1628
+ depth, pieces = _js_depth1_pieces(code, depth)
1629
+ cont = (kind, top, depth) if depth > 0 else None
1630
+ if kind == "list":
1631
+ hit = any(
1632
+ _js_export_item_binds(p, name, declarations_only) for p in pieces
1633
+ )
1634
+ else:
1635
+ hit = any(binds_key(p, number) for p in pieces)
1636
+ if hit:
1637
+ found.append((number, top))
1638
+ continue
1639
+ top = _is_column_0(text)
1640
+ # every rule that matches a line needs the name in it
1641
+ named = name in code
1642
+ if named and is_statement(code, number):
1643
+ found.append((number, top))
1644
+ continue
1645
+ m = _JS_VAR_DECL_RE.match(code)
1646
+ if m:
1647
+ carry, pieces = declarators(m.group(1))
1648
+ if carry is not None:
1649
+ decls.append((top, *carry))
1650
+ if named and any(declares(p, number) for p in pieces):
1651
+ # a later declarator: `export const a = 1, NAME = 2;`
1652
+ found.append((number, top))
1653
+ continue
1654
+ m = _JS_SPLIT_DECL_RE.match(code)
1655
+ if m:
1656
+ split = (m.group("kind"), top)
1657
+ continue
1658
+ m = _JS_EXPORT_LIST_RE.match(code)
1659
+ if m:
1660
+ depth, pieces = _js_depth1_pieces(m.group(1), 1)
1661
+ if depth > 0:
1662
+ cont = ("list", top, depth)
1663
+ if any(_js_export_item_binds(p, name, declarations_only) for p in pieces):
1664
+ found.append((number, top))
1665
+ continue
1666
+ m = _JS_MODULE_EXPORTS_OBJECT_RE.match(code)
1667
+ if m:
1668
+ depth, pieces = _js_depth1_pieces(m.group(1), 1)
1669
+ if depth > 0:
1670
+ cont = ("object", top, depth)
1671
+ if any(binds_key(p, number) for p in pieces):
1672
+ found.append((number, top))
1673
+ continue
1674
+ if not dotted:
1675
+ continue
1676
+ m = member.match(code)
1677
+ if not m:
1678
+ continue
1679
+ tail = m.group("tail")
1680
+ if tail == "(":
1681
+ if not m.group("mods") and _js_paren_tail_is_call(code[m.end():]):
1682
+ continue
1683
+ elif code.endswith(","):
1684
+ continue # an object-literal entry or a destructuring default
1685
+ elif tail == "=" and code[m.end() : m.end() + 1] in ("=", ">"):
1686
+ continue
1687
+ found.append((number, top))
1688
+ return found
1689
+
1690
+
1691
+ def _prefer_module_level(found: list[tuple[int, bool]], dotted: bool) -> list[int]:
1692
+ """The definition lines to report. For an undotted name, a column-0
1693
+ match drops the indented ones, so a local shadow never outranks the
1694
+ module-level definition; a dotted name keeps every match."""
1695
+ if not dotted:
1696
+ top = [n for n, at_column_0 in found if at_column_0]
1697
+ if top:
1698
+ return sorted(set(top))
1699
+ return sorted({n for n, _ in found})
1700
+
1701
+
1702
+ def line_rule_applies(source_file: str, export_type: object) -> bool:
1703
+ """True when the definition-line rules cover this entry: a Python or
1704
+ TS/JS file, and an export that is not a whole module or package."""
1705
+ if isinstance(export_type, str) and (
1706
+ export_type.strip().lower() in NO_DEFINITION_EXPORT_TYPES
1707
+ ):
1708
+ return False
1709
+ suffix = Path(source_file.strip().replace("\\", "/")).suffix.lower()
1710
+ return suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
1711
+
1712
+
1713
+ def _definition_matches(
1714
+ source_file: str,
1715
+ export_name: str,
1716
+ source_root: Path,
1717
+ cache: _SourceCache | None = None,
1718
+ declarations_only: bool = False,
1719
+ ) -> tuple[list[int], set[int]] | None:
1720
+ """(definition lines to report, every matching line) for an export.
1721
+
1722
+ The first item applies the column-0 rule; the second keeps the indented
1723
+ matches that rule dropped. None when the file's extension has no
1724
+ definition rule or the file cannot be read.
1725
+
1726
+ `declarations_only` (classify-stale's `defined_at` lookup) keeps the
1727
+ lines that declare the name at column 0, both items alike: no plain
1728
+ import, star import or re-export (`_python_definition_lines` and
1729
+ `_tsjs_definition_lines` say which), and nothing in a Python file that
1730
+ does not tokenize.
1731
+ """
1732
+ rel = source_file.strip().replace("\\", "/")
1733
+ suffix = Path(rel).suffix.lower()
1734
+ if suffix not in PYTHON_EXTENSIONS and suffix not in TSJS_EXTENSIONS:
1735
+ return None
1736
+ cache = cache if cache is not None else _SourceCache()
1737
+ path = (source_root / rel).resolve()
1738
+ lines = cache.lines(path)
1739
+ if lines is None:
1740
+ return None
1741
+ return _line_matches(suffix, path, lines, export_name, cache, declarations_only)
1742
+
1743
+
1744
+ def _line_matches(
1745
+ suffix: str,
1746
+ path: Path,
1747
+ lines: list[str],
1748
+ export_name: str,
1749
+ cache: _SourceCache,
1750
+ declarations_only: bool = False,
1751
+ ) -> tuple[list[int], set[int]]:
1752
+ """`_definition_matches` over the lines of a Python or TS/JS file already
1753
+ read: `suffix`, the cited file's extension, picks the rules, and `path`
1754
+ keys `cache`."""
1755
+ name = definition_name(export_name)
1756
+ if not name:
1757
+ return [], set()
1758
+ dotted = "." in export_name
1759
+ if suffix in PYTHON_EXTENSIONS:
1760
+ starts = cache.py_statement_starts(path, lines)
1761
+ if declarations_only and starts is None:
1762
+ # every line would be a candidate, a docstring line included
1763
+ return [], set()
1764
+ found = _python_definition_lines(lines, name, starts, declarations_only)
1765
+ else:
1766
+ code_lines = cache.js_code_lines(path, lines)
1767
+ found = _tsjs_definition_lines(
1768
+ lines, code_lines, name, dotted, cache.js_declarators, declarations_only,
1769
+ cache.js_quiet_lines(path, code_lines),
1770
+ )
1771
+ if declarations_only:
1772
+ top = {n for n, at_column_0 in found if at_column_0}
1773
+ return sorted(top), top
1774
+ return _prefer_module_level(found, dotted), {n for n, _ in found}
1775
+
1776
+
1777
+ def find_definition_lines(
1778
+ source_file: str,
1779
+ export_name: str,
1780
+ source_root: Path,
1781
+ cache: _SourceCache | None = None,
1782
+ ) -> list[int] | None:
1783
+ """Every line of `source_file` that defines `export_name`, in order,
1784
+ after the column-0 rule.
1785
+
1786
+ Returns None when the file's extension has no definition rule or the
1787
+ file cannot be read.
1788
+ """
1789
+ matches = _definition_matches(source_file, export_name, source_root, cache)
1790
+ return None if matches is None else matches[0]
1791
+
1792
+
1793
+ def definition_lines_report(
1794
+ source_file: str,
1795
+ export_name: str,
1796
+ source_root: Path,
1797
+ line: int | None = None,
1798
+ export_type: str | None = None,
1799
+ ) -> dict:
1800
+ """What `verify` would say about one entry's line (the definition-lines
1801
+ subcommand). Raises ValueError when the file exists but cannot be read.
1802
+
1803
+ `line_check` is `file-missing`, `skipped-export-type` (a `module` or
1804
+ `package`, no definition line), `skipped-language` (no rule for the
1805
+ extension) or `checked`. With `line`, a recorded line that is an
1806
+ indented match the column-0 rule dropped joins `definition_lines`, as in
1807
+ `verify`, so the answer carries several lines.
1808
+ """
1809
+ report: dict = {
1810
+ "source_file": source_file,
1811
+ "export_name": export_name,
1812
+ "line_check": "checked",
1813
+ "definition_lines": None,
1814
+ "source_line": line,
1815
+ "line_is_definition": None,
1816
+ }
1817
+ rel = source_file.strip().replace("\\", "/")
1818
+ if not rel or not (source_root / rel).is_file():
1819
+ report["line_check"] = "file-missing"
1820
+ return report
1821
+ if not line_rule_applies(source_file, export_type):
1822
+ suffix = Path(rel).suffix.lower()
1823
+ covered = suffix in PYTHON_EXTENSIONS or suffix in TSJS_EXTENSIONS
1824
+ report["line_check"] = "skipped-export-type" if covered else "skipped-language"
1825
+ return report
1826
+ matches = _definition_matches(source_file, export_name, source_root)
1827
+ if matches is None:
1828
+ raise ValueError(f"failed to read source file {source_root / rel}")
1829
+ defs, every = matches
1830
+ if line is not None:
1831
+ report["line_is_definition"] = line in defs
1832
+ if line not in defs and line in every:
1833
+ defs = sorted(set(defs) | {line})
1834
+ report["definition_lines"] = defs
1835
+ return report
1836
+
1837
+
1838
+ # --------------------------------------------------------------------------
1839
+ # Stale names (classify-stale)
1840
+ # --------------------------------------------------------------------------
1841
+
1842
+ STALE_NOT_DEFINED = "not-defined"
1843
+ STALE_DEFINED = "defined"
1844
+ STALE_UNCHECKED = "unchecked"
1845
+ STALE_NO_ENTRY = "no-entry"
1846
+
1847
+
1848
+ def stale_names(data: object) -> list[str]:
1849
+ """The names `--names` holds: reconcile-coverage.py's `stale`, or a JSON
1850
+ array of names, each once in the order given. Raises ValueError."""
1851
+ names = data.get("stale") if isinstance(data, dict) else data
1852
+ if not isinstance(names, list) or not all(isinstance(n, str) for n in names):
1853
+ raise ValueError(
1854
+ "--names must hold a `stale` list of names or a JSON array of names"
1855
+ )
1856
+ return list(dict.fromkeys(n for n in names if n))
1857
+
1858
+
1859
+ def _entry_file(entry: dict) -> str | None:
1860
+ """The entry's `source_file`, None when it records none."""
1861
+ source_file = entry.get("source_file")
1862
+ if not isinstance(source_file, str) or not source_file.strip():
1863
+ return None
1864
+ return source_file
1865
+
1866
+
1867
+ def _entry_citation(entry: dict) -> str | None:
1868
+ source_file = _entry_file(entry)
1869
+ if source_file is None:
1870
+ return None
1871
+ line = _coerce_line(entry.get("source_line"))
1872
+ return f"{source_file}:{line}" if isinstance(line, int) else source_file
1873
+
1874
+
1875
+ def _stale_line_check(
1876
+ entry: dict, name: str, source_root: Path
1877
+ ) -> tuple[str, list[int] | None]:
1878
+ """(line_check, definition_lines) of one provenance entry of a stale name."""
1879
+ source_file = _entry_file(entry)
1880
+ if source_file is None:
1881
+ # no file to check: it neither proves the name fabricated nor clears it
1882
+ return "no-file", None
1883
+ line = _coerce_line(entry.get("source_line"))
1884
+ export_type = entry.get("export_type")
1885
+ try:
1886
+ report = definition_lines_report(
1887
+ source_file,
1888
+ name,
1889
+ source_root,
1890
+ line if isinstance(line, int) else None,
1891
+ export_type if isinstance(export_type, str) else None,
1892
+ )
1893
+ except ValueError:
1894
+ # the file exists but cannot be read: nothing proves it
1895
+ return "unreadable", None
1896
+ return report["line_check"], report["definition_lines"]
1897
+
1898
+
1899
+ # The folders and files the `defined_at` walk leaves out, judged
1900
+ # case-insensitively on the path below its walk root: generated, built and
1901
+ # installed output, a vendored copy of another package, tests and their
1902
+ # configuration, docs, examples and benchmarks.
1903
+ # Keep identical to EXCLUDED_DIR_NAMES in skf-resolve-authoritative-files.py.
1904
+ EXCLUDED_DIR_NAMES = {
1905
+ "node_modules", "__pycache__", "dist", "build", ".webpack",
1906
+ "target", ".next", ".nuxt", "out", "coverage", ".git",
1907
+ ".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache",
1908
+ ".ruff_cache", ".gradle", ".idea", ".vscode",
1909
+ }
1910
+ # Keep identical to _VENDORED_SEGMENTS in skf-detect-language.py.
1911
+ _VENDORED_SEGMENTS = frozenset(
1912
+ {"vendor", "vendored", "third_party", "third-party", "thirdparty"}
1913
+ )
1914
+ # Keep identical to TEST_FOLDERS in skf-disqualify-candidates.py.
1915
+ TEST_FOLDERS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
1916
+ # Keep identical to TEST_CONFIG_NAMES in skf-disqualify-candidates.py.
1917
+ TEST_CONFIG_NAMES = frozenset({
1918
+ "conftest.py", "pytest.ini", "phpunit.xml", "phpunit.xml.dist", ".rspec",
1919
+ })
1920
+ # Keep identical to TEST_CONFIG_PREFIXES in skf-disqualify-candidates.py.
1921
+ TEST_CONFIG_PREFIXES = (
1922
+ "jest.config.", "vitest.config.", "karma.conf.", "playwright.config.",
1923
+ "cypress.config.",
1924
+ )
1925
+ # Keep identical to TEST_FILE_RE in skf-disqualify-candidates.py.
1926
+ TEST_FILE_RE = re.compile(
1927
+ r"(\.(test|spec)\.[^./]+$)|(_test\.[^./]+$)|(^test_.+\.py$)|(_spec\.rb$)"
1928
+ )
1929
+ # Keep identical to PY_NON_CORE_TOPS in skf-extract-public-api.py.
1930
+ PY_NON_CORE_TOPS = frozenset({"docs", "examples", "benchmarks"})
1931
+ # The walk's own additions: pip's and setuptools' `_vendor`, test helpers
1932
+ # (`testing`, `fixtures`, `__mocks__`), installed packages (`site-packages`,
1933
+ # a virtualenv named `env`, `bower_components`) and the ESM, CommonJS and
1934
+ # UMD builds a JS package commits.
1935
+ LOOKUP_EXTRA_FOLDERS = frozenset({
1936
+ "_vendor", "testing", "fixtures", "__mocks__", "site-packages", "env",
1937
+ "bower_components", "esm", "cjs", "umd",
1938
+ })
1939
+ LOOKUP_SKIPPED_FOLDERS = frozenset(
1940
+ EXCLUDED_DIR_NAMES | _VENDORED_SEGMENTS | TEST_FOLDERS | PY_NON_CORE_TOPS
1941
+ | LOOKUP_EXTRA_FOLDERS
1942
+ )
1943
+ # The test folders the walk alone skips wherever they lie (end-to-end tests,
1944
+ # Jest's `__fixtures__` beside the code): a file the map cites in one is
1945
+ # still read.
1946
+ LOOKUP_WALK_FOLDERS = frozenset({"e2e", "__fixtures__"})
1947
+ # The folders the walk alone skips as the first folder below its walk root
1948
+ # or below a package root it passes through (`_package_root`'s roots):
1949
+ # examples and demos, samples, scripts, playgrounds, benchmarks and
1950
+ # migrations, which hold code that uses or changes the package rather than
1951
+ # its API. Deeper in a package such a folder can be live package code
1952
+ # (cognee's `cognee/modules/migrations/`), so it is walked there; one that
1953
+ # holds a file the map cites is walked too, and that file is still read.
1954
+ LOOKUP_TOP_FOLDERS = frozenset({
1955
+ "example", "demo", "demos", "sample", "samples", "scripts", "playground",
1956
+ "bench", "benchmark", "migrations",
1957
+ })
1958
+ # The kinds of a documented inventory (validate-inventory.py's `kind`) that a
1959
+ # module cannot be: such a name takes no line 1 from a module so named. A
1960
+ # `method` is left out: reconcile-coverage.py never lists a method as stale,
1961
+ # and a module may share a method's name.
1962
+ NO_MODULE_KINDS = frozenset({"function", "hook", "class", "interface", "enum", "type"})
1963
+ # A TS/JS file that is itself the export: its module so named declares the
1964
+ # name whatever its kind.
1965
+ _SELF_EXPORT_RE = re.compile(r"^(?:export\s+default\b|module\.exports\s*=(?!=))", re.M)
1966
+ # A minified bundle: built output, whatever folder holds it.
1967
+ LOOKUP_SKIPPED_SUFFIXES = (".min.js",)
1968
+ # The errors a lookup gives up on: every `defined_at` is then null.
1969
+ _LOOKUP_ERRORS = (OSError, ValueError, RuntimeError)
1970
+
1971
+
1972
+ def _rule_family(path: str) -> str | None:
1973
+ """The definition rules a file's extension picks: `python`, `tsjs` or
1974
+ None."""
1975
+ suffix = posixpath.splitext(path)[1].lower()
1976
+ if suffix in PYTHON_EXTENSIONS:
1977
+ return "python"
1978
+ return "tsjs" if suffix in TSJS_EXTENSIONS else None
1979
+
1980
+
1981
+ def _lookup_skips_folder(segment: str) -> bool:
1982
+ """True when the `defined_at` walk never goes into this folder."""
1983
+ segment = segment.lower()
1984
+ return segment.startswith(".") or segment in LOOKUP_SKIPPED_FOLDERS
1985
+
1986
+
1987
+ def _lookup_skips_file(filename: str) -> bool:
1988
+ """True when the `defined_at` lookup never reads this file: a test file,
1989
+ as skf-disqualify-candidates.py's `_is_test_file` judges its name (a
1990
+ test configuration file included), or a minified bundle."""
1991
+ name = filename.lower()
1992
+ return (
1993
+ name in TEST_CONFIG_NAMES
1994
+ or name.startswith(TEST_CONFIG_PREFIXES)
1995
+ or TEST_FILE_RE.search(name) is not None
1996
+ or name.endswith(LOOKUP_SKIPPED_SUFFIXES)
1997
+ )
1998
+
1999
+
2000
+ def _symlinked(root: Path, parts: tuple[str, ...]) -> bool:
2001
+ """True when a path below `root` goes through a symlink, itself
2002
+ included."""
2003
+ return any(root.joinpath(*parts[:i]).is_symlink() for i in range(1, len(parts) + 1))
2004
+
2005
+
2006
+ def _readable_below(root: Path, parts: tuple[str, ...], real_root: Path | None = None) -> bool:
2007
+ """True when the `defined_at` lookup may read the file at `parts` below
2008
+ `root`: not a file it skips (`_lookup_skips_file`), no symlink on its
2009
+ path, and its real path a file inside the root (`real_root`, the root
2010
+ resolved)."""
2011
+ if _lookup_skips_file(parts[-1]) or _symlinked(root, parts):
2012
+ return False
2013
+ real = root.joinpath(*parts).resolve()
2014
+ return real.is_relative_to(real_root or root.resolve()) and real.is_file()
2015
+
2016
+
2017
+ def _cited_parts(source_file: str) -> tuple[str, ...] | None:
2018
+ """A recorded `source_file` as path segments below the source root, None
2019
+ when it is absolute or climbs out of the root."""
2020
+ path = normalize_path(source_file)
2021
+ if not path or path == "." or posixpath.isabs(path):
2022
+ return None
2023
+ parts = tuple(path.split("/"))
2024
+ return None if parts[0] == ".." or ":" in parts[0] else parts
2025
+
2026
+
2027
+ # The file whose folder is a package, by rule family.
2028
+ _PACKAGE_MARKERS = {"python": "__init__.py", "tsjs": "package.json"}
2029
+ # Type stubs: a declaration file beside the module it types.
2030
+ _STUB_SUFFIXES = {".pyi": "python", ".d.ts": "tsjs", ".d.mts": "tsjs", ".d.cts": "tsjs"}
2031
+
2032
+
2033
+ def _stub_of(path: str) -> tuple[str, str] | None:
2034
+ """(the path without its suffix, family) of a type stub (`NAME.pyi`,
2035
+ `NAME.d.ts`), None for any other file."""
2036
+ lower = path.lower()
2037
+ for suffix, family in _STUB_SUFFIXES.items():
2038
+ if lower.endswith(suffix):
2039
+ return path[: -len(suffix)], family
2040
+ return None
2041
+
2042
+
2043
+ def _without_stubs(hits: list[str]) -> list[str]:
2044
+ """`file:line` items less each stub whose implementation (`NAME.py`
2045
+ beside `NAME.pyi`, a TS/JS `NAME.<ext>` beside `NAME.d.ts`) is an item
2046
+ too: the two are one declaring file, the implementation."""
2047
+ files = [h.rsplit(":", 1)[0] for h in hits]
2048
+ implementations = {
2049
+ (posixpath.splitext(f)[0], _rule_family(f))
2050
+ for f in files
2051
+ if _stub_of(f) is None and _rule_family(f) is not None
2052
+ }
2053
+ return [h for h, f in zip(hits, files) if _stub_of(f) not in implementations]
2054
+
2055
+
2056
+ def _module_name(parts: tuple[str, ...]) -> str | None:
2057
+ """The name a file is the module or package of, which it declares at
2058
+ its line 1: `NAME.py`, `NAME/__init__.py`, a TS/JS `NAME.<ext>` or
2059
+ `NAME/index.<ext>`. None for any other file, and for a package file
2060
+ with no folder above it below the source root."""
2061
+ stem, ext = posixpath.splitext(parts[-1])
2062
+ ext = ext.lower()
2063
+ if ext == ".py":
2064
+ package = "__init__"
2065
+ elif ext in TSJS_EXTENSIONS:
2066
+ package = "index"
2067
+ else:
2068
+ return None
2069
+ if stem != package:
2070
+ return stem
2071
+ return parts[-2] if len(parts) > 1 else None
2072
+
2073
+
2074
+ def _exports_itself(parts: tuple[str, ...], text: str) -> bool:
2075
+ """True for a TS/JS file that is itself the export, with a column-0
2076
+ `export default` or `module.exports =`: its module so named declares
2077
+ the name whatever kind the inventory gives it."""
2078
+ return _rule_family(parts[-1]) == "tsjs" and _SELF_EXPORT_RE.search(text) is not None
2079
+
2080
+
2081
+ def declared_line(
2082
+ source_file: str | tuple[str, ...],
2083
+ name: str,
2084
+ source_root: Path | None = None,
2085
+ *,
2086
+ lines: list[str] | None = None,
2087
+ cache: _SourceCache | None = None,
2088
+ module_fallback: bool = True,
2089
+ ) -> int | None:
2090
+ """The line at which one file declares `name`, by classify-stale's
2091
+ `defined_at` rule (see the module docstring): its first line at column
2092
+ 0 that the definition-line rules match with imports left out (a `def`,
2093
+ `class`, assignment, annotation, type alias, TS/JS declaration, or an
2094
+ alias that renames, such as `from X import Y as NAME`; a plain or star
2095
+ import, an `as` that keeps the imported name, a re-export and an
2096
+ import-valued binding never count), else, with `module_fallback`, its
2097
+ line 1 when it is the module or package so named (`_module_name`).
2098
+
2099
+ `source_file` is the file's path below the source root, as recorded
2100
+ (one that is absolute or climbs out of the root gives None) or as its
2101
+ segments. `lines` are its lines when the caller read them already;
2102
+ otherwise the file is read below `source_root` under the lookup's
2103
+ guards (`_readable_below`: no symlink on its path, its real path
2104
+ inside the root, not a file the lookup skips). Pass one `cache` for
2105
+ the names of one source root, and each file is read and parsed once.
2106
+ None when no line declares the name, the name is empty or dotted, the
2107
+ file is not a Python or TS/JS one, or it cannot be read.
2108
+ """
2109
+ parts = _cited_parts(source_file) if isinstance(source_file, str) else tuple(source_file)
2110
+ if not name or "." in name or not parts or _rule_family(parts[-1]) is None:
2111
+ return None
2112
+ cache = cache if cache is not None else _SourceCache()
2113
+ if lines is None:
2114
+ if source_root is None:
2115
+ raise ValueError("declared_line reads the file below source_root when no lines are given")
2116
+ if not _readable_below(source_root, parts):
2117
+ return None
2118
+ lines = cache.lines(source_root.joinpath(*parts))
2119
+ if lines is None:
2120
+ return None
2121
+ path = Path(*parts)
2122
+ top, _ = _line_matches(path.suffix.lower(), path, lines, name, cache, True)
2123
+ if top:
2124
+ return top[0]
2125
+ return 1 if module_fallback and _module_name(parts) == name else None
2126
+
2127
+
2128
+ # declared_line's `cache`, public for a script that loads this one.
2129
+ SourceCache = _SourceCache
2130
+
2131
+
2132
+ def _baseline_path(path: str) -> str:
2133
+ """A cited path as an extraction writes it: forward slashes, no
2134
+ leading `./`."""
2135
+ rel = path.strip().replace("\\", "/")
2136
+ while rel.startswith("./"):
2137
+ rel = rel[2:]
2138
+ return rel
2139
+
2140
+
2141
+ def _declaration_refused(source_root: Path, file: str) -> bool:
2142
+ """True when `declared_line` would not read a file that is there: a
2143
+ path outside the source root, a file the lookup skips (a test file, a
2144
+ minified bundle) or one with a symlink on its path. A file that is gone
2145
+ is not refused: its entries are deleted."""
2146
+ parts = _cited_parts(file)
2147
+ if parts is None:
2148
+ return True
2149
+ path = source_root.joinpath(*parts)
2150
+ return os.path.lexists(path) and not _readable_below(source_root, parts)
2151
+
2152
+
2153
+ def baseline_gaps(
2154
+ provenance: dict,
2155
+ source_root: Path,
2156
+ held: set[tuple[str, str]],
2157
+ files: set[str],
2158
+ listed: set[tuple] | frozenset = frozenset(),
2159
+ *,
2160
+ cache: _SourceCache | None = None,
2161
+ ) -> tuple[list[dict], list[dict]]:
2162
+ """The provenance entries an extraction left out whose cited file still
2163
+ declares them: the recipe runner never reports a module, an alias that
2164
+ renames, a dunder such as `__version__` or another underscore name, so
2165
+ a diff would read each as removed.
2166
+
2167
+ `held` holds the (name, file) of every export the extraction found,
2168
+ `files` the files whose entries are checked (the ones read), and
2169
+ `listed` the (name, file) of the gaps already listed elsewhere (the
2170
+ runner's own extraction gaps), each path with forward slashes. An
2171
+ entry is left out when its name is dotted (a member) and it is no
2172
+ `module` or `package` entry, when its file is not in `files`, when the
2173
+ extraction holds the name or its re-export target (`reexport_map`,
2174
+ else the entry's `reexported_as`) at that file, or when it is listed
2175
+ already. Each name and file is taken once.
2176
+
2177
+ Returns (gaps, unchecked). A gap is {"name", "file", "line", "entry":
2178
+ null}, plus the entry's `export_type` when it has one, at the line
2179
+ `declared_line` gives (only a `module` or `package` entry takes the
2180
+ module rule); an entry whose file declares it no more is no gap.
2181
+ `unchecked` holds {"name", "file", "export_type"} for each entry no
2182
+ rule can check, to read by eye: one whose file is neither Python nor
2183
+ TS/JS, one whose file the lookup will not read (outside the source
2184
+ root, a test file or a minified bundle, a symlink on its path), and a
2185
+ dotted `module` or `package` entry (`api.v1`). The entries are read
2186
+ file by file over one cache, so each file is read and parsed once."""
2187
+ checkable = PYTHON_EXTENSIONS | TSJS_EXTENSIONS
2188
+ renamed = extract_reexport_map(provenance)
2189
+ seen = set(listed)
2190
+ by_file: dict[str, list[tuple[str, object]]] = {}
2191
+ unchecked: list[dict] = []
2192
+ entries = provenance.get("entries")
2193
+ for entry in entries if isinstance(entries, list) else []:
2194
+ if not isinstance(entry, dict):
2195
+ continue
2196
+ name, cited = entry.get("export_name"), entry.get("source_file")
2197
+ if not isinstance(name, str) or not isinstance(cited, str):
2198
+ continue
2199
+ name, file, kind = name.strip(), _baseline_path(cited), entry.get("export_type")
2200
+ module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2201
+ if not name or ("." in name and not module) or (name, file) in seen:
2202
+ continue
2203
+ if cited.replace("\\", "/") not in files and file not in files:
2204
+ continue
2205
+ if (name, file) in held or (renamed.get(name), file) in held:
2206
+ continue
2207
+ seen.add((name, file))
2208
+ if "." in name or Path(file).suffix.lower() not in checkable or _declaration_refused(source_root, file):
2209
+ unchecked.append({"name": name, "file": file, "export_type": kind})
2210
+ continue
2211
+ by_file.setdefault(file, []).append((name, kind))
2212
+ cache = cache if cache is not None else _SourceCache()
2213
+ gaps = []
2214
+ for file, named in by_file.items():
2215
+ for name, kind in named:
2216
+ module = isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2217
+ line = declared_line(file, name, source_root, cache=cache, module_fallback=module)
2218
+ if line is None:
2219
+ continue
2220
+ gap = {"name": name, "file": file, "line": line, "entry": None}
2221
+ if kind is not None:
2222
+ gap["export_type"] = kind
2223
+ gaps.append(gap)
2224
+ return gaps, unchecked
2225
+
2226
+
2227
+ class _DeclarationLookup:
2228
+ """Where the source declares the stale names (classify-stale's
2229
+ `defined_at`): a column-0 declaration, an alias that renames included, a
2230
+ plain or star import and a re-export never counting.
2231
+
2232
+ A module or package named after the name (`_module_name`) declares it,
2233
+ at its line 1 when no line of it does, unless `find` takes the name as
2234
+ `strict` and the file is not a TS/JS one that is itself the export
2235
+ (`_exports_itself`). A file the name's entries cite that declares it
2236
+ gives its first such line. Otherwise the walk lists every file that
2237
+ declares it (`declared_in`), and exactly one must: the walk reads,
2238
+ sorted, the files below the walk root (`_walk_root`), leaving out the
2239
+ folders `_lookup_skips_folder` names, the `LOOKUP_WALK_FOLDERS`, the
2240
+ `LOOKUP_TOP_FOLDERS` right below the walk root and a package root
2241
+ (`_is_package_root`) that hold no cited file, the files
2242
+ `_lookup_skips_file` names and every symlink. Only the rule families
2243
+ (Python, TS/JS) the entries cite are read (both when no entry cites a
2244
+ file). A cited file is read inside the source root only, with no symlink
2245
+ on its path and the same skips but the walk-only ones, judged below the
2246
+ walk root when it lies there. The walk runs once for all the names no
2247
+ cited file settles: each file is read once, and only a file whose text
2248
+ holds a name goes through the rules. Any error of the filesystem raises
2249
+ (`_LOOKUP_ERRORS`)."""
2250
+
2251
+ def __init__(self, entries: list[dict], source_root: Path) -> None:
2252
+ self.root = source_root
2253
+ self.real_root = source_root.resolve()
2254
+ recorded = [f for f in map(_entry_file, entries) if f is not None]
2255
+ # a map that cites no file names no language: both families
2256
+ families = {_rule_family(f) for f in recorded}
2257
+ self.families = families - {None} if recorded else {"python", "tsjs"}
2258
+ self.folder = self._walk_root(recorded)
2259
+ # the folders that hold a cited file: a LOOKUP_TOP_FOLDERS one is walked
2260
+ self._cited_folders = {
2261
+ parts[:i]
2262
+ for parts in filter(None, map(_cited_parts, recorded))
2263
+ for i in range(1, len(parts))
2264
+ }
2265
+ # the cited files read, kept for the walk: a few files at most
2266
+ self._cited: dict[tuple[str, ...], list[str] | None] = {}
2267
+ self._cited_cache = _SourceCache()
2268
+
2269
+ # -- the walk root ------------------------------------------------------
2270
+
2271
+ def _symlinked(self, parts: tuple[str, ...]) -> bool:
2272
+ """True when a path below the source root goes through a symlink,
2273
+ itself included."""
2274
+ return _symlinked(self.root, parts)
2275
+
2276
+ def _real_folder(self, parts: tuple[str, ...]) -> tuple[str, ...]:
2277
+ """The deepest folder of a path below the source root that is one,
2278
+ with no symlink on its path."""
2279
+ while parts and (
2280
+ self._symlinked(parts) or not self.root.joinpath(*parts).is_dir()
2281
+ ):
2282
+ parts = parts[:-1]
2283
+ return parts
2284
+
2285
+ def _holds(self, folder: tuple[str, ...], filename: str) -> bool:
2286
+ return (self.root.joinpath(*folder) / filename).is_file()
2287
+
2288
+ def _package_root(
2289
+ self, folder: tuple[str, ...], family: str
2290
+ ) -> tuple[str, ...] | None:
2291
+ """The root of the package a folder below the source root lies in,
2292
+ None when it lies in none. Python: the nearest folder at or above it
2293
+ that holds `__init__.py`, then up while the parent holds one, never
2294
+ the source root itself (a stray `__init__.py` there would make the
2295
+ whole tree one package). TS/JS: the nearest folder at or above it
2296
+ that holds `package.json`."""
2297
+ marker = _PACKAGE_MARKERS[family]
2298
+ python = family == "python"
2299
+ here = folder
2300
+ while not (self._holds(here, marker) and (here or not python)):
2301
+ if not here:
2302
+ return None
2303
+ here = here[:-1]
2304
+ if python:
2305
+ while len(here) > 1 and self._holds(here[:-1], marker):
2306
+ here = here[:-1]
2307
+ return here
2308
+
2309
+ def _walk_root(self, recorded: list[str]) -> tuple[str, ...]:
2310
+ """The folder the walk reads: the deepest folder the entries' files
2311
+ share, widened to its package root, leaving out an entry outside
2312
+ every package (the source root when no entry lies in one)."""
2313
+ folders: list[tuple[str, ...]] = []
2314
+ widen: set[str] = set()
2315
+ for source_file in recorded:
2316
+ parts = _cited_parts(source_file)
2317
+ family = _rule_family(source_file) if parts else None
2318
+ if family is None:
2319
+ continue
2320
+ if self._package_root(self._real_folder(parts[:-1]), family) is None:
2321
+ continue # outside every package: a script, a docs page
2322
+ folders.append(parts[:-1])
2323
+ widen.add(family)
2324
+ if not folders:
2325
+ return ()
2326
+ shared = self._real_folder(tuple(os.path.commonprefix(folders)))
2327
+ roots = [self._package_root(shared, family) for family in sorted(widen)]
2328
+ return min((r for r in roots if r is not None), key=len, default=shared)
2329
+
2330
+ # -- reading ------------------------------------------------------------
2331
+
2332
+ def _below_walk_root(self, parts: tuple[str, ...]) -> tuple[str, ...]:
2333
+ depth = len(self.folder)
2334
+ return parts[depth:] if parts[:depth] == self.folder else parts
2335
+
2336
+ def _reads_cited(self, parts: tuple[str, ...]) -> bool:
2337
+ """True when the lookup reads a cited file: one of a family the
2338
+ entries cite, in no skipped place, with no symlink on its path and
2339
+ inside the source root."""
2340
+ if _rule_family(parts[-1]) not in self.families:
2341
+ return False
2342
+ below = self._below_walk_root(parts)
2343
+ if any(_lookup_skips_folder(s) for s in below[:-1]):
2344
+ return False
2345
+ return _readable_below(self.root, parts, self.real_root)
2346
+
2347
+ def _cited_lines(self, parts: tuple[str, ...]) -> list[str] | None:
2348
+ if parts not in self._cited:
2349
+ readable = self._reads_cited(parts)
2350
+ path = self.root.joinpath(*parts)
2351
+ self._cited[parts] = _text_lines(_read_source(path)) if readable else None
2352
+ return self._cited[parts]
2353
+
2354
+ def _is_package_root(self, folder: tuple[str, ...], filenames: list[str]) -> bool:
2355
+ """True when a walked folder (its `filenames`) is a package root of a
2356
+ family the walk reads, as `_package_root` finds one: TS/JS, a folder
2357
+ that holds `package.json`; Python, the top of an `__init__.py`
2358
+ chain, never the source root."""
2359
+ if "tsjs" in self.families and "package.json" in filenames:
2360
+ return True
2361
+ return (
2362
+ "python" in self.families and bool(folder) and "__init__.py" in filenames
2363
+ and (len(folder) == 1 or not self._holds(folder[:-1], "__init__.py"))
2364
+ )
2365
+
2366
+ def _tree(self) -> list[tuple[str, ...]]:
2367
+ """The files the walk reads, as segments below the source root,
2368
+ sorted."""
2369
+ top = self.root.joinpath(*self.folder)
2370
+ found: list[tuple[str, ...]] = []
2371
+
2372
+ def fail(exc: OSError) -> None:
2373
+ raise exc
2374
+
2375
+ for dirpath, dirnames, filenames in os.walk(top, onerror=fail):
2376
+ here = Path(dirpath)
2377
+ inner = here.relative_to(top).parts
2378
+ below = self.folder + inner
2379
+ # a LOOKUP_TOP_FOLDERS name is skipped right below the walk root
2380
+ # and a package root, unless it holds a cited file
2381
+ root = not inner or self._is_package_root(below, filenames)
2382
+ dirnames[:] = [
2383
+ d for d in dirnames
2384
+ if not _lookup_skips_folder(d)
2385
+ and d.lower() not in LOOKUP_WALK_FOLDERS
2386
+ and not (
2387
+ root and d.lower() in LOOKUP_TOP_FOLDERS
2388
+ and (*below, d) not in self._cited_folders
2389
+ )
2390
+ and not (here / d).is_symlink()
2391
+ ]
2392
+ for filename in filenames:
2393
+ path = here / filename
2394
+ if (
2395
+ _rule_family(filename) in self.families
2396
+ and not _lookup_skips_file(filename)
2397
+ and not path.is_symlink()
2398
+ and path.is_file()
2399
+ ):
2400
+ found.append((*below, filename))
2401
+ return sorted(found)
2402
+
2403
+ @staticmethod
2404
+ def _first(
2405
+ parts: tuple[str, ...],
2406
+ lines: list[str],
2407
+ name: str,
2408
+ cache: _SourceCache,
2409
+ module_fallback: bool = True,
2410
+ ) -> str | None:
2411
+ """`file:line` of the first line of a file that declares `name`, else,
2412
+ with `module_fallback`, its line 1 when it is the module or package
2413
+ so named (`declared_line`)."""
2414
+ line = declared_line(
2415
+ parts, name, lines=lines, cache=cache, module_fallback=module_fallback
2416
+ )
2417
+ return f"{'/'.join(parts)}:{line}" if line is not None else None
2418
+
2419
+ # -- the answer ---------------------------------------------------------
2420
+
2421
+ def find(
2422
+ self, cited: dict[str, list[str]], strict: frozenset[str] = frozenset()
2423
+ ) -> tuple[dict[str, str], dict[str, list[str]]]:
2424
+ """For the names of `cited` (name -> the source_file values its
2425
+ entries record): name -> `file:line` for each one the source
2426
+ declares, and name -> the sorted `file:line` of every walked file
2427
+ that declares it, for each name the walk looked for (the names no
2428
+ cited file settles). A dotted name is never looked up, and a name of
2429
+ `strict` takes no line 1 from a module or package so named that is
2430
+ not itself the export (`_exports_itself`), in a cited file or in
2431
+ the walk."""
2432
+ found: dict[str, str] = {}
2433
+ if not self.families:
2434
+ return found, {}
2435
+ pending: list[str] = []
2436
+ for name, files in cited.items():
2437
+ if not name or "." in name:
2438
+ continue
2439
+ for parts in dict.fromkeys(filter(None, map(_cited_parts, files))):
2440
+ lines = self._cited_lines(parts)
2441
+ hit = None if lines is None else self._first(
2442
+ parts, lines, name, self._cited_cache,
2443
+ name not in strict or _exports_itself(parts, "\n".join(lines)),
2444
+ )
2445
+ if hit is not None:
2446
+ found[name] = hit
2447
+ break
2448
+ else:
2449
+ pending.append(name)
2450
+ declared, unsure = self._walk(pending, strict) if pending else ({}, set())
2451
+ # exactly one walked file must declare a name, and none it holds
2452
+ # in an untokenizable Python file may
2453
+ found.update({
2454
+ name: h[0] for name, h in declared.items()
2455
+ if len(h) == 1 and name not in unsure
2456
+ })
2457
+ return found, declared
2458
+
2459
+ def _walk(
2460
+ self, names: list[str], strict: frozenset[str] = frozenset()
2461
+ ) -> tuple[dict[str, list[str]], set[str]]:
2462
+ """name -> the sorted `file:line` of each walked file that declares
2463
+ it, at the file's first declaration (a stub beside its
2464
+ implementation left out: `_without_stubs`; a module so named
2465
+ declaring no name of `strict` unless it is the export), and the
2466
+ names a Python file that
2467
+ does not tokenize holds, which it may declare."""
2468
+ hits: dict[str, list[str]] = {name: [] for name in names}
2469
+ unsure: set[str] = set()
2470
+ for parts in self._tree():
2471
+ module = _module_name(parts)
2472
+ lines = self._cited.get(parts)
2473
+ text = (
2474
+ _read_source(self.root.joinpath(*parts))
2475
+ if lines is None
2476
+ else "\n".join(lines) # a cited file, read already
2477
+ )
2478
+ # a strict name's module so named counts when it is the export
2479
+ fallback = module not in strict or _exports_itself(parts, text)
2480
+ # the substring test spares the rules most files and names
2481
+ here = sorted(
2482
+ n for n in names if n in text or (n == module and fallback)
2483
+ )
2484
+ if not here:
2485
+ continue
2486
+ if lines is None:
2487
+ lines = _text_lines(text)
2488
+ cache = _SourceCache() # this file's rules only: nothing kept
2489
+ path = Path(*parts)
2490
+ if path.suffix.lower() in PYTHON_EXTENSIONS and (
2491
+ cache.py_statement_starts(path, lines) is None
2492
+ ):
2493
+ unsure.update(n for n in here if n in text)
2494
+ for name in here:
2495
+ hit = self._first(parts, lines, name, cache, fallback)
2496
+ if hit is not None:
2497
+ hits[name].append(hit)
2498
+ return {name: sorted(_without_stubs(h)) for name, h in hits.items()}, unsure
2499
+
2500
+
2501
+ def inventory_kinds(data: object, path: str) -> dict[str, frozenset[str]]:
2502
+ """name -> the kinds a documented inventory gives it (`--inventory`):
2503
+ validate-inventory.py's `--output` file, `{"exports": [...]}`, or its
2504
+ whole result, the inventory under `inventory`. An item without a string
2505
+ `name` and `kind` is left out, and a kind is kept in lower case. Raises
2506
+ ValueError when no `exports` list is there."""
2507
+ if isinstance(data, dict) and isinstance(data.get("inventory"), dict):
2508
+ data = data["inventory"] # validate-inventory.py's whole result
2509
+ if not isinstance(data, dict) or not isinstance(data.get("exports"), list):
2510
+ raise ValueError(f"--inventory {path} holds no `exports` list")
2511
+ kinds: dict[str, set[str]] = {}
2512
+ for item in data["exports"]:
2513
+ if not isinstance(item, dict):
2514
+ continue
2515
+ name, kind = item.get("name"), item.get("kind")
2516
+ if isinstance(name, str) and isinstance(kind, str):
2517
+ kinds.setdefault(name, set()).add(kind.strip().lower())
2518
+ return {name: frozenset(k) for name, k in kinds.items()}
2519
+
2520
+
2521
+ def _module_entry(entry: dict) -> bool:
2522
+ """True for a `module` or `package` entry."""
2523
+ kind = entry.get("export_type")
2524
+ return isinstance(kind, str) and kind.strip().lower() in NO_DEFINITION_EXPORT_TYPES
2525
+
2526
+
2527
+ def classify_stale(
2528
+ names: list[str],
2529
+ prov: dict,
2530
+ source_root: Path,
2531
+ kinds: dict[str, frozenset[str]] | None = None,
2532
+ ) -> list[dict]:
2533
+ """The fabricated-signature test for each stale name: the definition-lines
2534
+ rules at every provenance entry whose raw `export_name` is the name, and
2535
+ for a name that is not fabricated, where the source declares it
2536
+ (`defined_at`, and `declared_in` when the walk looked for it). The
2537
+ classify-stale section of the module docstring gives each reason, what
2538
+ a `no-file` entry means, which entry gives `source` and how
2539
+ `defined_at` and `declared_in` are found. `kinds` (`inventory_kinds`,
2540
+ None without `--inventory`) keeps a name of a `NO_MODULE_KINDS` kind
2541
+ from a module or package so named, unless an entry of the name is a
2542
+ `module` or `package` one.
2543
+ """
2544
+ entries = [e for e in prov.get("entries") or [] if isinstance(e, dict)]
2545
+ results: list[dict] = []
2546
+ # the names to look up (never a fabricated one): name -> its cited files
2547
+ cited: dict[str, list[str]] = {}
2548
+ # the names a module so named does not declare
2549
+ strict: set[str] = set()
2550
+ for name in names:
2551
+ mine = [e for e in entries if e.get("export_name") == name]
2552
+ checks: list[dict] = []
2553
+ for entry in mine:
2554
+ line_check, defs = _stale_line_check(entry, name, source_root)
2555
+ checks.append(
2556
+ {
2557
+ "source_file": entry.get("source_file"),
2558
+ "source_line": entry.get("source_line"),
2559
+ "line_check": line_check,
2560
+ "definition_lines": defs,
2561
+ }
2562
+ )
2563
+ if not checks:
2564
+ reason = STALE_NO_ENTRY
2565
+ elif all(
2566
+ c["line_check"] == "file-missing"
2567
+ or (c["line_check"] == "checked" and not c["definition_lines"])
2568
+ for c in checks
2569
+ ):
2570
+ reason = STALE_NOT_DEFINED
2571
+ elif any(
2572
+ c["line_check"] == "checked" and c["definition_lines"] for c in checks
2573
+ ):
2574
+ reason = STALE_DEFINED
2575
+ else:
2576
+ reason = STALE_UNCHECKED
2577
+ source = next(filter(None, map(_entry_citation, mine)), None)
2578
+ fabricated = reason == STALE_NOT_DEFINED
2579
+ if not fabricated:
2580
+ cited[name] = [f for f in map(_entry_file, mine) if f is not None]
2581
+ documented = (kinds or {}).get(name, frozenset())
2582
+ if documented & NO_MODULE_KINDS and not any(map(_module_entry, mine)):
2583
+ strict.add(name)
2584
+ results.append(
2585
+ {
2586
+ "name": name,
2587
+ "fabricated": fabricated,
2588
+ "source": source,
2589
+ "defined_at": None,
2590
+ "declared_in": None,
2591
+ "reason": reason,
2592
+ "entries": checks,
2593
+ }
2594
+ )
2595
+ try:
2596
+ found, declared = _DeclarationLookup(entries, source_root).find(
2597
+ cited, frozenset(strict)
2598
+ )
2599
+ except _LOOKUP_ERRORS:
2600
+ # a lookup that fails proves nothing: every name stays null
2601
+ found, declared = {}, {}
2602
+ for item in results:
2603
+ item["defined_at"] = found.get(item["name"])
2604
+ item["declared_in"] = declared.get(item["name"])
2605
+ return results
2606
+
2607
+
2608
+ # --------------------------------------------------------------------------
2609
+ # Skill citations ([AST:] / [SRC:] prefixes)
2610
+ # --------------------------------------------------------------------------
2611
+
2612
+ # `[AST:path:L12]`, `[SRC:path:L12-34]`, `[SRC:path:L12-L34]`. A citation
2613
+ # with no `:L<n>` part does not match and is ignored. Groups: prefix, path,
2614
+ # first line, the `L` of a range's end (may be empty), the range's end.
2615
+ _SKILL_CITATION_RE = re.compile(
2616
+ r"\[(AST|SRC):([^\[\]\n]+?):L(\d+)(?:-(L?)(\d+))?\]"
2617
+ )
2618
+
2619
+
2620
+ def normalize_path(path: str) -> str:
2621
+ """Normalize a cited or recorded source path for comparison."""
2622
+ p = path.strip().replace("\\", "/")
2623
+ while p.startswith("./"):
2624
+ p = p[2:]
2625
+ return posixpath.normpath(p) if p else p
2626
+
2627
+
2628
+ def skill_markdown_files(skill_dir: Path) -> list[Path]:
2629
+ """`SKILL.md` then every `references/**/*.md`, in a stable order."""
2630
+ files: list[Path] = []
2631
+ skill_md = skill_dir / "SKILL.md"
2632
+ if skill_md.is_file():
2633
+ files.append(skill_md)
2634
+ refs = skill_dir / "references"
2635
+ if refs.is_dir():
2636
+ files.extend(sorted(p for p in refs.rglob("*.md") if p.is_file()))
2637
+ return files
2638
+
2639
+
2640
+ def scan_skill_citations(skill_dir: Path) -> list[dict]:
2641
+ """Every `[AST:]` / `[SRC:]` citation with a line, in file order."""
2642
+ found: list[dict] = []
2643
+ for md in skill_markdown_files(skill_dir):
2644
+ rel = md.relative_to(skill_dir).as_posix()
2645
+ lines = _read_lines(md) or []
2646
+ for number, text in enumerate(lines, start=1):
2647
+ for m in _SKILL_CITATION_RE.finditer(text):
2648
+ found.append(
2649
+ {
2650
+ "file": rel,
2651
+ "line": number,
2652
+ "column": m.start() + 1,
2653
+ "citation": m.group(0),
2654
+ "prefix": m.group(1),
2655
+ "cited_file": m.group(2).strip(),
2656
+ "cited_line": int(m.group(3)),
2657
+ }
2658
+ )
2659
+ return found
2660
+
2661
+
2662
+ def _extraction_method(entry: dict) -> str | None:
2663
+ """The entry's `extraction_method`, or None when it is not a string
2664
+ (a list or object there reads as an unknown method, never a crash)."""
2665
+ method = entry.get("extraction_method")
2666
+ return method if isinstance(method, str) else None
2667
+
2668
+
2669
+ def check_skill_citations(
2670
+ prov: dict, skill_dir: Path
2671
+ ) -> tuple[list[dict], int, int]:
2672
+ """Compare each skill citation's prefix with the entry it cites.
2673
+
2674
+ Returns (findings, citations_scanned, citations_matched): a citation is
2675
+ matched when its path and line name at least one provenance entry.
2676
+ """
2677
+ entries = prov.get("entries")
2678
+ entries = entries if isinstance(entries, list) else []
2679
+ by_location: dict[tuple[str, int], list[dict]] = {}
2680
+ has_ast_entry = False
2681
+ # `ast-without-ast-grep` needs proof that no entry came from ast-grep:
2682
+ # every entry with a source line must carry a known method.
2683
+ methods_known = True
2684
+ for entry in entries:
2685
+ if not isinstance(entry, dict):
2686
+ continue
2687
+ method = _extraction_method(entry)
2688
+ if method in AST_METHODS:
2689
+ has_ast_entry = True
2690
+ sf = entry.get("source_file")
2691
+ line = _coerce_line(entry.get("source_line"))
2692
+ if not isinstance(sf, str) or not sf.strip() or not isinstance(line, int):
2693
+ continue
2694
+ if method not in KNOWN_METHODS:
2695
+ methods_known = False
2696
+ by_location.setdefault((normalize_path(sf), line), []).append(entry)
2697
+
2698
+ scanned = sorted(
2699
+ scan_skill_citations(skill_dir),
2700
+ key=lambda c: (c["file"], c["line"], c["column"]),
2701
+ )
2702
+ findings: list[dict] = []
2703
+ matched_count = 0
2704
+ for cit in scanned:
2705
+ prefix = cit["prefix"]
2706
+ other = "SRC" if prefix == "AST" else "AST"
2707
+ matched = by_location.get(
2708
+ (normalize_path(cit["cited_file"]), cit["cited_line"]), []
2709
+ )
2710
+ if matched:
2711
+ matched_count += 1
2712
+ implied = [(METHOD_PREFIX.get(_extraction_method(e)), e) for e in matched]
2713
+ agrees = any(p == prefix for p, _ in implied)
2714
+ mismatch = next((e for p, e in implied if p == other), None)
2715
+ reason = None
2716
+ export_name = None
2717
+ if not agrees and mismatch is not None:
2718
+ reason = CITATION_PREFIX_MISMATCH
2719
+ export_name = mismatch.get("export_name")
2720
+ elif prefix == "AST" and not has_ast_entry and methods_known:
2721
+ reason = CITATION_AST_WITHOUT_AST_GREP
2722
+ if matched:
2723
+ export_name = matched[0].get("export_name")
2724
+ if reason is None:
2725
+ continue
2726
+ item = {k: v for k, v in cit.items() if k != "column"}
2727
+ item.update(
2728
+ {
2729
+ "reason": reason,
2730
+ "expected_prefix": other,
2731
+ "export_name": export_name
2732
+ if isinstance(export_name, str) and export_name
2733
+ else None,
2734
+ }
2735
+ )
2736
+ findings.append(item)
2737
+ return findings, len(scanned), matched_count
2738
+
2739
+
2740
+ # --------------------------------------------------------------------------
2741
+ # Node kinds (--check-node-kinds)
2742
+ # --------------------------------------------------------------------------
2743
+
2744
+
2745
+ def _resolve_outside_cwd(command: str) -> str | None:
2746
+ """shutil.which with a CWD-shim guard. Returns the resolved path or None.
2747
+
2748
+ shutil.which on Windows searches the current directory ahead of PATH,
2749
+ and CWD here may be a repository SKF does not control: a bare-name
2750
+ lookup resolving into CWD would execute a planted shim (e.g.
2751
+ ast-grep.cmd). Such a resolution is treated as not-found. Explicit
2752
+ paths supplied by callers (containing a separator) are honored as-is.
2753
+ Keep the code identical to the sibling guards in
2754
+ skf-merge-ccc-exclusions.py, skf-detect-tools.py,
2755
+ skf-qmd-classify-collections.py, skf-ccc-git-hygiene.py,
2756
+ skf-source-tree.py and skf-tessl-review.py
2757
+ (test/test-skf-verify-provenance-completeness.py pins it against
2758
+ skf-merge-ccc-exclusions.py).
2759
+ """
2760
+ resolved = shutil.which(command)
2761
+ if resolved is None:
2762
+ return None
2763
+ if os.sep in command or (os.altsep and os.altsep in command):
2764
+ return resolved
2765
+ resolved_dir = os.path.dirname(resolved)
2766
+ if resolved_dir:
2767
+ cwd = os.path.normcase(os.path.abspath(os.getcwd()))
2768
+ if os.path.normcase(os.path.abspath(resolved_dir)) == cwd:
2769
+ return None
2770
+ return resolved
2771
+
2772
+
2773
+ def ast_grep_language(source_file: object) -> str | None:
2774
+ """The ast-grep language for `source_file`'s extension, or None."""
2775
+ if not isinstance(source_file, str) or not source_file.strip():
2776
+ return None
2777
+ ext = posixpath.splitext(normalize_path(source_file))[1].lower()
2778
+ return AST_GREP_LANGUAGES.get(ext)
2779
+
2780
+
2781
+ def ast_grep_judges_kind(exe: str, language: str, kind: str, folder: str) -> str:
2782
+ """Ask ast-grep whether `kind` is a node kind of `language`.
2783
+
2784
+ Runs one rule that matches only `kind` over an empty stdin: ast-grep
2785
+ parses the rule before it reads input and rejects a kind the grammar
2786
+ does not have. Returns "valid", "invalid" (the rule was rejected and the
2787
+ error names an invalid kind), "timeout", or "unknown" (a failed spawn or
2788
+ any other error, which proves nothing about the kind). `folder` holds
2789
+ the minimal sgconfig.yml passed with `--config` and is the working
2790
+ folder, so no other sgconfig.yml is read. The rule is JSON, which YAML
2791
+ reads as a flow mapping.
2792
+ """
2793
+ rule = json.dumps(
2794
+ {"id": "skf-node-kind", "language": language, "rule": {"kind": kind}}
2795
+ )
2796
+ config = os.path.join(folder, "sgconfig.yml")
2797
+ try:
2798
+ result = subprocess.run(
2799
+ [exe, "scan", "--config", config, "--inline-rules", rule, "--stdin"],
2800
+ input="",
2801
+ capture_output=True,
2802
+ encoding="utf-8",
2803
+ errors="replace",
2804
+ timeout=NODE_KIND_TIMEOUT_SEC,
2805
+ cwd=folder,
2806
+ check=False,
2807
+ )
2808
+ except subprocess.TimeoutExpired:
2809
+ return "timeout"
2810
+ except (OSError, subprocess.SubprocessError):
2811
+ return "unknown"
2812
+ if result.returncode == 0:
2813
+ return "valid"
2814
+ if "invalid kind" in (result.stderr or "").lower():
2815
+ return "invalid"
2816
+ return "unknown"
2817
+
2818
+
2819
+ def _judge_kinds(
2820
+ exe: str, pairs: list[tuple[str, str]]
2821
+ ) -> tuple[str, dict[tuple[str, str], str]]:
2822
+ """Ask ast-grep about each (language, kind) pair, in order.
2823
+
2824
+ Returns (node_kind_check, verdicts): each verdict is "valid", "invalid"
2825
+ or the reason the pair went unjudged. ast-grep is first asked about a
2826
+ kind no grammar has; unless it answers "invalid", its answers cannot be
2827
+ read and the check is "skipped-unrecognized-ast-grep" with no verdicts.
2828
+ After one timeout the remaining pairs are not asked. When the temporary
2829
+ folder cannot be made or written, the pairs not yet asked go unjudged.
2830
+ """
2831
+ verdicts: dict[tuple[str, str], str] = {}
2832
+ try:
2833
+ with tempfile.TemporaryDirectory(
2834
+ prefix="skf-node-kind-", ignore_cleanup_errors=True
2835
+ ) as folder:
2836
+ with open(
2837
+ os.path.join(folder, "sgconfig.yml"), "w", encoding="utf-8"
2838
+ ) as fh:
2839
+ fh.write(MINIMAL_SGCONFIG)
2840
+ calibration = ast_grep_judges_kind(
2841
+ exe, pairs[0][0], BOGUS_NODE_KIND, folder
2842
+ )
2843
+ if calibration != "invalid":
2844
+ return "skipped-unrecognized-ast-grep", {}
2845
+ timed_out = False
2846
+ for language, kind in pairs:
2847
+ if timed_out:
2848
+ verdicts[(language, kind)] = UNCHECKED_TIMEOUT
2849
+ continue
2850
+ verdict = ast_grep_judges_kind(exe, language, kind, folder)
2851
+ if verdict == "timeout":
2852
+ timed_out = True
2853
+ verdict = UNCHECKED_TIMEOUT
2854
+ elif verdict == "unknown":
2855
+ verdict = UNCHECKED_AST_GREP_ERROR
2856
+ verdicts[(language, kind)] = verdict
2857
+ except OSError:
2858
+ for pair in pairs:
2859
+ verdicts.setdefault(pair, UNCHECKED_TEMP_FOLDER)
2860
+ return "checked", verdicts
2861
+
2862
+
2863
+ def _node_kind_item(
2864
+ index: int, entry: dict, kind: str, language: str, reason: str
2865
+ ) -> dict:
2866
+ name = entry.get("export_name")
2867
+ return {
2868
+ "export_name": name if isinstance(name, str) and name else None,
2869
+ "entry_index": index,
2870
+ "source_file": entry.get("source_file"),
2871
+ "source_line": entry.get("source_line"),
2872
+ "ast_node_type": kind,
2873
+ "language": language,
2874
+ "reason": reason,
2875
+ }
2876
+
2877
+
2878
+ def check_node_kinds(prov: dict) -> tuple[list[dict], list[dict], dict]:
2879
+ """Check each ast-grep entry's `ast_node_type` against ast-grep itself.
2880
+
2881
+ Returns (findings, unchecked, summary fields). `ERROR` and a kind that
2882
+ is not kind-shaped are judged without ast-grep, whatever the binary
2883
+ lookup finds; every other kind is asked once per (language, kind).
2884
+ """
2885
+ entries = prov.get("entries")
2886
+ entries = entries if isinstance(entries, list) else []
2887
+ findings: list[dict] = []
2888
+ unchecked: list[dict] = []
2889
+ pending: list[tuple[int, dict, str, str]] = []
2890
+ judged = 0
2891
+ skipped = 0
2892
+ for index, entry in enumerate(entries):
2893
+ if not isinstance(entry, dict):
2894
+ continue
2895
+ if _extraction_method(entry) not in AST_METHODS:
2896
+ continue
2897
+ kind = entry.get("ast_node_type")
2898
+ if not isinstance(kind, str) or not kind.strip():
2899
+ continue
2900
+ language = ast_grep_language(entry.get("source_file"))
2901
+ if language is None:
2902
+ skipped += 1
2903
+ continue
2904
+ if kind == ERROR_NODE_KIND:
2905
+ reason = NODE_KIND_ERROR
2906
+ elif not NODE_KIND_SHAPE.fullmatch(kind):
2907
+ reason = NODE_KIND_INVALID
2908
+ else:
2909
+ pending.append((index, entry, language, kind))
2910
+ continue
2911
+ judged += 1
2912
+ findings.append(_node_kind_item(index, entry, kind, language, reason))
2913
+
2914
+ check = "checked"
2915
+ if pending:
2916
+ exe = _resolve_outside_cwd("ast-grep")
2917
+ if exe is None:
2918
+ check = "skipped-no-ast-grep"
2919
+ else:
2920
+ pairs = sorted({(lang, kind) for _, _, lang, kind in pending})
2921
+ check, verdicts = _judge_kinds(exe, pairs)
2922
+ if check == "checked":
2923
+ for index, entry, language, kind in pending:
2924
+ verdict = verdicts[(language, kind)]
2925
+ item = _node_kind_item(index, entry, kind, language, verdict)
2926
+ if verdict == "valid":
2927
+ judged += 1
2928
+ elif verdict == "invalid":
2929
+ judged += 1
2930
+ item["reason"] = NODE_KIND_INVALID
2931
+ findings.append(item)
2932
+ else:
2933
+ unchecked.append(item)
2934
+
2935
+ def order(item: dict) -> tuple:
2936
+ return (item["export_name"] or "", str(item["source_file"]),
2937
+ item["entry_index"])
2938
+
2939
+ findings.sort(key=order)
2940
+ unchecked.sort(key=order)
2941
+ summary = {
2942
+ "node_kind_check": check,
2943
+ "node_kinds_checked": judged,
2944
+ "node_kind_check_skipped": skipped,
2945
+ "node_kind_check_errors": len(unchecked),
2946
+ "node_kind_findings_count": len(findings),
2947
+ }
2948
+ return findings, unchecked, summary
2949
+
2950
+
2951
+ # --------------------------------------------------------------------------
2952
+ # The recipe kind at a line (kind-at)
2953
+ # --------------------------------------------------------------------------
2954
+
2955
+
2956
+ def _is_recipe(doc: object) -> bool:
2957
+ """A recipe: a mapping with a string `id` and `language`, and a `rule`
2958
+ mapping that names its node `kind`."""
2959
+ if not isinstance(doc, dict):
2960
+ return False
2961
+ rule = doc.get("rule")
2962
+ return (
2963
+ isinstance(doc.get("id"), str)
2964
+ and isinstance(doc.get("language"), str)
2965
+ and isinstance(rule, dict)
2966
+ and isinstance(rule.get("kind"), str)
2967
+ )
2968
+
2969
+
2970
+ def load_recipes(path: Path) -> list[dict]:
2971
+ """Every ast-grep recipe in `path`, first copy of each (id, language).
2972
+
2973
+ A markdown file (`.md`) gives the recipes in its ```yaml fences; any
2974
+ other file is read as YAML: one recipe per document, a list of recipes,
2975
+ or a mapping whose `recipes` key holds that list. Raises ValueError when
2976
+ the file cannot be read, and ImportError without PyYAML.
2977
+ """
2978
+ import yaml # only kind-at reads recipes; `uv run` installs it
2979
+
2980
+ try:
2981
+ text = path.read_text(encoding="utf-8")
2982
+ except (OSError, UnicodeDecodeError) as exc:
2983
+ raise ValueError(f"failed to read recipes {path}: {exc}") from exc
2984
+ docs: list[object] = []
2985
+ if path.suffix.lower() in (".md", ".markdown"):
2986
+ for block in _MD_YAML_FENCE_RE.findall(text):
2987
+ try:
2988
+ docs.append(yaml.safe_load(block))
2989
+ except yaml.YAMLError:
2990
+ continue # a template, not a recipe
2991
+ else:
2992
+ try:
2993
+ docs = list(yaml.safe_load_all(text))
2994
+ except yaml.YAMLError as exc:
2995
+ raise ValueError(f"malformed YAML in recipes {path}: {exc}") from exc
2996
+ candidates: list[object] = []
2997
+ for doc in docs:
2998
+ if isinstance(doc, dict) and isinstance(doc.get("recipes"), list):
2999
+ candidates.extend(doc["recipes"])
3000
+ elif isinstance(doc, list):
3001
+ candidates.extend(doc)
3002
+ else:
3003
+ candidates.append(doc)
3004
+ recipes: list[dict] = []
3005
+ seen: set[tuple[str, str]] = set()
3006
+ for doc in candidates:
3007
+ if not _is_recipe(doc):
3008
+ continue
3009
+ key = (doc["id"], doc["language"].strip().lower())
3010
+ if key not in seen:
3011
+ seen.add(key)
3012
+ recipes.append(doc)
3013
+ return recipes
3014
+
3015
+
3016
+ def recipe_runs(recipes: list[dict], language: str) -> list[dict]:
3017
+ """The recipes to run on a file of `language`, each with its `language`
3018
+ set to it: an id's own form for that language, else its form for the
3019
+ first language `RECIPE_LANGUAGE_FALLBACKS` lists (in recipe order)."""
3020
+ forms: dict[str, dict[str, dict]] = {}
3021
+ for recipe in recipes:
3022
+ lang = recipe["language"].strip().lower()
3023
+ forms.setdefault(recipe["id"], {}).setdefault(lang, recipe)
3024
+ runs: list[dict] = []
3025
+ for recipe_id, by_language in forms.items():
3026
+ for lang in (language, *RECIPE_LANGUAGE_FALLBACKS.get(language, ())):
3027
+ if lang in by_language:
3028
+ run = dict(by_language[lang], language=language)
3029
+ run["_rewritten"] = lang != language
3030
+ runs.append(run)
3031
+ break
3032
+ return runs
3033
+
3034
+
3035
+ def _scan_recipe(
3036
+ exe: str, run: dict, source: bytes, folder: str
3037
+ ) -> tuple[str, list[tuple[int, str]] | str]:
3038
+ """Run one recipe over `source` on stdin.
3039
+
3040
+ Returns ("ok", [($NAME's 1-based line, $NAME's text), ...]),
3041
+ ("not-applicable", []) for a recipe whose `language` was rewritten and
3042
+ whose kinds that grammar lacks, ("timeout", detail) or ("error",
3043
+ detail). The rule goes in a file as JSON (YAML reads it as a flow
3044
+ mapping), never on the command line, where an npm `.cmd` shim on
3045
+ Windows would hand its `|` and `^` to cmd.exe.
3046
+ """
3047
+ rule = {k: v for k, v in run.items() if not k.startswith("_")}
3048
+ rule_path = os.path.join(folder, "recipe.yml")
3049
+ with open(rule_path, "w", encoding="utf-8") as fh:
3050
+ fh.write(json.dumps(rule, ensure_ascii=False))
3051
+ config = os.path.join(folder, "sgconfig.yml")
3052
+ try:
3053
+ result = subprocess.run(
3054
+ [exe, "scan", "--config", config, "-r", rule_path, "--stdin",
3055
+ "--json=stream"],
3056
+ input=source,
3057
+ capture_output=True,
3058
+ timeout=KIND_AT_TIMEOUT_SEC,
3059
+ cwd=folder,
3060
+ check=False,
3061
+ )
3062
+ except subprocess.TimeoutExpired:
3063
+ return "timeout", f"no answer in {KIND_AT_TIMEOUT_SEC}s"
3064
+ except (OSError, subprocess.SubprocessError) as exc:
3065
+ return "error", f"{type(exc).__name__}: {exc}"
3066
+ stdout = (result.stdout or b"").decode("utf-8", errors="replace")
3067
+ stderr = (result.stderr or b"").decode("utf-8", errors="replace")
3068
+ if result.returncode != 0 and not stdout.strip():
3069
+ if run.get("_rewritten") and "invalid kind" in stderr.lower():
3070
+ return "not-applicable", []
3071
+ detail = next((ln.strip() for ln in stderr.splitlines() if ln.strip()), "")
3072
+ return "error", f"exit {result.returncode}: {detail}"
3073
+ found: list[tuple[int, str]] = []
3074
+ for raw in stdout.splitlines():
3075
+ if not raw.strip():
3076
+ continue
3077
+ try:
3078
+ match = json.loads(raw)
3079
+ except ValueError:
3080
+ return "error", "output is not JSON lines"
3081
+ single = (match.get("metaVariables") or {}).get("single") or {}
3082
+ name = single.get("NAME")
3083
+ if not isinstance(name, dict):
3084
+ continue
3085
+ start = (name.get("range") or {}).get("start") or {}
3086
+ line = start.get("line")
3087
+ if isinstance(line, int) and not isinstance(line, bool):
3088
+ found.append((line + 1, str(name.get("text") or "")))
3089
+ return "ok", found
3090
+
3091
+
3092
+ def _unquote(name: str) -> str:
3093
+ """A string name without its quotes (`"a-b"` -> `a-b`)."""
3094
+ if len(name) >= 2 and name[0] == name[-1] and name[0] in "'\"":
3095
+ return name[1:-1]
3096
+ return name
3097
+
3098
+
3099
+ def kinds_at(
3100
+ source: bytes,
3101
+ language: str | None,
3102
+ targets: list[tuple[int, str | None]],
3103
+ recipes: list[dict],
3104
+ ) -> list[dict]:
3105
+ """The node kind the recipes give each (line, export name or None)
3106
+ target of one file, running each recipe once.
3107
+
3108
+ Each result: {line, name, status, kind, matches[], errors[]}. A match is
3109
+ a recipe whose `$NAME` is on the target line (and, with a name, is that
3110
+ name's last segment). `status` is `found` (one kind among the matches),
3111
+ `ambiguous` (several), `no-match`, `incomplete` (a recipe run failed, so
3112
+ no kind is given), `skipped-no-language`, `skipped-no-recipes` or
3113
+ `skipped-no-ast-grep`.
3114
+ """
3115
+ results = [
3116
+ {"line": line, "name": name, "status": "", "kind": None,
3117
+ "matches": [], "errors": []}
3118
+ for line, name in targets
3119
+ ]
3120
+
3121
+ def skip(status: str) -> list[dict]:
3122
+ for result in results:
3123
+ result["status"] = status
3124
+ return results
3125
+
3126
+ if language is None:
3127
+ return skip("skipped-no-language")
3128
+ runs = recipe_runs(recipes, language)
3129
+ if not runs:
3130
+ return skip("skipped-no-recipes")
3131
+ exe = _resolve_outside_cwd("ast-grep")
3132
+ if exe is None:
3133
+ return skip("skipped-no-ast-grep")
3134
+
3135
+ matched: list[tuple[dict, list[tuple[int, str]]]] = []
3136
+ errors: list[dict] = []
3137
+ try:
3138
+ with tempfile.TemporaryDirectory(
3139
+ prefix="skf-kind-at-", ignore_cleanup_errors=True
3140
+ ) as folder:
3141
+ with open(
3142
+ os.path.join(folder, "sgconfig.yml"), "w", encoding="utf-8"
3143
+ ) as fh:
3144
+ fh.write(MINIMAL_SGCONFIG)
3145
+ timed_out = False
3146
+ for run in runs:
3147
+ if timed_out:
3148
+ errors.append({"recipe": run["id"], "language": language,
3149
+ "reason": UNCHECKED_TIMEOUT, "detail": ""})
3150
+ continue
3151
+ outcome, found = _scan_recipe(exe, run, source, folder)
3152
+ if outcome == "ok":
3153
+ matched.append((run, found))
3154
+ elif outcome in ("timeout", "error"):
3155
+ timed_out = outcome == "timeout"
3156
+ errors.append({
3157
+ "recipe": run["id"],
3158
+ "language": language,
3159
+ "reason": UNCHECKED_TIMEOUT if timed_out
3160
+ else UNCHECKED_AST_GREP_ERROR,
3161
+ "detail": found,
3162
+ })
3163
+ except OSError as exc:
3164
+ errors.append({"recipe": None, "language": language,
3165
+ "reason": UNCHECKED_TEMP_FOLDER, "detail": str(exc)})
3166
+
3167
+ for result in results:
3168
+ wanted = definition_name(result["name"]) if result["name"] else None
3169
+ for run, found in matched:
3170
+ for line, text in found:
3171
+ if line != result["line"]:
3172
+ continue
3173
+ if wanted is not None and _unquote(text) != wanted:
3174
+ continue
3175
+ item = {"recipe": run["id"], "kind": run["rule"]["kind"],
3176
+ "name": text}
3177
+ if item not in result["matches"]:
3178
+ result["matches"].append(item)
3179
+ result["matches"].sort(key=lambda m: (m["recipe"], m["name"]))
3180
+ result["errors"] = [dict(e) for e in errors]
3181
+ kinds = sorted({m["kind"] for m in result["matches"]})
3182
+ if errors:
3183
+ result["status"] = "incomplete"
3184
+ elif not kinds:
3185
+ result["status"] = "no-match"
3186
+ elif len(kinds) == 1:
3187
+ result["status"] = "found"
3188
+ result["kind"] = kinds[0]
3189
+ else:
3190
+ result["status"] = "ambiguous"
3191
+ return results
3192
+
3193
+
3194
+ # --------------------------------------------------------------------------
3195
+ # Verification
3196
+ # --------------------------------------------------------------------------
3197
+
3198
+
3199
+ def resolve_source_root(prov: dict, override: str | None) -> Path | None:
3200
+ """Pick the source root for citation resolution.
3201
+
3202
+ `--source-root` override wins; otherwise the provenance map's top-level
3203
+ `source_root`. Returns None when neither resolves to an on-disk dir.
3204
+ """
3205
+ candidate: str | None = None
3206
+ if override:
3207
+ candidate = override
3208
+ else:
3209
+ sr = prov.get("source_root")
3210
+ if isinstance(sr, str) and sr:
3211
+ candidate = sr
3212
+ if not candidate:
3213
+ return None
3214
+ root = Path(candidate)
3215
+ return root if root.is_dir() else None
3216
+
3217
+
3218
+ def verify(
3219
+ metadata: dict,
3220
+ prov: dict,
3221
+ source_root: Path | None,
3222
+ skill_dir: Path | None = None,
3223
+ check_kinds: bool = False,
3224
+ ) -> dict:
3225
+ """Compute the completeness / orphan / stale / citation / node-kind
3226
+ report."""
3227
+ reexport_map = extract_reexport_map(prov)
3228
+
3229
+ documented = extract_export_names(metadata)
3230
+ documented_canon = {canon(n, reexport_map) for n in documented}
3231
+
3232
+ raw_entry_names = provenance_entry_names(prov)
3233
+ entry_canon = {canon(n, reexport_map) for n in raw_entry_names}
3234
+
3235
+ # A reference app documents no exports (`exports[]` is empty by design)
3236
+ # while its entries are per-citation, so the two sets measure different
3237
+ # surfaces and their difference means nothing.
3238
+ set_diff_applies = not is_reference_app(metadata)
3239
+ if set_diff_applies:
3240
+ missing = sorted(documented_canon - entry_canon)
3241
+ orphaned = sorted(entry_canon - documented_canon)
3242
+ else:
3243
+ missing, orphaned = [], []
3244
+
3245
+ stale: list[dict] = []
3246
+ citations_checked = 0
3247
+ line_check_skipped = 0
3248
+ if source_root is not None:
3249
+ cache = _SourceCache()
3250
+ entries = prov.get("entries")
3251
+ if isinstance(entries, list):
3252
+ for index, entry in enumerate(entries):
3253
+ if not isinstance(entry, dict):
3254
+ continue
3255
+ name = entry.get("export_name")
3256
+ if not isinstance(name, str) or not name:
3257
+ continue
3258
+ sf = entry.get("source_file")
3259
+ sl = entry.get("source_line")
3260
+ line = _coerce_line(sl)
3261
+ # Only count a resolvable (non-null) citation as "checked".
3262
+ if isinstance(sf, str) and sf.strip() and line is not None:
3263
+ citations_checked += 1
3264
+ reason = check_citation(sf, sl, source_root)
3265
+ item = {
3266
+ "export_name": name,
3267
+ "entry_index": index,
3268
+ "source_file": sf if isinstance(sf, str) else None,
3269
+ "source_line": sl if isinstance(sl, int)
3270
+ and not isinstance(sl, bool) else sl,
3271
+ }
3272
+ if reason is not None:
3273
+ item["reason"] = reason
3274
+ stale.append(item)
3275
+ continue
3276
+ # The citation resolves (or is tolerated): check that the
3277
+ # line defines the export. A tolerated null or blank
3278
+ # citation has no line to check and is not counted.
3279
+ if (
3280
+ not isinstance(sf, str)
3281
+ or not sf.strip()
3282
+ or not isinstance(line, int)
3283
+ ):
3284
+ continue
3285
+ if not line_rule_applies(sf, entry.get("export_type")):
3286
+ line_check_skipped += 1
3287
+ continue
3288
+ matches = _definition_matches(sf, name, source_root, cache)
3289
+ if matches is None:
3290
+ line_check_skipped += 1
3291
+ continue
3292
+ defs, every = matches
3293
+ if line not in defs:
3294
+ if line in every:
3295
+ # The recorded line is an indented match the
3296
+ # column-0 rule dropped: keep it, so the finding
3297
+ # lists several lines and nobody auto-moves it.
3298
+ defs = sorted(set(defs) | {line})
3299
+ item["reason"] = STALE_LINE_NOT_DEFINITION
3300
+ item["definition_lines"] = defs
3301
+ stale.append(item)
3302
+ stale.sort(key=lambda s: (s["export_name"], s.get("source_file") or ""))
3303
+
3304
+ citations: list[dict] = []
3305
+ skill_citations_scanned = 0
3306
+ skill_citations_matched = 0
3307
+ if skill_dir is not None:
3308
+ citations, skill_citations_scanned, skill_citations_matched = (
3309
+ check_skill_citations(prov, skill_dir)
3310
+ )
3311
+
3312
+ node_kinds: list[dict] = []
3313
+ node_kinds_unchecked: list[dict] = []
3314
+ kind_summary: dict = {"node_kind_check": "not-requested"}
3315
+ if check_kinds:
3316
+ node_kinds, node_kinds_unchecked, kind_summary = check_node_kinds(prov)
3317
+
3318
+ status = (
3319
+ "pass"
3320
+ if not (missing or orphaned or stale or citations or node_kinds)
3321
+ else "findings"
3322
+ )
3323
+ result: dict = {
3324
+ "status": status,
3325
+ "missing": missing,
3326
+ "orphaned": orphaned,
3327
+ "stale": stale,
3328
+ "citations": citations,
3329
+ }
3330
+ if check_kinds:
3331
+ result["node_kinds"] = node_kinds
3332
+ result["node_kinds_unchecked"] = node_kinds_unchecked
3333
+ result["summary"] = {
3334
+ "exports_checked": len(documented_canon),
3335
+ "entries_checked": len(raw_entry_names),
3336
+ "missing_count": len(missing),
3337
+ "orphaned_count": len(orphaned),
3338
+ "set_diff": "checked" if set_diff_applies else "not-applicable",
3339
+ "stale_count": len(stale),
3340
+ "citations_checked": citations_checked,
3341
+ "line_check_skipped": line_check_skipped,
3342
+ "stale_check": "checked"
3343
+ if source_root is not None
3344
+ else "skipped-no-source-root",
3345
+ "skill_citations_scanned": skill_citations_scanned,
3346
+ "skill_citations_matched": skill_citations_matched,
3347
+ "citation_findings_count": len(citations),
3348
+ "citation_check": "checked"
3349
+ if skill_dir is not None
3350
+ else "skipped-no-skill-dir",
3351
+ **kind_summary,
3352
+ }
3353
+ return result
3354
+
3355
+
3356
+ # --------------------------------------------------------------------------
3357
+ # Fixes with one answer (fix)
3358
+ # --------------------------------------------------------------------------
3359
+
3360
+
3361
+ def manual_interiors(text: str) -> list[tuple[int, int]]:
3362
+ """(start, end) offsets of the interior of every [MANUAL] block, paired
3363
+ as skf-hash-content.py pairs them: an opening marker with the earliest
3364
+ following close marker of the same name. An unclosed marker is
3365
+ skipped."""
3366
+ closes = [
3367
+ (m.start(), m.group(1).strip()) for m in _MANUAL_CLOSE_RE.finditer(text)
3368
+ ]
3369
+ spans: list[tuple[int, int]] = []
3370
+ for om in _MANUAL_OPEN_RE.finditer(text):
3371
+ name = om.group(1).strip()
3372
+ close = min(
3373
+ (start for start, n in closes if start >= om.end() and n == name),
3374
+ default=None,
3375
+ )
3376
+ if close is not None:
3377
+ spans.append((om.end(), close))
3378
+ return spans
3379
+
3380
+
3381
+ class _SkillFile:
3382
+ """One skill markdown file: its text, its [MANUAL] interiors and every
3383
+ citation in it, found line by line as `scan_skill_citations` finds them.
3384
+ Bytes that are not UTF-8 survive the round trip (surrogateescape)."""
3385
+
3386
+ def __init__(self, skill_dir: Path, rel: str) -> None:
3387
+ self.rel = rel
3388
+ self.path = skill_dir / rel
3389
+ self.text = self.path.read_bytes().decode("utf-8", errors="surrogateescape")
3390
+ self.manual = manual_interiors(self.text)
3391
+ self.citations: list[dict] = []
3392
+ offset = 0
3393
+ for number, chunk in enumerate(_LINE_WITH_END_RE.findall(self.text), 1):
3394
+ content = chunk.rstrip("\r\n")
3395
+ for m in _SKILL_CITATION_RE.finditer(content):
3396
+ self.citations.append(
3397
+ {
3398
+ "line": number,
3399
+ "start": offset + m.start(),
3400
+ "end": offset + m.end(),
3401
+ "match": m,
3402
+ "text": m.group(0),
3403
+ "prefix": m.group(1),
3404
+ "cited_file": m.group(2).strip(),
3405
+ "cited_line": int(m.group(3)),
3406
+ }
3407
+ )
3408
+ offset += len(chunk)
3409
+
3410
+ def in_manual(self, offset: int) -> bool:
3411
+ return any(start <= offset < end for start, end in self.manual)
3412
+
3413
+ def fixed_text(self) -> str | None:
3414
+ """The text with every changed citation rewritten, or None when no
3415
+ citation changed."""
3416
+ edits = [
3417
+ (c["start"], c["end"], _rewritten_citation(c))
3418
+ for c in self.citations
3419
+ if c.get("new_prefix") or c.get("shift")
3420
+ ]
3421
+ if not edits:
3422
+ return None
3423
+ text = self.text
3424
+ for start, end, new in sorted(edits, reverse=True):
3425
+ text = text[:start] + new + text[end:]
3426
+ return text
3427
+
3428
+
3429
+ def _rewritten_citation(cit: dict) -> str:
3430
+ """A citation with its new prefix and its lines shifted: a range moves
3431
+ both ends by the same amount (`L10-12` to `L13-15`)."""
3432
+ m = cit["match"]
3433
+ base = m.start()
3434
+ subs: list[tuple[int, int, str]] = []
3435
+ if cit.get("new_prefix"):
3436
+ subs.append((m.start(1) - base, m.end(1) - base, cit["new_prefix"]))
3437
+ shift = cit.get("shift", 0)
3438
+ if shift:
3439
+ subs.append((m.start(3) - base, m.end(3) - base, str(int(m.group(3)) + shift)))
3440
+ if m.group(5) is not None:
3441
+ subs.append(
3442
+ (m.start(5) - base, m.end(5) - base, str(int(m.group(5)) + shift))
3443
+ )
3444
+ text = m.group(0)
3445
+ for start, end, new in sorted(subs, reverse=True):
3446
+ text = text[:start] + new + text[end:]
3447
+ return text
3448
+
3449
+
3450
+ def _dict_items(value: object) -> list[dict]:
3451
+ return [v for v in value if isinstance(v, dict)] if isinstance(value, list) else []
3452
+
3453
+
3454
+ def _stale_entry_indices(entries: list, item: dict) -> list[int]:
3455
+ """The provenance entries a `stale[]` item names: the one at its
3456
+ `entry_index` when that entry still has the item's name, file and line,
3457
+ else every entry that has them."""
3458
+ name = item.get("export_name")
3459
+ source_file = item.get("source_file")
3460
+ line = _coerce_line(item.get("source_line"))
3461
+
3462
+ def same(entry: object) -> bool:
3463
+ return (
3464
+ isinstance(entry, dict)
3465
+ and entry.get("export_name") == name
3466
+ and isinstance(source_file, str)
3467
+ and bool(source_file.strip())
3468
+ and entry.get("source_file") == source_file
3469
+ and isinstance(line, int)
3470
+ and _coerce_line(entry.get("source_line")) == line
3471
+ )
3472
+
3473
+ index = item.get("entry_index")
3474
+ if (
3475
+ isinstance(index, int)
3476
+ and not isinstance(index, bool)
3477
+ and 0 <= index < len(entries)
3478
+ and same(entries[index])
3479
+ ):
3480
+ return [index]
3481
+ return [i for i, entry in enumerate(entries) if same(entry)]
3482
+
3483
+
3484
+ def plan_fixes(
3485
+ verify_result: dict, prov: dict, skill_dir: Path, line_moves: bool = True
3486
+ ) -> dict:
3487
+ """Plan the fixes a `verify` result has one answer for.
3488
+
3489
+ 1. Citation prefixes: each `citations[]` item's citation, found by its
3490
+ file, line and text, takes `expected_prefix`.
3491
+ 2. Source lines: each `line-not-definition` item with exactly one
3492
+ `definition_lines` value moves its entry's `source_line` there, and
3493
+ the citations of either prefix whose normalized path and first line
3494
+ are the entry's (as they stand after step 1, before any move) move
3495
+ with it, all in one pass, so a citation moves at most once. A
3496
+ citation of a line another entry also records in that file stays.
3497
+
3498
+ A citation inside a [MANUAL] block never changes. Returns {applied,
3499
+ left_as_warn, markdown: {rel: new text}, provenance: the new map or
3500
+ None}; nothing is written.
3501
+ """
3502
+ rels = [
3503
+ p.relative_to(skill_dir).as_posix() for p in skill_markdown_files(skill_dir)
3504
+ ]
3505
+ files = {rel: _SkillFile(skill_dir, rel) for rel in rels}
3506
+ applied: list[dict] = []
3507
+ left: list[dict] = []
3508
+ prefix_fixes: list[tuple[dict, dict]] = []
3509
+
3510
+ # 1. Citation prefixes.
3511
+ groups: dict[tuple[str, int, str, str], list[dict]] = {}
3512
+ for item in _dict_items(verify_result.get("citations")):
3513
+ key = (item.get("file"), item.get("line"), item.get("citation"),
3514
+ item.get("expected_prefix"))
3515
+ if (
3516
+ not isinstance(key[0], str)
3517
+ or not isinstance(key[1], int)
3518
+ or not isinstance(key[2], str)
3519
+ or key[3] not in ("AST", "SRC")
3520
+ ):
3521
+ left.append(dict(item, kind=FIX_CITATION_PREFIX, why="citation-not-found"))
3522
+ continue
3523
+ groups.setdefault(key, []).append(item)
3524
+ for (rel, line, text, expected), items in groups.items():
3525
+ doc = files.get(rel)
3526
+ found = [] if doc is None else [
3527
+ c for c in doc.citations if c["line"] == line and c["text"] == text
3528
+ ]
3529
+ if not found:
3530
+ for item in items:
3531
+ left.append(
3532
+ dict(item, kind=FIX_CITATION_PREFIX, why="citation-not-found")
3533
+ )
3534
+ continue
3535
+ for cit in found:
3536
+ if doc.in_manual(cit["start"]):
3537
+ left.append(dict(items[0], kind=FIX_CITATION_PREFIX,
3538
+ why="inside-manual-block"))
3539
+ elif cit["prefix"] != expected:
3540
+ cit["new_prefix"] = expected
3541
+ prefix_fixes.append((doc, cit))
3542
+
3543
+ # 2. Source lines: plan every move before making any.
3544
+ entries = prov.get("entries") if isinstance(prov.get("entries"), list) else []
3545
+ moves: dict[int, int] = {}
3546
+ for item in _dict_items(verify_result.get("stale")):
3547
+ defs = item.get("definition_lines")
3548
+ new = defs[0] if isinstance(defs, list) and len(defs) == 1 else None
3549
+ if item.get("reason") != STALE_LINE_NOT_DEFINITION:
3550
+ why = "needs-decision"
3551
+ elif not isinstance(defs, list) or not defs:
3552
+ why = "no-definition-line"
3553
+ elif len(defs) > 1:
3554
+ why = "several-definition-lines"
3555
+ elif not isinstance(new, int) or isinstance(new, bool) or new < 1:
3556
+ why = "needs-decision"
3557
+ elif not line_moves:
3558
+ why = "line-moves-skipped"
3559
+ else:
3560
+ indices = _stale_entry_indices(entries, item)
3561
+ if indices:
3562
+ for index in indices:
3563
+ moves[index] = new
3564
+ continue
3565
+ why = "entry-not-found"
3566
+ left.append(dict(item, kind=FIX_SOURCE_LINE, why=why))
3567
+
3568
+ recorded: dict[tuple[str, int], set[int]] = {}
3569
+ for index, entry in enumerate(entries):
3570
+ if not isinstance(entry, dict):
3571
+ continue
3572
+ sf = entry.get("source_file")
3573
+ line = _coerce_line(entry.get("source_line"))
3574
+ if isinstance(sf, str) and sf.strip() and isinstance(line, int):
3575
+ recorded.setdefault((normalize_path(sf), line), set()).add(index)
3576
+ movers: dict[tuple[str, int], dict[int, int]] = {}
3577
+ for index, new in moves.items():
3578
+ entry = entries[index]
3579
+ key = (normalize_path(entry["source_file"]), _coerce_line(entry["source_line"]))
3580
+ movers.setdefault(key, {})[index] = new
3581
+
3582
+ line_fixes: list[tuple[dict, dict, list[str]]] = []
3583
+ for rel in rels:
3584
+ doc = files[rel]
3585
+ for cit in doc.citations:
3586
+ key = (normalize_path(cit["cited_file"]), cit["cited_line"])
3587
+ moving = movers.get(key)
3588
+ if not moving:
3589
+ continue
3590
+ here = recorded.get(key, set()) | set(moving)
3591
+ names = sorted(
3592
+ {n for n in (entries[i].get("export_name") for i in here)
3593
+ if isinstance(n, str) and n}
3594
+ )
3595
+ targets = set(moving.values())
3596
+ if here - set(moving) or len(targets) > 1:
3597
+ why = "shared-line" # it may cite the other entry's export
3598
+ elif doc.in_manual(cit["start"]):
3599
+ why = "inside-manual-block"
3600
+ else:
3601
+ cit["shift"] = targets.pop() - cit["cited_line"]
3602
+ line_fixes.append((doc, cit, names))
3603
+ continue
3604
+ left.append({
3605
+ "kind": FIX_CITATION_LINE,
3606
+ "file": rel,
3607
+ "line": cit["line"],
3608
+ "citation": cit["text"],
3609
+ "export_names": names,
3610
+ "why": why,
3611
+ })
3612
+
3613
+ for doc, cit in prefix_fixes:
3614
+ applied.append({
3615
+ "kind": FIX_CITATION_PREFIX,
3616
+ "file": doc.rel,
3617
+ "line": cit["line"],
3618
+ "citation": cit["text"],
3619
+ "fixed": _rewritten_citation(cit),
3620
+ })
3621
+ new_prov = None
3622
+ if moves:
3623
+ new_prov = copy.deepcopy(prov)
3624
+ for index in sorted(moves):
3625
+ entry = new_prov["entries"][index]
3626
+ applied.append({
3627
+ "kind": FIX_SOURCE_LINE,
3628
+ "entry_index": index,
3629
+ "export_name": entry.get("export_name"),
3630
+ "source_file": entry.get("source_file"),
3631
+ "from": _coerce_line(entry.get("source_line")),
3632
+ "to": moves[index],
3633
+ })
3634
+ entry["source_line"] = moves[index]
3635
+ for doc, cit, names in line_fixes:
3636
+ applied.append({
3637
+ "kind": FIX_CITATION_LINE,
3638
+ "file": doc.rel,
3639
+ "line": cit["line"],
3640
+ "citation": cit["text"],
3641
+ "fixed": _rewritten_citation(cit),
3642
+ "export_names": names,
3643
+ })
3644
+
3645
+ markdown = {}
3646
+ for rel in rels:
3647
+ text = files[rel].fixed_text()
3648
+ if text is not None:
3649
+ markdown[rel] = text
3650
+ return {
3651
+ "applied": applied,
3652
+ "left_as_warn": left,
3653
+ "markdown": markdown,
3654
+ "provenance": new_prov,
3655
+ }
3656
+
3657
+
3658
+ def _sibling_helper(filename: str) -> Path | None:
3659
+ """A helper installed beside this script (dev `src/shared/scripts/`
3660
+ and installed `_bmad/skf/shared/scripts/` alike), or None."""
3661
+ path = Path(__file__).resolve().parent / filename
3662
+ return path if path.is_file() else None
3663
+
3664
+
3665
+ def _load_sibling_module(filename: str, module_name: str):
3666
+ """Import a sibling helper as a module. Raises ImportError."""
3667
+ path = _sibling_helper(filename)
3668
+ spec = (
3669
+ importlib.util.spec_from_file_location(module_name, path)
3670
+ if path is not None
3671
+ else None
3672
+ )
3673
+ if spec is None or spec.loader is None:
3674
+ raise ImportError(f"{filename} not found beside {Path(__file__).name}")
3675
+ module = importlib.util.module_from_spec(spec)
3676
+ spec.loader.exec_module(module)
3677
+ return module
3678
+
3679
+
3680
+ def atomic_write(helper: Path, target: Path, data: bytes) -> None:
3681
+ """Write `data` to `target` through `skf-atomic-write.py write`.
3682
+ Raises OSError when the helper fails."""
3683
+ try:
3684
+ result = subprocess.run(
3685
+ [sys.executable, str(helper), "write", "--target", str(target)],
3686
+ input=data,
3687
+ capture_output=True,
3688
+ timeout=ATOMIC_WRITE_TIMEOUT_SEC,
3689
+ check=False,
3690
+ )
3691
+ except (OSError, subprocess.SubprocessError) as exc:
3692
+ raise OSError(f"atomic write of {target} failed: {exc}") from exc
3693
+ if result.returncode != 0:
3694
+ detail = (result.stderr or b"").decode("utf-8", errors="replace").strip()
3695
+ raise OSError(f"atomic write of {target} failed: {detail}")
3696
+
3697
+
3698
+ # --------------------------------------------------------------------------
3699
+ # CLI
3700
+ # --------------------------------------------------------------------------
3701
+
3702
+
3703
+ def _skill_dir_error(skill_dir: Path) -> str | None:
3704
+ """Why `skill_dir` is not a skill package folder, or None."""
3705
+ if not skill_dir.exists():
3706
+ return f"skill dir not found: {skill_dir}"
3707
+ if not skill_dir.is_dir():
3708
+ return f"skill dir is not a directory: {skill_dir}"
3709
+ if not (skill_dir / "SKILL.md").is_file():
3710
+ return f"skill dir has no SKILL.md: {skill_dir}"
3711
+ return None
3712
+
3713
+
3714
+ def _unexpected(what: str, exc: Exception, written: list[str] | None = None) -> int:
3715
+ """Report an unexpected failure on one stderr line; exit code 2. With
3716
+ `written` (fix), the line also names the files already written."""
3717
+ detail = " ".join(str(exc).split())
3718
+ note = ""
3719
+ if written is not None:
3720
+ note = f"; already written: {', '.join(written) if written else 'nothing'}"
3721
+ print(
3722
+ f"error: {what} failed: {type(exc).__name__}: {detail}{note}",
3723
+ file=sys.stderr,
3724
+ )
3725
+ return 2
3726
+
3727
+
3728
+ def _emit(result: dict, output: str | None, written: list[str] | None = None) -> bool:
3729
+ """Write the JSON result to `output`, or to stdout when it is absent or
3730
+ `-`. False, after one stderr line (naming `written`, as `_unexpected`
3731
+ does), when it cannot be written."""
3732
+ out_text = json.dumps(result, indent=2) + "\n"
3733
+ to_file = bool(output) and output != "-"
3734
+ try:
3735
+ if to_file:
3736
+ Path(output).write_text(out_text, encoding="utf-8")
3737
+ else:
3738
+ sys.stdout.write(out_text)
3739
+ except OSError as exc:
3740
+ _unexpected(f"writing {output if to_file else 'stdout'}", exc, written)
3741
+ return False
3742
+ return True
3743
+
3744
+
3745
+ def _cmd_verify(args: argparse.Namespace) -> int:
3746
+ meta_path = Path(args.metadata)
3747
+ prov_path = Path(args.provenance)
3748
+ if not meta_path.is_file():
3749
+ print(f"error: metadata not found: {meta_path}", file=sys.stderr)
3750
+ return 2
3751
+ if not prov_path.is_file():
3752
+ print(f"error: provenance map not found: {prov_path}", file=sys.stderr)
3753
+ return 2
3754
+ try:
3755
+ metadata = load_json_object(meta_path, "metadata")
3756
+ prov = load_json_object(prov_path, "provenance map")
3757
+ except ValueError as exc:
3758
+ print(f"error: {exc}", file=sys.stderr)
3759
+ return 2
3760
+
3761
+ skill_dir: Path | None = None
3762
+ if args.skill_dir is not None:
3763
+ skill_dir = Path(args.skill_dir)
3764
+ problem = _skill_dir_error(skill_dir)
3765
+ if problem is not None:
3766
+ print(f"error: {problem}", file=sys.stderr)
3767
+ return 2
3768
+
3769
+ source_root = resolve_source_root(prov, args.source_root)
3770
+ if args.verbose:
3771
+ if source_root is None:
3772
+ print(
3773
+ "verbose: no source root resolved — skipping stale-citation "
3774
+ "check (completeness + orphan diffs only)",
3775
+ file=sys.stderr,
3776
+ )
3777
+ else:
3778
+ print(f"verbose: resolving citations under {source_root}", file=sys.stderr)
3779
+
3780
+ try:
3781
+ result = verify(
3782
+ metadata, prov, source_root, skill_dir, args.check_node_kinds
3783
+ )
3784
+ except Exception as exc: # noqa: BLE001 - exit 2, never read as findings
3785
+ return _unexpected("verification", exc)
3786
+
3787
+ if not _emit(result, args.output):
3788
+ return 2
3789
+ return 0 if result["status"] == "pass" else 1
3790
+
3791
+
3792
+ def _cmd_definition_lines(args: argparse.Namespace) -> int:
3793
+ root = Path(args.source_root) if args.source_root else Path.cwd()
3794
+ if not root.is_dir():
3795
+ print(f"error: source root not found: {root}", file=sys.stderr)
3796
+ return 2
3797
+ try:
3798
+ report = definition_lines_report(
3799
+ args.file, args.name, root, args.line, args.export_type
3800
+ )
3801
+ except ValueError as exc:
3802
+ print(f"error: {exc}", file=sys.stderr)
3803
+ return 2
3804
+ except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
3805
+ return _unexpected("definition lookup", exc)
3806
+ return 0 if _emit(report, args.output) else 2
3807
+
3808
+
3809
+ def _cmd_classify_stale(args: argparse.Namespace) -> int:
3810
+ names_path, prov_path = Path(args.names), Path(args.provenance)
3811
+ inv_path = Path(args.inventory) if args.inventory is not None else None
3812
+ root = Path(args.source_root)
3813
+ inputs = [("names", names_path), ("provenance map", prov_path)]
3814
+ if inv_path is not None:
3815
+ inputs.append(("inventory", inv_path))
3816
+ for label, path in inputs:
3817
+ if not path.is_file():
3818
+ print(f"error: {label} not found: {path}", file=sys.stderr)
3819
+ return 2
3820
+ if not root.is_dir():
3821
+ print(f"error: source root not found: {root}", file=sys.stderr)
3822
+ return 2
3823
+ try:
3824
+ text = names_path.read_text(encoding="utf-8-sig")
3825
+ except (OSError, UnicodeDecodeError) as exc:
3826
+ print(f"error: failed to read names {names_path}: {exc}", file=sys.stderr)
3827
+ return 2
3828
+ try:
3829
+ data = json.loads(text)
3830
+ except json.JSONDecodeError as exc:
3831
+ print(f"error: malformed JSON in names {names_path}: {exc}", file=sys.stderr)
3832
+ return 2
3833
+ try:
3834
+ names = stale_names(data)
3835
+ prov = load_json_object(prov_path, "provenance map")
3836
+ kinds = None if inv_path is None else inventory_kinds(
3837
+ load_json_object(inv_path, "inventory"), args.inventory
3838
+ )
3839
+ except ValueError as exc:
3840
+ print(f"error: {exc}", file=sys.stderr)
3841
+ return 2
3842
+ try:
3843
+ result = classify_stale(names, prov, root, kinds)
3844
+ except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
3845
+ return _unexpected("stale-name classification", exc)
3846
+ return 0 if _emit(result, args.output) else 2
3847
+
3848
+
3849
+ def _cmd_kind_at(args: argparse.Namespace) -> int:
3850
+ # a `source_file` as a provenance entry records it, as `verify` reads it
3851
+ path = Path(args.file.strip().replace("\\", "/"))
3852
+ if args.source_root and not path.is_absolute():
3853
+ path = Path(args.source_root) / path
3854
+ if not path.is_file():
3855
+ print(f"error: source file not found: {path}", file=sys.stderr)
3856
+ return 2
3857
+ recipes_path = Path(args.recipes)
3858
+ if not recipes_path.is_file():
3859
+ print(f"error: recipes not found: {recipes_path}", file=sys.stderr)
3860
+ return 2
3861
+ try:
3862
+ recipes = load_recipes(recipes_path)
3863
+ source = path.read_bytes()
3864
+ except ImportError:
3865
+ print(
3866
+ "error: kind-at reads recipes with PyYAML: run the helper with "
3867
+ "`uv run`, which installs it",
3868
+ file=sys.stderr,
3869
+ )
3870
+ return 2
3871
+ except (ValueError, OSError) as exc:
3872
+ print(f"error: {exc}", file=sys.stderr)
3873
+ return 2
3874
+ if not recipes:
3875
+ print(f"error: no ast-grep recipe in {recipes_path}", file=sys.stderr)
3876
+ return 2
3877
+ language = (
3878
+ args.language.strip().lower() if args.language
3879
+ else ast_grep_language(args.file)
3880
+ )
3881
+ try:
3882
+ (result,) = kinds_at(source, language, [(args.line, args.name)], recipes)
3883
+ except Exception as exc: # noqa: BLE001 - exit 2, never read as an answer
3884
+ return _unexpected("kind lookup", exc)
3885
+ report = {"file": args.file, "language": language, **result}
3886
+ if not _emit(report, args.output):
3887
+ return 2
3888
+ return 0 if report["status"] == "found" else 1
3889
+
3890
+
3891
+ def _cmd_fix(args: argparse.Namespace) -> int:
3892
+ verify_path = Path(args.verify)
3893
+ prov_path = Path(args.provenance)
3894
+ skill_dir = Path(args.skill_dir)
3895
+ for path, label in ((verify_path, "verify result"), (prov_path, "provenance map")):
3896
+ if not path.is_file():
3897
+ print(f"error: {label} not found: {path}", file=sys.stderr)
3898
+ return 2
3899
+ problem = _skill_dir_error(skill_dir)
3900
+ if problem is not None:
3901
+ print(f"error: {problem}", file=sys.stderr)
3902
+ return 2
3903
+ try:
3904
+ verify_result = load_json_object(verify_path, "verify result")
3905
+ prov = load_json_object(prov_path, "provenance map")
3906
+ except ValueError as exc:
3907
+ print(f"error: {exc}", file=sys.stderr)
3908
+ return 2
3909
+ if not all(
3910
+ isinstance(verify_result.get(key), list) for key in ("stale", "citations")
3911
+ ):
3912
+ print(
3913
+ f"error: {verify_path} is not a verify result (no stale[] and "
3914
+ f"citations[])",
3915
+ file=sys.stderr,
3916
+ )
3917
+ return 2
3918
+
3919
+ inventory: list[dict] | None = None
3920
+ hash_content = None
3921
+ if args.manual_inventory is not None:
3922
+ try:
3923
+ hash_content = _load_sibling_module(
3924
+ HASH_CONTENT_HELPER, "skf_hash_content"
3925
+ )
3926
+ inventory = hash_content.load_inventory_blocks(Path(args.manual_inventory))
3927
+ except (ImportError, ValueError) as exc:
3928
+ print(f"error: {exc}", file=sys.stderr)
3929
+ return 2
3930
+
3931
+ try:
3932
+ plan = plan_fixes(verify_result, prov, skill_dir, not args.no_line_moves)
3933
+ except Exception as exc: # noqa: BLE001 - exit 2, never read as a fix
3934
+ return _unexpected("fix", exc)
3935
+
3936
+ changes: list[tuple[Path, bytes]] = [
3937
+ (skill_dir / rel, text.encode("utf-8", errors="surrogateescape"))
3938
+ for rel, text in sorted(plan["markdown"].items())
3939
+ ]
3940
+ if plan["provenance"] is not None:
3941
+ text = json.dumps(plan["provenance"], indent=2, ensure_ascii=False) + "\n"
3942
+ changes.append((prov_path, text.encode("utf-8")))
3943
+ written: list[str] = []
3944
+ if changes and not args.dry_run:
3945
+ helper = _sibling_helper(ATOMIC_WRITE_HELPER)
3946
+ if helper is None:
3947
+ print(
3948
+ f"error: {ATOMIC_WRITE_HELPER} not found beside "
3949
+ f"{Path(__file__).name}; nothing written",
3950
+ file=sys.stderr,
3951
+ )
3952
+ return 2
3953
+ for target, data in changes:
3954
+ try:
3955
+ atomic_write(helper, target, data)
3956
+ except OSError as exc:
3957
+ print(
3958
+ f"error: {exc}; already written: "
3959
+ f"{', '.join(written) if written else 'nothing'}",
3960
+ file=sys.stderr,
3961
+ )
3962
+ return 2
3963
+ written.append(str(target))
3964
+
3965
+ manual = None
3966
+ if hash_content is not None and inventory is not None and not args.dry_run:
3967
+ try:
3968
+ current = hash_content.find_manual_blocks(
3969
+ (skill_dir / "SKILL.md").read_bytes()
3970
+ )
3971
+ manual = hash_content.classify_manual_blocks(inventory, current)
3972
+ except Exception as exc: # noqa: BLE001 - exit 2, never read as a WARN
3973
+ return _unexpected("the [MANUAL] re-check", exc, written)
3974
+
3975
+ applied = plan["applied"]
3976
+ left = plan["left_as_warn"]
3977
+ result = {
3978
+ "applied": applied,
3979
+ "left_as_warn": left,
3980
+ "files_changed": [str(target) for target, _ in changes],
3981
+ "files_written": written,
3982
+ "dry_run": bool(args.dry_run),
3983
+ "manual_verify": manual,
3984
+ "summary": {
3985
+ "citation_prefixes_fixed": sum(
3986
+ 1 for a in applied if a["kind"] == FIX_CITATION_PREFIX
3987
+ ),
3988
+ "source_lines_moved": sum(
3989
+ 1 for a in applied if a["kind"] == FIX_SOURCE_LINE
3990
+ ),
3991
+ "citations_moved": sum(
3992
+ 1 for a in applied if a["kind"] == FIX_CITATION_LINE
3993
+ ),
3994
+ "left_as_warn_count": len(left),
3995
+ },
3996
+ }
3997
+ if not _emit(result, args.output, written):
3998
+ return 2
3999
+ return 0 if not left and (manual is None or manual.get("ok")) else 1
4000
+
4001
+
4002
+ def _build_parser() -> argparse.ArgumentParser:
4003
+ parser = argparse.ArgumentParser(
4004
+ prog="skf-verify-provenance-completeness",
4005
+ description=(
4006
+ "Cross-reference documented exports (metadata.json) against "
4007
+ "provenance-map entries: completeness, orphans, stale or "
4008
+ "non-definition file:line citations, (with --skill-dir) "
4009
+ "citation prefixes and (with --check-node-kinds) ast-grep node "
4010
+ "kinds, emitting findings as JSON; fix the findings with one "
4011
+ "answer; look up an export's definition lines, the stale names "
4012
+ "no cited file defines, or the recipe kind at a line."
4013
+ ),
4014
+ )
4015
+ sub = parser.add_subparsers(dest="cmd", required=True)
4016
+ p = sub.add_parser(
4017
+ "verify",
4018
+ help=(
4019
+ "emit provenance completeness / orphan / stale-citation / "
4020
+ "citation-prefix / node-kind findings"
4021
+ ),
4022
+ )
4023
+ p.add_argument("--metadata", required=True, help="path to metadata.json")
4024
+ p.add_argument(
4025
+ "--provenance", required=True, help="path to provenance-map.json"
4026
+ )
4027
+ p.add_argument(
4028
+ "--source-root",
4029
+ default=None,
4030
+ help=(
4031
+ "source tree root for citation resolution; overrides the "
4032
+ "provenance map's source_root. Stale check is skipped when no "
4033
+ "root resolves on disk."
4034
+ ),
4035
+ )
4036
+ p.add_argument(
4037
+ "--skill-dir",
4038
+ default=None,
4039
+ help=(
4040
+ "skill package folder holding a SKILL.md; scans it and "
4041
+ "references/**/*.md for [AST:]/[SRC:] citations and checks each "
4042
+ "prefix against the provenance entry it cites. Citation check is "
4043
+ "skipped when absent."
4044
+ ),
4045
+ )
4046
+ p.add_argument(
4047
+ "--check-node-kinds",
4048
+ action="store_true",
4049
+ help=(
4050
+ "ask the ast-grep CLI whether each ast-grep entry's ast_node_type "
4051
+ "is a node kind of its file's language; skipped when ast-grep is "
4052
+ "not found."
4053
+ ),
4054
+ )
4055
+ p.add_argument(
4056
+ "-o",
4057
+ "--output",
4058
+ default=None,
4059
+ help="write JSON to this file instead of stdout ('-' for stdout)",
4060
+ )
4061
+ p.add_argument(
4062
+ "--verbose", action="store_true", help="diagnostics to stderr"
4063
+ )
4064
+ p.set_defaults(func=_cmd_verify)
4065
+
4066
+ p = sub.add_parser(
4067
+ "fix",
4068
+ help=(
4069
+ "apply the citation-prefix and source-line fixes a verify result "
4070
+ "has one answer for"
4071
+ ),
4072
+ )
4073
+ p.add_argument(
4074
+ "--verify", required=True, help="path to the JSON a verify run wrote"
4075
+ )
4076
+ p.add_argument(
4077
+ "--provenance",
4078
+ required=True,
4079
+ help="path to the provenance-map.json that verify run read",
4080
+ )
4081
+ p.add_argument(
4082
+ "--skill-dir",
4083
+ required=True,
4084
+ help="skill package folder holding a SKILL.md and references/",
4085
+ )
4086
+ p.add_argument(
4087
+ "--manual-inventory",
4088
+ default=None,
4089
+ help=(
4090
+ "a [MANUAL] inventory of SKILL.md (skf-hash-content.py "
4091
+ "manual-inventory); checked again after the writes"
4092
+ ),
4093
+ )
4094
+ p.add_argument(
4095
+ "--no-line-moves",
4096
+ action="store_true",
4097
+ help="fix citation prefixes only; list every source-line finding",
4098
+ )
4099
+ p.add_argument(
4100
+ "--dry-run", action="store_true", help="plan the fixes, write nothing"
4101
+ )
4102
+ p.add_argument(
4103
+ "-o",
4104
+ "--output",
4105
+ default=None,
4106
+ help="write JSON to this file instead of stdout ('-' for stdout)",
4107
+ )
4108
+ p.set_defaults(func=_cmd_fix)
4109
+
4110
+ p = sub.add_parser(
4111
+ "definition-lines",
4112
+ help="emit the lines of a source file that define an export",
4113
+ )
4114
+ p.add_argument(
4115
+ "--file", required=True, help="source file, relative to --source-root"
4116
+ )
4117
+ p.add_argument("--name", required=True, help="the entry's export_name")
4118
+ p.add_argument(
4119
+ "--source-root",
4120
+ default=None,
4121
+ help="source tree root (default: the current folder)",
4122
+ )
4123
+ p.add_argument(
4124
+ "--line",
4125
+ type=int,
4126
+ default=None,
4127
+ help="the recorded source_line: report whether it defines the export",
4128
+ )
4129
+ p.add_argument(
4130
+ "--export-type",
4131
+ default=None,
4132
+ help="the entry's export_type (module and package have no line)",
4133
+ )
4134
+ p.add_argument(
4135
+ "-o",
4136
+ "--output",
4137
+ default=None,
4138
+ help="write JSON to this file instead of stdout ('-' for stdout)",
4139
+ )
4140
+ p.set_defaults(func=_cmd_definition_lines)
4141
+
4142
+ p = sub.add_parser(
4143
+ "classify-stale",
4144
+ help=(
4145
+ "decide which stale names are fabricated signatures (no cited "
4146
+ "file defines them) and where the source declares the others"
4147
+ ),
4148
+ )
4149
+ p.add_argument(
4150
+ "--names",
4151
+ required=True,
4152
+ help=(
4153
+ "reconcile-coverage.py's result (its `stale` names) or a JSON "
4154
+ "array of names"
4155
+ ),
4156
+ )
4157
+ p.add_argument(
4158
+ "--provenance", required=True, help="path to provenance-map.json"
4159
+ )
4160
+ p.add_argument(
4161
+ "--source-root",
4162
+ required=True,
4163
+ help="source tree root the entries' source_file paths are under",
4164
+ )
4165
+ p.add_argument(
4166
+ "--inventory",
4167
+ default=None,
4168
+ help=(
4169
+ "validate-inventory.py's documented inventory: a name it gives a "
4170
+ "function, hook, class, interface, enum or type kind takes no "
4171
+ "line 1 from a module so named, unless a provenance map entry of "
4172
+ "the name is a module or package one or the module is a TS/JS "
4173
+ "file that is itself the export"
4174
+ ),
4175
+ )
4176
+ p.add_argument(
4177
+ "-o",
4178
+ "--output",
4179
+ default=None,
4180
+ help="write JSON to this file instead of stdout ('-' for stdout)",
4181
+ )
4182
+ p.set_defaults(func=_cmd_classify_stale)
4183
+
4184
+ p = sub.add_parser(
4185
+ "kind-at",
4186
+ help="emit the node kind of the recipe that matches a source line",
4187
+ )
4188
+ p.add_argument(
4189
+ "--file",
4190
+ required=True,
4191
+ help="source file, relative to --source-root when that is given",
4192
+ )
4193
+ p.add_argument(
4194
+ "--line", type=int, required=True, help="the line $NAME must be on"
4195
+ )
4196
+ p.add_argument(
4197
+ "--recipes",
4198
+ required=True,
4199
+ help=(
4200
+ "the recipes: a markdown file (its ```yaml fences, such as "
4201
+ "extraction-patterns.md) or a YAML file"
4202
+ ),
4203
+ )
4204
+ p.add_argument(
4205
+ "--language",
4206
+ default=None,
4207
+ help="ast-grep language (default: from the file's extension)",
4208
+ )
4209
+ p.add_argument(
4210
+ "--name",
4211
+ default=None,
4212
+ help="export name: count only matches whose $NAME is its last segment",
4213
+ )
4214
+ p.add_argument("--source-root", default=None, help="source tree root")
4215
+ p.add_argument(
4216
+ "-o",
4217
+ "--output",
4218
+ default=None,
4219
+ help="write JSON to this file instead of stdout ('-' for stdout)",
4220
+ )
4221
+ p.set_defaults(func=_cmd_kind_at)
429
4222
  return parser
430
4223
 
431
4224