bmad-module-skill-forge 2.2.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (437) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +49 -49
  3. package/docs/404.md +10 -8
  4. package/docs/_data/pinned.yaml +22 -13
  5. package/docs/_internal/STABILITY.md +58 -35
  6. package/docs/agents.md +39 -53
  7. package/docs/architecture.md +65 -51
  8. package/docs/bmad-synergy.md +53 -51
  9. package/docs/campaign.md +127 -75
  10. package/docs/concepts.md +34 -28
  11. package/docs/examples.md +70 -68
  12. package/docs/forge-auto.md +34 -31
  13. package/docs/getting-started.md +118 -57
  14. package/docs/how-it-works.md +25 -21
  15. package/docs/index.md +15 -15
  16. package/docs/skill-model.md +70 -59
  17. package/docs/troubleshooting.md +143 -18
  18. package/docs/verifying-a-skill.md +87 -57
  19. package/docs/why-skf.md +24 -22
  20. package/docs/workflows.md +274 -110
  21. package/package.json +18 -10
  22. package/src/forger/preferences.yaml +8 -1
  23. package/src/knowledge/agentskills-spec.md +2 -0
  24. package/src/knowledge/ccc-bridge.md +40 -24
  25. package/src/knowledge/confidence-tiers.md +9 -5
  26. package/src/knowledge/overview.md +3 -3
  27. package/src/knowledge/progressive-capability.md +8 -8
  28. package/src/knowledge/provenance-tracking.md +9 -5
  29. package/src/knowledge/qmd-registry.md +3 -3
  30. package/src/knowledge/skf-knowledge-index.csv +2 -2
  31. package/src/knowledge/skill-lifecycle.md +18 -11
  32. package/src/knowledge/split-body-strategy.md +0 -3
  33. package/src/knowledge/tool-resolution.md +4 -2
  34. package/src/knowledge/version-paths.md +101 -15
  35. package/src/module-help.csv +3 -3
  36. package/src/module.yaml +10 -0
  37. package/src/shared/data/ast-grep-recipes.yaml +608 -0
  38. package/src/shared/data/ide-context-files.json +30 -0
  39. package/src/shared/health-check.md +31 -59
  40. package/src/shared/references/description-guard-protocol.md +12 -9
  41. package/src/shared/references/feasibility-report-schema.md +19 -13
  42. package/src/shared/references/headless-gate-convention.md +55 -3
  43. package/src/shared/references/output-contract-schema.md +19 -8
  44. package/src/shared/references/pipeline-contracts.md +94 -29
  45. package/src/shared/references/tessl-review.md +77 -0
  46. package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
  47. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
  48. package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
  49. package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
  50. package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
  51. package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
  52. package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
  53. package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
  54. package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
  55. package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
  56. package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
  57. package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
  58. package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
  59. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
  60. package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
  61. package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
  62. package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
  63. package/src/shared/scripts/skf-atomic-write.py +233 -64
  64. package/src/shared/scripts/skf-build-change-manifest.py +2753 -28
  65. package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
  66. package/src/shared/scripts/skf-check-unit-records.py +184 -0
  67. package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
  68. package/src/shared/scripts/skf-classify-changed-files.py +772 -0
  69. package/src/shared/scripts/skf-comention-pairs.py +1333 -123
  70. package/src/shared/scripts/skf-compare-file-hashes.py +267 -105
  71. package/src/shared/scripts/skf-count-imports.py +1867 -0
  72. package/src/shared/scripts/skf-count-tokens.py +1 -1
  73. package/src/shared/scripts/skf-derive-assembly-shape.py +21 -3
  74. package/src/shared/scripts/skf-description-guard.py +99 -11
  75. package/src/shared/scripts/skf-detect-docs.py +694 -2
  76. package/src/shared/scripts/skf-detect-language.py +486 -107
  77. package/src/shared/scripts/skf-detect-registry.py +887 -0
  78. package/src/shared/scripts/skf-detect-scripts-assets.py +197 -55
  79. package/src/shared/scripts/skf-detect-tools.py +209 -27
  80. package/src/shared/scripts/skf-detect-workspaces.py +506 -30
  81. package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
  82. package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
  83. package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
  84. package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
  85. package/src/shared/scripts/skf-extract-public-api.py +5247 -96
  86. package/src/shared/scripts/skf-extraction-inventory.py +1280 -0
  87. package/src/shared/scripts/skf-extraction-snapshot.py +664 -0
  88. package/src/shared/scripts/skf-fetch-temporal.py +674 -0
  89. package/src/shared/scripts/skf-find-cycles.py +97 -29
  90. package/src/shared/scripts/skf-find-test-report.py +476 -0
  91. package/src/shared/scripts/skf-forge-tier-rw.py +469 -205
  92. package/src/shared/scripts/skf-github-fetch.py +365 -0
  93. package/src/shared/scripts/skf-github-probe.py +700 -0
  94. package/src/shared/scripts/skf-hash-content.py +180 -20
  95. package/src/shared/scripts/skf-language-corpora.py +1 -1
  96. package/src/shared/scripts/skf-load-provenance.py +161 -27
  97. package/src/shared/scripts/skf-manifest-ops.py +1 -1
  98. package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
  99. package/src/shared/scripts/skf-merge-doc-urls.py +143 -20
  100. package/src/shared/scripts/skf-names-present.py +221 -0
  101. package/src/shared/scripts/skf-pair-intersect.py +119 -28
  102. package/src/shared/scripts/skf-parse-gaps.py +1384 -0
  103. package/src/shared/scripts/skf-preapply.py +1 -1
  104. package/src/shared/scripts/skf-preflight.py +176 -28
  105. package/src/shared/scripts/skf-promote-staged.py +430 -0
  106. package/src/shared/scripts/skf-provenance-gap-dispatch.py +64 -8
  107. package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
  108. package/src/shared/scripts/skf-quick-batch.py +774 -0
  109. package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
  110. package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
  111. package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
  112. package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
  113. package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
  114. package/src/shared/scripts/skf-resolve-authoritative-files.py +155 -14
  115. package/src/shared/scripts/skf-resolve-package.py +523 -106
  116. package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
  117. package/src/shared/scripts/skf-run-lock.py +577 -0
  118. package/src/shared/scripts/skf-scan-manifests.py +998 -57
  119. package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
  120. package/src/shared/scripts/skf-severity-classify.py +712 -108
  121. package/src/shared/scripts/skf-shape-detect.py +298 -204
  122. package/src/shared/scripts/skf-shard-body.py +9 -8
  123. package/src/shared/scripts/skf-skill-inventory.py +2412 -89
  124. package/src/shared/scripts/skf-skills-module.py +472 -0
  125. package/src/shared/scripts/skf-source-tree.py +1796 -0
  126. package/src/shared/scripts/skf-structural-diff.py +787 -121
  127. package/src/shared/scripts/skf-tessl-review.py +706 -0
  128. package/src/shared/scripts/skf-update-active-symlink.py +120 -67
  129. package/src/shared/scripts/skf-update-run-state.py +624 -0
  130. package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
  131. package/src/shared/scripts/skf-validate-brief-schema.py +23 -2
  132. package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
  133. package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
  134. package/src/shared/scripts/skf-validate-output.py +712 -64
  135. package/src/shared/scripts/skf-validate-pins.py +1 -1
  136. package/src/shared/scripts/skf-verify-no-trace.py +116 -10
  137. package/src/shared/scripts/skf-verify-provenance-completeness.py +3945 -152
  138. package/src/shared/scripts/skf-write-skill-brief.py +406 -25
  139. package/src/shared/tool-requirements.yaml +201 -0
  140. package/src/skf-analyze-source/SKILL.md +41 -36
  141. package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
  142. package/src/skf-analyze-source/customize.toml +36 -24
  143. package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
  144. package/src/skf-analyze-source/references/continue.md +2 -13
  145. package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
  146. package/src/skf-analyze-source/references/generate-briefs.md +121 -55
  147. package/src/skf-analyze-source/references/headless-contract.md +42 -15
  148. package/src/skf-analyze-source/references/identify-units.md +96 -110
  149. package/src/skf-analyze-source/references/init.md +73 -71
  150. package/src/skf-analyze-source/references/map-and-detect.md +123 -105
  151. package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
  152. package/src/skf-analyze-source/references/recommend.md +51 -70
  153. package/src/skf-analyze-source/references/scan-project.md +60 -84
  154. package/src/skf-analyze-source/references/scan-root.md +39 -0
  155. package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
  156. package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
  157. package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
  158. package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
  159. package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
  160. package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
  161. package/src/skf-audit-skill/SKILL.md +43 -51
  162. package/src/skf-audit-skill/assets/drift-report-template.md +28 -3
  163. package/src/skf-audit-skill/customize.toml +41 -20
  164. package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
  165. package/src/skf-audit-skill/references/doc-drift.md +141 -0
  166. package/src/skf-audit-skill/references/headless-contract.md +29 -0
  167. package/src/skf-audit-skill/references/health-check.md +2 -0
  168. package/src/skf-audit-skill/references/init.md +149 -210
  169. package/src/skf-audit-skill/references/re-index.md +103 -110
  170. package/src/skf-audit-skill/references/report.md +96 -33
  171. package/src/skf-audit-skill/references/semantic-diff.md +34 -25
  172. package/src/skf-audit-skill/references/severity-classify.md +79 -65
  173. package/src/skf-audit-skill/references/severity-rules.md +15 -6
  174. package/src/skf-audit-skill/references/structural-diff.md +80 -90
  175. package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
  176. package/src/skf-audit-skill/scripts/render-drift-tables.py +712 -0
  177. package/src/skf-brief-skill/SKILL.md +37 -26
  178. package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
  179. package/src/skf-brief-skill/assets/scope-templates.md +18 -25
  180. package/src/skf-brief-skill/assets/skill-brief-schema.md +30 -23
  181. package/src/skf-brief-skill/customize.toml +48 -20
  182. package/src/skf-brief-skill/references/analyze-target.md +144 -127
  183. package/src/skf-brief-skill/references/confirm-brief.md +11 -9
  184. package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
  185. package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
  186. package/src/skf-brief-skill/references/gather-intent.md +87 -182
  187. package/src/skf-brief-skill/references/headless-args.md +84 -16
  188. package/src/skf-brief-skill/references/health-check.md +1 -1
  189. package/src/skf-brief-skill/references/invocation-contract.md +18 -13
  190. package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
  191. package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
  192. package/src/skf-brief-skill/references/scope-definition.md +123 -47
  193. package/src/skf-brief-skill/references/step-auto-brief.md +26 -100
  194. package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
  195. package/src/skf-brief-skill/references/version-resolution.md +7 -10
  196. package/src/skf-brief-skill/references/write-brief.md +66 -57
  197. package/src/skf-campaign/SKILL.md +31 -34
  198. package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
  199. package/src/skf-campaign/customize.toml +47 -16
  200. package/src/skf-campaign/references/campaign-contracts.md +39 -29
  201. package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
  202. package/src/skf-campaign/references/health-check.md +7 -16
  203. package/src/skf-campaign/references/step-01-setup.md +46 -84
  204. package/src/skf-campaign/references/step-02-strategy.md +31 -30
  205. package/src/skf-campaign/references/step-03-pins.md +26 -18
  206. package/src/skf-campaign/references/step-04-provenance.md +24 -23
  207. package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
  208. package/src/skf-campaign/references/step-06-batch.md +41 -34
  209. package/src/skf-campaign/references/step-07-capstone.md +40 -25
  210. package/src/skf-campaign/references/step-08-verify.md +28 -29
  211. package/src/skf-campaign/references/step-09-refine.md +29 -30
  212. package/src/skf-campaign/references/step-10-export.md +48 -36
  213. package/src/skf-campaign/references/step-11-maintenance.md +31 -34
  214. package/src/skf-campaign/references/step-resume.md +37 -68
  215. package/src/skf-campaign/scripts/campaign-deps.py +57 -18
  216. package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
  217. package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
  218. package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
  219. package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
  220. package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
  221. package/src/skf-campaign/scripts/campaign-report.py +204 -37
  222. package/src/skf-campaign/scripts/campaign-state.py +1128 -0
  223. package/src/skf-campaign/scripts/campaign-status.py +10 -74
  224. package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
  225. package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
  226. package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
  227. package/src/skf-campaign/templates/campaign-report-template.md +5 -1
  228. package/src/skf-campaign/templates/kickoff-template.md +0 -14
  229. package/src/skf-create-skill/SKILL.md +27 -20
  230. package/src/skf-create-skill/assets/compile-assembly-rules.md +11 -8
  231. package/src/skf-create-skill/assets/skill-sections.md +21 -11
  232. package/src/skf-create-skill/customize.toml +20 -8
  233. package/src/skf-create-skill/references/authoritative-files-protocol.md +36 -58
  234. package/src/skf-create-skill/references/batch-mode.md +81 -0
  235. package/src/skf-create-skill/references/ccc-index-check.md +27 -0
  236. package/src/skf-create-skill/references/compile.md +119 -70
  237. package/src/skf-create-skill/references/component-extraction.md +113 -91
  238. package/src/skf-create-skill/references/enrich.md +41 -23
  239. package/src/skf-create-skill/references/entry-points-by-hand.md +37 -0
  240. package/src/skf-create-skill/references/extract.md +204 -162
  241. package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
  242. package/src/skf-create-skill/references/extraction-patterns-tracing.md +11 -11
  243. package/src/skf-create-skill/references/extraction-patterns.md +588 -201
  244. package/src/skf-create-skill/references/generate-artifacts.md +145 -93
  245. package/src/skf-create-skill/references/health-check.md +1 -1
  246. package/src/skf-create-skill/references/load-brief.md +53 -36
  247. package/src/skf-create-skill/references/relabel-rule.md +12 -0
  248. package/src/skf-create-skill/references/report.md +37 -57
  249. package/src/skf-create-skill/references/source-resolution-protocols.md +58 -169
  250. package/src/skf-create-skill/references/step-auto-shard.md +20 -93
  251. package/src/skf-create-skill/references/step-doc-rot.md +44 -60
  252. package/src/skf-create-skill/references/step-doc-sources.md +28 -19
  253. package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
  254. package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
  255. package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
  256. package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
  257. package/src/skf-create-skill/references/validate.md +173 -99
  258. package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
  259. package/src/skf-create-stack-skill/SKILL.md +29 -56
  260. package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
  261. package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
  262. package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
  263. package/src/skf-create-stack-skill/customize.toml +23 -12
  264. package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
  265. package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
  266. package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
  267. package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
  268. package/src/skf-create-stack-skill/references/generate-output.md +138 -128
  269. package/src/skf-create-stack-skill/references/init.md +90 -32
  270. package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
  271. package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
  272. package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
  273. package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
  274. package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
  275. package/src/skf-create-stack-skill/references/report.md +39 -33
  276. package/src/skf-create-stack-skill/references/validate.md +20 -39
  277. package/src/skf-drop-skill/SKILL.md +28 -51
  278. package/src/skf-drop-skill/customize.toml +44 -30
  279. package/src/skf-drop-skill/references/execute.md +83 -146
  280. package/src/skf-drop-skill/references/health-check.md +2 -0
  281. package/src/skf-drop-skill/references/invocation-contract.md +64 -0
  282. package/src/skf-drop-skill/references/report.md +46 -30
  283. package/src/skf-drop-skill/references/select.md +122 -124
  284. package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
  285. package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
  286. package/src/skf-export-skill/SKILL.md +40 -62
  287. package/src/skf-export-skill/assets/managed-section-format.md +14 -50
  288. package/src/skf-export-skill/assets/snippet-format.md +3 -1
  289. package/src/skf-export-skill/customize.toml +42 -13
  290. package/src/skf-export-skill/references/generate-snippet.md +54 -46
  291. package/src/skf-export-skill/references/health-check.md +2 -0
  292. package/src/skf-export-skill/references/invocation-contract.md +15 -0
  293. package/src/skf-export-skill/references/load-skill.md +88 -62
  294. package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
  295. package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
  296. package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
  297. package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
  298. package/src/skf-export-skill/references/package.md +15 -73
  299. package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
  300. package/src/skf-export-skill/references/result-envelope.md +31 -16
  301. package/src/skf-export-skill/references/summary.md +41 -17
  302. package/src/skf-export-skill/references/token-report.md +4 -12
  303. package/src/skf-export-skill/references/update-context.md +118 -237
  304. package/src/skf-forger/SKILL.md +61 -37
  305. package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
  306. package/src/skf-forger/customize.toml +43 -0
  307. package/src/skf-forger/references/pipeline-mode.md +73 -18
  308. package/src/skf-forger/scripts/forge-status.py +299 -0
  309. package/src/skf-forger/scripts/parse-pipeline.py +322 -54
  310. package/src/skf-forger/scripts/pipeline-gate.py +428 -0
  311. package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
  312. package/src/skf-quick-skill/SKILL.md +54 -53
  313. package/src/skf-quick-skill/assets/skill-template.md +4 -2
  314. package/src/skf-quick-skill/customize.toml +51 -20
  315. package/src/skf-quick-skill/references/batch-mode.md +74 -21
  316. package/src/skf-quick-skill/references/compile.md +30 -31
  317. package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
  318. package/src/skf-quick-skill/references/finalize.md +69 -18
  319. package/src/skf-quick-skill/references/halt-contract.md +41 -29
  320. package/src/skf-quick-skill/references/health-check.md +5 -3
  321. package/src/skf-quick-skill/references/quick-extract.md +114 -45
  322. package/src/skf-quick-skill/references/registry-resolution.md +9 -64
  323. package/src/skf-quick-skill/references/resolve-target.md +116 -66
  324. package/src/skf-quick-skill/references/write-and-validate.md +79 -40
  325. package/src/skf-refine-architecture/SKILL.md +43 -53
  326. package/src/skf-refine-architecture/customize.toml +49 -20
  327. package/src/skf-refine-architecture/references/compile.md +97 -54
  328. package/src/skf-refine-architecture/references/exit-codes.md +30 -0
  329. package/src/skf-refine-architecture/references/finding-storage.md +21 -0
  330. package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
  331. package/src/skf-refine-architecture/references/improvements.md +14 -20
  332. package/src/skf-refine-architecture/references/init.md +98 -57
  333. package/src/skf-refine-architecture/references/issue-detection.md +32 -39
  334. package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
  335. package/src/skf-refine-architecture/references/report.md +65 -54
  336. package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
  337. package/src/skf-rename-skill/SKILL.md +19 -54
  338. package/src/skf-rename-skill/customize.toml +32 -26
  339. package/src/skf-rename-skill/references/execute.md +230 -184
  340. package/src/skf-rename-skill/references/exit-codes.md +7 -7
  341. package/src/skf-rename-skill/references/health-check.md +1 -7
  342. package/src/skf-rename-skill/references/invocation-contract.md +25 -0
  343. package/src/skf-rename-skill/references/report.md +63 -52
  344. package/src/skf-rename-skill/references/select.md +197 -115
  345. package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
  346. package/src/skf-setup/SKILL.md +26 -26
  347. package/src/skf-setup/customize.toml +14 -10
  348. package/src/skf-setup/references/auto-index.md +50 -63
  349. package/src/skf-setup/references/ccc-index.md +29 -90
  350. package/src/skf-setup/references/detect-and-tier.md +58 -67
  351. package/src/skf-setup/references/health-check.md +2 -0
  352. package/src/skf-setup/references/invocation-contract.md +14 -0
  353. package/src/skf-setup/references/report.md +43 -166
  354. package/src/skf-setup/references/tier-rules.md +5 -5
  355. package/src/skf-setup/references/write-config.md +34 -57
  356. package/src/skf-test-skill/SKILL.md +17 -57
  357. package/src/skf-test-skill/assets/output-section-formats.md +33 -34
  358. package/src/skf-test-skill/customize.toml +27 -13
  359. package/src/skf-test-skill/references/coherence-check.md +134 -62
  360. package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
  361. package/src/skf-test-skill/references/coverage-check.md +209 -264
  362. package/src/skf-test-skill/references/detect-mode.md +5 -5
  363. package/src/skf-test-skill/references/external-validators.md +94 -82
  364. package/src/skf-test-skill/references/health-check.md +6 -7
  365. package/src/skf-test-skill/references/init.md +164 -91
  366. package/src/skf-test-skill/references/invocation-contract.md +66 -0
  367. package/src/skf-test-skill/references/migration-section-rules.md +5 -2
  368. package/src/skf-test-skill/references/report.md +164 -142
  369. package/src/skf-test-skill/references/score.md +80 -107
  370. package/src/skf-test-skill/references/scoring-rules.md +42 -155
  371. package/src/skf-test-skill/references/source-access-protocol.md +18 -94
  372. package/src/skf-test-skill/references/step-hard-gate.md +51 -33
  373. package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
  374. package/src/skf-test-skill/scripts/build-result-context.py +380 -0
  375. package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
  376. package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
  377. package/src/skf-test-skill/scripts/compute-score.py +223 -45
  378. package/src/skf-test-skill/scripts/gap-ledger.py +1666 -0
  379. package/src/skf-test-skill/scripts/hard-gate.py +203 -0
  380. package/src/skf-test-skill/scripts/load-coverage-inputs.py +1198 -0
  381. package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
  382. package/src/skf-test-skill/scripts/reconcile-coverage.py +547 -79
  383. package/src/skf-test-skill/scripts/score-signatures.py +479 -0
  384. package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
  385. package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
  386. package/src/skf-test-skill/scripts/verify-declared-numerator.py +133 -56
  387. package/src/skf-test-skill/templates/test-report-template.md +21 -15
  388. package/src/skf-update-skill/SKILL.md +38 -25
  389. package/src/skf-update-skill/customize.toml +27 -12
  390. package/src/skf-update-skill/references/detect-changes.md +253 -223
  391. package/src/skf-update-skill/references/gap-driven.md +336 -0
  392. package/src/skf-update-skill/references/health-check.md +31 -5
  393. package/src/skf-update-skill/references/init.md +254 -84
  394. package/src/skf-update-skill/references/invocation-contract.md +12 -0
  395. package/src/skf-update-skill/references/manual-section-rules.md +6 -23
  396. package/src/skf-update-skill/references/merge.md +182 -49
  397. package/src/skf-update-skill/references/re-extract.md +53 -262
  398. package/src/skf-update-skill/references/report.md +102 -35
  399. package/src/skf-update-skill/references/write.md +357 -155
  400. package/src/skf-update-skill/scripts/skf-new-file-diff.py +183 -46
  401. package/src/skf-verify-stack/SKILL.md +23 -42
  402. package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
  403. package/src/skf-verify-stack/customize.toml +55 -22
  404. package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
  405. package/src/skf-verify-stack/references/coverage.md +67 -64
  406. package/src/skf-verify-stack/references/exit-codes.md +21 -8
  407. package/src/skf-verify-stack/references/init.md +107 -62
  408. package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
  409. package/src/skf-verify-stack/references/integrations.md +106 -65
  410. package/src/skf-verify-stack/references/report.md +59 -42
  411. package/src/skf-verify-stack/references/requirements.md +32 -9
  412. package/src/skf-verify-stack/references/synthesize.md +53 -39
  413. package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
  414. package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
  415. package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
  416. package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
  417. package/tools/cli/commands/install.js +11 -4
  418. package/tools/cli/commands/status.js +15 -3
  419. package/tools/cli/commands/update.js +11 -4
  420. package/tools/cli/lib/tool-check.js +320 -0
  421. package/tools/cli/lib/version-check.js +22 -9
  422. package/tools/cli/skf-cli.js +4 -1
  423. package/.gitattributes +0 -16
  424. package/.nvmrc +0 -1
  425. package/docs/_internal/RELEASING.md +0 -679
  426. package/release-audits/v1.0.0-launch-audit.md +0 -728
  427. package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
  428. package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
  429. package/src/skf-campaign/manifest.yaml +0 -15
  430. package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
  431. package/src/skf-create-skill/references/ecosystem-check.md +0 -74
  432. package/src/skf-drop-skill/references/headless-contract.md +0 -18
  433. package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
  434. package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
  435. package/src/skf-update-skill/references/validate.md +0 -124
  436. package/tools/validate-docs-drift.js +0 -233
  437. package/tools/validate-docs-links.js +0 -665
@@ -1,117 +1,216 @@
1
1
  # /// script
2
- # requires-python = ">=3.10"
2
+ # requires-python = ">=3.11"
3
3
  # dependencies = []
4
4
  # ///
5
- """SKF Disqualify Candidates — deterministic filter for skillable-unit boundaries.
6
-
7
- `identify-units.md` step 2 §D asks the LLM to evaluate each detected boundary
8
- against a fixed disqualification list (too-few-files, too-low-loc,
9
- generated-code, auto-generated-tag, etc.). The boundary checks are fully
10
- deterministic — file counts, line counts, path-substring matches against a
11
- known generated-code roster, header sentinel scans — and prior runs have
12
- shown LLMs miscount files, miss generated-code paths inside nested
13
- `node_modules/`, and forget to scan for `@generated`. This script collapses
14
- that work into one call.
15
-
16
- Canonical disqualification rules (kept in sync with
5
+ """SKF Disqualify Candidates: size, generated-code and signal evidence per boundary.
6
+
7
+ `identify-units.md` section 2 hands this helper every detected boundary, and
8
+ the helper lists each boundary's files itself. What it checks has one right
9
+ answer per file list (file and
10
+ line counts, generated or vendored directory segments, generated-code header
11
+ sentinels, which signal files exist, what the boundary's own manifest names),
12
+ and prior runs showed models miscount files, miss generated paths inside
13
+ nested `node_modules/` and forget to scan for `@generated`, so one call does
14
+ it all. What the evidence means for a boundary with only some generated files
15
+ is the model's call.
16
+
17
+ Which files count:
18
+
19
+ - A lockfile (`Cargo.lock`, `package-lock.json`, `yarn.lock`, `go.sum` and
20
+ the rest of LOCKFILE_NAMES and LOCKFILE_SUFFIXES) and a non-source file
21
+ (documentation, data and configuration, media, archives and binaries:
22
+ NON_SOURCE_EXTENSIONS, NON_SOURCE_NAMES, every dotfile; build and tool
23
+ configuration: every manifest skf-scan-manifests.py scans, setup.py,
24
+ build.gradle and Package.swift included, the Gradle and Maven wrappers,
25
+ Bazel files and `<tool>.config.<ext>` files such as vite.config.ts) is
26
+ never a source file. It is never scanned for a header sentinel and
27
+ never counted. Every other file is a source file, so a language these
28
+ tables do not know still counts.
29
+ - A source file is generated when its path below the boundary path holds
30
+ a vendored or cache folder at any depth (node_modules/, vendor/,
31
+ __pycache__/, .next/, .gradle/), or a build output folder (dist/,
32
+ build/, target/) that is the first folder below the boundary or sits in
33
+ a folder holding a manifest of the file list (a member package's own
34
+ dist/); or else when one of its first 10 lines holds a generated-code
35
+ sentinel (`@generated`, `// GENERATED CODE`, `# AUTOGENERATED`,
36
+ `<!-- generated -->`, matched case-insensitively). So a package that
37
+ itself lives in a folder named build/ is not generated, and neither is
38
+ a source folder of that name further down (src/build/, the Java package
39
+ folder com/acme/build/).
40
+
41
+ Rules, first match wins (kept in sync with
17
42
  `src/skf-analyze-source/references/unit-detection-heuristics.md`):
18
43
 
19
- - too-few-files : files_count < {MIN_FILES} (default 3)
20
- - too-low-loc : loc_total < {MIN_LOC} (default 100)
21
- - generated-code : any file path contains a known generated/vendor
22
- directory segment (node_modules/, vendor/, dist/,
23
- build/, __pycache__/, .next/, target/, .gradle/)
24
- - auto-generated-tag: any file's first 10 lines contain a known
25
- "auto-generated" sentinel string (`@generated`,
26
- `// GENERATED CODE`, `# AUTOGENERATED`,
27
- `<!-- generated -->`); case-insensitive matching
28
- on the full sentinel phrases.
29
-
30
- Rules NOT enforced by this script (still LLM-judged in step 2 §D):
31
- - pure-configuration
32
- - test-only
33
- - already-skilled (requires existing_skills cross-reference)
34
-
35
- LoC counting: per file, count lines with NON-empty content. A "blank line"
36
- is one whose stripped form is empty (whitespace-only lines count as blank).
37
- This matches the spec phrasing "lines with content (strip empty lines)".
38
-
39
- The check order matters because heuristics are reported as a single
40
- `reason`; the first match wins. Order is:
41
- 1. generated-code (path-based, cheapest, often the dominant signal)
42
- 2. auto-generated-tag (header scan)
43
- 3. too-few-files (file count after the above)
44
- 4. too-low-loc (loc after the above)
44
+ 1. generated-code / auto-generated-tag: every source file of the boundary
45
+ is generated (and it has at least one). generated-code when a
46
+ directory segment marked any of them, auto-generated-tag when only
47
+ header sentinels did.
48
+ 2. too-few-files: files_count < --min-files (default 3)
49
+ 3. too-low-loc: loc_total < --min-loc (default 100)
50
+
51
+ `files_count` and `loc_total` count the source files that are not
52
+ generated; a line counts when its stripped form is not empty. A boundary
53
+ with some generated files is kept when it passes rules 2 and 3, and its
54
+ `generated_files` and `generated_ratio` are evidence the model weighs.
55
+ `passes_with_generated` says whether rules 2 and 3 pass with the generated
56
+ files counted too, so a boundary whose flagged files the model judges to be
57
+ its own code can be restored without counting them by hand.
58
+
59
+ Rules NOT enforced by this script (judged in identify-units section 2):
60
+ pure-configuration, test-only, already-skilled.
61
+
62
+ Manifest. Every record carries `manifest`, the boundary's own manifest: of
63
+ the manifests at the boundary root (the file names skf-scan-manifests.py
64
+ scans, its MANIFEST_ECOSYSTEMS), the first in path order that names a
65
+ package, else the first; null when there is none. It is
66
+ `{"path", "ecosystem", "name", "private"}`, `name` and `private` as that
67
+ script's identity readers report them, which skf-skill-inventory.py
68
+ derive-name takes to name the unit.
69
+
70
+ Signals. Every record carries `signals`, the detection signals of
71
+ unit-detection-heuristics.md that the file list shows, each the first file
72
+ that shows it (shallowest, then in path order) or null:
73
+
74
+ own_manifest the `manifest` path
75
+ entry_point an entry file (ENTRY_POINT_NAMES: index.ts, main.go,
76
+ lib.rs, __init__.py, ...) at most two folders below
77
+ the boundary root
78
+ service_definition a Dockerfile, Containerfile, compose file or
79
+ serverless.yml, at any depth (skf-scan-manifests.py's
80
+ is_service_definition, whose `services[]` lists them)
81
+ readme a README at the boundary root
82
+ test_suite a file in a test folder (test/, tests/, __tests__/,
83
+ spec/, specs/), a test file (*.test.*, *.spec.*,
84
+ *_test.*, test_*.py, *_spec.rb) or a test runner
85
+ configuration
86
+ ci_reference a CI configuration under --source-root (CI_FILE_GLOBS)
87
+ that names the boundary path; null for a boundary at
88
+ the root
89
+
90
+ and `large_directory`: true when the file list holds 50 or more files.
91
+
92
+ Boundary paths. Each `path` is normalized once, so `./packages/auth`,
93
+ `packages/auth/` and `packages/./auth` all read as `packages/auth` (and
94
+ `""` or `./` as `.`, the root); the record carries the normalized path. An
95
+ absolute path, or one that leaves the source root (`..`), is an error.
96
+
97
+ Listing. A boundary without `files` gets the files under
98
+ <source-root>/<path>: those git lists when the source root is in a git
99
+ work tree (tracked, and untracked but not ignored, so .gitignore holds and
100
+ an installed node_modules/ or a build's target/ stays out), else every file
101
+ of a walk that skips node_modules/ and __pycache__/. The walk also runs
102
+ when git lists no file there (a folder the outer work tree ignores, such
103
+ as an extracted tarball under an ignored vendor/). Hidden entries and
104
+ symbolic links are left out either way. Paths are relative to the source
105
+ root, forward-slash, sorted, each once. A boundary path that is no folder
106
+ under the source root is an error. With --tree-dir, each boundary's file
107
+ list (its listing, or the `files` it was given) is written there, one path
108
+ per line, as `boundary-<n>.txt` (n its 1-based place in the input), and
109
+ its record carries `tree_file`: the listing skf-detect-language.py
110
+ --tree-file reads for that unit, so no file list passes through the
111
+ caller. A file name that is not UTF-8 is written with `?` for each byte
112
+ that is not, so the listing stays one UTF-8 file.
45
113
 
46
114
  Subcommand:
47
115
  filter --boundaries <json-file-or--for-stdin> [--source-root <path>]
48
- [--min-files N] [--min-loc N]
116
+ [--tree-dir <folder>] [--min-files N] [--min-loc N]
49
117
 
50
118
  Read a boundaries JSON of the shape:
51
119
  [
52
120
  {"name": "<unit-name>",
53
121
  "path": "<rel-from-source-root>",
54
- "files": ["<rel-path>", ...]},
122
+ "files": ["<rel-path>", ...]}, // optional: listed when absent
55
123
  ...
56
124
  ]
57
125
  and emit:
58
126
  {
59
- "kept": [{"name": "...", "path": "...",
60
- "files_count": N, "loc_total": L}, ...],
61
- "dropped": [{"name": "...",
62
- "reason": "<too-few-files|too-low-loc|generated-code|"
63
- "auto-generated-tag>",
64
- "context": {"files_count": N, "loc_total": L,
65
- "first_match": "..."}}, ...],
66
- "stats": {"kept": N, "dropped": N,
67
- "by_reason": {"<reason>": N, ...}}
127
+ "kept": [<record>, ...],
128
+ "dropped": [<record> + {"reason": "<too-few-files|too-low-loc|"
129
+ "generated-code|"
130
+ "auto-generated-tag>",
131
+ "context": {...}}, ...],
132
+ "stats": {"kept": N, "dropped": N, "mixed": N,
133
+ "by_reason": {"<reason>": N, ...}}
68
134
  }
135
+ where every <record> is
136
+ {"name": "...", "path": "...",
137
+ "files_count": N, "loc_total": L, "non_source_count": K,
138
+ "generated_files": [{"path": "...",
139
+ "reason": "generated-code|auto-generated-tag",
140
+ "match": "<segment or sentinel>"}, ...],
141
+ "generated_ratio": R, "passes_with_generated": B,
142
+ "manifest": {"path", "ecosystem", "name", "private"} | null,
143
+ "signals": {"own_manifest": "<path>|null", ..., "large_directory": B},
144
+ "tree_file": "<path>"} // with --tree-dir only
145
+ `generated_files` is sorted by path. `generated_ratio` is the share of
146
+ the source files that are generated (0.0 with no source file), to
147
+ three decimals. `context` holds the rule's own evidence: `threshold`
148
+ for too-few-files and too-low-loc, `first_match` and
149
+ `matched_segment` or `matched_sentinel` for the generated rules.
150
+ `stats.mixed` counts the kept boundaries with generated files: the
151
+ mixed cases the model judges.
69
152
 
70
153
  --source-root defaults to the current working directory. File paths in
71
- the `files[]` array are resolved against `<source-root>/<boundary-path>`
72
- first (to match `path` from the earlier scan-project step), and falling
73
- back to `<source-root>/<file-rel>` if that doesn't exist. Missing files
74
- are treated as 0-line zero-byte files; the boundary is still subject to
75
- count/loc thresholds.
154
+ the `files[]` array are resolved against `<source-root>/<file-rel>` first
155
+ and, when that does not exist, against
156
+ `<source-root>/<boundary-path>/<file-rel>`. A file listed twice counts
157
+ once. Missing files read as empty files; the boundary is still subject
158
+ to the count and LoC thresholds.
76
159
 
77
160
  CLI examples:
78
161
  uv run skf-disqualify-candidates.py filter --boundaries boundaries.json
79
162
  cat boundaries.json | uv run skf-disqualify-candidates.py filter \\
80
- --boundaries - --source-root /path/to/project
163
+ --boundaries - --source-root /path/to/project --tree-dir /tmp/trees
81
164
 
82
165
  Exit codes:
83
- 0 — operation succeeded (including: empty input → empty kept/dropped)
84
- 1 — user error (bad JSON, missing required keys, malformed entry)
166
+ 0 operation succeeded (including: empty input, which gives empty kept
167
+ and dropped lists)
168
+ 1 user error (bad JSON, missing required keys, malformed entry, a
169
+ boundary path that is absolute or leaves the source root, a boundary
170
+ path to list that is no folder, a --tree-dir that cannot be
171
+ written), or no skf-scan-manifests.py beside this script that loads
172
+ (it needs Python 3.11 for tomllib, hence this script's floor)
85
173
  """
86
174
 
87
175
  from __future__ import annotations
88
176
 
89
177
  import argparse
178
+ import importlib.util
90
179
  import json
180
+ import os
181
+ import posixpath
182
+ import re
91
183
  import sys
92
184
  from pathlib import Path
93
185
 
94
186
 
95
187
  # --------------------------------------------------------------------------
96
- # Constants — disqualification rules (single source of truth)
188
+ # Constants: disqualification rules (single source of truth)
97
189
  # --------------------------------------------------------------------------
98
190
 
99
191
 
100
192
  MIN_FILES = 3
101
193
  MIN_LOC = 100
102
194
 
103
- # Path segments that mark generated / vendored / build-output trees.
104
- # Trailing slash is significant — we match the segment as a directory.
105
- GENERATED_PATH_SEGMENTS = (
195
+ # Path segments that mark vendored code and tool caches, generated at any
196
+ # depth below the boundary. Trailing slash is significant: we match the
197
+ # segment as a directory.
198
+ VENDORED_PATH_SEGMENTS = (
106
199
  "node_modules/",
107
200
  "vendor/",
108
- "dist/",
109
- "build/",
110
201
  "__pycache__/",
111
202
  ".next/",
112
- "target/",
113
203
  ".gradle/",
114
204
  )
205
+ # Build output folders: generated only as the first folder below the
206
+ # boundary or inside a folder that holds a manifest, since a source folder
207
+ # can carry the same name (src/build/, the Java package com/acme/build/).
208
+ OUTPUT_PATH_SEGMENTS = (
209
+ "dist/",
210
+ "build/",
211
+ "target/",
212
+ )
213
+ GENERATED_PATH_SEGMENTS = VENDORED_PATH_SEGMENTS + OUTPUT_PATH_SEGMENTS
115
214
 
116
215
  # Header-sentinel scan: case-insensitive substring match on each of the
117
216
  # first AUTO_GEN_HEADER_LINES of a file. The strings are spec-canonical;
@@ -125,8 +224,127 @@ AUTO_GEN_SENTINELS = (
125
224
 
126
225
  AUTO_GEN_HEADER_LINES = 10
127
226
 
227
+ # Lockfiles: never source, never scanned (a committed Cargo.lock opens with
228
+ # "@generated"). Compared on the lowercased file name.
229
+ LOCKFILE_NAMES = frozenset({
230
+ "package-lock.json",
231
+ "npm-shrinkwrap.json",
232
+ "pnpm-lock.yaml",
233
+ "go.sum",
234
+ "go.work.sum",
235
+ "package.resolved",
236
+ "packages.lock.json",
237
+ })
238
+ # Cargo.lock, yarn.lock, poetry.lock, Pipfile.lock, uv.lock, composer.lock,
239
+ # Gemfile.lock, mix.lock, pubspec.lock, flake.lock, bun.lockb, gradle.lockfile
240
+ LOCKFILE_SUFFIXES = (".lock", ".lockb", ".lockfile")
241
+
242
+ # Non-source files: documentation, data and configuration, media, archives
243
+ # and binaries. Compared on the lowercased extension.
244
+ NON_SOURCE_EXTENSIONS = frozenset({
245
+ # documentation and text
246
+ ".md", ".mdx", ".markdown", ".rst", ".txt", ".adoc", ".rtf",
247
+ # data and configuration
248
+ ".json", ".jsonc", ".json5", ".yaml", ".yml", ".toml", ".ini", ".cfg",
249
+ ".conf", ".properties", ".env", ".csv", ".tsv", ".xml", ".plist",
250
+ # media and fonts
251
+ ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp", ".bmp",
252
+ ".tif", ".tiff", ".avif", ".mp3", ".mp4", ".wav", ".ogg", ".webm",
253
+ ".mov", ".woff", ".woff2", ".ttf", ".otf", ".eot",
254
+ # documents, archives and binaries
255
+ ".pdf", ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".jar",
256
+ ".war", ".class", ".so", ".dll", ".dylib", ".a", ".o", ".obj", ".lib",
257
+ ".exe", ".bin", ".pyc", ".pyo", ".wasm", ".node",
258
+ # by-products
259
+ ".map", ".snap", ".log",
260
+ })
261
+ # Non-source files named without a source extension, lowercased. Every
262
+ # manifest skf-scan-manifests.py scans is non-source too.
263
+ NON_SOURCE_NAMES = frozenset({
264
+ "license", "licence", "copying", "notice", "authors", "contributors",
265
+ "changelog", "changes", "history", "readme", "codeowners", "owners",
266
+ "makefile", "gnumakefile", "dockerfile", "containerfile", "procfile",
267
+ "gemfile", "pipfile", "podfile", "brewfile", "vagrantfile",
268
+ "jenkinsfile", "go.mod", "go.work",
269
+ # build tools: Gradle and Maven wrappers and settings, Bazel files
270
+ "settings.gradle", "settings.gradle.kts", "gradlew", "gradlew.bat",
271
+ "mvnw", "mvnw.cmd", "build", "build.bazel", "workspace",
272
+ "workspace.bazel", "module.bazel",
273
+ })
274
+ # LICENSE-MIT, LICENSE-APACHE, Dockerfile.dev
275
+ NON_SOURCE_PREFIXES = ("license-", "licence-", "dockerfile.")
276
+ # Tool configuration named <tool>.config.<ext>: vite.config.ts,
277
+ # jest.config.js, babel.config.cjs, webpack.config.mjs
278
+ NON_SOURCE_INFIXES = (".config.",)
279
+
280
+ # The manifest scanner beside this script: own_manifest counts the file
281
+ # names it scans and `manifest` reads them with its identity readers, so
282
+ # the two never disagree on what a manifest is or what it names.
283
+ SCAN_MANIFESTS = Path(__file__).resolve().parent / "skf-scan-manifests.py"
284
+ _SCAN_MODULE = None
285
+ _MANIFEST_FILENAMES: frozenset[str] = frozenset()
286
+ _MANIFEST_FILENAMES_LC: frozenset[str] = frozenset()
287
+
288
+ # The git runner beside this script (no hooks, no git location variables,
289
+ # never a git shim in the current folder), which lists a boundary's files.
290
+ SOURCE_TREE = Path(__file__).resolve().parent / "skf-source-tree.py"
291
+ _SOURCE_TREE_MODULE = None
292
+
293
+ # Folders a walk outside a git work tree skips: installed dependencies and
294
+ # caches, never a boundary's own code, and often thousands of files.
295
+ WALK_SKIPPED_FOLDERS = frozenset({"node_modules", "__pycache__"})
296
+
297
+ # Entry files, lowercased; one counts at most ENTRY_POINT_MAX_DEPTH folders
298
+ # below the boundary root (src/index.ts, src/pkg/__init__.py, cmd/x/main.go).
299
+ ENTRY_POINT_NAMES = frozenset({
300
+ "index.js", "index.jsx", "index.mjs", "index.cjs", "index.ts",
301
+ "index.tsx", "index.mts", "index.cts", "index.php",
302
+ "main.js", "main.mjs", "main.cjs", "main.ts", "main.go", "main.rs",
303
+ "main.py", "main.swift", "main.java", "main.kt", "main.c", "main.cpp",
304
+ "main.dart",
305
+ "app.js", "app.ts", "app.py",
306
+ "lib.rs", "mod.ts", "__init__.py", "__main__.py",
307
+ })
308
+ ENTRY_POINT_MAX_DEPTH = 2
309
+
310
+ TEST_FOLDERS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
311
+ TEST_CONFIG_NAMES = frozenset({
312
+ "conftest.py", "pytest.ini", "phpunit.xml", "phpunit.xml.dist", ".rspec",
313
+ })
314
+ TEST_CONFIG_PREFIXES = (
315
+ "jest.config.", "vitest.config.", "karma.conf.", "playwright.config.",
316
+ "cypress.config.",
317
+ )
318
+ TEST_FILE_RE = re.compile(
319
+ r"(\.(test|spec)\.[^./]+$)|(_test\.[^./]+$)|(^test_.+\.py$)|(_spec\.rb$)"
320
+ )
128
321
 
129
- # Reason codes — exported as module-level constants so callers (and tests)
322
+ # CI configurations read for ci_reference, relative to --source-root.
323
+ CI_FILE_GLOBS = (
324
+ ".github/workflows/*.yml",
325
+ ".github/workflows/*.yaml",
326
+ ".gitlab-ci.yml",
327
+ ".circleci/config.yml",
328
+ ".travis.yml",
329
+ "azure-pipelines.yml",
330
+ "bitbucket-pipelines.yml",
331
+ ".buildkite/pipeline.yml",
332
+ "Jenkinsfile",
333
+ )
334
+
335
+ LARGE_DIRECTORY_FILES = 50
336
+
337
+ SIGNAL_KEYS = (
338
+ "own_manifest",
339
+ "entry_point",
340
+ "service_definition",
341
+ "readme",
342
+ "test_suite",
343
+ "ci_reference",
344
+ )
345
+
346
+
347
+ # Reason codes: exported as module-level constants so callers (and tests)
130
348
  # can reference them without string typos.
131
349
  REASON_TOO_FEW_FILES = "too-few-files"
132
350
  REASON_TOO_LOW_LOC = "too-low-loc"
@@ -146,11 +364,30 @@ ALL_REASONS = (
146
364
  # --------------------------------------------------------------------------
147
365
 
148
366
 
367
+ def normalize_boundary_path(name: str, path: str) -> str:
368
+ """`path` as one forward-slash path relative to the source root: `.`
369
+ for the root, no `./`, `//` or trailing slash. Raises ValueError for an
370
+ absolute path or one that leaves the source root."""
371
+ norm = posixpath.normpath(path.replace("\\", "/") or ".")
372
+ if (
373
+ norm.startswith("/")
374
+ or norm == ".."
375
+ or norm.startswith("../")
376
+ or re.match(r"^[A-Za-z]:", norm)
377
+ ):
378
+ raise ValueError(
379
+ f"boundary {name!r}: path {path!r} is not a folder below the source root"
380
+ )
381
+ return norm
382
+
383
+
149
384
  def validate_boundaries(boundaries: object) -> list[dict]:
150
- """Validate the boundaries input is a list of {name, path, files} records.
385
+ """Validate the boundaries input is a list of {name, path, files?} records.
151
386
 
152
- Returns the list with each entry's `files` normalized to forward-slash
153
- strings. Raises ValueError on any structural defect.
387
+ Returns the list with each entry's `path` normalized
388
+ (normalize_boundary_path) and its `files` normalized to forward-slash
389
+ strings, or None when the entry gives none (the helper lists it).
390
+ Raises ValueError on any structural defect.
154
391
  """
155
392
  if not isinstance(boundaries, list):
156
393
  raise ValueError(
@@ -172,10 +409,15 @@ def validate_boundaries(boundaries: object) -> list[dict]:
172
409
  raise ValueError(
173
410
  f"boundary entry {name!r} missing required `path` field"
174
411
  )
412
+ path = normalize_boundary_path(name, path)
175
413
  files = entry.get("files")
414
+ if files is None:
415
+ # Listed by the helper (list_boundary_files).
416
+ normalized.append({"name": name, "path": path, "files": None})
417
+ continue
176
418
  if not isinstance(files, list):
177
419
  raise ValueError(
178
- f"boundary entry {name!r} missing required `files` array"
420
+ f"boundary entry {name!r} has a `files` value that is not an array"
179
421
  )
180
422
  norm_files: list[str] = []
181
423
  for f_idx, f in enumerate(files):
@@ -186,35 +428,204 @@ def validate_boundaries(boundaries: object) -> list[dict]:
186
428
  norm_files.append(f.replace("\\", "/"))
187
429
  normalized.append({
188
430
  "name": name,
189
- "path": path.replace("\\", "/"),
431
+ "path": path,
190
432
  "files": norm_files,
191
433
  })
192
434
  return normalized
193
435
 
194
436
 
437
+ # --------------------------------------------------------------------------
438
+ # Listing a boundary's files
439
+ # --------------------------------------------------------------------------
440
+
441
+
442
+ def _source_tree_module():
443
+ """skf-source-tree.py, loaded once from beside this script, or None
444
+ when it cannot be loaded (the listing then walks the folder)."""
445
+ global _SOURCE_TREE_MODULE
446
+ if _SOURCE_TREE_MODULE is None:
447
+ spec = importlib.util.spec_from_file_location("skf_source_tree", SOURCE_TREE)
448
+ if spec is None or spec.loader is None or not SOURCE_TREE.is_file():
449
+ return None
450
+ module = importlib.util.module_from_spec(spec)
451
+ try:
452
+ spec.loader.exec_module(module)
453
+ except (OSError, ImportError, SyntaxError):
454
+ return None
455
+ _SOURCE_TREE_MODULE = module
456
+ return _SOURCE_TREE_MODULE
457
+
458
+
459
+ def _hidden(rel: str) -> bool:
460
+ return any(part.startswith(".") for part in rel.split("/"))
461
+
462
+
463
+ def _git_files(source_root: Path) -> list[str] | None:
464
+ """The files git lists under `source_root` (tracked, and untracked but
465
+ not ignored), relative to it, as git prints them (an unmerged path once
466
+ per stage); None when git is missing or the folder is in no work tree."""
467
+ module = _source_tree_module()
468
+ if module is None:
469
+ return None
470
+ res = module._git(source_root, "ls-files", "-z", "--cached", "--others", "--exclude-standard")
471
+ if res is None or res[0] != 0:
472
+ return None
473
+ files = []
474
+ for raw in res[1].split(b"\0"):
475
+ rel = os.fsdecode(raw).replace("\\", "/") if raw else ""
476
+ if rel and (source_root / rel).is_file() and not (source_root / rel).is_symlink():
477
+ files.append(rel)
478
+ return files
479
+
480
+
481
+ def _walked_files(source_root: Path) -> list[str]:
482
+ files = []
483
+ for dirpath, dirnames, filenames in os.walk(source_root):
484
+ rel_dir = os.path.relpath(dirpath, source_root).replace(os.sep, "/")
485
+ rel_dir = "" if rel_dir == "." else rel_dir
486
+ dirnames[:] = sorted(
487
+ d for d in dirnames
488
+ if not d.startswith(".") and d not in WALK_SKIPPED_FOLDERS
489
+ and not os.path.islink(os.path.join(dirpath, d))
490
+ )
491
+ for name in filenames:
492
+ if not os.path.islink(os.path.join(dirpath, name)):
493
+ files.append(f"{rel_dir}/{name}" if rel_dir else name)
494
+ return files
495
+
496
+
497
+ def list_source_files(source_root: Path) -> list[str]:
498
+ """Every file under `source_root` a boundary may hold, each once; see
499
+ the module docstring's Listing."""
500
+ listed = [rel for rel in _git_files(source_root) or [] if not _hidden(rel)]
501
+ if not listed:
502
+ # No git, no work tree, or a folder the outer work tree ignores
503
+ # (git lists nothing there): walk it.
504
+ listed = [rel for rel in _walked_files(source_root) if not _hidden(rel)]
505
+ # An unmerged path is listed once per stage during a conflicted merge.
506
+ return sorted(set(listed))
507
+
508
+
509
+ def list_boundary_files(source_files: list[str], boundary_path: str) -> list[str]:
510
+ """The files of `source_files` below the boundary folder (all of them
511
+ for a boundary at the root)."""
512
+ root = boundary_path.strip("/")
513
+ if root in ("", "."):
514
+ return list(source_files)
515
+ prefix = root + "/"
516
+ return [f for f in source_files if f.startswith(prefix)]
517
+
518
+
519
+ # --------------------------------------------------------------------------
520
+ # File kinds: lockfiles and non-source files
521
+ # --------------------------------------------------------------------------
522
+
523
+
524
+ def scan_manifests_module():
525
+ """skf-scan-manifests.py, loaded once from beside this script.
526
+ Raises ImportError when it cannot be loaded.
527
+ """
528
+ global _SCAN_MODULE, _MANIFEST_FILENAMES, _MANIFEST_FILENAMES_LC
529
+ if _SCAN_MODULE is None:
530
+ spec = importlib.util.spec_from_file_location(
531
+ "skf_scan_manifests", SCAN_MANIFESTS
532
+ )
533
+ if spec is None or spec.loader is None or not SCAN_MANIFESTS.is_file():
534
+ raise ImportError(f"cannot load {SCAN_MANIFESTS}")
535
+ module = importlib.util.module_from_spec(spec)
536
+ spec.loader.exec_module(module)
537
+ if not hasattr(module, "IDENTITY_READERS"):
538
+ raise ImportError(f"{SCAN_MANIFESTS} has no IDENTITY_READERS")
539
+ _MANIFEST_FILENAMES = frozenset(module.MANIFEST_ECOSYSTEMS)
540
+ _MANIFEST_FILENAMES_LC = frozenset(n.lower() for n in _MANIFEST_FILENAMES)
541
+ _SCAN_MODULE = module
542
+ return _SCAN_MODULE
543
+
544
+
545
+ def manifest_filenames() -> frozenset[str]:
546
+ """The manifest file names skf-scan-manifests.py scans (the keys of its
547
+ MANIFEST_ECOSYSTEMS). Raises ImportError when it cannot be loaded.
548
+ """
549
+ scan_manifests_module()
550
+ return _MANIFEST_FILENAMES
551
+
552
+
553
+ def _basename(file_path: str) -> str:
554
+ return file_path.replace("\\", "/").rsplit("/", 1)[-1]
555
+
556
+
557
+ def is_lockfile(file_path: str) -> bool:
558
+ """True for a dependency lockfile (by file name)."""
559
+ name = _basename(file_path).lower()
560
+ return name in LOCKFILE_NAMES or name.endswith(LOCKFILE_SUFFIXES)
561
+
562
+
563
+ def is_non_source(file_path: str) -> bool:
564
+ """True for documentation, data, configuration, media and binary files,
565
+ and for build and tool configuration (every manifest included).
566
+
567
+ Lockfiles are non-source too. A file with an extension the tables do
568
+ not list is a source file.
569
+ """
570
+ name = _basename(file_path).lower()
571
+ if not name or name.startswith("."):
572
+ return True
573
+ scan_manifests_module()
574
+ if (
575
+ is_lockfile(name)
576
+ or name in NON_SOURCE_NAMES
577
+ or name in _MANIFEST_FILENAMES_LC
578
+ or name.startswith(NON_SOURCE_PREFIXES)
579
+ or name.endswith(".dockerfile")
580
+ or any(infix in name for infix in NON_SOURCE_INFIXES)
581
+ ):
582
+ return True
583
+ ext = posixpath.splitext(name)[1]
584
+ return ext in NON_SOURCE_EXTENSIONS
585
+
586
+
195
587
  # --------------------------------------------------------------------------
196
588
  # Path-based detection (generated-code)
197
589
  # --------------------------------------------------------------------------
198
590
 
199
591
 
200
- def find_generated_segment(file_path: str) -> str | None:
201
- """Return the first generated-path segment that appears in `file_path`,
202
- or None if none match.
592
+ def find_generated_segment(
593
+ file_path: str, manifest_dirs: frozenset[str] | set[str] = frozenset()
594
+ ) -> str | None:
595
+ """The generated-folder segment that marks `file_path`, a path below the
596
+ boundary, or None.
203
597
 
204
- The match is on the raw string with forward slashes. We compare against
205
- `<segment>` at any position (start or interior); a `node_modules/` at
206
- the root counts the same as one nested deeper.
598
+ A vendored or cache folder counts at any depth (`node_modules/foo.js`
599
+ and `pkg/node_modules/foo.js` both hit). A build output folder counts
600
+ as the first folder, or inside a folder of `manifest_dirs`: the folders
601
+ below the boundary that hold a manifest ("" for the boundary root). The
602
+ leftmost folder that counts is returned.
207
603
  """
208
- norm = file_path.replace("\\", "/")
209
- # accept matches at the very start (no leading slash needed) too —
210
- # so `node_modules/foo.js` and `pkg/node_modules/foo.js` both hit.
211
- bracketed = "/" + norm # so segments can match at the head consistently
212
- for seg in GENERATED_PATH_SEGMENTS:
213
- if "/" + seg in bracketed:
604
+ parts = [p for p in file_path.replace("\\", "/").split("/")[:-1] if p != "."]
605
+ for depth, part in enumerate(parts):
606
+ seg = part + "/"
607
+ if seg in VENDORED_PATH_SEGMENTS:
608
+ return seg
609
+ if seg in OUTPUT_PATH_SEGMENTS and (
610
+ depth == 0 or "/".join(parts[:depth]) in manifest_dirs
611
+ ):
214
612
  return seg
215
613
  return None
216
614
 
217
615
 
616
+ def below_boundary(file_path: str, boundary_path: str) -> str:
617
+ """The part of `file_path` below the boundary folder.
618
+
619
+ A path that does not start with the boundary path (a boundary-relative
620
+ entry, or a boundary at the root) is returned whole.
621
+ """
622
+ root = boundary_path.strip("/")
623
+ if root in ("", "."):
624
+ return file_path
625
+ prefix = root + "/"
626
+ return file_path[len(prefix):] if file_path.startswith(prefix) else file_path
627
+
628
+
218
629
  # --------------------------------------------------------------------------
219
630
  # Header-sentinel scan (auto-generated-tag)
220
631
  # --------------------------------------------------------------------------
@@ -228,8 +639,8 @@ def has_auto_generated_marker(file_abs: Path) -> str | None:
228
639
  the AUTO_GEN_SENTINELS (case-insensitive substring match).
229
640
 
230
641
  Returns the matched sentinel (lowercased) on first hit, or None.
231
- Unreadable / non-UTF-8 files return None — header-sentinel detection
232
- is opportunistic; binary blobs will simply not match.
642
+ Unreadable / non-UTF-8 files return None: header-sentinel detection
643
+ is opportunistic, and binary blobs will simply not match.
233
644
  """
234
645
  try:
235
646
  with file_abs.open("r", encoding="utf-8", errors="replace") as fh:
@@ -265,6 +676,133 @@ def count_nonblank_lines(file_abs: Path) -> int:
265
676
  return 0
266
677
 
267
678
 
679
+ # --------------------------------------------------------------------------
680
+ # Signals
681
+ # --------------------------------------------------------------------------
682
+
683
+
684
+ def load_ci_files(source_root: Path) -> list[tuple[str, str]]:
685
+ """Read the CI configurations under `source_root`: (rel path, text) pairs,
686
+ sorted by path. Unreadable files are left out.
687
+ """
688
+ found: dict[str, str] = {}
689
+ for pattern in CI_FILE_GLOBS:
690
+ for path in source_root.glob(pattern):
691
+ if not path.is_file():
692
+ continue
693
+ try:
694
+ text = path.read_bytes().decode("utf-8", errors="replace")
695
+ except OSError:
696
+ continue
697
+ found[path.relative_to(source_root).as_posix()] = text
698
+ return sorted(found.items())
699
+
700
+
701
+ def _names_path(text: str, boundary_path: str) -> bool:
702
+ """True when `text` names `boundary_path` as a whole path: not as part
703
+ of a longer name (`packages/auth` is not named by `packages/auth-ui`).
704
+ """
705
+ pattern = r"(?<![\w-])" + re.escape(boundary_path) + r"(?![\w.-])"
706
+ return re.search(pattern, text) is not None
707
+
708
+
709
+ def _is_test_file(rel: str) -> bool:
710
+ parts = rel.split("/")
711
+ if any(part.lower() in TEST_FOLDERS for part in parts[:-1]):
712
+ return True
713
+ name = parts[-1].lower()
714
+ if name in TEST_CONFIG_NAMES or name.startswith(TEST_CONFIG_PREFIXES):
715
+ return True
716
+ return TEST_FILE_RE.search(name) is not None
717
+
718
+
719
+ def collect_signals(
720
+ files: list[str],
721
+ boundary_path: str,
722
+ ci_files: list[tuple[str, str]],
723
+ ) -> dict:
724
+ """The presence flags of one boundary; see the module docstring."""
725
+ signals: dict = {key: None for key in SIGNAL_KEYS}
726
+ manifests = manifest_filenames()
727
+ entry_depth: int | None = None
728
+ for f in sorted(files):
729
+ rel = below_boundary(f, boundary_path)
730
+ depth = rel.count("/")
731
+ name = _basename(rel)
732
+ lname = name.lower()
733
+ if depth == 0 and signals["own_manifest"] is None and name in manifests:
734
+ signals["own_manifest"] = f
735
+ if depth == 0 and signals["readme"] is None and lname.startswith("readme"):
736
+ signals["readme"] = f
737
+ if (
738
+ lname in ENTRY_POINT_NAMES
739
+ and depth <= ENTRY_POINT_MAX_DEPTH
740
+ and (entry_depth is None or depth < entry_depth)
741
+ ):
742
+ signals["entry_point"] = f
743
+ entry_depth = depth
744
+ if (
745
+ signals["service_definition"] is None
746
+ and scan_manifests_module().is_service_definition(name)
747
+ ):
748
+ signals["service_definition"] = f
749
+ if signals["test_suite"] is None and _is_test_file(rel):
750
+ signals["test_suite"] = f
751
+ root = boundary_path.strip("/")
752
+ if root not in ("", "."):
753
+ for ci_path, text in ci_files:
754
+ if _names_path(text, root):
755
+ signals["ci_reference"] = ci_path
756
+ break
757
+ signals["large_directory"] = len(files) >= LARGE_DIRECTORY_FILES
758
+ return signals
759
+
760
+
761
+ # --------------------------------------------------------------------------
762
+ # Own manifest
763
+ # --------------------------------------------------------------------------
764
+
765
+
766
+ def describe_manifest(file_abs: Path, file_rel: str) -> dict:
767
+ """`{path, ecosystem, name, private}` of one manifest, `name` and
768
+ `private` as skf-scan-manifests.py's identity reader for its file name
769
+ reports them (null when the file is unreadable or names no package).
770
+ """
771
+ module = scan_manifests_module()
772
+ file_name = _basename(file_rel)
773
+ record = {
774
+ "path": file_rel,
775
+ "ecosystem": module.MANIFEST_ECOSYSTEMS[file_name],
776
+ "name": None,
777
+ "private": None,
778
+ }
779
+ reader = module.IDENTITY_READERS.get(file_name)
780
+ if reader is None:
781
+ return record
782
+ try:
783
+ text = file_abs.read_bytes().decode("utf-8", errors="replace")
784
+ record["name"], record["private"] = reader(text)
785
+ except Exception: # noqa: BLE001 (a manifest that does not parse names nothing)
786
+ pass
787
+ return record
788
+
789
+
790
+ def own_manifest(
791
+ files: list[str], boundary_path: str, source_root: Path
792
+ ) -> dict | None:
793
+ """The boundary's own manifest: of the manifests at its root, the first
794
+ in path order that names a package, else the first; None without one.
795
+ """
796
+ names = manifest_filenames()
797
+ described = [
798
+ describe_manifest(_resolve_file(source_root, boundary_path, f), f)
799
+ for f in sorted(files)
800
+ if "/" not in below_boundary(f, boundary_path) and _basename(f) in names
801
+ ]
802
+ named = [m for m in described if m["name"]]
803
+ return (named or described or [None])[0]
804
+
805
+
268
806
  # --------------------------------------------------------------------------
269
807
  # Per-boundary evaluation
270
808
  # --------------------------------------------------------------------------
@@ -286,114 +824,141 @@ def _resolve_file(source_root: Path, boundary_path: str, file_rel: str) -> Path:
286
824
  candidate_b = source_root / boundary_path / file_rel
287
825
  if candidate_b.is_file():
288
826
  return candidate_b
289
- # neither exists — return the source-root form for missing-file accounting
827
+ # neither exists: return the source-root form for missing-file accounting
290
828
  return candidate_a
291
829
 
292
830
 
831
+ def classify_file(
832
+ file_rel: str,
833
+ boundary_path: str,
834
+ source_root: Path,
835
+ manifest_dirs: frozenset[str] | set[str] = frozenset(),
836
+ ) -> dict | None:
837
+ """The generated-file entry for one source file, or None when the file
838
+ is not generated. Call it for source files only (see is_non_source);
839
+ `manifest_dirs` is find_generated_segment's.
840
+ """
841
+ rel = below_boundary(file_rel, boundary_path)
842
+ seg = find_generated_segment(rel, manifest_dirs)
843
+ if seg:
844
+ return {"path": file_rel, "reason": REASON_GENERATED_CODE, "match": seg}
845
+ sentinel = has_auto_generated_marker(
846
+ _resolve_file(source_root, boundary_path, file_rel)
847
+ )
848
+ if sentinel:
849
+ return {
850
+ "path": file_rel, "reason": REASON_AUTO_GENERATED_TAG, "match": sentinel,
851
+ }
852
+ return None
853
+
854
+
293
855
  def evaluate_boundary(
294
856
  boundary: dict,
295
857
  *,
296
858
  source_root: Path,
297
859
  min_files: int = MIN_FILES,
298
860
  min_loc: int = MIN_LOC,
861
+ ci_files: list[tuple[str, str]] | None = None,
299
862
  ) -> dict:
300
863
  """Evaluate a single boundary against all disqualification rules.
301
864
 
302
- Returns a record:
303
- - kept: {"name", "path", "files_count", "loc_total"}
304
- - dropped: {"name", "reason", "context": {...}}
865
+ Returns {"kind": "kept" | "dropped", "record": {...}}; the record shape
866
+ is the module docstring's <record>, and a dropped one adds `reason` and
867
+ `context`.
305
868
 
306
869
  Check order:
307
- 1. generated-code (path segment match)
308
- 2. auto-generated-tag (header sentinel scan)
309
- 3. too-few-files (count < min_files)
310
- 4. too-low-loc (sum < min_loc)
870
+ 1. generated-code / auto-generated-tag (every source file generated)
871
+ 2. too-few-files (files_count < min_files)
872
+ 3. too-low-loc (loc_total < min_loc)
311
873
  """
312
874
  name = boundary["name"]
313
875
  path = boundary["path"]
314
- files = boundary["files"]
315
- files_count = len(files)
316
-
317
- # 1. generated-code — path-substring match (cheapest signal)
318
- for f in files:
319
- seg = find_generated_segment(f)
320
- if seg:
321
- return {
322
- "kind": "dropped",
323
- "record": {
324
- "name": name,
325
- "reason": REASON_GENERATED_CODE,
326
- "context": {
327
- "files_count": files_count,
328
- "first_match": f,
329
- "matched_segment": seg,
330
- },
331
- },
332
- }
333
-
334
- # 2. auto-generated-tag — header sentinel scan
335
- for f in files:
336
- abs_path = _resolve_file(source_root, path, f)
337
- sentinel = has_auto_generated_marker(abs_path)
338
- if sentinel:
339
- return {
340
- "kind": "dropped",
341
- "record": {
342
- "name": name,
343
- "reason": REASON_AUTO_GENERATED_TAG,
344
- "context": {
345
- "files_count": files_count,
346
- "first_match": f,
347
- "matched_sentinel": sentinel,
348
- },
349
- },
350
- }
351
-
352
- # Compute loc_total for the remaining checks (and for kept records).
876
+ files = list(dict.fromkeys(boundary["files"])) # a file listed twice counts once
877
+ if ci_files is None:
878
+ ci_files = load_ci_files(source_root)
879
+
880
+ manifests = manifest_filenames()
881
+ manifest_dirs = {
882
+ posixpath.dirname(below_boundary(f, path))
883
+ for f in files
884
+ if _basename(f) in manifests
885
+ }
886
+ sources = [f for f in files if not is_non_source(f)]
887
+ generated: list[dict] = []
888
+ plain: list[str] = []
889
+ for f in sources:
890
+ entry = classify_file(f, path, source_root, manifest_dirs)
891
+ if entry:
892
+ generated.append(entry)
893
+ else:
894
+ plain.append(f)
895
+ generated.sort(key=lambda g: g["path"])
896
+
353
897
  loc_total = 0
354
- for f in files:
355
- abs_path = _resolve_file(source_root, path, f)
356
- loc_total += count_nonblank_lines(abs_path)
898
+ for f in plain:
899
+ loc_total += count_nonblank_lines(_resolve_file(source_root, path, f))
900
+
901
+ # Would rules 2 and 3 pass with the generated files counted too? Their
902
+ # lines are read only when that decides it.
903
+ passes = len(plain) >= min_files and loc_total >= min_loc
904
+ if not passes and len(sources) >= min_files:
905
+ loc = loc_total
906
+ for g in generated:
907
+ if loc >= min_loc:
908
+ break
909
+ loc += count_nonblank_lines(_resolve_file(source_root, path, g["path"]))
910
+ passes = loc >= min_loc
911
+
912
+ manifest = own_manifest(files, path, source_root)
913
+ signals = collect_signals(files, path, ci_files)
914
+ signals["own_manifest"] = manifest["path"] if manifest else None
915
+ record = {
916
+ "name": name,
917
+ "path": path,
918
+ "files_count": len(plain),
919
+ "loc_total": loc_total,
920
+ "non_source_count": len(files) - len(sources),
921
+ "generated_files": generated,
922
+ "generated_ratio": (
923
+ round(len(generated) / len(sources), 3) if sources else 0.0
924
+ ),
925
+ "passes_with_generated": passes,
926
+ "manifest": manifest,
927
+ "signals": signals,
928
+ }
929
+
930
+ # 1. every source file generated
931
+ if sources and not plain:
932
+ by_path = [g for g in generated if g["reason"] == REASON_GENERATED_CODE]
933
+ if by_path:
934
+ reason, first = REASON_GENERATED_CODE, by_path[0]
935
+ key = "matched_segment"
936
+ else:
937
+ reason, first = REASON_AUTO_GENERATED_TAG, generated[0]
938
+ key = "matched_sentinel"
939
+ context = {"first_match": first["path"], key: first["match"]}
940
+ return {
941
+ "kind": "dropped",
942
+ "record": {**record, "reason": reason, "context": context},
943
+ }
357
944
 
358
- # 3. too-few-files
359
- if files_count < min_files:
945
+ # 2. too-few-files
946
+ if len(plain) < min_files:
360
947
  return {
361
948
  "kind": "dropped",
362
- "record": {
363
- "name": name,
364
- "reason": REASON_TOO_FEW_FILES,
365
- "context": {
366
- "files_count": files_count,
367
- "loc_total": loc_total,
368
- "threshold": min_files,
369
- },
370
- },
949
+ "record": {**record, "reason": REASON_TOO_FEW_FILES,
950
+ "context": {"threshold": min_files}},
371
951
  }
372
952
 
373
- # 4. too-low-loc
953
+ # 3. too-low-loc
374
954
  if loc_total < min_loc:
375
955
  return {
376
956
  "kind": "dropped",
377
- "record": {
378
- "name": name,
379
- "reason": REASON_TOO_LOW_LOC,
380
- "context": {
381
- "files_count": files_count,
382
- "loc_total": loc_total,
383
- "threshold": min_loc,
384
- },
385
- },
957
+ "record": {**record, "reason": REASON_TOO_LOW_LOC,
958
+ "context": {"threshold": min_loc}},
386
959
  }
387
960
 
388
- return {
389
- "kind": "kept",
390
- "record": {
391
- "name": name,
392
- "path": path,
393
- "files_count": files_count,
394
- "loc_total": loc_total,
395
- },
396
- }
961
+ return {"kind": "kept", "record": record}
397
962
 
398
963
 
399
964
  # --------------------------------------------------------------------------
@@ -401,32 +966,77 @@ def evaluate_boundary(
401
966
  # --------------------------------------------------------------------------
402
967
 
403
968
 
969
+ def fill_files(boundaries: list[dict], source_root: Path) -> list[dict]:
970
+ """The boundaries, each one without `files` given the files listed under
971
+ its path. Raises ValueError for such a boundary whose path is no folder
972
+ under `source_root`."""
973
+ source_files: list[str] | None = None
974
+ filled = []
975
+ for b in boundaries:
976
+ if b["files"] is None:
977
+ root = b["path"].strip("/")
978
+ if not (source_root / root).is_dir():
979
+ raise ValueError(
980
+ f"boundary {b['name']!r}: {root or '.'} is not a folder under the source root"
981
+ )
982
+ if source_files is None:
983
+ source_files = list_source_files(source_root)
984
+ b = {**b, "files": list_boundary_files(source_files, b["path"])}
985
+ filled.append(b)
986
+ return filled
987
+
988
+
989
+ def write_tree_file(tree_dir: Path, index: int, files: list[str]) -> Path:
990
+ """Write one boundary's file list, one path per line, as
991
+ `boundary-<index>.txt`; returns its path. A name that is not UTF-8
992
+ (decoded with surrogateescape) gets `?` for each byte that is not, so
993
+ the file stays UTF-8 for skf-detect-language.py."""
994
+ path = tree_dir / f"boundary-{index}.txt"
995
+ text = "".join(f"{f}\n" for f in sorted(set(files)))
996
+ path.write_bytes(text.encode("utf-8", errors="replace"))
997
+ return path
998
+
999
+
404
1000
  def filter_boundaries(
405
1001
  boundaries: list[dict],
406
1002
  *,
407
1003
  source_root: Path,
408
1004
  min_files: int = MIN_FILES,
409
1005
  min_loc: int = MIN_LOC,
1006
+ tree_dir: Path | None = None,
410
1007
  ) -> dict:
411
- """Run evaluate_boundary on each entry; aggregate kept/dropped/stats."""
1008
+ """Run evaluate_boundary on each entry; aggregate kept/dropped/stats.
1009
+
1010
+ A boundary without `files` is listed first (fill_files). With
1011
+ `tree_dir`, each boundary's file list is written there and its record
1012
+ names it in `tree_file`. Raises ValueError for a boundary path to list
1013
+ that is no folder, and OSError for a tree file that cannot be written.
1014
+ """
412
1015
  kept: list[dict] = []
413
1016
  dropped: list[dict] = []
414
1017
  by_reason: dict[str, int] = {r: 0 for r in ALL_REASONS}
1018
+ boundaries = fill_files(boundaries, source_root)
1019
+ ci_files = load_ci_files(source_root) if boundaries else []
1020
+ if tree_dir is not None and boundaries:
1021
+ tree_dir.mkdir(parents=True, exist_ok=True)
415
1022
 
416
- for b in boundaries:
1023
+ for index, b in enumerate(boundaries, 1):
417
1024
  result = evaluate_boundary(
418
1025
  b,
419
1026
  source_root=source_root,
420
1027
  min_files=min_files,
421
1028
  min_loc=min_loc,
1029
+ ci_files=ci_files,
422
1030
  )
1031
+ if tree_dir is not None:
1032
+ result["record"]["tree_file"] = write_tree_file(tree_dir, index, b["files"]).as_posix()
423
1033
  if result["kind"] == "kept":
424
1034
  kept.append(result["record"])
425
1035
  else:
426
1036
  dropped.append(result["record"])
427
1037
  by_reason[result["record"]["reason"]] += 1
428
1038
 
429
- # prune zero-count reasons from by_reason — keeps the report tight
1039
+ # prune zero-count reasons from by_reason: keeps the report tight
430
1040
  by_reason_compact = {r: n for r, n in by_reason.items() if n > 0}
431
1041
 
432
1042
  return {
@@ -435,6 +1045,7 @@ def filter_boundaries(
435
1045
  "stats": {
436
1046
  "kept": len(kept),
437
1047
  "dropped": len(dropped),
1048
+ "mixed": sum(1 for r in kept if r["generated_files"]),
438
1049
  "by_reason": by_reason_compact,
439
1050
  },
440
1051
  }
@@ -498,12 +1109,27 @@ def _cmd_filter(args: argparse.Namespace) -> int:
498
1109
  )
499
1110
  return 1
500
1111
 
501
- result = filter_boundaries(
502
- boundaries,
503
- source_root=source_root,
504
- min_files=args.min_files,
505
- min_loc=args.min_loc,
506
- )
1112
+ try:
1113
+ manifest_filenames()
1114
+ except (ImportError, OSError, SyntaxError, AttributeError) as exc:
1115
+ print(f"error: {exc}", file=sys.stderr)
1116
+ return 1
1117
+
1118
+ tree_dir = Path(args.tree_dir).resolve() if args.tree_dir else None
1119
+ try:
1120
+ result = filter_boundaries(
1121
+ boundaries,
1122
+ source_root=source_root,
1123
+ min_files=args.min_files,
1124
+ min_loc=args.min_loc,
1125
+ tree_dir=tree_dir,
1126
+ )
1127
+ except ValueError as exc:
1128
+ print(f"error: {exc}", file=sys.stderr)
1129
+ return 1
1130
+ except OSError as exc:
1131
+ print(f"error: cannot write a tree file under {tree_dir}: {exc}", file=sys.stderr)
1132
+ return 1
507
1133
  json.dump(result, sys.stdout, indent=2)
508
1134
  sys.stdout.write("\n")
509
1135
  return 0
@@ -514,8 +1140,10 @@ def _build_parser() -> argparse.ArgumentParser:
514
1140
  prog="skf-disqualify-candidates",
515
1141
  description=(
516
1142
  "Filter detected boundaries against the unit-detection "
517
- "disqualification list (too-few-files, too-low-loc, "
518
- "generated-code, auto-generated-tag)."
1143
+ "disqualification list (generated-code and auto-generated-tag "
1144
+ "when every source file is generated, too-few-files, "
1145
+ "too-low-loc), with each boundary's generated files and "
1146
+ "detection signals as evidence."
519
1147
  ),
520
1148
  )
521
1149
  sub = parser.add_subparsers(dest="cmd", required=True)
@@ -530,26 +1158,37 @@ def _build_parser() -> argparse.ArgumentParser:
530
1158
  help=(
531
1159
  "path to boundaries JSON, or '-' to read from stdin. "
532
1160
  "Shape: [{\"name\": \"<unit>\", \"path\": \"<rel>\", "
533
- "\"files\": [\"<rel>\", ...]}, ...]"
1161
+ "\"files\": [\"<rel>\", ...]}, ...]; without files the "
1162
+ "helper lists the files under the path"
534
1163
  ),
535
1164
  )
536
1165
  p_filter.add_argument(
537
1166
  "--source-root",
538
1167
  default=None,
539
1168
  help=(
540
- "project root for resolving file paths in `files[]` "
1169
+ "project root for listing a boundary's files, resolving file "
1170
+ "paths in `files[]` and the CI configurations ci_reference reads "
541
1171
  "(default: current working directory). Each file is resolved as "
542
1172
  "<source-root>/<file-rel> first, falling back to "
543
1173
  "<source-root>/<boundary-path>/<file-rel>."
544
1174
  ),
545
1175
  )
1176
+ p_filter.add_argument(
1177
+ "--tree-dir",
1178
+ default=None,
1179
+ help=(
1180
+ "folder to write each boundary's file list to, one path per line "
1181
+ "(boundary-<n>.txt, n its place in the input); each record names "
1182
+ "its file in tree_file"
1183
+ ),
1184
+ )
546
1185
  p_filter.add_argument(
547
1186
  "--min-files",
548
1187
  type=int,
549
1188
  default=MIN_FILES,
550
1189
  help=(
551
- f"minimum files for too-few-files (default {MIN_FILES}; "
552
- "matches unit-detection-heuristics.md)"
1190
+ f"minimum source files (not generated) for too-few-files "
1191
+ f"(default {MIN_FILES}; matches unit-detection-heuristics.md)"
553
1192
  ),
554
1193
  )
555
1194
  p_filter.add_argument(
@@ -557,8 +1196,8 @@ def _build_parser() -> argparse.ArgumentParser:
557
1196
  type=int,
558
1197
  default=MIN_LOC,
559
1198
  help=(
560
- f"minimum non-blank LoC for too-low-loc (default {MIN_LOC}; "
561
- "matches unit-detection-heuristics.md)"
1199
+ f"minimum non-blank LoC of those files for too-low-loc "
1200
+ f"(default {MIN_LOC}; matches unit-detection-heuristics.md)"
562
1201
  ),
563
1202
  )
564
1203
  p_filter.set_defaults(func=_cmd_filter)