bmad-module-skill-forge 2.2.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +49 -49
  3. package/docs/404.md +10 -8
  4. package/docs/_data/pinned.yaml +22 -13
  5. package/docs/_internal/STABILITY.md +58 -35
  6. package/docs/agents.md +39 -53
  7. package/docs/architecture.md +65 -51
  8. package/docs/bmad-synergy.md +53 -51
  9. package/docs/campaign.md +127 -75
  10. package/docs/concepts.md +34 -28
  11. package/docs/examples.md +70 -68
  12. package/docs/forge-auto.md +34 -31
  13. package/docs/getting-started.md +118 -57
  14. package/docs/how-it-works.md +25 -21
  15. package/docs/index.md +15 -15
  16. package/docs/skill-model.md +70 -59
  17. package/docs/troubleshooting.md +135 -18
  18. package/docs/verifying-a-skill.md +83 -57
  19. package/docs/why-skf.md +24 -22
  20. package/docs/workflows.md +274 -110
  21. package/package.json +18 -10
  22. package/src/forger/preferences.yaml +8 -1
  23. package/src/knowledge/agentskills-spec.md +2 -0
  24. package/src/knowledge/ccc-bridge.md +40 -24
  25. package/src/knowledge/confidence-tiers.md +9 -5
  26. package/src/knowledge/overview.md +3 -3
  27. package/src/knowledge/progressive-capability.md +8 -8
  28. package/src/knowledge/provenance-tracking.md +9 -5
  29. package/src/knowledge/qmd-registry.md +2 -2
  30. package/src/knowledge/skf-knowledge-index.csv +2 -2
  31. package/src/knowledge/skill-lifecycle.md +18 -11
  32. package/src/knowledge/split-body-strategy.md +0 -3
  33. package/src/knowledge/tool-resolution.md +4 -2
  34. package/src/knowledge/version-paths.md +101 -15
  35. package/src/module-help.csv +3 -3
  36. package/src/module.yaml +10 -0
  37. package/src/shared/data/ast-grep-recipes.yaml +608 -0
  38. package/src/shared/data/ide-context-files.json +30 -0
  39. package/src/shared/health-check.md +32 -12
  40. package/src/shared/references/description-guard-protocol.md +12 -9
  41. package/src/shared/references/feasibility-report-schema.md +19 -13
  42. package/src/shared/references/headless-gate-convention.md +55 -3
  43. package/src/shared/references/output-contract-schema.md +19 -8
  44. package/src/shared/references/pipeline-contracts.md +94 -29
  45. package/src/shared/references/tessl-review.md +77 -0
  46. package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
  47. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
  48. package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
  49. package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
  50. package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
  51. package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
  52. package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
  53. package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
  54. package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
  55. package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
  56. package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
  57. package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
  58. package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
  59. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
  60. package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
  61. package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
  62. package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
  63. package/src/shared/scripts/skf-atomic-write.py +30 -14
  64. package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
  65. package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
  66. package/src/shared/scripts/skf-check-unit-records.py +184 -0
  67. package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
  68. package/src/shared/scripts/skf-classify-changed-files.py +694 -0
  69. package/src/shared/scripts/skf-comention-pairs.py +1333 -123
  70. package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
  71. package/src/shared/scripts/skf-count-imports.py +1867 -0
  72. package/src/shared/scripts/skf-count-tokens.py +1 -1
  73. package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
  74. package/src/shared/scripts/skf-description-guard.py +99 -11
  75. package/src/shared/scripts/skf-detect-docs.py +694 -2
  76. package/src/shared/scripts/skf-detect-language.py +486 -107
  77. package/src/shared/scripts/skf-detect-registry.py +887 -0
  78. package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
  79. package/src/shared/scripts/skf-detect-tools.py +209 -27
  80. package/src/shared/scripts/skf-detect-workspaces.py +506 -30
  81. package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
  82. package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
  83. package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
  84. package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
  85. package/src/shared/scripts/skf-extract-public-api.py +4084 -85
  86. package/src/shared/scripts/skf-extraction-inventory.py +746 -0
  87. package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
  88. package/src/shared/scripts/skf-fetch-temporal.py +674 -0
  89. package/src/shared/scripts/skf-find-cycles.py +97 -29
  90. package/src/shared/scripts/skf-find-test-report.py +476 -0
  91. package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
  92. package/src/shared/scripts/skf-github-fetch.py +365 -0
  93. package/src/shared/scripts/skf-github-probe.py +661 -0
  94. package/src/shared/scripts/skf-hash-content.py +166 -12
  95. package/src/shared/scripts/skf-language-corpora.py +1 -1
  96. package/src/shared/scripts/skf-load-provenance.py +159 -27
  97. package/src/shared/scripts/skf-manifest-ops.py +1 -1
  98. package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
  99. package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
  100. package/src/shared/scripts/skf-names-present.py +221 -0
  101. package/src/shared/scripts/skf-pair-intersect.py +119 -28
  102. package/src/shared/scripts/skf-parse-gaps.py +1384 -0
  103. package/src/shared/scripts/skf-preapply.py +1 -1
  104. package/src/shared/scripts/skf-preflight.py +161 -27
  105. package/src/shared/scripts/skf-promote-staged.py +430 -0
  106. package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
  107. package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
  108. package/src/shared/scripts/skf-quick-batch.py +774 -0
  109. package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
  110. package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
  111. package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
  112. package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
  113. package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
  114. package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
  115. package/src/shared/scripts/skf-resolve-package.py +523 -106
  116. package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
  117. package/src/shared/scripts/skf-run-lock.py +577 -0
  118. package/src/shared/scripts/skf-scan-manifests.py +998 -57
  119. package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
  120. package/src/shared/scripts/skf-severity-classify.py +712 -108
  121. package/src/shared/scripts/skf-shape-detect.py +298 -204
  122. package/src/shared/scripts/skf-shard-body.py +9 -8
  123. package/src/shared/scripts/skf-skill-inventory.py +2412 -89
  124. package/src/shared/scripts/skf-skills-module.py +472 -0
  125. package/src/shared/scripts/skf-source-tree.py +1796 -0
  126. package/src/shared/scripts/skf-structural-diff.py +670 -119
  127. package/src/shared/scripts/skf-tessl-review.py +706 -0
  128. package/src/shared/scripts/skf-update-active-symlink.py +5 -5
  129. package/src/shared/scripts/skf-update-run-state.py +624 -0
  130. package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
  131. package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
  132. package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
  133. package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
  134. package/src/shared/scripts/skf-validate-output.py +712 -64
  135. package/src/shared/scripts/skf-validate-pins.py +1 -1
  136. package/src/shared/scripts/skf-verify-no-trace.py +116 -10
  137. package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
  138. package/src/shared/scripts/skf-write-skill-brief.py +367 -19
  139. package/src/shared/tool-requirements.yaml +201 -0
  140. package/src/skf-analyze-source/SKILL.md +41 -36
  141. package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
  142. package/src/skf-analyze-source/customize.toml +36 -24
  143. package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
  144. package/src/skf-analyze-source/references/continue.md +2 -13
  145. package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
  146. package/src/skf-analyze-source/references/generate-briefs.md +121 -55
  147. package/src/skf-analyze-source/references/headless-contract.md +42 -15
  148. package/src/skf-analyze-source/references/identify-units.md +96 -110
  149. package/src/skf-analyze-source/references/init.md +73 -71
  150. package/src/skf-analyze-source/references/map-and-detect.md +123 -105
  151. package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
  152. package/src/skf-analyze-source/references/recommend.md +51 -70
  153. package/src/skf-analyze-source/references/scan-project.md +60 -84
  154. package/src/skf-analyze-source/references/scan-root.md +39 -0
  155. package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
  156. package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
  157. package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
  158. package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
  159. package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
  160. package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
  161. package/src/skf-audit-skill/SKILL.md +43 -51
  162. package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
  163. package/src/skf-audit-skill/customize.toml +41 -20
  164. package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
  165. package/src/skf-audit-skill/references/doc-drift.md +141 -0
  166. package/src/skf-audit-skill/references/headless-contract.md +29 -0
  167. package/src/skf-audit-skill/references/init.md +148 -210
  168. package/src/skf-audit-skill/references/re-index.md +96 -108
  169. package/src/skf-audit-skill/references/report.md +96 -33
  170. package/src/skf-audit-skill/references/semantic-diff.md +27 -18
  171. package/src/skf-audit-skill/references/severity-classify.md +79 -65
  172. package/src/skf-audit-skill/references/severity-rules.md +15 -6
  173. package/src/skf-audit-skill/references/structural-diff.md +75 -90
  174. package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
  175. package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
  176. package/src/skf-brief-skill/SKILL.md +36 -25
  177. package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
  178. package/src/skf-brief-skill/assets/scope-templates.md +17 -24
  179. package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
  180. package/src/skf-brief-skill/customize.toml +48 -20
  181. package/src/skf-brief-skill/references/analyze-target.md +142 -125
  182. package/src/skf-brief-skill/references/confirm-brief.md +10 -8
  183. package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
  184. package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
  185. package/src/skf-brief-skill/references/gather-intent.md +87 -182
  186. package/src/skf-brief-skill/references/headless-args.md +84 -16
  187. package/src/skf-brief-skill/references/health-check.md +1 -1
  188. package/src/skf-brief-skill/references/invocation-contract.md +18 -13
  189. package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
  190. package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
  191. package/src/skf-brief-skill/references/scope-definition.md +123 -47
  192. package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
  193. package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
  194. package/src/skf-brief-skill/references/version-resolution.md +3 -2
  195. package/src/skf-brief-skill/references/write-brief.md +66 -57
  196. package/src/skf-campaign/SKILL.md +31 -34
  197. package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
  198. package/src/skf-campaign/customize.toml +47 -16
  199. package/src/skf-campaign/references/campaign-contracts.md +39 -29
  200. package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
  201. package/src/skf-campaign/references/health-check.md +7 -16
  202. package/src/skf-campaign/references/step-01-setup.md +46 -84
  203. package/src/skf-campaign/references/step-02-strategy.md +31 -30
  204. package/src/skf-campaign/references/step-03-pins.md +26 -18
  205. package/src/skf-campaign/references/step-04-provenance.md +24 -23
  206. package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
  207. package/src/skf-campaign/references/step-06-batch.md +41 -34
  208. package/src/skf-campaign/references/step-07-capstone.md +40 -25
  209. package/src/skf-campaign/references/step-08-verify.md +28 -29
  210. package/src/skf-campaign/references/step-09-refine.md +29 -30
  211. package/src/skf-campaign/references/step-10-export.md +48 -36
  212. package/src/skf-campaign/references/step-11-maintenance.md +31 -34
  213. package/src/skf-campaign/references/step-resume.md +37 -68
  214. package/src/skf-campaign/scripts/campaign-deps.py +57 -18
  215. package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
  216. package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
  217. package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
  218. package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
  219. package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
  220. package/src/skf-campaign/scripts/campaign-report.py +204 -37
  221. package/src/skf-campaign/scripts/campaign-state.py +1128 -0
  222. package/src/skf-campaign/scripts/campaign-status.py +10 -74
  223. package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
  224. package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
  225. package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
  226. package/src/skf-campaign/templates/campaign-report-template.md +5 -1
  227. package/src/skf-campaign/templates/kickoff-template.md +0 -14
  228. package/src/skf-create-skill/SKILL.md +27 -20
  229. package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
  230. package/src/skf-create-skill/assets/skill-sections.md +15 -5
  231. package/src/skf-create-skill/customize.toml +20 -8
  232. package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
  233. package/src/skf-create-skill/references/batch-mode.md +81 -0
  234. package/src/skf-create-skill/references/ccc-index-check.md +27 -0
  235. package/src/skf-create-skill/references/compile.md +107 -29
  236. package/src/skf-create-skill/references/component-extraction.md +113 -91
  237. package/src/skf-create-skill/references/enrich.md +41 -23
  238. package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
  239. package/src/skf-create-skill/references/extract.md +200 -161
  240. package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
  241. package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
  242. package/src/skf-create-skill/references/extraction-patterns.md +584 -196
  243. package/src/skf-create-skill/references/generate-artifacts.md +145 -93
  244. package/src/skf-create-skill/references/health-check.md +1 -1
  245. package/src/skf-create-skill/references/load-brief.md +53 -36
  246. package/src/skf-create-skill/references/relabel-rule.md +12 -0
  247. package/src/skf-create-skill/references/report.md +28 -59
  248. package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
  249. package/src/skf-create-skill/references/step-auto-shard.md +20 -93
  250. package/src/skf-create-skill/references/step-doc-rot.md +44 -60
  251. package/src/skf-create-skill/references/step-doc-sources.md +28 -19
  252. package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
  253. package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
  254. package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
  255. package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
  256. package/src/skf-create-skill/references/validate.md +173 -99
  257. package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
  258. package/src/skf-create-stack-skill/SKILL.md +29 -56
  259. package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
  260. package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
  261. package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
  262. package/src/skf-create-stack-skill/customize.toml +23 -12
  263. package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
  264. package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
  265. package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
  266. package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
  267. package/src/skf-create-stack-skill/references/generate-output.md +138 -128
  268. package/src/skf-create-stack-skill/references/init.md +90 -32
  269. package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
  270. package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
  271. package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
  272. package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
  273. package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
  274. package/src/skf-create-stack-skill/references/report.md +39 -33
  275. package/src/skf-create-stack-skill/references/validate.md +20 -39
  276. package/src/skf-drop-skill/SKILL.md +28 -51
  277. package/src/skf-drop-skill/customize.toml +44 -30
  278. package/src/skf-drop-skill/references/execute.md +79 -146
  279. package/src/skf-drop-skill/references/invocation-contract.md +64 -0
  280. package/src/skf-drop-skill/references/report.md +46 -30
  281. package/src/skf-drop-skill/references/select.md +118 -124
  282. package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
  283. package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
  284. package/src/skf-export-skill/SKILL.md +40 -62
  285. package/src/skf-export-skill/assets/managed-section-format.md +14 -50
  286. package/src/skf-export-skill/assets/snippet-format.md +3 -1
  287. package/src/skf-export-skill/customize.toml +42 -13
  288. package/src/skf-export-skill/references/generate-snippet.md +54 -46
  289. package/src/skf-export-skill/references/health-check.md +2 -0
  290. package/src/skf-export-skill/references/invocation-contract.md +15 -0
  291. package/src/skf-export-skill/references/load-skill.md +88 -62
  292. package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
  293. package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
  294. package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
  295. package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
  296. package/src/skf-export-skill/references/package.md +15 -73
  297. package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
  298. package/src/skf-export-skill/references/result-envelope.md +31 -16
  299. package/src/skf-export-skill/references/summary.md +41 -17
  300. package/src/skf-export-skill/references/token-report.md +4 -12
  301. package/src/skf-export-skill/references/update-context.md +118 -237
  302. package/src/skf-forger/SKILL.md +61 -37
  303. package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
  304. package/src/skf-forger/customize.toml +43 -0
  305. package/src/skf-forger/references/pipeline-mode.md +73 -18
  306. package/src/skf-forger/scripts/forge-status.py +299 -0
  307. package/src/skf-forger/scripts/parse-pipeline.py +322 -54
  308. package/src/skf-forger/scripts/pipeline-gate.py +428 -0
  309. package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
  310. package/src/skf-quick-skill/SKILL.md +54 -53
  311. package/src/skf-quick-skill/assets/skill-template.md +4 -2
  312. package/src/skf-quick-skill/customize.toml +51 -20
  313. package/src/skf-quick-skill/references/batch-mode.md +74 -21
  314. package/src/skf-quick-skill/references/compile.md +30 -31
  315. package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
  316. package/src/skf-quick-skill/references/finalize.md +69 -18
  317. package/src/skf-quick-skill/references/halt-contract.md +41 -29
  318. package/src/skf-quick-skill/references/health-check.md +5 -3
  319. package/src/skf-quick-skill/references/quick-extract.md +114 -45
  320. package/src/skf-quick-skill/references/registry-resolution.md +9 -64
  321. package/src/skf-quick-skill/references/resolve-target.md +116 -66
  322. package/src/skf-quick-skill/references/write-and-validate.md +79 -40
  323. package/src/skf-refine-architecture/SKILL.md +43 -53
  324. package/src/skf-refine-architecture/customize.toml +49 -20
  325. package/src/skf-refine-architecture/references/compile.md +97 -54
  326. package/src/skf-refine-architecture/references/exit-codes.md +30 -0
  327. package/src/skf-refine-architecture/references/finding-storage.md +21 -0
  328. package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
  329. package/src/skf-refine-architecture/references/improvements.md +14 -20
  330. package/src/skf-refine-architecture/references/init.md +98 -57
  331. package/src/skf-refine-architecture/references/issue-detection.md +32 -39
  332. package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
  333. package/src/skf-refine-architecture/references/report.md +65 -54
  334. package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
  335. package/src/skf-rename-skill/SKILL.md +19 -54
  336. package/src/skf-rename-skill/customize.toml +32 -26
  337. package/src/skf-rename-skill/references/execute.md +230 -184
  338. package/src/skf-rename-skill/references/exit-codes.md +7 -7
  339. package/src/skf-rename-skill/references/health-check.md +1 -7
  340. package/src/skf-rename-skill/references/invocation-contract.md +25 -0
  341. package/src/skf-rename-skill/references/report.md +63 -52
  342. package/src/skf-rename-skill/references/select.md +197 -115
  343. package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
  344. package/src/skf-setup/SKILL.md +26 -26
  345. package/src/skf-setup/customize.toml +14 -10
  346. package/src/skf-setup/references/auto-index.md +50 -63
  347. package/src/skf-setup/references/ccc-index.md +29 -90
  348. package/src/skf-setup/references/detect-and-tier.md +58 -67
  349. package/src/skf-setup/references/health-check.md +2 -0
  350. package/src/skf-setup/references/invocation-contract.md +14 -0
  351. package/src/skf-setup/references/report.md +43 -166
  352. package/src/skf-setup/references/tier-rules.md +5 -5
  353. package/src/skf-setup/references/write-config.md +34 -57
  354. package/src/skf-test-skill/SKILL.md +17 -57
  355. package/src/skf-test-skill/assets/output-section-formats.md +33 -34
  356. package/src/skf-test-skill/customize.toml +27 -13
  357. package/src/skf-test-skill/references/coherence-check.md +134 -62
  358. package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
  359. package/src/skf-test-skill/references/coverage-check.md +217 -254
  360. package/src/skf-test-skill/references/detect-mode.md +5 -5
  361. package/src/skf-test-skill/references/external-validators.md +94 -82
  362. package/src/skf-test-skill/references/health-check.md +6 -7
  363. package/src/skf-test-skill/references/init.md +163 -88
  364. package/src/skf-test-skill/references/invocation-contract.md +66 -0
  365. package/src/skf-test-skill/references/migration-section-rules.md +5 -2
  366. package/src/skf-test-skill/references/report.md +162 -142
  367. package/src/skf-test-skill/references/score.md +80 -107
  368. package/src/skf-test-skill/references/scoring-rules.md +41 -155
  369. package/src/skf-test-skill/references/source-access-protocol.md +19 -92
  370. package/src/skf-test-skill/references/step-hard-gate.md +51 -33
  371. package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
  372. package/src/skf-test-skill/scripts/build-result-context.py +287 -0
  373. package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
  374. package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
  375. package/src/skf-test-skill/scripts/compute-score.py +223 -45
  376. package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
  377. package/src/skf-test-skill/scripts/hard-gate.py +203 -0
  378. package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
  379. package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
  380. package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
  381. package/src/skf-test-skill/scripts/score-signatures.py +459 -0
  382. package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
  383. package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
  384. package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
  385. package/src/skf-test-skill/templates/test-report-template.md +21 -15
  386. package/src/skf-update-skill/SKILL.md +38 -25
  387. package/src/skf-update-skill/customize.toml +27 -12
  388. package/src/skf-update-skill/references/detect-changes.md +233 -222
  389. package/src/skf-update-skill/references/gap-driven.md +336 -0
  390. package/src/skf-update-skill/references/health-check.md +31 -5
  391. package/src/skf-update-skill/references/init.md +250 -84
  392. package/src/skf-update-skill/references/invocation-contract.md +12 -0
  393. package/src/skf-update-skill/references/manual-section-rules.md +6 -23
  394. package/src/skf-update-skill/references/merge.md +180 -47
  395. package/src/skf-update-skill/references/re-extract.md +53 -262
  396. package/src/skf-update-skill/references/report.md +100 -35
  397. package/src/skf-update-skill/references/write.md +331 -149
  398. package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
  399. package/src/skf-verify-stack/SKILL.md +23 -42
  400. package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
  401. package/src/skf-verify-stack/customize.toml +55 -22
  402. package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
  403. package/src/skf-verify-stack/references/coverage.md +67 -64
  404. package/src/skf-verify-stack/references/exit-codes.md +21 -8
  405. package/src/skf-verify-stack/references/init.md +107 -62
  406. package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
  407. package/src/skf-verify-stack/references/integrations.md +106 -65
  408. package/src/skf-verify-stack/references/report.md +59 -42
  409. package/src/skf-verify-stack/references/requirements.md +32 -9
  410. package/src/skf-verify-stack/references/synthesize.md +53 -39
  411. package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
  412. package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
  413. package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
  414. package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
  415. package/tools/cli/commands/install.js +11 -4
  416. package/tools/cli/commands/status.js +15 -3
  417. package/tools/cli/commands/update.js +11 -4
  418. package/tools/cli/lib/tool-check.js +320 -0
  419. package/tools/cli/lib/version-check.js +22 -9
  420. package/tools/cli/skf-cli.js +4 -1
  421. package/.gitattributes +0 -16
  422. package/.nvmrc +0 -1
  423. package/docs/_internal/RELEASING.md +0 -679
  424. package/release-audits/v1.0.0-launch-audit.md +0 -728
  425. package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
  426. package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
  427. package/src/skf-campaign/manifest.yaml +0 -15
  428. package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
  429. package/src/skf-create-skill/references/ecosystem-check.md +0 -74
  430. package/src/skf-drop-skill/references/headless-contract.md +0 -18
  431. package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
  432. package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
  433. package/src/skf-update-skill/references/validate.md +0 -124
  434. package/tools/validate-docs-drift.js +0 -233
  435. package/tools/validate-docs-links.js +0 -665
@@ -1,48 +1,26 @@
1
1
  # /// script
2
- # requires-python = ">=3.10"
2
+ # requires-python = ">=3.11"
3
3
  # dependencies = ["pyyaml"]
4
4
  # ///
5
- """SKF Merge CCC Exclusions — Set-union merge of SKF exclusion patterns.
6
-
7
- Replaces the prose-driven validation + merge logic in `src/skf-setup/
8
- references/ccc-index.md` §2b with one Python invocation. Reads
9
- `.cocoindex_code/settings.yml`, applies PR #248's config-value
10
- validation rules to reject inputs that would silently produce a
11
- malformed glob (the empty-value-becomes-`**/`-matches-everything bug
12
- that would silently exclude the entire repo from indexing), and
13
- performs an idempotent set-union merge of the SKF exclusion list into
14
- the existing `exclude_patterns` array.
15
-
16
- Always-included patterns (4 hardcoded — no config needed):
17
-
18
- **/_bmad SKF framework module (workflows, agents, knowledge)
19
- **/_bmad-output Build output artifacts
20
- **/.claude Claude Code configuration
21
- **/_skf-learn SKF learning materials
22
-
23
- Conditionally-included patterns (2 from config — validated before use):
24
-
25
- **/{skills_output_folder} Generated skill files
26
- **/{forge_data_folder} Compilation workspace
27
-
28
- Validation rules for the 2 conditional values (PR #248):
29
-
30
- - REJECT empty or whitespace-only values
31
- → would produce `**/`, matching every path → ccc indexes nothing
32
- - REJECT values starting with `/`, `~/`, or `./`
33
- → produces `**//abs/path`, `**/~/x`, or `**/./rel` — malformed glob
34
- - REJECT values containing glob meta-characters (`*`, `?`, `[`)
35
- → interpolation collides with the surrounding pattern syntax
36
- - REJECT values containing unresolved `{project-root}`-style placeholders
37
- (any `{` or `}` character) — produces `**/{project-root}/x`, which
38
- cocoindex accepts as a literal-segment glob and silently never matches.
39
- This is the bug from issue #293: the step file was passing template
40
- strings straight through.
41
-
42
- Rejected values are SKIPPED (the pattern is not added) and a warning
43
- is appended to the output. The 4 always-include patterns are applied
44
- unconditionally — config-value rejection cannot disable indexing
45
- entirely.
5
+ """SKF Merge CCC Exclusions — prepare .cocoindex_code/settings.yml and keep
6
+ the SKF exclusion patterns in it current.
7
+
8
+ One invocation from setup step 1b (`src/skf-setup/references/ccc-index.md`)
9
+ owns every decision about the project's ccc settings file, so none of it
10
+ lives in step prose:
11
+
12
+ 1. run `ccc init -f` when settings.yml is missing, and rebuild a file that
13
+ lacks the ccc default exclusions;
14
+ 2. validate the two folder values from `_bmad/skf/config.yaml`;
15
+ 3. exclude each folder whole, or entry by entry when it also holds
16
+ content SKF did not generate; leave out a folder with no SKF output,
17
+ and leave alone a folder the user already excluded;
18
+ 4. merge the SKF patterns and prune the SKF-owned patterns the current
19
+ config no longer produces;
20
+ 5. warn when `/.cocoindex_code/` is not gitignored;
21
+ 6. return one `index_action` telling the step whether to run `ccc index`,
22
+ and with --build-index act on it: run `ccc index` itself, read the
23
+ file count from `ccc status` and stamp the time (Index build, below).
46
24
 
47
25
  CLI — invoke via `uv run` so the PEP 723 PyYAML dependency declared
48
26
  above is auto-resolved on first call and cached. `docs/getting-started.md`
@@ -53,43 +31,432 @@ on a fresh interpreter:
53
31
  uv run skf-merge-ccc-exclusions.py \\
54
32
  --project-root /abs/path \\
55
33
  --skills-output-folder skills \\
56
- --forge-data-folder _bmad-output/forge-data
34
+ --forge-data-folder _bmad-output/forge-data \\
35
+ --prior-state-from /abs/path/_bmad/_memory/forger-sidecar/forge-tier.yaml \\
36
+ --index-fresh false \\
37
+ --skip-index false
38
+
39
+ uv run skf-merge-ccc-exclusions.py \\
40
+ --project-root /abs/path \\
41
+ --config /abs/path/_bmad/skf/config.yaml \\
42
+ --prior-state-from /abs/path/_bmad/_memory/forger-sidecar/forge-tier.yaml \\
43
+ --index-fresh false --skip-index false \\
44
+ --build-index --result-to /abs/run/ccc-exclusions.json
45
+
46
+ Flags:
47
+
48
+ --project-root required; the directory holding .cocoindex_code/
49
+ --skills-output-folder raw skills_output_folder config value, forwarded
50
+ verbatim (`{project-root}/...` is resolved here)
51
+ --forge-data-folder raw forge_data_folder config value, same handling
52
+ --config instead of the two folder flags: the
53
+ `_bmad/skf/config.yaml` to read both raw values
54
+ from, so no caller forwards them by hand (a key
55
+ the file lacks reads as an empty value)
56
+ --prior-state-from forge-tier.yaml; its ccc_index.exclude_patterns is
57
+ the record of the patterns SKF owns
58
+ --index-fresh true|false (case-insensitive); the prior index is
59
+ still fresh, so an unchanged settings.yml keeps it
60
+ --skip-index true|false; the setup run opted out of indexing
61
+ --defer-index a due index build waits for a later run: setup
62
+ passes it when the run missed its --require-tier,
63
+ which ends tier_failure (Index decision, below)
64
+ --no-ccc-init never run ccc init (tests and diagnostics)
65
+ --build-index setup mode only: act on index_action and add the
66
+ `index` result (Index build, below)
67
+ --result-to also write the JSON result, or the JSON error of a
68
+ run that fails, to this file, so the caller's
69
+ next steps read it from there; with
70
+ --build-index it is written before `ccc index`
71
+ runs too (Index build, below)
72
+ --clone-root instead of --project-root: an SKF workspace clone
73
+ (clone mode, below)
74
+ --include-ext clone mode only, repeatable: a file extension whose
75
+ files ccc must index (`ex`, `.ex`)
76
+
77
+ Clone mode (create-skill step 3 and step 7) prepares the settings.yml of an
78
+ SKF workspace clone, a source repository rather than an SKF project, so the
79
+ list merges its step prose used to make by hand live here too:
80
+
81
+ uv run skf-merge-ccc-exclusions.py --clone-root /abs/clone \\
82
+ [--include-ext ex --include-ext exs]
83
+
84
+ 1. run `ccc init -f` when settings.yml is missing, so the clone gets a
85
+ project of its own instead of an enclosing one's;
86
+ 2. quote each bare `- *...` list item an earlier hand edit left, which
87
+ YAML reads as an alias, so ccc can load the file again;
88
+ 3. append each standard build and dependency exclusion (CLONE_EXCLUDES)
89
+ that exclude_patterns lacks, keeping every entry already there;
90
+ 4. with --include-ext, append `**/*.<ext>` to include_patterns for each
91
+ extension no include_patterns entry matches, reading each entry as
92
+ ccc's glob library does.
93
+
94
+ A file with no exclude_patterns list, or no include_patterns list, gets
95
+ nothing added to that list: a list written from scratch replaces the ccc
96
+ defaults. Clone mode never removes an entry, never rebuilds the file, and
97
+ takes none of the folder, record or index flags of setup mode. Its writes
98
+ go through a temporary file of their own, so two runs merging the same
99
+ clone never tear the file; they add the same entries.
100
+
101
+ Settings states (target = {project-root}/.cocoindex_code/settings.yml):
102
+
103
+ missing `ccc init -f` creates it (ccc_init "created"). With
104
+ --no-ccc-init, or when ccc init leaves no file (ccc_init
105
+ "failed"), settings_ready is false and index_action is "fail":
106
+ a later `ccc index` would auto-initialise at the enclosing git
107
+ root and index it without the SKF exclusions. When the created
108
+ file has no exclude_patterns list, SKF adds nothing to it.
109
+ rebuild the file is empty, has no `exclude_patterns` key, or has
110
+ neither an `include_patterns` key nor `**/.cocoindex_code` in
111
+ its exclusions — every file ccc writes has both, so such a file
112
+ suppresses the ccc defaults (hidden folders, node_modules, the
113
+ ccc database). The original is moved aside to
114
+ `settings.yml.skf-repair`, `ccc init -f` writes a fresh file,
115
+ and the result is the fresh file plus every original top-level
116
+ key plus every original exclude entry not already there
117
+ (ccc_init "rebuilt"). The backup is removed only after the
118
+ write succeeds; a leftover backup found on startup is restored
119
+ and the repair runs again. If ccc init produces no usable file,
120
+ the original bytes are restored, ccc_init is "failed", and no
121
+ SKF change is applied that run; a fresh prior index is kept, but
122
+ no new index is built from that file (index_action "keep" or
123
+ "fail"). With --no-ccc-init the rebuild is skipped with a
124
+ warning and the file is left alone.
125
+ ready any other file is reconciled in place (ccc_init "not_needed").
126
+
127
+ The script never writes settings.yml from scratch. `ccc init -f` is the
128
+ only creator: `-f` merely skips ccc's parent-marker check, so one call
129
+ covers the git-root, nested-folder and repair cases, and on an existing
130
+ file ccc prints "Project already initialized." and changes nothing. At the
131
+ top of a git checkout ccc also adds `/.cocoindex_code/` to `.gitignore`;
132
+ SKF never writes `.gitignore` itself.
133
+
134
+ Patterns SKF owns:
57
135
 
58
- Output (single JSON document on stdout):
136
+ **/_bmad SKF framework module (workflows, agents, knowledge)
137
+ **/_bmad-output Build output artifacts
138
+ **/.claude Claude Code configuration
139
+ **/_skf-learn SKF learning materials
140
+ {skills_output_folder} Generated skill files (default `skills`)
141
+ {forge_data_folder} Compilation workspace
142
+
143
+ The four `**/` patterns are always produced. The two folder patterns are
144
+ the project-relative value itself, anchored to the project root: ccc reads
145
+ `skills` as that one root folder, while `**/skills` would also drop every
146
+ nested folder with the same name.
147
+
148
+ A folder that also holds content SKF did not generate gets per-entry
149
+ patterns instead of the bare value, each anchored the same way:
150
+
151
+ {folder}/<entry> each SKF-owned first-level directory or
152
+ root file, named as it is on disk
153
+ {folder}/<fixed entry> always, present or not: `_batch` and
154
+ `.export-manifest.json` (skills),
155
+ `_campaign` and `improvement-queue` (forge)
156
+ {folder}/<stem>-result*.json always for the `export-skill` and
157
+ `drop-skill` stems (skills), and for the
158
+ stem of any other SKF result at the root
159
+ {folder}/<prefix>* forge only, one per SKF report prefix
160
+
161
+ Glob characters in an entry name are written as one-character classes
162
+ (`[x]` becomes `[[]x[]]`); a backslash escape is never used, because ccc's
163
+ matcher reads a backslash as `/` on Windows. A name holding `'`, `\\`, a
164
+ control character, U+FFFD or an undecodable byte cannot go through a
165
+ quoted shell payload, so that entry stays indexed with a warning. No
166
+ pattern is ever `{folder}/*` or a `!` negation: ccc applies a negation
167
+ against every pattern, its own defaults included, and a `!{folder}/...`
168
+ also cancels a bare `{folder}` pattern.
169
+
170
+ A `!` entry of the user's cancels an SKF pattern the same way. ccc walks
171
+ into an excluded folder when a `!` entry matches the folder, matches the
172
+ one child name ccc probes it with (in practice a wildcard child: `!*/*`
173
+ and `!**/skills/*` re-open `skills`, `!*/x` does not), or starts with
174
+ `{folder}/`, and then indexes everything below it that no other pattern
175
+ excludes. So once the patterns are merged, each folder with the bare
176
+ value or per-entry patterns gets one warning naming the entries that
177
+ re-open one of its patterns, when ccc, reading the exclude_patterns this
178
+ run writes, walks down to a file of SKF output git lists below that
179
+ pattern (or, with none there yet, into the child it probes). Result and
180
+ report families are not checked. Inside a hidden folder ccc's own `**/.*`
181
+ still excludes that output, so an entry there warns only when it
182
+ re-opens every level down to it, and the bare value's warning says all
183
+ SKF output is back only when every SKF entry there is reached. A folder
184
+ inside an always-excluded one never switches to per-entry patterns, so
185
+ its warning offers no folder of the user's own there. A backslash in an
186
+ entry is read as ccc's glob library reads it on the machine the helper
187
+ runs on: `/` on Windows, an escape elsewhere. The warning repeats on
188
+ every run while the entry stays, including after a folder that held
189
+ other content goes back to the bare value; SKF never removes the entry.
190
+
191
+ Folder values are normalized first: surrounding whitespace is stripped,
192
+ `\\` becomes `/`, a leading `{project-root}/` is dropped, and `./`, `//`
193
+ and a trailing `/` collapse. A normalized value is then refused, with a
194
+ warning naming the key and the file to fix, when it is:
195
+
196
+ - empty or `.` (would exclude the whole repository)
197
+ - absolute or anchored: a leading `/`, `~` or `./`, a drive letter, or
198
+ a trailing `/` (absolute paths are never rewritten)
199
+ - carrying a `..` segment (outside the project)
200
+ - starting with `!` (ccc reads it as a negation)
201
+ - carrying `*`, `?`, `[`, `]` or `\\` (ccc reads them as glob syntax)
202
+ - carrying `{` or `}` (an unresolved template placeholder)
203
+ - carrying a control character (does not survive a quoted payload)
204
+ - carrying `'` (breaks a single-quoted payload)
205
+
206
+ A refused value is left out; the four `**/` patterns still merge.
207
+
208
+ Folder check: for each accepted folder that exists, the script lists it
209
+ with `git --literal-pathspecs ls-files -z --cached --others
210
+ --exclude-per-directory=.gitignore -- <folder>`: tracked files plus
211
+ untracked files no `.gitignore` excludes, the only ignore files ccc reads
212
+ (`.git/info/exclude` and a global excludes file hide nothing from ccc, so
213
+ they hide nothing here either). A tracked file deleted from the worktree
214
+ is dropped. The paths are grouped by first-level entry. A group is SKF
215
+ output by the same rule the workflows use before they write into, move or
216
+ delete a skill's folders (skf-skill-inventory.py):
217
+
218
+ skills it is `_batch` or its name contains `.skf-`, or its
219
+ `metadata.json` or a `<v>/<n>/metadata.json` carries an SKF
220
+ marker (generated_by, tool_versions.skf, or skill_type with
221
+ forge_tier / confidence_tier); a linked group never is. The
222
+ versioned layout, an `active` pointer, a manifest key or a
223
+ `*-result*.json` alone do not count, so a module's skills that an
224
+ earlier SKF moved into the versioned layout stay indexed
225
+ forge its name contains `.skf-`, it is `_campaign` or
226
+ `improvement-queue`, or it holds a `skill-brief.yaml*`,
227
+ `.brief-draft.json` or `*-result*.json` file directly, or a
228
+ provenance map, evidence report, extraction rules or
229
+ `*-result*.json` file directly in a folder of it; a linked group,
230
+ folder or file never counts. Like the skills rule it reads the
231
+ group on disk, so a file `.gitignore` hides still counts
232
+
233
+ Root files named `*-result*.json`, `.export-manifest.json` (skills) or
234
+ with an SKF report prefix (forge) are SKF output. Neutral entries never
235
+ count: `.gitkeep`, `.gitignore`, `.gitattributes`, `.DS_Store`,
236
+ `Thumbs.db`, `desktop.ini` and README files, plus any other dot-named
237
+ entry when `**/.*` (the ccc default) is in exclude_patterns and no entry
238
+ there starts with `!`. Everything else is foreign. A group that is, or
239
+ contains, the other configured SKF folder counts as SKF output but gets no
240
+ pattern of its own: that folder is reconciled separately, deepest first.
241
+ When both settings name the same folder, an entry either kind accepts is
242
+ SKF output. A submodule or nested repository is classified too: when the
243
+ folder itself is one, its own files are listed from inside it, and one
244
+ deeper down is a group of its own.
245
+
246
+ Each folder gets one verdict:
247
+
248
+ user the folder value is already in exclude_patterns and SKF does
249
+ not own it (see the record below): no pattern, no warning
250
+ bare no foreign group, and either SKF output or no other root file
251
+ (loose notes beside SKF output are tolerated): the bare value
252
+ entries foreign entries next to SKF output, or a folder nested inside
253
+ it that is itself `entries` or `left_out`: the per-entry
254
+ patterns, with a note naming the foreign entries when there
255
+ are any
256
+ left_out foreign entries and no SKF output: nothing, with a warning
257
+ naming the setting to change
258
+
259
+ The check is skipped, giving the bare value, silently when git is missing,
260
+ the project is not a git repository, the folder does not exist, or an
261
+ always-included `**/` pattern already covers the folder; it is skipped
262
+ with a warning when git times out or fails in any other way (for example
263
+ a repository git refuses as unsafe). Every git and ccc child runs with
264
+ the git location variables (GIT_DIR, GIT_INDEX_FILE, ...) removed from
265
+ its environment, so a git hook's index never leaks into the check; git
266
+ also runs with LC_ALL=C so its messages are recognised in any locale.
267
+
268
+ Ownership record and pruning: `ccc_index.exclude_patterns` in the
269
+ forge-tier.yaml passed through --prior-state-from is the set of patterns
270
+ SKF owned after its last reconcile, per-entry patterns included. A
271
+ recorded pattern the current run does not produce is removed from
272
+ settings.yml, so a dropped or renamed skill loses its pattern and a folder
273
+ that changes verdict swaps its bare value for per-entry patterns or back;
274
+ entries SKF never recorded (user entries and the ccc defaults) are never
275
+ removed, and the four `**/` patterns are always produced so are never
276
+ pruned. A record is legacy when forge-tier.yaml exists, the record names
277
+ no folder pattern (it holds at most the four `**/` patterns), and it is
278
+ empty or settings.yml still holds a `**/{value}` form of an accepted
279
+ folder value; those `**/{value}` forms then count as owned, so installs
280
+ migrate to the anchored forms. A `**/skills` added by the user after a
281
+ folder pattern is recorded survives, and with no forge-tier.yaml at all
282
+ SKF owns nothing yet. A record is trusted when forge-tier.yaml exists and
283
+ the record is not legacy: with a trusted record, a folder value already in
284
+ exclude_patterns that the record does not hold is the user's (verdict
285
+ `user`), unless the record holds `{value}/...` per-entry patterns and none
286
+ of them is still in exclude_patterns. SKF then swapped them for the bare
287
+ value in a run that stopped before its record was saved, so the value is
288
+ SKF's and is classified again. Without a trusted record the folder is
289
+ still classified: an all-SKF folder adopts the bare value already there,
290
+ and a folder with foreign entries keeps it untouched with a one-time
291
+ warning (the next run has a trusted record). When a folder value is refused, pruning is blocked
292
+ for the whole run and the recorded patterns that would have been removed
293
+ stay (warning): a config typo must not silently un-exclude real output. If
294
+ that run also used the legacy fallback, effective_patterns is null so the
295
+ record stays folder-free and the next valid run can still migrate.
296
+
297
+ Index decision: "fail" when no settings.yml exists; otherwise "skip"
298
+ under --skip-index true (with a warning when settings.yml changed);
299
+ otherwise, after a failed rebuild, "keep" when the prior index is fresh
300
+ and "fail" when it is not; otherwise "index" when settings.yml changed or
301
+ the prior index is not fresh, or "defer" in its place under --defer-index;
302
+ otherwise "keep". --defer-index changes only a due build: a fresh index
303
+ is still kept, its record carried forward, which --skip-index would null.
304
+
305
+ After a reconcile, `/.cocoindex_code/` coverage is checked with
306
+ `git check-ignore -q --no-index -- .cocoindex_code/target_sqlite.db`; an
307
+ uncovered path adds a warning. `gitignore_updated` compares the
308
+ `.gitignore` bytes before and after the run (only ccc init writes it).
309
+
310
+ Index build (--build-index): the script acts on index_action itself and
311
+ returns the index state the run ends with, so no step reads `ccc status`
312
+ or types a timestamp:
313
+
314
+ keep status "fresh"; last_indexed and file_count carry over from the
315
+ prior forge-tier.yaml (--prior-state-from)
316
+ skip status "skipped"
317
+ defer status "skipped", as for skip: the `ccc index` run that "index"
318
+ would make waits for a later run, so the next setup run builds it
319
+ fail status "failed", with not_ready_reason as failed_reason
320
+ index `ccc index` runs in the project root (up to CCC_INDEX_TIMEOUT_SEC).
321
+ Then `ccc status` gives the file count. While it prints an
322
+ `Indexing in progress:` line, `ccc index` runs again (it waits
323
+ for the running pass, then makes a quick incremental one), up to
324
+ CCC_INDEX_RERUNS times; a pass still running after that adds a
325
+ warning and the count read last. Status "created", stamped with
326
+ this script's UTC clock, when ccc indexed at least one file;
327
+ status "failed", with the reason, when `ccc index` or `ccc status`
328
+ fails, the status shows no file count, or the count is 0.
329
+ Before `ccc index` starts, --result-to already holds the merge
330
+ result with the index "failed" ("ccc index did not finish"),
331
+ so a call its host stops during a long index still leaves this
332
+ run's exclusion record for the next steps; the final result
333
+ replaces it.
334
+
335
+ indexed_path is the --project-root value exactly as given, which is what
336
+ skf-detect-tools.py compares the next run's --project-root with.
337
+
338
+ Output (single JSON document on stdout, ASCII only):
59
339
 
60
340
  {
61
341
  "status": "ok",
62
- "version": "v1",
63
- "settings_yml_existed": bool,
64
- "settings_yml_path": "/abs/path/.cocoindex_code/settings.yml",
65
- "patterns_added": int,
66
- "patterns_added_list": ["**/_bmad", ...],
67
- "patterns_already_present": int,
68
- "written": bool,
69
- "warnings": ["string", ...]
342
+ "version": "v2",
343
+ "settings_yml_path": "/abs/path/.cocoindex_code/settings.yml",
344
+ "settings_yml_existed": bool, after crash recovery, before init
345
+ "ccc_init": "not_needed" | "created" | "rebuilt"
346
+ | "failed",
347
+ "settings_ready": bool, false only when no usable
348
+ settings.yml exists
349
+ "not_ready_reason": str | null, why index_action is
350
+ "fail" (null otherwise)
351
+ "patterns_added": int,
352
+ "patterns_added_list": [str], produced patterns not in the
353
+ pre-run list
354
+ "patterns_removed": int,
355
+ "patterns_removed_list": [str], SKF-owned patterns pruned
356
+ "patterns_already_present": int,
357
+ "effective_patterns": [str] | null, sorted SKF-owned set to
358
+ record, per-entry patterns
359
+ included (copy every entry
360
+ exactly); null when nothing was
361
+ reconciled, or a legacy-record run
362
+ with a refused value (keep the
363
+ old record)
364
+ "written": bool, settings.yml changed this run
365
+ (ccc init create, rebuild, or
366
+ SKF edit)
367
+ "gitignore_updated": bool,
368
+ "index_action": "index" | "keep" | "skip" | "defer"
369
+ | "fail",
370
+ "index": only with --build-index:
371
+ {"status": "fresh" | "created" | "skipped"
372
+ | "failed",
373
+ "indexed_path": str | null,
374
+ "last_indexed": str | null, ISO-8601 UTC
375
+ "file_count": int | null,
376
+ "failed_reason": str | null},
377
+ "warnings": [str] refused values and other notes for
378
+ the report, a user `!` entry that
379
+ cancels an SKF pattern included
70
380
  }
71
381
 
72
- `written` is false when no new patterns needed adding (idempotent
73
- re-run — no I/O wasted). Non-zero `warnings` does NOT prevent the
74
- write — always-include patterns still merge.
382
+ Clone mode output (single JSON document on stdout, ASCII only):
75
383
 
76
- Atomic writes via temp + fsync + rename (mirrors skf-atomic-write.py).
77
- Concurrent writers must coordinate via external `flock` (the typical
78
- pattern for shared-file mutation in this module).
384
+ {
385
+ "status": "ok",
386
+ "version": "v2",
387
+ "settings_yml_path": "/abs/clone/.cocoindex_code/settings.yml",
388
+ "settings_yml_existed": bool,
389
+ "ccc_init": "not_needed" | "created" | "failed",
390
+ "settings_ready": bool, false only when no settings.yml
391
+ exists after the ccc init
392
+ "not_ready_reason": str | null,
393
+ "patterns_added": int,
394
+ "patterns_added_list": [str], standard exclusions appended
395
+ "patterns_already_present": int,
396
+ "includes_added_list": [str], `**/*.<ext>` entries appended
397
+ "includes_covered_list": [str], extensions an include_patterns
398
+ entry already matched
399
+ "written": bool, settings.yml changed this run
400
+ "warnings": [str]
401
+ }
402
+
403
+ Payload safety: setup's steps once embedded these strings in
404
+ single-quoted `echo '...'` payloads, where dash's `echo` rewrites
405
+ backslash escapes; they now read the --result-to file, and the strings
406
+ stay safe for any caller that still quotes them. So every warning,
407
+ `not_ready_reason` and error message has `'` replaced by a
408
+ backtick, `\\` by `/`, and control characters and lone surrogates by `?`;
409
+ folder values carrying `'`, `\\` or a control character are refused, and
410
+ an entry name carrying one gets no pattern. No pattern SKF produces, and
411
+ so no effective_patterns entry, holds any of them. Data fields such as
412
+ `settings_yml_path` are emitted as they are.
413
+
414
+ Paths in messages: when a warning or `not_ready_reason` names the
415
+ project root in SKF's own words, it writes the literal `{project-root}`
416
+ placeholder, never the absolute path. Setup's step 4 banner renders a
417
+ warning's placeholder like its own `{project-root}` paths, and the
418
+ envelope keeps it verbatim. The unresolved-placeholder refusal names the
419
+ placeholder this script resolves in words ("the project-root
420
+ placeholder"), so a `{project-root}` in SKF's own words always means the
421
+ project root. Output and errors quoted from `ccc init` or git can carry
422
+ paths of their own, the project root included. Error messages on stderr
423
+ name absolute paths.
424
+
425
+ Writes to settings.yml use temp + fsync + rename (mirrors
426
+ skf-atomic-write.py), emit ASCII-only YAML (non-ASCII escaped, as ccc
427
+ writes it — ccc reads the file with the platform default encoding), and
428
+ drop YAML comments; ccc ignores them and keeps unknown top-level keys,
429
+ which are preserved. Concurrent writers must
430
+ coordinate via external `flock` (the typical pattern for shared-file
431
+ mutation in this module).
79
432
 
80
433
  Exit codes:
81
- 0 success (warnings, if any, are still on the success path)
82
- 1 user error (bad args, missing required flag, malformed settings.yml)
83
- 2 internal error (write failure)
434
+ 0 success, including "not ready" (warnings stay on the success path)
435
+ 1 --project-root (or --clone-root) is not a directory; settings.yml
436
+ cannot be parsed, is not a mapping, or has an exclude_patterns (clone
437
+ mode: or include_patterns) that is neither a list nor null (JSON error
438
+ on stderr)
439
+ 2 usage error (including a non-boolean --index-fresh / --skip-index, an
440
+ --include-ext that is not a plain extension, and a setup flag or
441
+ --include-ext given in the other mode), a write, move-aside or restore
442
+ failure, or any unexpected internal error (always JSON on stderr, never
443
+ a traceback)
84
444
  """
85
445
 
86
446
  from __future__ import annotations
87
447
 
88
448
  import argparse
449
+ import functools
89
450
  import json
90
451
  import os
452
+ import re
453
+ import shutil
454
+ import subprocess
91
455
  import sys
92
- from pathlib import Path
456
+ import unicodedata
457
+ from datetime import datetime, timezone
458
+ from pathlib import Path, PurePosixPath
459
+ from typing import NamedTuple
93
460
 
94
461
  import yaml
95
462
 
@@ -100,29 +467,178 @@ ALWAYS_INCLUDE = (
100
467
  "**/.claude",
101
468
  "**/_skf-learn",
102
469
  )
103
- GLOB_META_CHARS = set("*?[")
470
+ GLOB_META_CHARS = set("*?[]\\")
104
471
  PLACEHOLDER_CHARS = set("{}")
472
+ CCC_SELF_EXCLUDE = "**/.cocoindex_code"
473
+ CCC_INIT_TIMEOUT_SEC = 75
474
+ # A first index of a large codebase can take many minutes.
475
+ CCC_INDEX_TIMEOUT_SEC = 3600
476
+ CCC_STATUS_TIMEOUT_SEC = 75
477
+ CCC_INDEX_RERUNS = 3
478
+ # The index failed_reason --result-to holds while `ccc index` runs.
479
+ INDEX_UNFINISHED = "ccc index did not finish"
480
+ CCC_IN_PROGRESS = "Indexing in progress:"
481
+ CCC_FILES_RE = re.compile(r"^\s*Files:\s*(\d+)\s*$", re.MULTILINE)
482
+ GIT_TIMEOUT_SEC = 10
483
+ SAMPLE_SIZE = 3
484
+ INDEX_ACTIONS = ("index", "keep", "skip", "defer", "fail")
485
+ # Keep identical to GIT_LOCATION_VARS in skf-check-workspace-drift.py.
486
+ GIT_LOCATION_VARS = (
487
+ "GIT_DIR",
488
+ "GIT_WORK_TREE",
489
+ "GIT_INDEX_FILE",
490
+ "GIT_PREFIX",
491
+ "GIT_COMMON_DIR",
492
+ "GIT_OBJECT_DIRECTORY",
493
+ "GIT_ALTERNATE_OBJECT_DIRECTORIES",
494
+ "GIT_NAMESPACE",
495
+ )
496
+ # Keep identical to RESULT_JSON_RE in skf-skill-inventory.py
497
+ # (test/test-skf-skill-inventory.py pins the copies).
498
+ RESULT_JSON_RE = re.compile(r"^([a-z0-9][a-z0-9-]*)-result(-latest|-\d[^/]*)?\.json$")
499
+ README_RE = re.compile(r"^readme(\..+)?$", re.IGNORECASE)
500
+ NEUTRAL_ROOT_FILES = frozenset({".gitkeep", ".gitignore", ".gitattributes", ".DS_Store",
501
+ "Thumbs.db", "desktop.ini"})
502
+ SKF_FIXED_ENTRIES = {
503
+ "skills": ("_batch", ".export-manifest.json"),
504
+ "forge": ("_campaign", "improvement-queue"),
505
+ }
506
+ SKF_ROOT_RESULT_STEMS = {"skills": ("export-skill", "drop-skill"), "forge": ()}
507
+ # Keep identical to SKF_GENERATORS in skf-skill-inventory.py
508
+ # (test/test-skf-skill-inventory.py pins the copies).
509
+ SKF_GENERATORS = frozenset({"quick-skill", "create-skill", "create-stack-skill"})
510
+ FORGE_ROOT_PREFIXES = (
511
+ "analyze-source-",
512
+ "feasibility-report-",
513
+ "verify-stack-result-",
514
+ "ra-state-",
515
+ "refine-architecture-result-",
516
+ "refined-architecture-",
517
+ )
518
+ # Keep identical to FORGE_GROUP_DIRS in skf-skill-inventory.py
519
+ # (test/test-skf-skill-inventory.py pins the copies).
520
+ FORGE_GROUP_DIRS = frozenset({"_campaign", "improvement-queue"})
521
+ # Keep identical to FORGE_VERSION_ANCHORS in skf-skill-inventory.py
522
+ # (test/test-skf-skill-inventory.py pins the copies).
523
+ FORGE_VERSION_ANCHORS = frozenset({"provenance-map.json", "evidence-report.md", "extraction-rules.yaml"})
524
+ FOLDER_KEYS = (("skills_output_folder", "skills"), ("forge_data_folder", "forge"))
525
+ # Glob characters written as one-character classes in per-entry patterns,
526
+ # as ccc's glob library escapes them; its backslash escaping is off on Windows.
527
+ CCC_CLASS_ESCAPES = frozenset("*?[]{}")
528
+
529
+ SETTINGS_DIR = ".cocoindex_code"
530
+ SETTINGS_NAME = "settings.yml"
531
+ BACKUP_NAME = "settings.yml.skf-repair"
532
+ FIX_HINT = "fix the value in {project-root}/_bmad/skf/config.yaml"
533
+
534
+ # Clone mode: the build and dependency folders create-skill keeps out of the
535
+ # index of a workspace clone. They are generic artifact patterns, not SKF
536
+ # paths, in the `**/name` form (ccc matches it at any depth; a trailing-slash
537
+ # form such as `build/` matches nothing).
538
+ CLONE_EXCLUDES = (
539
+ "**/node_modules",
540
+ "**/dist",
541
+ "**/build",
542
+ "**/.git",
543
+ "**/vendor",
544
+ "**/__pycache__",
545
+ "**/.cache",
546
+ "**/.next",
547
+ "**/.nuxt",
548
+ "**/target",
549
+ "**/out",
550
+ "**/.venv",
551
+ "**/.tox",
552
+ )
553
+ # An extension --include-ext takes: letters, digits, `_`, `+` and `-` only,
554
+ # so `**/*.<ext>` is never a wider glob than one file type.
555
+ EXTENSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_+-]*$")
556
+ # A nested file with a given extension, matched against include_patterns.
557
+ INCLUDE_PROBE = "skf-probe/skf-probe.{ext}"
558
+ # A bare `- *...` list item: YAML reads the value as an alias. The value ends
559
+ # before trailing blanks or a comment (a `#` after a blank), which stay after it.
560
+ BARE_GLOB_ITEM_RE = re.compile(r"^([ \t]*-[ \t]+)(\*[^'\"\n]*?)((?:[ \t]+#[^\n]*)?|[ \t]*)$", re.MULTILINE)
561
+
562
+
563
+ class HelperError(Exception):
564
+ """A handled failure that ends the run with a JSON error and exit `code`."""
565
+
566
+ def __init__(self, code: int, message: str):
567
+ super().__init__(message)
568
+ self.code = code
569
+
570
+
571
+ def _payload_safe(text) -> str:
572
+ """Make text safe for a single-quoted `echo '...'` payload.
573
+
574
+ `'` becomes a backtick, a backslash becomes `/`, and control characters
575
+ and lone surrogates become `?`: dash's `echo` rewrites backslash
576
+ escapes, and a surrogate cannot be encoded.
577
+ """
578
+ return "".join(
579
+ "`" if ch == "'" else "/" if ch == "\\" else
580
+ "?" if ord(ch) < 0x20 or ord(ch) == 0x7F or 0xD800 <= ord(ch) <= 0xDFFF else ch
581
+ for ch in str(text))
105
582
 
106
583
 
107
584
  def _die(code: int, message: str) -> None:
108
- print(json.dumps({"status": "error", "message": message}), file=sys.stderr)
585
+ print(json.dumps({"status": "error", "message": _payload_safe(message)}), file=sys.stderr)
109
586
  sys.exit(code)
110
587
 
111
588
 
112
- def _ok(payload: dict) -> None:
113
- payload.setdefault("status", "ok")
114
- payload.setdefault("version", "v1")
115
- print(json.dumps(payload))
589
+ def _child_env() -> dict:
590
+ """os.environ without the git location variables, plus PYTHONUTF8=1.
116
591
 
117
-
118
- def _atomic_write(target: Path, content: str) -> None:
119
- """Crash-safe write via temp + fsync + rename. Mirrors skf-atomic-write.py."""
120
- target.parent.mkdir(parents=True, exist_ok=True)
592
+ A git hook (for example `git commit -a` running the test suite) exports
593
+ an absolute GIT_INDEX_FILE; inherited by `git -C <other repo>`, it would
594
+ make git read the hook's index instead of the project's.
595
+ """
596
+ env = {k: v for k, v in os.environ.items() if k not in GIT_LOCATION_VARS}
597
+ env["PYTHONUTF8"] = "1"
598
+ return env
599
+
600
+
601
+ def _resolve_outside_cwd(command: str) -> str | None:
602
+ """shutil.which with a CWD-shim guard. Returns the resolved path or None.
603
+
604
+ shutil.which on Windows searches the current directory ahead of PATH,
605
+ and CWD here is the repo under analysis — a bare-name lookup resolving
606
+ into CWD would execute a repo-planted shim (e.g. ccc.cmd). Such a
607
+ resolution is treated as not-found. Explicit paths supplied by callers
608
+ (containing a separator) are honored as-is. Keep identical to the
609
+ sibling guards in skf-detect-tools.py,
610
+ skf-qmd-classify-collections.py, skf-ccc-git-hygiene.py,
611
+ skf-source-tree.py, skf-tessl-review.py and
612
+ skf-verify-provenance-completeness.py.
613
+ """
614
+ resolved = shutil.which(command)
615
+ if resolved is None:
616
+ return None
617
+ if os.sep in command or (os.altsep and os.altsep in command):
618
+ return resolved
619
+ resolved_dir = os.path.dirname(resolved)
620
+ if resolved_dir:
621
+ cwd = os.path.normcase(os.path.abspath(os.getcwd()))
622
+ if os.path.normcase(os.path.abspath(resolved_dir)) == cwd:
623
+ return None
624
+ return resolved
625
+
626
+
627
+ def _atomic_write(target: Path, content: str, unique: bool = False) -> None:
628
+ """Crash-safe write via temp + fsync + rename. Mirrors skf-atomic-write.py.
629
+
630
+ With `unique`, the temp file is one of this call's own (clone mode,
631
+ where two runs can write the same file at once), else `<name>.skf-tmp`.
632
+ Raises HelperError(2) on any OS failure; the temp file is removed.
633
+ """
121
634
  tmp = target.with_name(target.name + ".skf-tmp")
635
+ if unique:
636
+ tmp = target.with_name(f"{target.name}.{os.getpid()}.{os.urandom(4).hex()}.skf-tmp")
122
637
  # O_BINARY (Windows only; 0 elsewhere) suppresses the text-mode \n -> \r\n
123
638
  # translation that would otherwise corrupt verbatim writes on Windows.
124
639
  flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0)
125
640
  try:
641
+ target.parent.mkdir(parents=True, exist_ok=True)
126
642
  fd = os.open(tmp, flags, 0o644)
127
643
  try:
128
644
  os.write(fd, content.encode("utf-8"))
@@ -131,75 +647,153 @@ def _atomic_write(target: Path, content: str) -> None:
131
647
  os.close(fd)
132
648
  os.replace(tmp, target)
133
649
  except OSError as e:
134
- if tmp.exists():
135
- try:
136
- tmp.unlink()
137
- except OSError:
138
- pass
139
- _die(2, f"atomic write failed for {target}: {e}")
650
+ try:
651
+ tmp.unlink()
652
+ except OSError:
653
+ pass
654
+ raise HelperError(2, f"atomic write failed for {target}: {e}")
655
+
656
+
657
+ # ─── External commands (git, ccc) ───────────────────────────────────────────
140
658
 
141
659
 
142
- # ─── Config-value validation (PR #248 rules) ────────────────────────────────
660
+ def _git(root: Path, *args: str, literal: bool = False) -> tuple[int, bytes, str] | None:
661
+ """Run `git -C root <args>`. Return (returncode, stdout bytes, stderr text).
143
662
 
663
+ Returns None when git cannot be resolved or fails to start, and
664
+ (-1, b"", "") on timeout. `literal=True` adds `--literal-pathspecs` so a
665
+ folder value such as `:odd` is never read as pathspec magic — only for
666
+ `ls-files`: `check-ignore` rejects that flag. git runs with LC_ALL=C so
667
+ its "not a git repository" message can be recognised in any locale.
668
+ """
669
+ exe = _resolve_outside_cwd("git")
670
+ if exe is None:
671
+ return None
672
+ argv = [exe, *(["--literal-pathspecs"] if literal else []), "-C", str(root), *args]
673
+ env = _child_env()
674
+ env["LC_ALL"] = "C"
675
+ try:
676
+ result = subprocess.run(
677
+ argv,
678
+ stdin=subprocess.DEVNULL,
679
+ capture_output=True,
680
+ env=env,
681
+ timeout=GIT_TIMEOUT_SEC,
682
+ )
683
+ except subprocess.TimeoutExpired:
684
+ return -1, b"", ""
685
+ except (OSError, ValueError):
686
+ return None
687
+ return result.returncode, result.stdout, result.stderr.decode("utf-8", errors="replace")
144
688
 
145
- def resolve_repo_relative(raw_value: str, project_root: Path) -> str:
146
- """Normalize a config value like '{project-root}/skills' to 'skills'.
147
689
 
148
- Accepts:
149
- - `{project-root}/<segments>` → returns `<segments>` (verbatim, no globs)
150
- - `<segments>` (already relative) → returns as-is, stripped
151
- - Other input → returns the stripped value (validate_config_value
152
- will catch absolute paths, placeholders, glob-meta).
690
+ def run_ccc_init(root: Path) -> tuple[bool, str]:
691
+ """Run `ccc init -f` in root. Return (exit code was 0, condensed output).
153
692
 
154
- Owning the substitution here removes ~150 tokens of "substitute and
155
- basename-or-keep-tail" reasoning from every step prompt that invokes
156
- this helper.
693
+ The caller judges success by whether settings.yml exists afterwards,
694
+ not by the exit code. Output is stdout plus stderr, decoded as UTF-8,
695
+ whitespace-collapsed and cut to 300 characters.
696
+ """
697
+ exe = _resolve_outside_cwd("ccc")
698
+ if exe is None:
699
+ return False, "ccc was not found on PATH"
700
+ try:
701
+ result = subprocess.run(
702
+ [exe, "init", "-f"],
703
+ cwd=str(root),
704
+ stdin=subprocess.DEVNULL,
705
+ capture_output=True,
706
+ env=_child_env(),
707
+ timeout=CCC_INIT_TIMEOUT_SEC,
708
+ )
709
+ except subprocess.TimeoutExpired:
710
+ return False, f"ccc init timed out after {CCC_INIT_TIMEOUT_SEC}s"
711
+ except (OSError, ValueError) as e:
712
+ return False, f"ccc init could not start: {e}"
713
+ output = (result.stdout + result.stderr).decode("utf-8", errors="replace")
714
+ return result.returncode == 0, " ".join(output.split())[:300]
715
+
716
+
717
+ # ─── Config-value normalization and validation ─────────────────────────────
718
+
719
+
720
+ def resolve_repo_relative(raw_value, project_root: Path) -> str:
721
+ """Normalize a config value like '{project-root}/skills/' to 'skills'.
722
+
723
+ Strips whitespace, turns `\\` into `/`, drops a leading
724
+ `{project-root}/` (`{project-root}` alone gives ""), and collapses
725
+ `./`, `//` and a trailing `/`. Absolute paths and `..` segments are
726
+ kept so validate_config_value can refuse them.
157
727
  """
158
728
  if raw_value is None:
159
729
  return ""
160
- value = str(raw_value).strip()
730
+ value = str(raw_value).strip().replace("\\", "/")
161
731
  if value.startswith("{project-root}/"):
162
732
  value = value[len("{project-root}/"):]
163
733
  elif value == "{project-root}":
164
734
  value = ""
165
- return value
735
+ return PurePosixPath(value).as_posix() if value else ""
166
736
 
167
737
 
168
- def validate_config_value(key: str, raw_value: str) -> tuple[str | None, str | None]:
169
- """Validate a config value used to interpolate `**/{value}`.
738
+ def validate_config_value(key: str, raw_value) -> tuple[str | None, str | None]:
739
+ """Validate a folder value used as a root-anchored ccc exclusion.
170
740
 
171
741
  Returns (cleaned_value, warning_or_None). When cleaned_value is None
172
- the value was rejected and the caller MUST NOT add the pattern.
742
+ the value was rejected and the caller must not add the pattern. The
743
+ first failing rule wins.
173
744
  """
174
- if raw_value is None or not str(raw_value).strip():
745
+ if raw_value is None or str(raw_value).strip() in ("", "."):
175
746
  return None, (
176
747
  f"{key} is empty or whitespace-only; refused for ccc exclusion because "
177
- f"interpolating it would produce `**/`, which matches every path and "
178
- f"would silently exclude the entire repository from indexing — fix the "
179
- f"value in {{project-root}}/_bmad/skf/config.yaml"
748
+ f"an empty pattern would exclude the entire repository from indexing; "
749
+ f"{FIX_HINT}"
180
750
  )
181
751
 
182
752
  value = str(raw_value).strip()
183
753
 
184
- if value.startswith("/") or value.startswith("~/") or value.startswith("./"):
754
+ if value.startswith(("/", "~", "./")) or re.match(r"^[A-Za-z]:", value) or value.endswith("/"):
185
755
  return None, (
186
756
  f"{key} is an absolute or anchored path; refused for ccc exclusion "
187
- f"because interpolating it would produce a malformed glob — fix the "
188
- f"value in {{project-root}}/_bmad/skf/config.yaml to a repo-relative path"
757
+ f"because SKF exclusions are relative to the project root; "
758
+ f"{FIX_HINT} to a repo-relative path"
759
+ )
760
+
761
+ if ".." in PurePosixPath(value).parts:
762
+ return None, (
763
+ f"{key} contains a .. segment; refused for ccc exclusion because the "
764
+ f"folder must sit inside the project; {FIX_HINT}"
765
+ )
766
+
767
+ if value.startswith("!"):
768
+ return None, (
769
+ f"{key} starts with !, which ccc reads as a negation; refused for ccc "
770
+ f"exclusion; {FIX_HINT}"
189
771
  )
190
772
 
191
773
  if any(ch in GLOB_META_CHARS for ch in value):
192
774
  return None, (
193
- f"{key} contains glob meta-character (*, ?, [); refused for ccc "
194
- f"exclusion because interpolation collides with the surrounding "
195
- f"pattern syntax — fix the value in {{project-root}}/_bmad/skf/config.yaml"
775
+ f"{key} contains a glob meta-character (*, ?, [, ] or a backslash); "
776
+ f"refused for ccc exclusion because ccc would read it as pattern "
777
+ f"syntax; {FIX_HINT}"
196
778
  )
197
779
 
198
780
  if any(ch in PLACEHOLDER_CHARS for ch in value):
199
781
  return None, (
200
782
  f"{key} contains an unresolved template placeholder ({{ or }}); "
201
783
  f"refused for ccc exclusion because the step is supposed to forward "
202
- f"the raw config value and let this script resolve {{project-root}}"
784
+ f"the raw config value and let this script resolve the project-root placeholder"
785
+ )
786
+
787
+ if any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in value):
788
+ return None, (
789
+ f"{key} contains a control character; refused for ccc exclusion "
790
+ f"because it does not survive a quoted shell payload; {FIX_HINT}"
791
+ )
792
+
793
+ if "'" in value:
794
+ return None, (
795
+ f"{key} contains a single quote, which breaks a single-quoted shell payload; "
796
+ f"refused for ccc exclusion; {FIX_HINT}"
203
797
  )
204
798
 
205
799
  return value, None
@@ -210,161 +804,1651 @@ def validate_config_value(key: str, raw_value: str) -> tuple[str | None, str | N
210
804
 
211
805
  def assemble_patterns(skills_output_folder: str, forge_data_folder: str
212
806
  ) -> tuple[list[str], list[str]]:
213
- """Return (validated_patterns_to_add, warnings).
807
+ """Return (patterns, warnings) without touching the filesystem.
214
808
 
215
- Validated patterns include the 4 unconditional ones plus any of the
216
- 2 config-driven ones whose value passed validation.
809
+ Patterns are the 4 unconditional ones plus each folder value that
810
+ passed validation, anchored as-is (a value shared by both folders
811
+ appears once). The merge itself also runs the collision check, which
812
+ this pure helper does not.
217
813
  """
218
814
  patterns = list(ALWAYS_INCLUDE)
219
815
  warnings: list[str] = []
816
+ for (key, _kind), raw in zip(FOLDER_KEYS, (skills_output_folder, forge_data_folder)):
817
+ cleaned, warn = validate_config_value(key, raw)
818
+ if cleaned is None:
819
+ warnings.append(warn)
820
+ elif cleaned not in patterns:
821
+ patterns.append(cleaned)
822
+ return patterns, warnings
220
823
 
221
- cleaned, warn = validate_config_value("skills_output_folder", skills_output_folder)
222
- if cleaned is not None:
223
- patterns.append(f"**/{cleaned}")
224
- elif warn:
225
- warnings.append(warn)
226
824
 
227
- cleaned, warn = validate_config_value("forge_data_folder", forge_data_folder)
228
- if cleaned is not None:
229
- patterns.append(f"**/{cleaned}")
230
- elif warn:
231
- warnings.append(warn)
825
+ # ─── Folder check ───────────────────────────────────────────────────────────
232
826
 
233
- return patterns, warnings
234
827
 
828
+ # Only `.gitignore` files: ccc reads no other ignore file, so an entry that
829
+ # `.git/info/exclude` or a global excludes file hides is still indexed.
830
+ _LS_FILES_ARGS = ("ls-files", "-z", "--cached", "--others",
831
+ "--exclude-per-directory=.gitignore")
235
832
 
236
- # ─── settings.yml read / merge / render ─────────────────────────────────────
237
833
 
834
+ def _ls_files(cwd: Path, *pathspec: str) -> tuple[list[str] | None, str | None]:
835
+ """Run `git ls-files` in cwd. Return (records, problem).
836
+
837
+ records None means the listing is unavailable. problem is None for a
838
+ silent skip (git missing, not a git repository) and otherwise a short
839
+ description for a warning (timeout, any other git failure such as a
840
+ repository git refuses as unsafe).
841
+ """
842
+ res = _git(cwd, *_LS_FILES_ARGS, *(("--", *pathspec) if pathspec else ()), literal=True)
843
+ if res is None:
844
+ return None, None
845
+ returncode, stdout, stderr = res
846
+ if returncode == -1:
847
+ return None, "git ls-files timed out"
848
+ if returncode != 0:
849
+ if "not a git repository" in stderr.lower():
850
+ return None, None
851
+ detail = next((line.strip() for line in stderr.splitlines() if line.strip()),
852
+ f"exit code {returncode}")
853
+ return None, f"git ls-files failed ({detail[:200]})"
854
+ # surrogateescape keeps an undecodable name equal to the name
855
+ # os.listdir returns, so the existence check and the disk match still
856
+ # find it; _unwritable then refuses it a pattern.
857
+ return [r.decode("utf-8", errors="surrogateescape") for r in stdout.split(b"\0") if r], None
858
+
859
+
860
+ def list_folder_paths(root: Path, folder: str) -> tuple[list[tuple[str, ...]] | None, str | None]:
861
+ """Return (paths relative to folder, problem).
862
+
863
+ Lists tracked files plus untracked files no `.gitignore` excludes (the
864
+ only ignore files ccc reads); a tracked file deleted from the worktree
865
+ is dropped, since ccc cannot see it. A missing folder gives ([], None).
866
+ None paths mean the check is skipped; problem (see _ls_files) says
867
+ whether that deserves a warning. When git reports the folder itself as
868
+ one entry — a submodule or a nested repository, which ccc indexes like
869
+ any other folder — its own files are listed from inside it. A nested
870
+ repository deeper down comes back as a single directory entry, which
871
+ the classifier treats as a group.
872
+ """
873
+ if not (root / folder).is_dir():
874
+ return [], None
875
+ records, problem = _ls_files(root, folder)
876
+ if records is None:
877
+ return None, problem
878
+ depth = len(PurePosixPath(folder).parts)
879
+ cwd = root
880
+ if any(len(PurePosixPath(r).parts) <= depth for r in records):
881
+ records, problem = _ls_files(root / folder)
882
+ if records is None:
883
+ return None, problem
884
+ depth = 0
885
+ cwd = root / folder
886
+ found: set[tuple[str, ...]] = set()
887
+ for record in records:
888
+ if not os.path.lexists(cwd / record):
889
+ continue
890
+ parts = PurePosixPath(record).parts[depth:]
891
+ if parts:
892
+ found.add(parts)
893
+ return sorted(found), None
238
894
 
239
- def read_settings_yml(path: Path) -> tuple[dict, bool]:
240
- """Return (parsed_dict, existed_before).
241
895
 
242
- Missing file → ({}, False). Existing file is parsed; non-mapping top-level
243
- or YAML errors exit with a clear message.
896
+ # Keep identical to _has_skf_metadata in skf-skill-inventory.py
897
+ # (test/test-skf-skill-inventory.py pins the copies).
898
+ def _has_skf_metadata(path: Path) -> bool:
899
+ """True when a flat skill's metadata.json carries an SKF marker."""
900
+ try:
901
+ data = json.loads(path.read_text(encoding="utf-8"))
902
+ except (OSError, ValueError):
903
+ return False
904
+ if not isinstance(data, dict):
905
+ return False
906
+ generated_by = data.get("generated_by")
907
+ if isinstance(generated_by, str) and generated_by in SKF_GENERATORS:
908
+ return True
909
+ tool_versions = data.get("tool_versions")
910
+ if isinstance(tool_versions, dict) and "skf" in tool_versions:
911
+ return True
912
+ return (data.get("skill_type") in ("single", "individual", "stack")
913
+ and ("forge_tier" in data or "confidence_tier" in data))
914
+
915
+
916
+ # Keep identical to _is_link_or_junction in skf-atomic-write.py
917
+ # (test/test-skf-skill-inventory.py pins the copies).
918
+ def _is_link_or_junction(p: Path) -> bool:
919
+ """True for POSIX symlinks AND Windows junctions/symlinks.
920
+
921
+ `Path.is_symlink()` is False for Windows junctions; os.readlink succeeds
922
+ for both symlinks and junctions (since CPython 3.8 on Windows). A regular
923
+ directory raises OSError on readlink, which is the signal we want to
924
+ refuse replacement. On Windows, any other reparse point (a cloud-sync
925
+ placeholder, a deduplicated file, an app execution alias) raises
926
+ ValueError: it does not redirect to another path, so it is not a link.
927
+ """
928
+ if p.is_symlink():
929
+ return True
930
+ if not p.exists() and not p.is_symlink():
931
+ return False
932
+ try:
933
+ os.readlink(p)
934
+ return True
935
+ except (OSError, ValueError):
936
+ return False
937
+
938
+
939
+ # Keep identical to _is_marked_version in skf-skill-inventory.py
940
+ # (test/test-skf-skill-inventory.py pins the copies).
941
+ def _is_marked_version(version_dir: Path, name: str) -> bool:
942
+ """True when `version_dir/name/metadata.json` carries an SKF marker.
943
+
944
+ SKF never links a version folder or the package inside one: a linked
945
+ one is not its output, whatever the metadata behind the link says.
946
+ """
947
+ package = version_dir / name
948
+ return (not _is_link_or_junction(version_dir) and not _is_link_or_junction(package)
949
+ and _has_skf_metadata(package / "metadata.json"))
950
+
951
+
952
+ # Keep identical to _has_skf_evidence in skf-skill-inventory.py
953
+ # (test/test-skf-skill-inventory.py pins the copies).
954
+ def _has_skf_evidence(group_dir: Path, name: str) -> bool:
955
+ """True when skill group `group_dir` holds evidence that SKF generated it.
956
+
957
+ Evidence is the `_batch` name, `.skf-` in the name, a marked root
958
+ `metadata.json`, or a marked version folder (`.skf-` staging folders
959
+ aside). The versioned layout, an `active` link, a manifest key or a
960
+ result file are not evidence on their own, and a linked group holds none.
961
+ """
962
+ if _is_link_or_junction(group_dir):
963
+ return False
964
+ if name == "_batch" or ".skf-" in name:
965
+ return True
966
+ if _has_skf_metadata(group_dir / "metadata.json"):
967
+ return True
968
+ try:
969
+ children = list(group_dir.iterdir())
970
+ except OSError:
971
+ return False
972
+ return any(".skf-" not in child.name and child.is_dir() and _is_marked_version(child, name)
973
+ for child in children)
974
+
975
+
976
+ # Keep identical to _has_forge_evidence in skf-skill-inventory.py
977
+ # (test/test-skf-skill-inventory.py pins the copies).
978
+ def _has_forge_evidence(group_dir: Path, name: str) -> bool:
979
+ """True when forge group `group_dir` holds evidence that SKF generated it.
980
+
981
+ Evidence is `.skf-` in the name, SKF's own `_campaign` or
982
+ `improvement-queue` folder, a skill brief (`skill-brief.yaml*` or
983
+ `.brief-draft.json`) or a `*-result*.json` file directly in the group, or
984
+ a provenance map, evidence report, extraction rules or `*-result*.json`
985
+ file directly in a folder of the group (`.skf-` staging folders aside).
986
+ A linked group, folder or file is never evidence.
987
+ """
988
+ if _is_link_or_junction(group_dir):
989
+ return False
990
+ if ".skf-" in name or name in FORGE_GROUP_DIRS:
991
+ return True
992
+ try:
993
+ children = list(group_dir.iterdir())
994
+ except OSError:
995
+ return False
996
+ for child in children:
997
+ entry = child.name
998
+ if _is_link_or_junction(child):
999
+ continue
1000
+ if child.is_file():
1001
+ if (entry.startswith("skill-brief.yaml") or entry == ".brief-draft.json"
1002
+ or RESULT_JSON_RE.match(entry)):
1003
+ return True
1004
+ elif child.is_dir() and ".skf-" not in entry:
1005
+ try:
1006
+ inner = list(child.iterdir())
1007
+ except OSError:
1008
+ continue
1009
+ if any((f.name in FORGE_VERSION_ANCHORS or RESULT_JSON_RE.match(f.name))
1010
+ and not _is_link_or_junction(f) and f.is_file() for f in inner):
1011
+ return True
1012
+ return False
1013
+
1014
+
1015
+ def _skf_group(base: Path, kind: str, group: str) -> bool:
1016
+ """True when first-level entry `group` under the folder is SKF output of `kind`.
1017
+
1018
+ Both kinds follow the rule the workflows use before they write into,
1019
+ move or delete a skill's folders (skf-skill-inventory.py), read from the
1020
+ group on disk, so ccc excludes exactly the groups the workflows treat as
1021
+ SKF output.
1022
+ """
1023
+ if kind == "skills":
1024
+ return _has_skf_evidence(base / group, group)
1025
+ return _has_forge_evidence(base / group, group)
1026
+
1027
+
1028
+ def _skf_root_file(kind: str, name: str) -> bool:
1029
+ """True when a file directly under the folder is SKF output."""
1030
+ return bool(
1031
+ ".skf-" in name
1032
+ or RESULT_JSON_RE.match(name)
1033
+ or (kind == "skills" and name == ".export-manifest.json")
1034
+ or (kind == "forge" and name.startswith(FORGE_ROOT_PREFIXES))
1035
+ )
1036
+
1037
+
1038
+ def _holds_other_folder(folder: str, group: str, other_folders) -> bool:
1039
+ """True when folder/group is, or contains, another configured SKF folder."""
1040
+ group_parts = PurePosixPath(folder, group).parts
1041
+ return any(PurePosixPath(other).parts[:len(group_parts)] == group_parts
1042
+ for other in other_folders)
1043
+
1044
+
1045
+ class FolderScan(NamedTuple):
1046
+ """What classify_folder found in one configured folder."""
1047
+
1048
+ foreign: list[str] # the entries SKF did not generate that decide the verdict
1049
+ owned_groups: list[str] # first-level directories SKF generated
1050
+ owned_files: list[str] # root files SKF generated
1051
+ has_output: bool # any SKF output, a group holding the other folder included
1052
+ listed: frozenset # every first-level name git listed
1053
+
1054
+
1055
+ def classify_folder(root: Path, folder: str, kind, paths, other_folders=(),
1056
+ hidden_neutral: bool = False) -> FolderScan:
1057
+ """Classify the first-level entries of a configured folder.
1058
+
1059
+ `kind` is "skills" or "forge", or a collection of both when the two
1060
+ folder values are the same folder; an entry is SKF output when any of
1061
+ the kinds accepts it. `paths` comes from list_folder_paths. A group
1062
+ that is, or contains, another configured SKF folder (`other_folders`)
1063
+ is SKF output but gets no pattern of its own: that folder is reconciled
1064
+ separately. A single-segment entry that is a directory on disk (a
1065
+ nested repository or submodule) is a group, not a root file.
1066
+ `foreign`: any foreign group gives the foreign groups, as `name/`, then
1067
+ the other root files; otherwise other root files count only when the
1068
+ folder holds no SKF output at all. Neutral entries never count; with
1069
+ `hidden_neutral` (ccc's `**/.*` default is active) neither does any
1070
+ other dot-named entry that is not SKF output.
1071
+ """
1072
+ kinds = {kind} if isinstance(kind, str) else set(kind)
1073
+ base = root / folder
1074
+ groups: dict[str, list[tuple[str, ...]]] = {}
1075
+ other_root: list[str] = []
1076
+ owned_files: list[str] = []
1077
+ for rel in paths:
1078
+ if len(rel) == 1 and not (base / rel[0]).is_dir():
1079
+ name = rel[0]
1080
+ if any(_skf_root_file(k, name) for k in kinds):
1081
+ owned_files.append(name)
1082
+ elif not (name in NEUTRAL_ROOT_FILES or README_RE.match(name)
1083
+ or (hidden_neutral and name.startswith("."))):
1084
+ other_root.append(name)
1085
+ continue
1086
+ members = groups.setdefault(rel[0], [])
1087
+ if len(rel) > 1:
1088
+ members.append(rel)
1089
+ foreign_groups: list[str] = []
1090
+ owned_groups: list[str] = []
1091
+ holds_other = False
1092
+ for group in sorted(groups):
1093
+ if _holds_other_folder(folder, group, other_folders):
1094
+ holds_other = True
1095
+ elif any(_skf_group(base, k, group) for k in kinds):
1096
+ owned_groups.append(group)
1097
+ elif hidden_neutral and group.startswith("."):
1098
+ continue
1099
+ else:
1100
+ foreign_groups.append(group + "/")
1101
+ has_output = bool(owned_files or owned_groups or holds_other)
1102
+ if foreign_groups:
1103
+ foreign = foreign_groups + other_root
1104
+ elif other_root and not has_output:
1105
+ foreign = other_root
1106
+ else:
1107
+ foreign = []
1108
+ listed = frozenset(groups) | frozenset(owned_files) | frozenset(other_root) | frozenset(
1109
+ rel[0] for rel in paths if len(rel) == 1)
1110
+ return FolderScan(foreign, owned_groups, sorted(owned_files), has_output, listed)
1111
+
1112
+
1113
+ def ccc_literal(name: str) -> str:
1114
+ """Return `name` as a ccc glob that matches only itself.
1115
+
1116
+ Each glob character becomes a one-character class, as the escape
1117
+ function of ccc's glob library (globset) does. Backslash escapes are
1118
+ never used: globset turns them off on Windows and reads a backslash
1119
+ there as `/`.
1120
+ """
1121
+ return "".join(f"[{ch}]" if ch in CCC_CLASS_ESCAPES else ch for ch in name)
1122
+
1123
+
1124
+ def _unwritable(name: str) -> bool:
1125
+ """True when a name cannot go into a pattern.
1126
+
1127
+ A `'` breaks a single-quoted shell payload, a backslash or a control
1128
+ character does not survive `echo` in dash, a replacement character
1129
+ stands for a name that could not be decoded, and a surrogate (bytes on
1130
+ disk that are not UTF-8) makes ccc fail on the whole pattern list.
1131
+ """
1132
+ return any(ch in "'\\\ufffd" or ord(ch) < 0x20 or ord(ch) == 0x7F
1133
+ or 0xD800 <= ord(ch) <= 0xDFFF for ch in name)
1134
+
1135
+
1136
+ def _norm(name: str) -> str:
1137
+ return unicodedata.normalize("NFC", name).casefold()
1138
+
1139
+
1140
+ def _on_disk(name: str, disk: list[str] | None, listed=frozenset()) -> list[str]:
1141
+ """The names ccc sees for `name`: itself when listed on disk, else the
1142
+ entries equal to it up to Unicode normalization and case (macOS,
1143
+ Windows), else none (the entry is gone). No listing: `name` itself."""
1144
+ if disk is None or name in disk:
1145
+ return [name]
1146
+ key = _norm(name)
1147
+ return [d for d in disk if d not in listed and _norm(d) == key]
1148
+
1149
+
1150
+ def entry_patterns(folder: str, kind, owned_groups, owned_files, disk, listed=frozenset()
1151
+ ) -> tuple[list[str], list[str]]:
1152
+ """Return (patterns, skipped names) that exclude the SKF entries one by one.
1153
+
1154
+ Each pattern is `{folder}/` plus one entry name as it is on disk (`disk`,
1155
+ from os.listdir; None when unreadable), glob characters written as
1156
+ classes. The fixed entries and result stems of each kind, and every
1157
+ forge report prefix, are always included, so output SKF writes before
1158
+ the next setup run is covered. Timestamped root results share one
1159
+ `{folder}/{stem}-result*.json` and forge reports one `{folder}/{prefix}*`.
1160
+ Never a `!` negation: ccc applies it to every pattern, its own defaults
1161
+ included. Skipped names are SKF entries `_unwritable` refuses.
1162
+ """
1163
+ kinds = {kind} if isinstance(kind, str) else set(kind)
1164
+ patterns: set[str] = set()
1165
+ skipped: set[str] = set()
1166
+ exact = list(owned_groups)
1167
+ for kind_ in kinds:
1168
+ exact.extend(SKF_FIXED_ENTRIES[kind_])
1169
+ for stem in SKF_ROOT_RESULT_STEMS[kind_]:
1170
+ patterns.add(f"{folder}/{stem}-result*.json")
1171
+ if kind_ == "forge":
1172
+ patterns.update(f"{folder}/{prefix}*" for prefix in FORGE_ROOT_PREFIXES)
1173
+ for name in owned_files:
1174
+ prefix = next((p for p in FORGE_ROOT_PREFIXES if name.startswith(p)), None)
1175
+ if "forge" in kinds and prefix:
1176
+ patterns.add(f"{folder}/{prefix}*")
1177
+ continue
1178
+ m = RESULT_JSON_RE.match(name)
1179
+ if m:
1180
+ patterns.add(f"{folder}/{m.group(1)}-result*.json")
1181
+ continue
1182
+ exact.append(name)
1183
+ fixed = {n for k in kinds for n in SKF_FIXED_ENTRIES[k]}
1184
+ for name in dict.fromkeys(exact):
1185
+ if name in fixed and (disk is None or name not in disk):
1186
+ patterns.add(f"{folder}/{ccc_literal(name)}")
1187
+ continue
1188
+ if _unwritable(name):
1189
+ skipped.add(name)
1190
+ continue
1191
+ for real in _on_disk(name, disk, listed):
1192
+ if _unwritable(real):
1193
+ skipped.add(real)
1194
+ else:
1195
+ patterns.add(f"{folder}/{ccc_literal(real)}")
1196
+ return sorted(patterns), sorted(skipped)
1197
+
1198
+
1199
+ # ─── settings.yml and the ownership record ─────────────────────────────────
1200
+
1201
+
1202
+ def load_settings(path: Path) -> dict:
1203
+ """Parse settings.yml. An empty file gives {}.
1204
+
1205
+ Raises HelperError(1) when the file cannot be read or parsed, is not a
1206
+ mapping, or has an exclude_patterns that is neither a list nor null.
244
1207
  """
245
- if not path.exists():
246
- return {}, False
247
1208
  try:
248
1209
  data = yaml.safe_load(path.read_text(encoding="utf-8"))
249
- except yaml.YAMLError as e:
250
- _die(1, f"failed to parse {path}: {e}")
1210
+ except (yaml.YAMLError, ValueError) as e:
1211
+ raise HelperError(1, f"failed to parse {path}: {e}")
1212
+ except OSError as e:
1213
+ raise HelperError(1, f"failed to read {path}: {e}")
251
1214
  if data is None:
252
- return {}, True
1215
+ return {}
253
1216
  if not isinstance(data, dict):
254
- _die(1, f"expected mapping at top of {path}, got {type(data).__name__}")
255
- return data, True
1217
+ raise HelperError(1, f"expected mapping at top of {path}, got {type(data).__name__}")
1218
+ excludes = data.get("exclude_patterns")
1219
+ if excludes is not None and not isinstance(excludes, list):
1220
+ raise HelperError(1, f"exclude_patterns in {path} is not a list "
1221
+ f"(got {type(excludes).__name__})")
1222
+ return data
256
1223
 
257
1224
 
258
- def merge_patterns(existing_patterns: list[str], to_add: list[str]
259
- ) -> tuple[list[str], list[str]]:
260
- """Append `to_add` entries to `existing_patterns` if not already present.
1225
+ def needs_rebuild(data: dict) -> bool:
1226
+ """True when the file lacks the ccc defaults and must be rebuilt.
1227
+
1228
+ Every file ccc writes has both `exclude_patterns` (including
1229
+ `**/.cocoindex_code`) and `include_patterns`.
1230
+ """
1231
+ if "exclude_patterns" not in data:
1232
+ return True
1233
+ excludes = [str(p) for p in (data.get("exclude_patterns") or [])]
1234
+ return "include_patterns" not in data and CCC_SELF_EXCLUDE not in excludes
1235
+
1236
+
1237
+ def read_owned_record(path, warnings: list[str]) -> list[str]:
1238
+ """Return `ccc_index.exclude_patterns` from forge-tier.yaml, or [].
1239
+
1240
+ A missing path or file, or a non-list value, gives [] silently; an
1241
+ unreadable or unparseable file gives [] plus a warning.
1242
+ """
1243
+ if path is None:
1244
+ return []
1245
+ path = Path(path)
1246
+ if not path.is_file():
1247
+ return []
1248
+ try:
1249
+ data = yaml.safe_load(path.read_text(encoding="utf-8"))
1250
+ except (OSError, ValueError, yaml.YAMLError):
1251
+ warnings.append("could not read the SKF exclusion record in forge-tier.yaml; "
1252
+ "treated it as empty")
1253
+ return []
1254
+ ccc = data.get("ccc_index") if isinstance(data, dict) else None
1255
+ record = ccc.get("exclude_patterns") if isinstance(ccc, dict) else None
1256
+ return [str(p) for p in record] if isinstance(record, list) else []
1257
+
1258
+
1259
+ def plan_exclusions(existing: list[str], produced: list[str], owned_prior, prune_allowed: bool
1260
+ ) -> tuple[list[str], list[str], list[str]]:
1261
+ """Return (merged, added, removed).
1262
+
1263
+ Removes the entries of `existing` that SKF owned and no longer produces
1264
+ (only when `prune_allowed`), keeps every other entry in order, and
1265
+ appends the produced patterns not already kept.
1266
+ """
1267
+ removable = (set(owned_prior) - set(produced)) if prune_allowed else set()
1268
+ kept = [e for e in existing if e not in removable]
1269
+ removed: list[str] = []
1270
+ for e in existing:
1271
+ if e in removable and e not in removed:
1272
+ removed.append(e)
1273
+ added: list[str] = []
1274
+ for p in produced:
1275
+ if p not in kept and p not in added:
1276
+ added.append(p)
1277
+ return kept + added, added, removed
1278
+
1279
+
1280
+ def decide_index_action(ready: bool, skip_index: bool, written: bool, index_fresh: bool,
1281
+ usable: bool = True, defer: bool = False) -> str:
1282
+ """Return one of INDEX_ACTIONS for the step to act on.
1283
+
1284
+ `usable` is False when settings.yml exists but still lacks the ccc
1285
+ defaults (a failed rebuild): a fresh index is kept, but no new index is
1286
+ built from that file. `defer` turns a due build ("index") into "defer".
1287
+ """
1288
+ if not ready:
1289
+ return "fail"
1290
+ if skip_index:
1291
+ return "skip"
1292
+ if not usable:
1293
+ return "keep" if index_fresh else "fail"
1294
+ if written or not index_fresh:
1295
+ return "defer" if defer else "index"
1296
+ return "keep"
1297
+
1298
+
1299
+ # ─── Merge ──────────────────────────────────────────────────────────────────
1300
+
1301
+
1302
+ def _read_bytes_or_none(path: Path) -> bytes | None:
1303
+ try:
1304
+ return path.read_bytes() if path.is_file() else None
1305
+ except OSError:
1306
+ return None
1307
+
1308
+
1309
+ def _plural(n: int, one: str, many: str) -> str:
1310
+ return f"{n} {one if n == 1 else many}"
261
1311
 
262
- Returns (merged_patterns, newly_added_patterns). Order of existing
263
- entries is preserved; new entries are appended in the order given.
1312
+
1313
+ def _output_hint(output: str) -> str:
1314
+ return output or "no output"
1315
+
1316
+
1317
+ def _prepare_settings(root: Path, target: Path, backup: Path, allow_ccc_init: bool,
1318
+ state: dict, warnings: list[str]) -> dict | None:
1319
+ """Bring settings.yml to a reconcilable state.
1320
+
1321
+ Returns the parsed data to reconcile, or None when this run must not
1322
+ touch the SKF patterns (not ready, rebuild skipped or failed, or ccc
1323
+ wrote no exclude_patterns list).
1324
+ """
1325
+ if not state["existed"]:
1326
+ if not allow_ccc_init:
1327
+ state.update(ready=False,
1328
+ reason="settings.yml is missing and --no-ccc-init was given")
1329
+ return None
1330
+ _ok, output = run_ccc_init(root)
1331
+ if not target.is_file():
1332
+ state.update(ready=False, ccc_init="failed",
1333
+ reason=("ccc init did not create .cocoindex_code/settings.yml "
1334
+ f"({_output_hint(output)})"))
1335
+ return None
1336
+ state.update(ccc_init="created", written=True)
1337
+ data = load_settings(target)
1338
+ if not isinstance(data.get("exclude_patterns"), list):
1339
+ warnings.append("ccc init wrote no exclude_patterns list, so SKF left its "
1340
+ "exclusions out rather than replace the ccc defaults")
1341
+ return None
1342
+ return data
1343
+
1344
+ data = load_settings(target)
1345
+ if not needs_rebuild(data):
1346
+ return data
1347
+ if not allow_ccc_init:
1348
+ warnings.append("settings.yml lacks the ccc default exclusions; rebuild skipped "
1349
+ "because --no-ccc-init was given")
1350
+ return None
1351
+
1352
+ original = data
1353
+ try:
1354
+ os.replace(target, backup)
1355
+ except OSError as e:
1356
+ raise HelperError(2, f"could not move {target} aside to {backup.name}: {e}")
1357
+ _ok, output = run_ccc_init(root)
1358
+ fresh = None
1359
+ if target.is_file():
1360
+ try:
1361
+ fresh = load_settings(target)
1362
+ except HelperError:
1363
+ fresh = None
1364
+ if fresh is None or not isinstance(fresh.get("exclude_patterns"), list):
1365
+ try:
1366
+ os.replace(backup, target)
1367
+ except OSError as e:
1368
+ raise HelperError(2, f"could not restore {target} from {backup.name}: {e}")
1369
+ state["ccc_init"] = "failed"
1370
+ state["index_block"] = ("settings.yml lacks the ccc default exclusions and ccc init "
1371
+ f"could not rebuild it ({_output_hint(output)})")
1372
+ warnings.append(
1373
+ "could not rebuild .cocoindex_code/settings.yml on the ccc defaults because "
1374
+ f"ccc init failed ({_output_hint(output)}); the file was left unchanged and "
1375
+ "SKF exclusions were not updated this run; fix ccc init in {project-root} "
1376
+ "and re-run /skf-setup"
1377
+ )
1378
+ return None
1379
+
1380
+ fresh_excludes = [str(p) for p in fresh["exclude_patterns"]]
1381
+ original_excludes = [str(p) for p in (original.get("exclude_patterns") or [])]
1382
+ data = {**fresh, **{k: v for k, v in original.items() if k != "exclude_patterns"}}
1383
+ data["exclude_patterns"] = fresh_excludes + [
1384
+ e for e in original_excludes if e not in fresh_excludes
1385
+ ]
1386
+ state.update(ccc_init="rebuilt", written=True)
1387
+ warnings.append(
1388
+ "rebuilt .cocoindex_code/settings.yml on the ccc defaults: it lacked them, so "
1389
+ "ccc was also indexing hidden folders, node_modules and its own database; your "
1390
+ "own entries were kept"
1391
+ )
1392
+ return data
1393
+
1394
+
1395
+ def _inside_always_excluded(value: str) -> bool:
1396
+ """True when an always-included `**/<name>` pattern already covers the folder."""
1397
+ return any(f"**/{part}" in ALWAYS_INCLUDE for part in PurePosixPath(value).parts)
1398
+
1399
+
1400
+ def _printable(name: str) -> str:
1401
+ """A name safe to show in a warning: backslashes and control characters become `?`."""
1402
+ return "".join("?" if ch == "\\" or ord(ch) < 0x20 or ord(ch) == 0x7F
1403
+ or 0xD800 <= ord(ch) <= 0xDFFF else ch for ch in name)
1404
+
1405
+
1406
+ def _sample(names: list[str]) -> str:
1407
+ return ", ".join(_printable(n) for n in names[:SAMPLE_SIZE])
1408
+
1409
+
1410
+ def _collision_warning(key: str, value: str, foreign: list[str], kinds) -> str:
1411
+ installed = (
1412
+ " If those are skills installed from elsewhere, keep the folder: once SKF has "
1413
+ "written its own output there, re-run /skf-setup and SKF excludes only its own "
1414
+ "entries."
1415
+ ) if "skills" in kinds else ""
1416
+ return (
1417
+ f"{key} {value} already holds {_plural(len(foreign), 'entry', 'entries')} SKF did "
1418
+ f"not generate and no SKF output (for example {_sample(foreign)}), so SKF left it "
1419
+ f"out of the ccc exclusions and that content stays indexed. If that is source you "
1420
+ f"work on, set {key} in {{project-root}}/_bmad/skf/config.yaml to a folder only SKF "
1421
+ f"uses and re-run /skf-setup.{installed} To exclude it anyway, add {value} to "
1422
+ f"exclude_patterns in .cocoindex_code/settings.yml yourself; SKF never removes "
1423
+ f"entries it did not add"
1424
+ )
1425
+
1426
+
1427
+ def _shared_folder_note(key: str, value: str, foreign: list[str], kinds) -> str:
1428
+ later = (
1429
+ "A skill SKF creates there later stays indexed, twice through its version folder "
1430
+ "and its active link, until you re-run /skf-setup."
1431
+ ) if "skills" in kinds else (
1432
+ "Output SKF writes there later stays indexed until you re-run /skf-setup."
1433
+ )
1434
+ return (
1435
+ f"{key} {value} also holds {_plural(len(foreign), 'entry', 'entries')} SKF did not "
1436
+ f"generate (for example {_sample(foreign)}), so SKF excluded only its own entries "
1437
+ f"there and those stay indexed. {later} To exclude the other entries too, add each "
1438
+ f"as {value}/<name> to exclude_patterns in .cocoindex_code/settings.yml yourself; "
1439
+ f"SKF never removes entries it did not add"
1440
+ )
1441
+
1442
+
1443
+ def _unowned_entry_warning(key: str, value: str, foreign: list[str]) -> str:
1444
+ return (
1445
+ f"{key} {value} is in exclude_patterns in .cocoindex_code/settings.yml but SKF has "
1446
+ f"no record of adding it, so SKF left it in place; it also excludes "
1447
+ f"{_plural(len(foreign), 'entry', 'entries')} SKF did not generate (for example "
1448
+ f"{_sample(foreign)}). If those should stay searchable, remove {value} from "
1449
+ f"exclude_patterns and re-run /skf-setup; SKF then excludes only its own output there"
1450
+ )
1451
+
1452
+
1453
+ def _unwritable_warning(key: str, value: str, skipped: list[str]) -> str:
1454
+ return (
1455
+ f"{key} {value} holds {_plural(len(skipped), 'SKF entry', 'SKF entries')} whose "
1456
+ f"name SKF cannot write as a ccc pattern (a quote, a backslash, a control character "
1457
+ f"or bytes that are not UTF-8; for example {_sample(skipped)}), so they stay indexed; "
1458
+ f"rename them, or add them to exclude_patterns in .cocoindex_code/settings.yml "
1459
+ f"yourself"
1460
+ )
1461
+
1462
+
1463
+ # ─── a user `!` entry that cancels an SKF pattern ──────────────────────────
1464
+
1465
+ # The child name ccc's matcher probes an excluded directory with when it
1466
+ # checks whether a `!` entry applies below it.
1467
+ CCC_DIR_PROBE = "__probe__"
1468
+ # Brace alternatives of one `!` entry checked at most, and `{` in one entry
1469
+ # expanded at most (guards, not limits any real entry reaches).
1470
+ MAX_BRACE_ALTERNATIVES = 256
1471
+ MAX_BRACE_GROUPS = 64
1472
+ # ccc reads settings.yml on the machine this helper runs on, and its glob
1473
+ # library reads a backslash as `/` on Windows and as an escape elsewhere.
1474
+ CCC_BACKSLASH_IS_SLASH = os.name == "nt"
1475
+
1476
+
1477
+ def _brace_alternatives(pattern: str) -> list[str]:
1478
+ """Expand the `{a,b}` groups of a glob as ccc does before its prefix check.
1479
+
1480
+ Nested groups recurse, braces and commas inside `[...]` are literal, and
1481
+ an unbalanced group leaves the pattern as it is. A pattern with more
1482
+ than MAX_BRACE_GROUPS `{` is left as it is too, so the recursion stays
1483
+ shallow whatever the entry holds.
1484
+ """
1485
+ if pattern.count("{") > MAX_BRACE_GROUPS:
1486
+ return [pattern]
1487
+ in_class = False
1488
+ for i, ch in enumerate(pattern):
1489
+ if ch == "[":
1490
+ in_class = True
1491
+ elif ch == "]":
1492
+ in_class = False
1493
+ elif ch == "{" and not in_class:
1494
+ depth, inner_class, start, alternatives, j = 1, False, i + 1, [], i + 1
1495
+ while j < len(pattern) and depth:
1496
+ c = pattern[j]
1497
+ if c == "[":
1498
+ inner_class = True
1499
+ elif c == "]":
1500
+ inner_class = False
1501
+ elif c == "{" and not inner_class:
1502
+ depth += 1
1503
+ elif c == "}" and not inner_class:
1504
+ depth -= 1
1505
+ if depth == 0:
1506
+ alternatives.append(pattern[start:j])
1507
+ elif c == "," and not inner_class and depth == 1:
1508
+ alternatives.append(pattern[start:j])
1509
+ start = j + 1
1510
+ j += 1
1511
+ if depth:
1512
+ return [pattern]
1513
+ out: list[str] = []
1514
+ for alternative in alternatives:
1515
+ out.extend(_brace_alternatives(pattern[:i] + alternative + pattern[j:]))
1516
+ if len(out) >= MAX_BRACE_ALTERNATIVES:
1517
+ return out[:MAX_BRACE_ALTERNATIVES]
1518
+ return out
1519
+ return [pattern]
1520
+
1521
+
1522
+ def _ccc_glob_regex(glob: str, backslash_escape: bool):
1523
+ """A regex for one brace-free glob as ccc's glob library reads it, or None.
1524
+
1525
+ `*` and `?` also match `/`; `**/` at the start, `/**/` inside and `/**`
1526
+ at the end span whole directories; `[...]` and `[!...]` or `[^...]` are
1527
+ classes. A backslash escapes the next character when `backslash_escape`
1528
+ is set (POSIX); otherwise the caller has already turned it into `/`
1529
+ (Windows).
264
1530
  """
265
- seen = set(existing_patterns)
266
- merged = list(existing_patterns)
267
- newly_added: list[str] = []
268
- for p in to_add:
269
- if p in seen:
1531
+ collapsed = None
1532
+ while collapsed != glob: # `**/**` spans what one `**` spans
1533
+ collapsed = glob
1534
+ glob = glob.replace("/**/**", "/**")
1535
+ if glob.startswith("**/**"):
1536
+ glob = glob[3:]
1537
+ if glob in ("**", "**/"):
1538
+ return re.compile(".*", re.S)
1539
+ out, i, n = [], 0, len(glob)
1540
+ if glob.startswith("**/"):
1541
+ out.append("(?:.*/)?")
1542
+ i = 3
1543
+ while i < n:
1544
+ ch = glob[i]
1545
+ if glob.startswith("/**/", i):
1546
+ out.append("(?:/|/.*/)")
1547
+ i += 4
1548
+ elif glob.startswith("/**", i) and i + 3 == n:
1549
+ out.append("/.*")
1550
+ i += 3
1551
+ elif ch == "*":
1552
+ while i < n and glob[i] == "*":
1553
+ i += 1
1554
+ out.append(".*")
1555
+ elif ch == "?":
1556
+ out.append(".")
1557
+ i += 1
1558
+ elif ch == "[":
1559
+ j = i + 1
1560
+ negate = j < n and glob[j] in "!^"
1561
+ if negate:
1562
+ j += 1
1563
+ k = j + 1 if j < n and glob[j] == "]" else j
1564
+ while k < n and glob[k] != "]":
1565
+ k += 1
1566
+ if k >= n:
1567
+ return None
1568
+ body = "".join("\\" + c if c in "\\^[]" else c for c in glob[j:k])
1569
+ out.append(f"[{'^' if negate else ''}{body}]")
1570
+ i = k + 1
1571
+ elif ch == "\\" and backslash_escape:
1572
+ if i + 1 >= n:
1573
+ return None
1574
+ out.append(re.escape(glob[i + 1]))
1575
+ i += 2
1576
+ else:
1577
+ out.append(re.escape(ch))
1578
+ i += 1
1579
+ try:
1580
+ return re.compile("".join(out), re.S)
1581
+ except re.error:
1582
+ return None
1583
+
1584
+
1585
+ def _ccc_glob(glob: str, backslash_is_slash: bool):
1586
+ """`_ccc_glob_regex` for one brace-free glob, its backslashes read as ccc
1587
+ reads them on Windows (`/`) or elsewhere (an escape)."""
1588
+ if backslash_is_slash:
1589
+ return _ccc_glob_regex(glob.replace("\\", "/"), False)
1590
+ return _ccc_glob_regex(glob, True)
1591
+
1592
+
1593
+ @functools.lru_cache(maxsize=512)
1594
+ def _negation_forms(body: str, backslash_is_slash: bool) -> tuple[tuple[str, ...], tuple]:
1595
+ """The brace alternatives of a `!` entry's body, and their regexes."""
1596
+ alternatives = tuple(_brace_alternatives(body))
1597
+ globs = (_ccc_glob(alt, backslash_is_slash) for alt in alternatives)
1598
+ return alternatives, tuple(rx for rx in globs if rx is not None)
1599
+
1600
+
1601
+ def _negation_reopens(entry: str, target: str, backslash_is_slash: bool | None = None) -> bool:
1602
+ """True when the `!` entry makes ccc walk into or index `target` again.
1603
+
1604
+ ccc keeps an excluded path when a `!` entry matches it, or matches the
1605
+ child it probes the path with (`target/__probe__`, so in practice a
1606
+ wildcard child), or when one of the entry's brace alternatives starts
1607
+ with `target/`: a directory on the way to a re-included path is walked.
1608
+ The glob is read as ccc's glob library reads it on this machine
1609
+ (CCC_BACKSLASH_IS_SLASH unless `backslash_is_slash` is given).
1610
+ """
1611
+ if backslash_is_slash is None:
1612
+ backslash_is_slash = CCC_BACKSLASH_IS_SLASH
1613
+ alternatives, regexes = _negation_forms(entry[1:], backslash_is_slash)
1614
+ if any(alt.startswith(target + "/") for alt in alternatives):
1615
+ return True
1616
+ probe = f"{target}/{CCC_DIR_PROBE}"
1617
+ return any(rx.fullmatch(target) or rx.fullmatch(probe) for rx in regexes)
1618
+
1619
+
1620
+ def _pattern_path(pattern: str) -> str | None:
1621
+ """The path a pattern SKF wrote names, or None for a real glob.
1622
+
1623
+ Undoes ccc_literal's one-character classes; a `*` left over marks a
1624
+ result or report family, which a `!` entry cannot re-open as a folder.
1625
+ """
1626
+ out, i = [], 0
1627
+ while i < len(pattern):
1628
+ ch = pattern[i]
1629
+ if (ch == "[" and i + 2 < len(pattern) and pattern[i + 2] == "]"
1630
+ and pattern[i + 1] in CCC_CLASS_ESCAPES):
1631
+ out.append(pattern[i + 1])
1632
+ i += 3
270
1633
  continue
271
- seen.add(p)
272
- merged.append(p)
273
- newly_added.append(p)
274
- return merged, newly_added
275
-
276
-
277
- def render_settings_yml(data: dict, original_text: str | None) -> str:
278
- """Render the updated settings dict as YAML.
279
-
280
- For first-time-write (no original text), emit a minimal file with
281
- just the exclude_patterns the script controls. For updates, dump the
282
- full data structure with PyYAML — this DROPS comments from the
283
- original file. The cocoindex CLI is the only consumer of settings.yml
284
- contents, and it doesn't care about comments. User-customized
285
- exclude_patterns entries are preserved (set-union, not overwrite);
286
- other top-level keys (if any) are also preserved.
287
- """
288
- return yaml.safe_dump(data, default_flow_style=False, sort_keys=False, allow_unicode=True)
289
-
290
-
291
- def cmd_merge(project_root: Path, skills_output_folder: str, forge_data_folder: str) -> None:
292
- target = project_root / ".cocoindex_code" / "settings.yml"
293
-
294
- # Resolve `{project-root}/...` template strings here so callers can
295
- # forward raw config values verbatim. This removes the "substitute and
296
- # basename" reasoning from step prose — the script owns the contract.
297
- skills_output_folder = resolve_repo_relative(skills_output_folder, project_root)
298
- forge_data_folder = resolve_repo_relative(forge_data_folder, project_root)
299
-
300
- patterns_to_add, warnings = assemble_patterns(skills_output_folder, forge_data_folder)
301
- data, existed = read_settings_yml(target)
302
-
303
- existing_excludes = data.get("exclude_patterns", []) or []
304
- if not isinstance(existing_excludes, list):
305
- _die(1, f"exclude_patterns in {target} is not a list "
306
- f"(got {type(existing_excludes).__name__})")
307
- # Normalize entries to strings (cocoindex tolerates non-string entries
308
- # but we want byte-identical comparison)
309
- existing_excludes = [str(p) for p in existing_excludes]
310
-
311
- merged, newly_added = merge_patterns(existing_excludes, patterns_to_add)
312
- patterns_already_present = len(patterns_to_add) - len(newly_added)
313
-
314
- if not newly_added:
315
- # Idempotent re-run — no write, no mtime change.
316
- _ok({
317
- "settings_yml_existed": existed,
318
- "settings_yml_path": str(target),
319
- "patterns_added": 0,
320
- "patterns_added_list": [],
321
- "patterns_already_present": patterns_already_present,
322
- "effective_patterns": sorted(set(patterns_to_add)),
323
- "written": False,
324
- "warnings": warnings,
325
- })
326
- return
1634
+ if ch in CCC_CLASS_ESCAPES:
1635
+ return None
1636
+ out.append(ch)
1637
+ i += 1
1638
+ return "".join(out)
1639
+
1640
+
1641
+ def _negation_warning(key: str, value: str, entries: list[str], whole: bool, kinds,
1642
+ everything: bool = True) -> str:
1643
+ shown = _sample(entries) + (
1644
+ f" and {len(entries) - SAMPLE_SIZE} more" if len(entries) > SAMPLE_SIZE else "")
1645
+ it = "that entry" if len(entries) == 1 else "those entries"
1646
+ if whole:
1647
+ twice = (", twice for each skill through its version folder and its active link"
1648
+ if "skills" in kinds else "")
1649
+ always = next((part for part in PurePosixPath(value).parts
1650
+ if f"**/{part}" in ALWAYS_INCLUDE), None)
1651
+ own = (
1652
+ f"SKF always keeps {always} out of ccc, so keep content you want searchable "
1653
+ f"outside it"
1654
+ ) if always else (
1655
+ f"To keep content of your own in {value} searchable, put it in a folder of its "
1656
+ f"own there and re-run /skf-setup; SKF then excludes its entries one by one and "
1657
+ f"leaves yours indexed"
1658
+ )
1659
+ amount = "all" if everything else "some"
1660
+ return (
1661
+ f"{key} {value} is excluded from ccc as one folder, but exclude_patterns in "
1662
+ f".cocoindex_code/settings.yml also lists {shown}: ccc applies a ! entry against "
1663
+ f"every exclusion, so {amount} SKF output in {value} is indexed again{twice}. "
1664
+ f"Remove {it}; SKF output never needs one. {own}"
1665
+ )
1666
+ return (
1667
+ f"{key} {value} is excluded from ccc one SKF entry at a time, but exclude_patterns "
1668
+ f"in .cocoindex_code/settings.yml also lists {shown}: ccc applies a ! entry against "
1669
+ f"every exclusion, so some SKF output there is indexed again. Remove {it}; your own "
1670
+ f"content in {value} stays indexed without it"
1671
+ )
327
1672
 
328
- data["exclude_patterns"] = merged
329
- rendered = render_settings_yml(data, None)
330
- _atomic_write(target, rendered)
331
1673
 
332
- _ok({
333
- "settings_yml_existed": existed,
334
- "settings_yml_path": str(target),
335
- "patterns_added": len(newly_added),
336
- "patterns_added_list": newly_added,
337
- "patterns_already_present": patterns_already_present,
338
- "effective_patterns": sorted(set(patterns_to_add)),
339
- "written": True,
340
- "warnings": warnings,
341
- })
1674
+ def _negation_warnings(root: Path, merged: list[str], by_value: dict, verdicts: dict,
1675
+ value_patterns: dict, listings: dict) -> list[str]:
1676
+ """One warning per folder a user `!` entry brings SKF output back into ccc for.
1677
+
1678
+ Only bare and per-entry verdicts are checked, against `merged` (the
1679
+ exclude_patterns this run writes). A pattern counts when ccc walks down
1680
+ to a file of SKF output below it that git lists (`listings` holds the
1681
+ verdict loop's (paths, scan) per folder; a folder inside an
1682
+ always-excluded one is listed here), or, with no such file, into the
1683
+ child it probes the pattern with, as output SKF writes there later
1684
+ would be. The `!` entries that re-open a pattern that counts are named;
1685
+ the bare value's warning says all SKF output is back only when every
1686
+ SKF entry there is reached.
1687
+ """
1688
+ negations = [e for e in merged if e.startswith("!")]
1689
+ if not negations:
1690
+ return []
1691
+ slash = CCC_BACKSLASH_IS_SLASH
1692
+ # A pattern with no glob character left (SKF's own, mostly) matches one path.
1693
+ literal = {path for path in (_pattern_path(p) for p in merged
1694
+ if not p.startswith("!") and "\\" not in p) if path}
1695
+ globs = [_ccc_glob(alt, slash) for p in merged
1696
+ if not p.startswith("!") and ("\\" in p or _pattern_path(p) is None)
1697
+ for alt in _brace_alternatives(p)]
1698
+ excludes = [rx for rx in globs if rx is not None]
1699
+ kept: dict[str, bool] = {}
1700
+
1701
+ def walked(path: str) -> bool:
1702
+ """True when ccc walks every directory down to `path`, and `path` itself:
1703
+ each is matched by no exclude pattern, or re-opened by a `!` entry."""
1704
+ parts = PurePosixPath(path).parts
1705
+ for depth in range(1, len(parts) + 1):
1706
+ step = "/".join(parts[:depth])
1707
+ if step not in kept:
1708
+ kept[step] = ((step not in literal
1709
+ and not any(rx.fullmatch(step) for rx in excludes))
1710
+ or any(_negation_reopens(n, step, slash) for n in negations))
1711
+ if not kept[step]:
1712
+ return False
1713
+ return True
1714
+
1715
+ out = []
1716
+ for value, entries in by_value.items():
1717
+ bare = verdicts[value] == "bare"
1718
+ if not bare and verdicts[value] != "entries":
1719
+ continue
1720
+ kinds = {kind for _, kind in entries}
1721
+ if value in listings:
1722
+ paths, scan = listings[value]
1723
+ else:
1724
+ paths = list_folder_paths(root, value)[0]
1725
+ others = [v for v in by_value if v != value]
1726
+ scan = classify_folder(root, value, kinds, paths, others) if paths else None
1727
+ owned = set(scan.owned_groups) | set(scan.owned_files) if scan else set()
1728
+ files: dict[str, list[str]] = {} # the listed files under each first-level entry
1729
+ for rel in paths or []:
1730
+ if not bare or rel[0] in owned:
1731
+ files.setdefault(rel[0], []).append("/".join((value, *rel)))
1732
+ reached, everything = [], True
1733
+ for target in filter(None, map(_pattern_path, value_patterns[value])):
1734
+ if target == value:
1735
+ hits = [any(map(walked, below)) for below in files.values()]
1736
+ else: # `{value}/<entry>`
1737
+ below = files.get(target[len(value) + 1:], [])
1738
+ hits = [any(map(walked, below))] if below else []
1739
+ if not hits:
1740
+ hits = [walked(f"{target}/{CCC_DIR_PROBE}")]
1741
+ everything = everything and all(hits)
1742
+ if any(hits):
1743
+ reached.append(target)
1744
+ cancelling = [e for e in negations
1745
+ if any(_negation_reopens(e, t, slash) for t in reached)]
1746
+ if cancelling:
1747
+ out.append(_negation_warning(" and ".join(k for k, _ in entries), value,
1748
+ cancelling, bare, kinds, everything))
1749
+ return out
1750
+
1751
+
1752
+ def _list_dir(path: Path) -> list[str] | None:
1753
+ """The names in `path` as the OS reports them (what ccc matches), or None."""
1754
+ try:
1755
+ return os.listdir(path)
1756
+ except OSError:
1757
+ return None
1758
+
1759
+
1760
+ def _contains(outer: str, inner: str) -> bool:
1761
+ """True when folder `inner` sits below folder `outer`."""
1762
+ o, i = PurePosixPath(outer).parts, PurePosixPath(inner).parts
1763
+ return i[:len(o)] == o and len(i) > len(o)
1764
+
1765
+
1766
+ def _stale_bare_values(existing: list[str], record: list[str], folders: list[str]) -> set[str]:
1767
+ """Bare folder values SKF wrote after its record was last saved.
1768
+
1769
+ Setup writes settings.yml before it saves the record, so a run stopped in
1770
+ between leaves a bare value the record does not hold. When the record
1771
+ holds `{value}/...` per-entry patterns for a folder and none of them is
1772
+ still in settings.yml, SKF swapped them for the bare value itself: that
1773
+ value is SKF's, not the user's. Patterns of another configured folder
1774
+ inside this one do not count.
1775
+ """
1776
+ stale: set[str] = set()
1777
+ for value in folders:
1778
+ if value not in existing or value in record:
1779
+ continue
1780
+ inner = [v for v in folders if _contains(value, v)]
1781
+ entries = [r for r in record if r.startswith(value + "/")
1782
+ and not any(r == v or r.startswith(v + "/") for v in inner)]
1783
+ if entries and not set(entries) & set(existing):
1784
+ stale.add(value)
1785
+ return stale
1786
+
1787
+
1788
+ def _reconcile(root: Path, target: Path, backup: Path, data: dict,
1789
+ values: dict[str, tuple[str, str]], record: list[str], prior_exists: bool,
1790
+ prune_allowed: bool, state: dict, warnings: list[str]) -> None:
1791
+ """Give each folder a verdict, merge and prune; write settings.yml when it changed.
1792
+
1793
+ Verdicts and record trust follow the module docstring. Folders are
1794
+ checked deepest first, so a folder holding another configured folder
1795
+ knows that folder's verdict: when it is `entries` or `left_out`, the
1796
+ outer folder uses per-entry patterns so its bare value never covers
1797
+ content left indexed on purpose.
1798
+ """
1799
+ existing = [str(p) for p in (data.get("exclude_patterns") or [])]
1800
+ # Legacy: a folder-free record (at most the always-included patterns)
1801
+ # that is empty or sits next to a `**/<value>` form predates the
1802
+ # anchored forms, so those forms count as owned and migrate. A
1803
+ # folder-free record without them comes from a run whose folders were
1804
+ # all user-excluded or left out, and is trusted like any other.
1805
+ folder_free = prior_exists and not (set(record) - set(ALWAYS_INCLUDE))
1806
+ legacy_forms = {f"**/{v}" for v, _k in values.values()} & set(existing)
1807
+ legacy = folder_free and (not record or bool(legacy_forms))
1808
+ owned_prior = set(record) | (legacy_forms if legacy else set())
1809
+ trusted = prior_exists and not legacy
1810
+ owned_prior |= _stale_bare_values(existing, record, [v for v, _k in values.values()])
1811
+ state["legacy"] = legacy
1812
+ hidden_neutral = "**/.*" in existing and not any(e.startswith("!") for e in existing)
1813
+ by_value: dict[str, list[tuple[str, str]]] = {}
1814
+ for key, (value, kind) in values.items():
1815
+ by_value.setdefault(value, []).append((key, kind))
1816
+ verdicts: dict[str, str] = {}
1817
+ value_patterns: dict[str, list[str]] = {}
1818
+ listings: dict[str, tuple] = {}
1819
+ for value in sorted(by_value, key=lambda v: -len(PurePosixPath(v).parts)):
1820
+ entries = by_value[value]
1821
+ keys = " and ".join(key for key, _kind in entries)
1822
+ if trusted and value in existing and value not in owned_prior:
1823
+ verdicts[value], value_patterns[value] = "user", []
1824
+ continue
1825
+ if _inside_always_excluded(value):
1826
+ verdicts[value], value_patterns[value] = "bare", [value]
1827
+ continue
1828
+ paths, problem = list_folder_paths(root, value)
1829
+ if problem:
1830
+ warnings.append(f"{problem} for {keys}; collision check skipped")
1831
+ others = [v for v in by_value if v != value]
1832
+ kinds = {kind for _key, kind in entries}
1833
+ scan = (classify_folder(root, value, kinds, paths, others, hidden_neutral)
1834
+ if paths else None)
1835
+ listings[value] = paths, scan
1836
+ deferred = any(_contains(value, v) and verdicts.get(v) in ("entries", "left_out")
1837
+ for v in others)
1838
+ if (scan is not None and (scan.foreign or deferred)
1839
+ and value in existing and value not in owned_prior):
1840
+ # No trusted record says who added the bare value: keep it, and
1841
+ # tell the user once (the next run has a trusted record).
1842
+ if scan.foreign:
1843
+ warnings.append(_unowned_entry_warning(keys, value, scan.foreign))
1844
+ verdicts[value], value_patterns[value] = "user", []
1845
+ continue
1846
+ if scan is not None and scan.foreign and not scan.has_output:
1847
+ warnings.append(_collision_warning(keys, value, scan.foreign, kinds))
1848
+ verdicts[value], value_patterns[value] = "left_out", []
1849
+ continue
1850
+ if scan is not None and (scan.foreign or deferred):
1851
+ patterns, skipped = entry_patterns(value, kinds, scan.owned_groups,
1852
+ scan.owned_files, _list_dir(root / value),
1853
+ scan.listed)
1854
+ verdicts[value], value_patterns[value] = "entries", patterns
1855
+ if scan.foreign:
1856
+ warnings.append(_shared_folder_note(keys, value, scan.foreign, kinds))
1857
+ if skipped:
1858
+ warnings.append(_unwritable_warning(keys, value, skipped))
1859
+ continue
1860
+ verdicts[value], value_patterns[value] = "bare", [value]
1861
+ produced = list(ALWAYS_INCLUDE)
1862
+ for value in by_value:
1863
+ for p in value_patterns[value]:
1864
+ if p not in produced:
1865
+ produced.append(p)
1866
+
1867
+ merged, added, removed = plan_exclusions(existing, produced, owned_prior, prune_allowed)
1868
+ warnings.extend(_negation_warnings(root, merged, by_value, verdicts, value_patterns,
1869
+ listings))
1870
+ blocked = [] if prune_allowed else sorted((owned_prior - set(produced)) & set(existing))
1871
+ if blocked:
1872
+ shown = ", ".join(blocked[:SAMPLE_SIZE]) + (
1873
+ f" and {len(blocked) - SAMPLE_SIZE} more" if len(blocked) > SAMPLE_SIZE else "")
1874
+ warnings.append(
1875
+ "kept previously recorded SKF exclusions (" + shown + ") because a "
1876
+ "folder value was refused; fix it and re-run /skf-setup to remove patterns SKF "
1877
+ "no longer needs"
1878
+ )
1879
+
1880
+ if merged != existing or state["written"]:
1881
+ data["exclude_patterns"] = merged
1882
+ # ASCII-only output (non-ASCII escaped), as ccc writes it: ccc reads
1883
+ # settings.yml with the platform default encoding.
1884
+ _atomic_write(target, yaml.safe_dump(data, default_flow_style=False, sort_keys=False))
1885
+ state["written"] = True
1886
+ if backup.is_file():
1887
+ try:
1888
+ backup.unlink()
1889
+ except OSError as e:
1890
+ raise HelperError(2, f"could not remove {backup}: {e}")
1891
+ state.update(added=added, removed=removed,
1892
+ present=len(produced) - len(added),
1893
+ effective=sorted(set(produced) | set(blocked)))
1894
+
1895
+
1896
+ def run_merge(project_root, skills_output_folder, forge_data_folder, prior_state_from=None,
1897
+ index_fresh: bool = False, skip_index: bool = False,
1898
+ allow_ccc_init: bool = True, defer_index: bool = False) -> dict:
1899
+ """Prepare settings.yml, reconcile the SKF patterns, decide the index action.
1900
+
1901
+ Returns the v2 payload documented in the module docstring. Raises
1902
+ HelperError for the exit-1 and exit-2 cases.
1903
+ """
1904
+ root = Path(project_root)
1905
+ target = root / SETTINGS_DIR / SETTINGS_NAME
1906
+ backup = target.with_name(BACKUP_NAME)
1907
+ gitignore = root / ".gitignore"
1908
+ warnings: list[str] = []
1909
+ gitignore_before = _read_bytes_or_none(gitignore)
1910
+
1911
+ # Crash recovery: a leftover backup is the pre-repair original.
1912
+ if backup.is_file():
1913
+ try:
1914
+ os.replace(backup, target)
1915
+ except OSError as e:
1916
+ raise HelperError(2, f"could not restore {target} from {backup.name}: {e}")
1917
+ warnings.append("restored .cocoindex_code/settings.yml from an interrupted SKF repair")
1918
+
1919
+ values: dict[str, tuple[str, str]] = {}
1920
+ prune_allowed = True
1921
+ for (key, kind), raw in zip(FOLDER_KEYS, (skills_output_folder, forge_data_folder)):
1922
+ value, warn = validate_config_value(key, resolve_repo_relative(raw, root))
1923
+ if value is None:
1924
+ prune_allowed = False
1925
+ warnings.append(warn)
1926
+ else:
1927
+ values[key] = (value, kind)
1928
+
1929
+ record = read_owned_record(prior_state_from, warnings)
1930
+ # With no forge-tier.yaml at all, SKF never finished a setup here and
1931
+ # owns nothing yet; _reconcile decides whether the record is legacy.
1932
+ prior_exists = prior_state_from is not None and Path(prior_state_from).is_file()
1933
+
1934
+ state = {
1935
+ "existed": target.is_file(),
1936
+ "ccc_init": "not_needed",
1937
+ "ready": True,
1938
+ "reason": None,
1939
+ "written": False,
1940
+ "added": [],
1941
+ "removed": [],
1942
+ "present": 0,
1943
+ "effective": None,
1944
+ "index_block": None,
1945
+ "legacy": False,
1946
+ }
1947
+
1948
+ data = _prepare_settings(root, target, backup, allow_ccc_init, state, warnings)
1949
+ if data is not None:
1950
+ _reconcile(root, target, backup, data, values, record, prior_exists, prune_allowed,
1951
+ state, warnings)
1952
+ if state["legacy"] and not prune_allowed:
1953
+ # Keep the folder-free record so the next valid run can still
1954
+ # migrate the legacy `**/<value>` forms of the refused value.
1955
+ state["effective"] = None
1956
+
1957
+ if state["ready"] and target.is_file():
1958
+ res = _git(root, "check-ignore", "-q", "--no-index", "--",
1959
+ ".cocoindex_code/target_sqlite.db")
1960
+ if res is not None and res[0] == 1:
1961
+ warnings.append(
1962
+ "/.cocoindex_code/ is not gitignored here, so git lists the ccc index "
1963
+ "database (tens of MB) as untracked; add /.cocoindex_code/ to "
1964
+ "{project-root}/.gitignore"
1965
+ )
1966
+
1967
+ action = decide_index_action(state["ready"], skip_index, state["written"], index_fresh,
1968
+ usable=state["index_block"] is None, defer=defer_index)
1969
+ reason = state["reason"] or (state["index_block"] if action == "fail" else None)
1970
+ if action == "skip" and state["written"]:
1971
+ warnings.append(
1972
+ f"--ccc-skip-index: settings.yml exclusions changed (+{len(state['added'])}, "
1973
+ f"-{len(state['removed'])}) but the index was not rebuilt; run ccc index in "
1974
+ "{project-root} or re-run /skf-setup without --ccc-skip-index"
1975
+ )
1976
+
1977
+ return {
1978
+ "status": "ok",
1979
+ "version": "v2",
1980
+ "settings_yml_path": str(target),
1981
+ "settings_yml_existed": state["existed"],
1982
+ "ccc_init": state["ccc_init"],
1983
+ "settings_ready": state["ready"],
1984
+ "not_ready_reason": _payload_safe(reason) if reason else None,
1985
+ "patterns_added": len(state["added"]),
1986
+ "patterns_added_list": state["added"],
1987
+ "patterns_removed": len(state["removed"]),
1988
+ "patterns_removed_list": state["removed"],
1989
+ "patterns_already_present": state["present"],
1990
+ "effective_patterns": state["effective"],
1991
+ "written": state["written"],
1992
+ "gitignore_updated": _read_bytes_or_none(gitignore) != gitignore_before,
1993
+ "index_action": action,
1994
+ "warnings": [_payload_safe(w) for w in warnings],
1995
+ }
1996
+
1997
+
1998
+ # ─── Clone mode ─────────────────────────────────────────────────────────────
1999
+
2000
+
2001
+ def quote_bare_glob_items(text: str) -> tuple[str, int]:
2002
+ """Single-quote each bare `- *...` list item; return (text, items quoted).
2003
+
2004
+ YAML reads a plain scalar that starts with `*` as an alias, so a
2005
+ `- **/build` item an earlier hand edit wrote stops ccc from loading the
2006
+ file. ccc writes these items quoted, as yaml.safe_dump does. A trailing
2007
+ comment stays a comment, after the closing quote.
2008
+ """
2009
+ return BARE_GLOB_ITEM_RE.subn(lambda m: f"{m.group(1)}'{m.group(2)}'{m.group(3)}", text)
2010
+
2011
+
2012
+ def _load_clone_settings(path: Path, warnings: list[str]) -> tuple[dict, bool]:
2013
+ """Parse a clone's settings.yml: (data, repaired).
2014
+
2015
+ A file YAML cannot read is read again with its bare glob items quoted,
2016
+ and `repaired` is then True so the caller writes it back. Raises
2017
+ HelperError(1) when it still cannot be read, is not a mapping, or holds
2018
+ an exclude_patterns or include_patterns that is neither a list nor null.
2019
+ """
2020
+ try:
2021
+ text = path.read_text(encoding="utf-8")
2022
+ except OSError as e:
2023
+ raise HelperError(1, f"failed to read {path}: {e}")
2024
+ repaired = False
2025
+ try:
2026
+ data = yaml.safe_load(text)
2027
+ except (yaml.YAMLError, ValueError) as e:
2028
+ fixed, count = quote_bare_glob_items(text)
2029
+ try:
2030
+ data = yaml.safe_load(fixed) if count else None
2031
+ except (yaml.YAMLError, ValueError):
2032
+ count = 0
2033
+ if not count:
2034
+ raise HelperError(1, f"failed to parse {path}: {e}")
2035
+ repaired = True
2036
+ warnings.append(f"quoted {_plural(count, 'list item', 'list items')} of settings.yml that YAML "
2037
+ "read as an alias, so ccc can load the file again")
2038
+ if data is None:
2039
+ return {}, repaired
2040
+ if not isinstance(data, dict):
2041
+ raise HelperError(1, f"expected mapping at top of {path}, got {type(data).__name__}")
2042
+ for key in ("exclude_patterns", "include_patterns"):
2043
+ value = data.get(key)
2044
+ if value is not None and not isinstance(value, list):
2045
+ raise HelperError(1, f"{key} in {path} is not a list (got {type(value).__name__})")
2046
+ return data, repaired
2047
+
2048
+
2049
+ def include_covers(pattern, ext: str) -> bool:
2050
+ """True when an include_patterns entry matches a nested `*.<ext>` file as ccc reads it."""
2051
+ if not isinstance(pattern, str):
2052
+ return False
2053
+ probe = INCLUDE_PROBE.format(ext=ext)
2054
+ for alternative in _brace_alternatives(pattern):
2055
+ rx = _ccc_glob(alternative, CCC_BACKSLASH_IS_SLASH)
2056
+ if rx is not None and rx.fullmatch(probe):
2057
+ return True
2058
+ return False
2059
+
2060
+
2061
+ # ─── Index build (--build-index) ────────────────────────────────────────────
2062
+
2063
+
2064
+ def _run_ccc(root: Path, args: tuple[str, ...], timeout: int) -> tuple[bool, str]:
2065
+ """Run `ccc <args>` in root. Return (exit code was 0, stdout plus stderr).
2066
+
2067
+ A ccc that cannot be found, started or finished in time returns False
2068
+ with the reason as its output.
2069
+ """
2070
+ exe = _resolve_outside_cwd("ccc")
2071
+ if exe is None:
2072
+ return False, "ccc was not found on PATH"
2073
+ try:
2074
+ result = subprocess.run(
2075
+ [exe, *args],
2076
+ cwd=str(root),
2077
+ stdin=subprocess.DEVNULL,
2078
+ capture_output=True,
2079
+ env=_child_env(),
2080
+ timeout=timeout,
2081
+ )
2082
+ except subprocess.TimeoutExpired:
2083
+ return False, f"ccc {args[0]} timed out after {timeout}s"
2084
+ except (OSError, ValueError) as e:
2085
+ return False, f"ccc {args[0]} could not start: {e}"
2086
+ output = (result.stdout + result.stderr).decode("utf-8", errors="replace")
2087
+ if result.returncode != 0:
2088
+ return False, f"ccc {args[0]} exited {result.returncode}: {' '.join(output.split())[:300]}"
2089
+ return True, output
2090
+
2091
+
2092
+ def parse_status_file_count(output: str) -> int | None:
2093
+ """The `Files:` count of `ccc status`, or None when it shows none (no index yet)."""
2094
+ match = CCC_FILES_RE.search(output)
2095
+ return int(match.group(1)) if match else None
2096
+
2097
+
2098
+ def _prior_index(prior_state_from) -> dict:
2099
+ """`ccc_index` of the prior forge-tier.yaml, {} when there is none to read."""
2100
+ if prior_state_from is None or not Path(prior_state_from).is_file():
2101
+ return {}
2102
+ try:
2103
+ data = yaml.safe_load(Path(prior_state_from).read_text(encoding="utf-8"))
2104
+ except (OSError, ValueError, yaml.YAMLError):
2105
+ return {}
2106
+ ccc = data.get("ccc_index") if isinstance(data, dict) else None
2107
+ return ccc if isinstance(ccc, dict) else {}
2108
+
2109
+
2110
+ def _utc_now() -> str:
2111
+ return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
2112
+
2113
+
2114
+ def build_index(root: Path, indexed_path: str, action: str, not_ready_reason,
2115
+ prior_state_from, warnings: list[str]) -> dict:
2116
+ """The index state a setup run ends with, after acting on `action`.
2117
+
2118
+ See "Index build" in the module docstring. `warnings` gains a note when
2119
+ a pass is still running after the reruns.
2120
+ """
2121
+ unset = {"indexed_path": None, "last_indexed": None, "file_count": None, "failed_reason": None}
2122
+ if action in ("skip", "defer"):
2123
+ return {"status": "skipped", **unset}
2124
+ if action == "fail":
2125
+ return {"status": "failed", **unset,
2126
+ "failed_reason": not_ready_reason or "no usable .cocoindex_code/settings.yml"}
2127
+ if action == "keep":
2128
+ prior = _prior_index(prior_state_from)
2129
+ last = prior.get("last_indexed")
2130
+ count = prior.get("file_count")
2131
+ return {"status": "fresh", **unset, "indexed_path": indexed_path,
2132
+ "last_indexed": last.isoformat() if isinstance(last, datetime) else last,
2133
+ "file_count": count if isinstance(count, int) and not isinstance(count, bool) else None}
2134
+
2135
+ def failed(reason: str) -> dict:
2136
+ return {"status": "failed", **unset, "failed_reason": _payload_safe(reason)}
2137
+
2138
+ ok, output = _run_ccc(root, ("index",), CCC_INDEX_TIMEOUT_SEC)
2139
+ if not ok:
2140
+ return failed(output)
2141
+ reruns = 0
2142
+ while True:
2143
+ ok, status = _run_ccc(root, ("status",), CCC_STATUS_TIMEOUT_SEC)
2144
+ if not ok:
2145
+ return failed(status)
2146
+ if CCC_IN_PROGRESS not in status or reruns == CCC_INDEX_RERUNS:
2147
+ break
2148
+ reruns += 1
2149
+ ok, output = _run_ccc(root, ("index",), CCC_INDEX_TIMEOUT_SEC)
2150
+ if not ok:
2151
+ return failed(output)
2152
+ if CCC_IN_PROGRESS in status:
2153
+ warnings.append(f"ccc still reports an indexing pass in progress after {CCC_INDEX_RERUNS} reruns "
2154
+ "of ccc index; the file count may grow")
2155
+ count = parse_status_file_count(status)
2156
+ if count is None:
2157
+ return failed("ccc index finished, but ccc status shows no index")
2158
+ if count == 0:
2159
+ return failed("ccc index finished, but ccc status counts no indexed file")
2160
+ return {"status": "created", **unset, "indexed_path": indexed_path, "last_indexed": _utc_now(),
2161
+ "file_count": count}
2162
+
2163
+
2164
+ def read_config_folders(path: Path) -> tuple[str, str]:
2165
+ """The raw skills_output_folder and forge_data_folder values of config.yaml."""
2166
+ try:
2167
+ data = yaml.safe_load(path.read_text(encoding="utf-8"))
2168
+ except FileNotFoundError:
2169
+ raise HelperError(1, f"--config does not exist: {path}")
2170
+ except (OSError, ValueError, yaml.YAMLError) as e:
2171
+ raise HelperError(1, f"--config cannot be read as YAML: {path}: {e}")
2172
+ if data is None:
2173
+ data = {}
2174
+ if not isinstance(data, dict):
2175
+ raise HelperError(1, f"--config is not a YAML mapping: {path}")
2176
+ values = (data.get("skills_output_folder"), data.get("forge_data_folder"))
2177
+ return tuple("" if v is None else str(v) for v in values)
2178
+
2179
+
2180
+ def run_clone_merge(clone_root, include_exts=(), allow_ccc_init: bool = True) -> dict:
2181
+ """Clone mode: prepare a workspace clone's settings.yml and merge into it.
2182
+
2183
+ Returns the clone-mode payload documented in the module docstring.
2184
+ Raises HelperError for the exit-1 and exit-2 cases.
2185
+ """
2186
+ root = Path(clone_root)
2187
+ target = root / SETTINGS_DIR / SETTINGS_NAME
2188
+ warnings: list[str] = []
2189
+ existed = target.is_file()
2190
+ ccc_init, ready, reason, written = "not_needed", True, None, False
2191
+ added: list[str] = []
2192
+ includes_added: list[str] = []
2193
+ covered: list[str] = []
2194
+ present = 0
2195
+
2196
+ if not existed:
2197
+ if not allow_ccc_init:
2198
+ ready, reason = False, "settings.yml is missing and --no-ccc-init was given"
2199
+ else:
2200
+ _ok, output = run_ccc_init(root)
2201
+ if target.is_file():
2202
+ ccc_init, written = "created", True
2203
+ else:
2204
+ ccc_init, ready = "failed", False
2205
+ reason = ("ccc init did not create .cocoindex_code/settings.yml "
2206
+ f"({_output_hint(output)})")
2207
+
2208
+ if ready:
2209
+ data, repaired = _load_clone_settings(target, warnings)
2210
+ excludes = data.get("exclude_patterns")
2211
+ if isinstance(excludes, list):
2212
+ existing = [str(p) for p in excludes]
2213
+ added = [p for p in CLONE_EXCLUDES if p not in existing]
2214
+ present = len(CLONE_EXCLUDES) - len(added)
2215
+ if added:
2216
+ data["exclude_patterns"] = [*excludes, *added]
2217
+ else:
2218
+ warnings.append("settings.yml has no exclude_patterns list, so SKF added no exclusion: "
2219
+ "a list written from scratch replaces the ccc default exclusions")
2220
+ includes = data.get("include_patterns")
2221
+ if include_exts and isinstance(includes, list):
2222
+ current = list(includes)
2223
+ for ext in dict.fromkeys(include_exts):
2224
+ if any(include_covers(p, ext) for p in current):
2225
+ covered.append(ext)
2226
+ else:
2227
+ current.append(f"**/*.{ext}")
2228
+ includes_added.append(f"**/*.{ext}")
2229
+ if includes_added:
2230
+ data["include_patterns"] = current
2231
+ elif include_exts:
2232
+ warnings.append("settings.yml has no include_patterns list, so SKF added no file type: "
2233
+ "a list written from scratch replaces the ccc default file types")
2234
+ if added or includes_added or repaired:
2235
+ # ASCII-only output (non-ASCII escaped), as ccc writes it.
2236
+ _atomic_write(target, yaml.safe_dump(data, default_flow_style=False, sort_keys=False),
2237
+ unique=True)
2238
+ written = True
2239
+
2240
+ return {
2241
+ "status": "ok",
2242
+ "version": "v2",
2243
+ "settings_yml_path": str(target),
2244
+ "settings_yml_existed": existed,
2245
+ "ccc_init": ccc_init,
2246
+ "settings_ready": ready,
2247
+ "not_ready_reason": _payload_safe(reason) if reason else None,
2248
+ "patterns_added": len(added),
2249
+ "patterns_added_list": added,
2250
+ "patterns_already_present": present,
2251
+ "includes_added_list": includes_added,
2252
+ "includes_covered_list": covered,
2253
+ "written": written,
2254
+ "warnings": [_payload_safe(w) for w in warnings],
2255
+ }
2256
+
2257
+
2258
+ def _ext_arg(value: str) -> str:
2259
+ """argparse type for --include-ext: a plain file extension, a leading `.` dropped."""
2260
+ ext = str(value).strip()
2261
+ ext = ext[1:] if ext.startswith(".") else ext
2262
+ if not EXTENSION_RE.match(ext):
2263
+ raise argparse.ArgumentTypeError(f"expected a file extension such as ex or .ex, got {value}")
2264
+ return ext
2265
+
2266
+
2267
+ def _bool_arg(value: str) -> bool:
2268
+ """argparse type for true|false (case-insensitive); anything else is a usage error."""
2269
+ normalized = str(value).strip().lower()
2270
+ if normalized in ("true", "false"):
2271
+ return normalized == "true"
2272
+ raise argparse.ArgumentTypeError(f"expected true or false, got {value}")
2273
+
2274
+
2275
+ class _JsonErrorParser(argparse.ArgumentParser):
2276
+ """Report usage errors as the same sanitized stderr JSON as every other exit.
2277
+
2278
+ A caller that reads stderr JSON may forward the message into a
2279
+ single-quoted payload; argparse's own text quotes the bad value.
2280
+ """
2281
+
2282
+ def error(self, message: str) -> None:
2283
+ _die(2, f"usage error: {message}")
342
2284
 
343
2285
 
344
2286
  def main() -> None:
345
- parser = argparse.ArgumentParser(
346
- description="Merge SKF exclusion patterns into .cocoindex_code/settings.yml.",
2287
+ parser = _JsonErrorParser(
2288
+ description="Prepare .cocoindex_code/settings.yml (running ccc init when needed), "
2289
+ "reconcile the SKF exclusion patterns in it, and decide whether the "
2290
+ "ccc index must be built.",
347
2291
  formatter_class=argparse.RawDescriptionHelpFormatter,
348
2292
  )
349
- parser.add_argument(
350
- "--project-root", type=Path, required=True,
351
- help="Absolute path to the project root. Script writes/reads "
352
- "{project-root}/.cocoindex_code/settings.yml.",
2293
+ roots = parser.add_mutually_exclusive_group(required=True)
2294
+ roots.add_argument(
2295
+ "--project-root",
2296
+ help="Absolute path to the project root. The script reads and writes "
2297
+ "{project-root}/.cocoindex_code/settings.yml and runs ccc init there. "
2298
+ "--build-index records the value as given as the index's indexed_path.",
2299
+ )
2300
+ roots.add_argument(
2301
+ "--clone-root", type=Path,
2302
+ help="Clone mode: absolute path to an SKF workspace clone. The script runs ccc init -f "
2303
+ "there when .cocoindex_code/settings.yml is missing, and merges the standard build "
2304
+ "and dependency exclusions (and the --include-ext file types) into it.",
353
2305
  )
354
2306
  parser.add_argument(
355
- "--skills-output-folder", default="",
356
- help="Raw value of skills_output_folder from {project-root}/_bmad/skf/config.yaml. "
357
- "Validated before being interpolated into `**/{value}`. Empty/absolute/glob-meta "
358
- "values are rejected with a warning.",
2307
+ "--skills-output-folder", default=None,
2308
+ help="Raw value of skills_output_folder from {project-root}/_bmad/skf/config.yaml, "
2309
+ "forwarded verbatim. Normalized, validated and checked, then excluded as a "
2310
+ "root-anchored pattern, or entry by entry when the folder also holds content "
2311
+ "SKF did not generate. Refused values produce a warning.",
359
2312
  )
360
2313
  parser.add_argument(
361
- "--forge-data-folder", default="",
2314
+ "--forge-data-folder", default=None,
362
2315
  help="Raw value of forge_data_folder from {project-root}/_bmad/skf/config.yaml. "
363
- "Same validation as --skills-output-folder.",
2316
+ "Same handling as --skills-output-folder.",
2317
+ )
2318
+ parser.add_argument(
2319
+ "--config", type=Path, default=None,
2320
+ help="Setup mode, instead of the two folder flags: the _bmad/skf/config.yaml "
2321
+ "to read the raw skills_output_folder and forge_data_folder values from.",
2322
+ )
2323
+ parser.add_argument(
2324
+ "--prior-state-from", type=Path, default=None,
2325
+ help="Path to forge-tier.yaml. Its ccc_index.exclude_patterns is the record of "
2326
+ "the patterns SKF owns, per-entry patterns included; recorded patterns the "
2327
+ "current config no longer produces are removed, and with this record a "
2328
+ "folder already in exclude_patterns that it does not hold is left to the "
2329
+ "user. Missing file: empty record.",
2330
+ )
2331
+ parser.add_argument(
2332
+ "--index-fresh", type=_bool_arg, default=None, metavar="true|false",
2333
+ help="Whether the prior ccc index is still fresh. With true and an unchanged "
2334
+ "settings.yml, index_action is keep. Default false.",
2335
+ )
2336
+ parser.add_argument(
2337
+ "--skip-index", type=_bool_arg, default=None, metavar="true|false",
2338
+ help="Whether the setup run opted out of indexing (--ccc-skip-index). With true, "
2339
+ "a ready settings.yml gives index_action skip. Default false.",
2340
+ )
2341
+ parser.add_argument(
2342
+ "--defer-index", action="store_true",
2343
+ help="Leave a due index build (index_action index) to a later run: index_action is defer "
2344
+ "and the index skipped. A fresh index is still kept. Setup passes it on a "
2345
+ "--require-tier miss.",
2346
+ )
2347
+ parser.add_argument(
2348
+ "--no-ccc-init", action="store_true",
2349
+ help="Never run ccc init (tests and diagnostics). A missing settings.yml is "
2350
+ "reported as not ready; a rebuild is skipped with a warning.",
2351
+ )
2352
+ parser.add_argument(
2353
+ "--build-index", action="store_true",
2354
+ help="Setup mode: act on index_action (run ccc index when it is index) and add "
2355
+ "the index result, stamped by this script, to the output.",
2356
+ )
2357
+ parser.add_argument(
2358
+ "--result-to", type=Path, default=None,
2359
+ help="Also write the JSON result, or the JSON error of a run that fails, to this file.",
2360
+ )
2361
+ parser.add_argument(
2362
+ "--include-ext", type=_ext_arg, action="append", default=[], metavar="EXT",
2363
+ help="Clone mode only, repeatable: a file extension (ex, or .ex) whose files ccc must "
2364
+ "index. Unless an include_patterns entry already matches it, **/*.EXT is appended "
2365
+ "to include_patterns.",
364
2366
  )
365
2367
  args = parser.parse_args()
366
2368
 
367
- cmd_merge(args.project_root, args.skills_output_folder, args.forge_data_folder)
2369
+ def fail(code: int, message: str) -> None:
2370
+ if args.result_to is not None:
2371
+ _write_result(args.result_to, {"status": "error", "message": _payload_safe(message)})
2372
+ _die(code, message)
2373
+
2374
+ def done(payload: dict) -> None:
2375
+ if args.result_to is not None:
2376
+ _write_result(args.result_to, payload)
2377
+ print(json.dumps(payload))
2378
+
2379
+ if args.clone_root is not None:
2380
+ # A setup flag is refused whatever its value: `--skip-index false` or an empty folder too.
2381
+ setup_flags = [flag for flag, value in (
2382
+ ("--skills-output-folder", args.skills_output_folder),
2383
+ ("--forge-data-folder", args.forge_data_folder),
2384
+ ("--config", args.config),
2385
+ ("--prior-state-from", args.prior_state_from),
2386
+ ("--index-fresh", args.index_fresh),
2387
+ ("--skip-index", args.skip_index),
2388
+ ("--build-index", args.build_index or None),
2389
+ ("--defer-index", args.defer_index or None),
2390
+ ) if value is not None]
2391
+ if setup_flags:
2392
+ _die(2, f"usage error: --clone-root takes no setup flag ({', '.join(setup_flags)})")
2393
+ if not args.clone_root.is_dir():
2394
+ fail(1, f"--clone-root is not a directory: {args.clone_root}")
2395
+ try:
2396
+ payload = run_clone_merge(args.clone_root, args.include_ext,
2397
+ allow_ccc_init=not args.no_ccc_init)
2398
+ except HelperError as e:
2399
+ fail(e.code, str(e))
2400
+ except Exception as e: # noqa: BLE001 (every failure must stay stderr JSON)
2401
+ fail(2, f"unexpected error: {type(e).__name__}: {e}")
2402
+ done(payload)
2403
+ return
2404
+ if args.include_ext:
2405
+ _die(2, "usage error: --include-ext needs --clone-root")
2406
+ if args.config is not None and (args.skills_output_folder is not None or args.forge_data_folder is not None):
2407
+ _die(2, "usage error: --config replaces --skills-output-folder and --forge-data-folder")
2408
+
2409
+ root = Path(args.project_root)
2410
+ if not root.is_dir():
2411
+ fail(1, f"--project-root is not a directory: {args.project_root}")
2412
+ try:
2413
+ skills_output_folder, forge_data_folder = (
2414
+ read_config_folders(args.config) if args.config is not None
2415
+ else (args.skills_output_folder or "", args.forge_data_folder or ""))
2416
+ payload = run_merge(
2417
+ root,
2418
+ skills_output_folder,
2419
+ forge_data_folder,
2420
+ prior_state_from=args.prior_state_from,
2421
+ index_fresh=bool(args.index_fresh),
2422
+ skip_index=bool(args.skip_index),
2423
+ allow_ccc_init=not args.no_ccc_init,
2424
+ defer_index=args.defer_index,
2425
+ )
2426
+ if args.build_index:
2427
+ if args.result_to is not None and payload["index_action"] == "index":
2428
+ # settings.yml is already rewritten, and `ccc index` can run for
2429
+ # an hour: a host that stops the call must still find this run's
2430
+ # exclusion record, with an index that did not finish.
2431
+ _write_result(args.result_to, {**payload, "index": {
2432
+ "status": "failed", "indexed_path": None, "last_indexed": None,
2433
+ "file_count": None, "failed_reason": INDEX_UNFINISHED}})
2434
+ warnings: list[str] = []
2435
+ payload["index"] = build_index(root, args.project_root, payload["index_action"],
2436
+ payload["not_ready_reason"], args.prior_state_from, warnings)
2437
+ payload["warnings"] += [_payload_safe(w) for w in warnings]
2438
+ except HelperError as e:
2439
+ fail(e.code, str(e))
2440
+ except Exception as e: # noqa: BLE001 — every failure must stay stderr JSON
2441
+ fail(2, f"unexpected error: {type(e).__name__}: {e}")
2442
+ done(payload)
2443
+
2444
+
2445
+ def _write_result(path: Path, value: dict) -> None:
2446
+ """Write --result-to; a file that cannot be written is one more stderr error."""
2447
+ try:
2448
+ _atomic_write(path, json.dumps(value) + "\n")
2449
+ except HelperError as e:
2450
+ print(json.dumps({"status": "error", "message": _payload_safe(str(e))}), file=sys.stderr)
2451
+ sys.exit(e.code)
368
2452
 
369
2453
 
370
2454
  if __name__ == "__main__":