bmad-module-skill-forge 2.2.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +49 -49
  3. package/docs/404.md +10 -8
  4. package/docs/_data/pinned.yaml +22 -13
  5. package/docs/_internal/STABILITY.md +58 -35
  6. package/docs/agents.md +39 -53
  7. package/docs/architecture.md +65 -51
  8. package/docs/bmad-synergy.md +53 -51
  9. package/docs/campaign.md +127 -75
  10. package/docs/concepts.md +34 -28
  11. package/docs/examples.md +70 -68
  12. package/docs/forge-auto.md +34 -31
  13. package/docs/getting-started.md +118 -57
  14. package/docs/how-it-works.md +25 -21
  15. package/docs/index.md +15 -15
  16. package/docs/skill-model.md +70 -59
  17. package/docs/troubleshooting.md +135 -18
  18. package/docs/verifying-a-skill.md +83 -57
  19. package/docs/why-skf.md +24 -22
  20. package/docs/workflows.md +274 -110
  21. package/package.json +18 -10
  22. package/src/forger/preferences.yaml +8 -1
  23. package/src/knowledge/agentskills-spec.md +2 -0
  24. package/src/knowledge/ccc-bridge.md +40 -24
  25. package/src/knowledge/confidence-tiers.md +9 -5
  26. package/src/knowledge/overview.md +3 -3
  27. package/src/knowledge/progressive-capability.md +8 -8
  28. package/src/knowledge/provenance-tracking.md +9 -5
  29. package/src/knowledge/qmd-registry.md +2 -2
  30. package/src/knowledge/skf-knowledge-index.csv +2 -2
  31. package/src/knowledge/skill-lifecycle.md +18 -11
  32. package/src/knowledge/split-body-strategy.md +0 -3
  33. package/src/knowledge/tool-resolution.md +4 -2
  34. package/src/knowledge/version-paths.md +101 -15
  35. package/src/module-help.csv +3 -3
  36. package/src/module.yaml +10 -0
  37. package/src/shared/data/ast-grep-recipes.yaml +608 -0
  38. package/src/shared/data/ide-context-files.json +30 -0
  39. package/src/shared/health-check.md +32 -12
  40. package/src/shared/references/description-guard-protocol.md +12 -9
  41. package/src/shared/references/feasibility-report-schema.md +19 -13
  42. package/src/shared/references/headless-gate-convention.md +55 -3
  43. package/src/shared/references/output-contract-schema.md +19 -8
  44. package/src/shared/references/pipeline-contracts.md +94 -29
  45. package/src/shared/references/tessl-review.md +77 -0
  46. package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
  47. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
  48. package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
  49. package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
  50. package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
  51. package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
  52. package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
  53. package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
  54. package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
  55. package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
  56. package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
  57. package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
  58. package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
  59. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
  60. package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
  61. package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
  62. package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
  63. package/src/shared/scripts/skf-atomic-write.py +30 -14
  64. package/src/shared/scripts/skf-build-change-manifest.py +2203 -24
  65. package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
  66. package/src/shared/scripts/skf-check-unit-records.py +184 -0
  67. package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
  68. package/src/shared/scripts/skf-classify-changed-files.py +694 -0
  69. package/src/shared/scripts/skf-comention-pairs.py +1333 -123
  70. package/src/shared/scripts/skf-compare-file-hashes.py +1 -1
  71. package/src/shared/scripts/skf-count-imports.py +1867 -0
  72. package/src/shared/scripts/skf-count-tokens.py +1 -1
  73. package/src/shared/scripts/skf-derive-assembly-shape.py +3 -2
  74. package/src/shared/scripts/skf-description-guard.py +99 -11
  75. package/src/shared/scripts/skf-detect-docs.py +694 -2
  76. package/src/shared/scripts/skf-detect-language.py +486 -107
  77. package/src/shared/scripts/skf-detect-registry.py +887 -0
  78. package/src/shared/scripts/skf-detect-scripts-assets.py +1 -1
  79. package/src/shared/scripts/skf-detect-tools.py +209 -27
  80. package/src/shared/scripts/skf-detect-workspaces.py +506 -30
  81. package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
  82. package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
  83. package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
  84. package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
  85. package/src/shared/scripts/skf-extract-public-api.py +4084 -85
  86. package/src/shared/scripts/skf-extraction-inventory.py +746 -0
  87. package/src/shared/scripts/skf-extraction-snapshot.py +539 -0
  88. package/src/shared/scripts/skf-fetch-temporal.py +674 -0
  89. package/src/shared/scripts/skf-find-cycles.py +97 -29
  90. package/src/shared/scripts/skf-find-test-report.py +476 -0
  91. package/src/shared/scripts/skf-forge-tier-rw.py +444 -203
  92. package/src/shared/scripts/skf-github-fetch.py +365 -0
  93. package/src/shared/scripts/skf-github-probe.py +661 -0
  94. package/src/shared/scripts/skf-hash-content.py +166 -12
  95. package/src/shared/scripts/skf-language-corpora.py +1 -1
  96. package/src/shared/scripts/skf-load-provenance.py +159 -27
  97. package/src/shared/scripts/skf-manifest-ops.py +1 -1
  98. package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
  99. package/src/shared/scripts/skf-merge-doc-urls.py +1 -1
  100. package/src/shared/scripts/skf-names-present.py +221 -0
  101. package/src/shared/scripts/skf-pair-intersect.py +119 -28
  102. package/src/shared/scripts/skf-parse-gaps.py +1384 -0
  103. package/src/shared/scripts/skf-preapply.py +1 -1
  104. package/src/shared/scripts/skf-preflight.py +161 -27
  105. package/src/shared/scripts/skf-promote-staged.py +430 -0
  106. package/src/shared/scripts/skf-provenance-gap-dispatch.py +39 -7
  107. package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
  108. package/src/shared/scripts/skf-quick-batch.py +774 -0
  109. package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
  110. package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
  111. package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
  112. package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
  113. package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
  114. package/src/shared/scripts/skf-resolve-authoritative-files.py +128 -13
  115. package/src/shared/scripts/skf-resolve-package.py +523 -106
  116. package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
  117. package/src/shared/scripts/skf-run-lock.py +577 -0
  118. package/src/shared/scripts/skf-scan-manifests.py +998 -57
  119. package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
  120. package/src/shared/scripts/skf-severity-classify.py +712 -108
  121. package/src/shared/scripts/skf-shape-detect.py +298 -204
  122. package/src/shared/scripts/skf-shard-body.py +9 -8
  123. package/src/shared/scripts/skf-skill-inventory.py +2412 -89
  124. package/src/shared/scripts/skf-skills-module.py +472 -0
  125. package/src/shared/scripts/skf-source-tree.py +1796 -0
  126. package/src/shared/scripts/skf-structural-diff.py +670 -119
  127. package/src/shared/scripts/skf-tessl-review.py +706 -0
  128. package/src/shared/scripts/skf-update-active-symlink.py +5 -5
  129. package/src/shared/scripts/skf-update-run-state.py +624 -0
  130. package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
  131. package/src/shared/scripts/skf-validate-brief-schema.py +4 -1
  132. package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
  133. package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
  134. package/src/shared/scripts/skf-validate-output.py +712 -64
  135. package/src/shared/scripts/skf-validate-pins.py +1 -1
  136. package/src/shared/scripts/skf-verify-no-trace.py +116 -10
  137. package/src/shared/scripts/skf-verify-provenance-completeness.py +2919 -54
  138. package/src/shared/scripts/skf-write-skill-brief.py +367 -19
  139. package/src/shared/tool-requirements.yaml +201 -0
  140. package/src/skf-analyze-source/SKILL.md +41 -36
  141. package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
  142. package/src/skf-analyze-source/customize.toml +36 -24
  143. package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
  144. package/src/skf-analyze-source/references/continue.md +2 -13
  145. package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
  146. package/src/skf-analyze-source/references/generate-briefs.md +121 -55
  147. package/src/skf-analyze-source/references/headless-contract.md +42 -15
  148. package/src/skf-analyze-source/references/identify-units.md +96 -110
  149. package/src/skf-analyze-source/references/init.md +73 -71
  150. package/src/skf-analyze-source/references/map-and-detect.md +123 -105
  151. package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
  152. package/src/skf-analyze-source/references/recommend.md +51 -70
  153. package/src/skf-analyze-source/references/scan-project.md +60 -84
  154. package/src/skf-analyze-source/references/scan-root.md +39 -0
  155. package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
  156. package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
  157. package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
  158. package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
  159. package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
  160. package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
  161. package/src/skf-audit-skill/SKILL.md +43 -51
  162. package/src/skf-audit-skill/assets/drift-report-template.md +27 -3
  163. package/src/skf-audit-skill/customize.toml +41 -20
  164. package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
  165. package/src/skf-audit-skill/references/doc-drift.md +141 -0
  166. package/src/skf-audit-skill/references/headless-contract.md +29 -0
  167. package/src/skf-audit-skill/references/init.md +148 -210
  168. package/src/skf-audit-skill/references/re-index.md +96 -108
  169. package/src/skf-audit-skill/references/report.md +96 -33
  170. package/src/skf-audit-skill/references/semantic-diff.md +27 -18
  171. package/src/skf-audit-skill/references/severity-classify.md +79 -65
  172. package/src/skf-audit-skill/references/severity-rules.md +15 -6
  173. package/src/skf-audit-skill/references/structural-diff.md +75 -90
  174. package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
  175. package/src/skf-audit-skill/scripts/render-drift-tables.py +685 -0
  176. package/src/skf-brief-skill/SKILL.md +36 -25
  177. package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
  178. package/src/skf-brief-skill/assets/scope-templates.md +17 -24
  179. package/src/skf-brief-skill/assets/skill-brief-schema.md +29 -15
  180. package/src/skf-brief-skill/customize.toml +48 -20
  181. package/src/skf-brief-skill/references/analyze-target.md +142 -125
  182. package/src/skf-brief-skill/references/confirm-brief.md +10 -8
  183. package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
  184. package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
  185. package/src/skf-brief-skill/references/gather-intent.md +87 -182
  186. package/src/skf-brief-skill/references/headless-args.md +84 -16
  187. package/src/skf-brief-skill/references/health-check.md +1 -1
  188. package/src/skf-brief-skill/references/invocation-contract.md +18 -13
  189. package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
  190. package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
  191. package/src/skf-brief-skill/references/scope-definition.md +123 -47
  192. package/src/skf-brief-skill/references/step-auto-brief.md +24 -80
  193. package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
  194. package/src/skf-brief-skill/references/version-resolution.md +3 -2
  195. package/src/skf-brief-skill/references/write-brief.md +66 -57
  196. package/src/skf-campaign/SKILL.md +31 -34
  197. package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
  198. package/src/skf-campaign/customize.toml +47 -16
  199. package/src/skf-campaign/references/campaign-contracts.md +39 -29
  200. package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
  201. package/src/skf-campaign/references/health-check.md +7 -16
  202. package/src/skf-campaign/references/step-01-setup.md +46 -84
  203. package/src/skf-campaign/references/step-02-strategy.md +31 -30
  204. package/src/skf-campaign/references/step-03-pins.md +26 -18
  205. package/src/skf-campaign/references/step-04-provenance.md +24 -23
  206. package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
  207. package/src/skf-campaign/references/step-06-batch.md +41 -34
  208. package/src/skf-campaign/references/step-07-capstone.md +40 -25
  209. package/src/skf-campaign/references/step-08-verify.md +28 -29
  210. package/src/skf-campaign/references/step-09-refine.md +29 -30
  211. package/src/skf-campaign/references/step-10-export.md +48 -36
  212. package/src/skf-campaign/references/step-11-maintenance.md +31 -34
  213. package/src/skf-campaign/references/step-resume.md +37 -68
  214. package/src/skf-campaign/scripts/campaign-deps.py +57 -18
  215. package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
  216. package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
  217. package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
  218. package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
  219. package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
  220. package/src/skf-campaign/scripts/campaign-report.py +204 -37
  221. package/src/skf-campaign/scripts/campaign-state.py +1128 -0
  222. package/src/skf-campaign/scripts/campaign-status.py +10 -74
  223. package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
  224. package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
  225. package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
  226. package/src/skf-campaign/templates/campaign-report-template.md +5 -1
  227. package/src/skf-campaign/templates/kickoff-template.md +0 -14
  228. package/src/skf-create-skill/SKILL.md +27 -20
  229. package/src/skf-create-skill/assets/compile-assembly-rules.md +6 -5
  230. package/src/skf-create-skill/assets/skill-sections.md +15 -5
  231. package/src/skf-create-skill/customize.toml +20 -8
  232. package/src/skf-create-skill/references/authoritative-files-protocol.md +41 -42
  233. package/src/skf-create-skill/references/batch-mode.md +81 -0
  234. package/src/skf-create-skill/references/ccc-index-check.md +27 -0
  235. package/src/skf-create-skill/references/compile.md +107 -29
  236. package/src/skf-create-skill/references/component-extraction.md +113 -91
  237. package/src/skf-create-skill/references/enrich.md +41 -23
  238. package/src/skf-create-skill/references/entry-points-by-hand.md +20 -0
  239. package/src/skf-create-skill/references/extract.md +200 -161
  240. package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
  241. package/src/skf-create-skill/references/extraction-patterns-tracing.md +2 -2
  242. package/src/skf-create-skill/references/extraction-patterns.md +584 -196
  243. package/src/skf-create-skill/references/generate-artifacts.md +145 -93
  244. package/src/skf-create-skill/references/health-check.md +1 -1
  245. package/src/skf-create-skill/references/load-brief.md +53 -36
  246. package/src/skf-create-skill/references/relabel-rule.md +12 -0
  247. package/src/skf-create-skill/references/report.md +28 -59
  248. package/src/skf-create-skill/references/source-resolution-protocols.md +49 -155
  249. package/src/skf-create-skill/references/step-auto-shard.md +20 -93
  250. package/src/skf-create-skill/references/step-doc-rot.md +44 -60
  251. package/src/skf-create-skill/references/step-doc-sources.md +28 -19
  252. package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
  253. package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
  254. package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
  255. package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
  256. package/src/skf-create-skill/references/validate.md +173 -99
  257. package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
  258. package/src/skf-create-stack-skill/SKILL.md +29 -56
  259. package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
  260. package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
  261. package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
  262. package/src/skf-create-stack-skill/customize.toml +23 -12
  263. package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
  264. package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
  265. package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
  266. package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
  267. package/src/skf-create-stack-skill/references/generate-output.md +138 -128
  268. package/src/skf-create-stack-skill/references/init.md +90 -32
  269. package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
  270. package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
  271. package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
  272. package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
  273. package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
  274. package/src/skf-create-stack-skill/references/report.md +39 -33
  275. package/src/skf-create-stack-skill/references/validate.md +20 -39
  276. package/src/skf-drop-skill/SKILL.md +28 -51
  277. package/src/skf-drop-skill/customize.toml +44 -30
  278. package/src/skf-drop-skill/references/execute.md +79 -146
  279. package/src/skf-drop-skill/references/invocation-contract.md +64 -0
  280. package/src/skf-drop-skill/references/report.md +46 -30
  281. package/src/skf-drop-skill/references/select.md +118 -124
  282. package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
  283. package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
  284. package/src/skf-export-skill/SKILL.md +40 -62
  285. package/src/skf-export-skill/assets/managed-section-format.md +14 -50
  286. package/src/skf-export-skill/assets/snippet-format.md +3 -1
  287. package/src/skf-export-skill/customize.toml +42 -13
  288. package/src/skf-export-skill/references/generate-snippet.md +54 -46
  289. package/src/skf-export-skill/references/health-check.md +2 -0
  290. package/src/skf-export-skill/references/invocation-contract.md +15 -0
  291. package/src/skf-export-skill/references/load-skill.md +88 -62
  292. package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
  293. package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
  294. package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
  295. package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
  296. package/src/skf-export-skill/references/package.md +15 -73
  297. package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
  298. package/src/skf-export-skill/references/result-envelope.md +31 -16
  299. package/src/skf-export-skill/references/summary.md +41 -17
  300. package/src/skf-export-skill/references/token-report.md +4 -12
  301. package/src/skf-export-skill/references/update-context.md +118 -237
  302. package/src/skf-forger/SKILL.md +61 -37
  303. package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
  304. package/src/skf-forger/customize.toml +43 -0
  305. package/src/skf-forger/references/pipeline-mode.md +73 -18
  306. package/src/skf-forger/scripts/forge-status.py +299 -0
  307. package/src/skf-forger/scripts/parse-pipeline.py +322 -54
  308. package/src/skf-forger/scripts/pipeline-gate.py +428 -0
  309. package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
  310. package/src/skf-quick-skill/SKILL.md +54 -53
  311. package/src/skf-quick-skill/assets/skill-template.md +4 -2
  312. package/src/skf-quick-skill/customize.toml +51 -20
  313. package/src/skf-quick-skill/references/batch-mode.md +74 -21
  314. package/src/skf-quick-skill/references/compile.md +30 -31
  315. package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
  316. package/src/skf-quick-skill/references/finalize.md +69 -18
  317. package/src/skf-quick-skill/references/halt-contract.md +41 -29
  318. package/src/skf-quick-skill/references/health-check.md +5 -3
  319. package/src/skf-quick-skill/references/quick-extract.md +114 -45
  320. package/src/skf-quick-skill/references/registry-resolution.md +9 -64
  321. package/src/skf-quick-skill/references/resolve-target.md +116 -66
  322. package/src/skf-quick-skill/references/write-and-validate.md +79 -40
  323. package/src/skf-refine-architecture/SKILL.md +43 -53
  324. package/src/skf-refine-architecture/customize.toml +49 -20
  325. package/src/skf-refine-architecture/references/compile.md +97 -54
  326. package/src/skf-refine-architecture/references/exit-codes.md +30 -0
  327. package/src/skf-refine-architecture/references/finding-storage.md +21 -0
  328. package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
  329. package/src/skf-refine-architecture/references/improvements.md +14 -20
  330. package/src/skf-refine-architecture/references/init.md +98 -57
  331. package/src/skf-refine-architecture/references/issue-detection.md +32 -39
  332. package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
  333. package/src/skf-refine-architecture/references/report.md +65 -54
  334. package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
  335. package/src/skf-rename-skill/SKILL.md +19 -54
  336. package/src/skf-rename-skill/customize.toml +32 -26
  337. package/src/skf-rename-skill/references/execute.md +230 -184
  338. package/src/skf-rename-skill/references/exit-codes.md +7 -7
  339. package/src/skf-rename-skill/references/health-check.md +1 -7
  340. package/src/skf-rename-skill/references/invocation-contract.md +25 -0
  341. package/src/skf-rename-skill/references/report.md +63 -52
  342. package/src/skf-rename-skill/references/select.md +197 -115
  343. package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
  344. package/src/skf-setup/SKILL.md +26 -26
  345. package/src/skf-setup/customize.toml +14 -10
  346. package/src/skf-setup/references/auto-index.md +50 -63
  347. package/src/skf-setup/references/ccc-index.md +29 -90
  348. package/src/skf-setup/references/detect-and-tier.md +58 -67
  349. package/src/skf-setup/references/health-check.md +2 -0
  350. package/src/skf-setup/references/invocation-contract.md +14 -0
  351. package/src/skf-setup/references/report.md +43 -166
  352. package/src/skf-setup/references/tier-rules.md +5 -5
  353. package/src/skf-setup/references/write-config.md +34 -57
  354. package/src/skf-test-skill/SKILL.md +17 -57
  355. package/src/skf-test-skill/assets/output-section-formats.md +33 -34
  356. package/src/skf-test-skill/customize.toml +27 -13
  357. package/src/skf-test-skill/references/coherence-check.md +134 -62
  358. package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
  359. package/src/skf-test-skill/references/coverage-check.md +217 -254
  360. package/src/skf-test-skill/references/detect-mode.md +5 -5
  361. package/src/skf-test-skill/references/external-validators.md +94 -82
  362. package/src/skf-test-skill/references/health-check.md +6 -7
  363. package/src/skf-test-skill/references/init.md +163 -88
  364. package/src/skf-test-skill/references/invocation-contract.md +66 -0
  365. package/src/skf-test-skill/references/migration-section-rules.md +5 -2
  366. package/src/skf-test-skill/references/report.md +162 -142
  367. package/src/skf-test-skill/references/score.md +80 -107
  368. package/src/skf-test-skill/references/scoring-rules.md +41 -155
  369. package/src/skf-test-skill/references/source-access-protocol.md +19 -92
  370. package/src/skf-test-skill/references/step-hard-gate.md +51 -33
  371. package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
  372. package/src/skf-test-skill/scripts/build-result-context.py +287 -0
  373. package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
  374. package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
  375. package/src/skf-test-skill/scripts/compute-score.py +223 -45
  376. package/src/skf-test-skill/scripts/gap-ledger.py +1381 -0
  377. package/src/skf-test-skill/scripts/hard-gate.py +203 -0
  378. package/src/skf-test-skill/scripts/load-coverage-inputs.py +760 -0
  379. package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
  380. package/src/skf-test-skill/scripts/reconcile-coverage.py +273 -79
  381. package/src/skf-test-skill/scripts/score-signatures.py +459 -0
  382. package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
  383. package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
  384. package/src/skf-test-skill/scripts/verify-declared-numerator.py +127 -52
  385. package/src/skf-test-skill/templates/test-report-template.md +21 -15
  386. package/src/skf-update-skill/SKILL.md +38 -25
  387. package/src/skf-update-skill/customize.toml +27 -12
  388. package/src/skf-update-skill/references/detect-changes.md +233 -222
  389. package/src/skf-update-skill/references/gap-driven.md +336 -0
  390. package/src/skf-update-skill/references/health-check.md +31 -5
  391. package/src/skf-update-skill/references/init.md +250 -84
  392. package/src/skf-update-skill/references/invocation-contract.md +12 -0
  393. package/src/skf-update-skill/references/manual-section-rules.md +6 -23
  394. package/src/skf-update-skill/references/merge.md +180 -47
  395. package/src/skf-update-skill/references/re-extract.md +53 -262
  396. package/src/skf-update-skill/references/report.md +100 -35
  397. package/src/skf-update-skill/references/write.md +331 -149
  398. package/src/skf-update-skill/scripts/skf-new-file-diff.py +29 -7
  399. package/src/skf-verify-stack/SKILL.md +23 -42
  400. package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
  401. package/src/skf-verify-stack/customize.toml +55 -22
  402. package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
  403. package/src/skf-verify-stack/references/coverage.md +67 -64
  404. package/src/skf-verify-stack/references/exit-codes.md +21 -8
  405. package/src/skf-verify-stack/references/init.md +107 -62
  406. package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
  407. package/src/skf-verify-stack/references/integrations.md +106 -65
  408. package/src/skf-verify-stack/references/report.md +59 -42
  409. package/src/skf-verify-stack/references/requirements.md +32 -9
  410. package/src/skf-verify-stack/references/synthesize.md +53 -39
  411. package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
  412. package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
  413. package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
  414. package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
  415. package/tools/cli/commands/install.js +11 -4
  416. package/tools/cli/commands/status.js +15 -3
  417. package/tools/cli/commands/update.js +11 -4
  418. package/tools/cli/lib/tool-check.js +320 -0
  419. package/tools/cli/lib/version-check.js +22 -9
  420. package/tools/cli/skf-cli.js +4 -1
  421. package/.gitattributes +0 -16
  422. package/.nvmrc +0 -1
  423. package/docs/_internal/RELEASING.md +0 -679
  424. package/release-audits/v1.0.0-launch-audit.md +0 -728
  425. package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
  426. package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
  427. package/src/skf-campaign/manifest.yaml +0 -15
  428. package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
  429. package/src/skf-create-skill/references/ecosystem-check.md +0 -74
  430. package/src/skf-drop-skill/references/headless-contract.md +0 -18
  431. package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
  432. package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
  433. package/src/skf-update-skill/references/validate.md +0 -124
  434. package/tools/validate-docs-drift.js +0 -233
  435. package/tools/validate-docs-links.js +0 -665
@@ -1,5 +1,5 @@
1
1
  # /// script
2
- # requires-python = ">=3.10"
2
+ # requires-python = ">=3.11"
3
3
  # dependencies = []
4
4
  # ///
5
5
  """SKF Build Change Manifest — aggregate category A/B/C/D detection results.
@@ -26,15 +26,107 @@ schema, plus the provenance map for the denominator.
26
26
 
27
27
  Subcommands:
28
28
 
29
- build [--input <file>] [--no-changes-on-empty]
29
+ build [--input <file>] [--category-a <file>] [--category-b-diff <file>]
30
+ [--category-c <file>] [--ccc-pairs <file>]
31
+ [--file-compare <file> --provenance-map <file>] [--new-files <file>]
30
32
  Reads category JSON from stdin or `--input <file>`, emits the
31
- unified manifest envelope.
33
+ unified manifest envelope (see "Manifest" below).
34
+
35
+ build --doc-hashes <file>
36
+ A docs-only skill's manifest, from skf-detect-docs.py
37
+ compare-hashes' output and nothing else (see "Docs-only" below).
32
38
 
33
39
  deletion-ratio --provenance-map <file> [--input <file>]
40
+ [--category-a <file>] [--category-b-diff <file>]
41
+ [--category-c <file>] [--ccc-pairs <file>]
34
42
  Same category JSON input. Reads provenance entries[] from
35
43
  <file>, computes the §2.2 trigger envelope. Auto-skips when
36
44
  degraded_mode or update_mode==gap-driven is set in the input.
37
45
 
46
+ rename-candidates --category-a <file> --provenance-map <file>
47
+ --tier Quick|Forge|Forge+|Deep [--category-b-diff <file>]
48
+ [--extraction <file>] [--export-details <file>]
49
+ [--source-root <dir> [--sizes-commit <commit>]] [-o <file>]
50
+ Category C by fixed rules (see "Rename candidates" below): pairs
51
+ Category A's deleted files with its added ones, and the diff's
52
+ removed exports with its added ones and the added files' exports.
53
+ Prints {"category_c": {...}, "evidence": {...}, "unpaired": {...}}.
54
+
55
+ apply --update-type incremental|gap-driven|full --skill-name <name>
56
+ --generation-date <iso> --confidence-tier <tier>
57
+ --manual-sections-preserved <n> -o <file> [inputs...]
58
+ The provenance map an update writes (see "Apply" below): the old
59
+ map with this run's changes, written to -o through a temporary file
60
+ and a rename. Prints a summary with the warnings for a person.
61
+
62
+ records [--extraction <file>] [--export-details <file>]
63
+ [--files-from <file>] [--patches <dir>] -o <file>
64
+ Step 3's re-extraction records (see "Records" below), written to -o
65
+ through a temporary file and a rename. Prints {"status": "written",
66
+ "output", "files_extracted", "exports_extracted",
67
+ "confidence_breakdown": {"T1", "T1-low", "T2"}, "warnings"}.
68
+
69
+ gap-records --manifest <file> --provenance-map <file> [--source-root <dir>]
70
+ [--drift-status ok|skipped|overridden] [--judgments <file>]
71
+ [--remediation-records <file>] [--evidence <file> --tier <tier>]
72
+ (--plan [--files-out <file>] | -o <file>)
73
+ A gap-driven repair's spot-checks, routing and verification records
74
+ (see "Gap records" below): --plan prints what to answer and which
75
+ files targeted re-extraction scans; without it, the records are
76
+ written to -o through a temporary file and a rename.
77
+
78
+ Helper files in place of typed slices (update-skill detect-changes §2.1):
79
+
80
+ --category-a <file> the output of skf-classify-changed-files.py
81
+ classify: its category_a is the Category A
82
+ slice, and each of its moved_files (a
83
+ same-content move) is a category_c.renamed_files
84
+ item, ahead of the input's own
85
+ --category-b-diff <file> the output of skf-structural-diff.py over the
86
+ modified files: the Category B slice, mapped
87
+ removed[] -> deleted_exports {name, file, old_line}
88
+ added[] -> new_exports {name, file, line}
89
+ changed[] -> modified_exports {name, file,
90
+ old_line, new_line} for an export
91
+ with a field other than `line`, or
92
+ one signature_unverified[] lists
93
+ (its lines null when no field
94
+ changed); moved_exports for an
95
+ export whose only field is `line`
96
+ moved[] -> moved_exports {name, file:
97
+ current_file, old_line:
98
+ previous_line, new_line: line},
99
+ unless it is a modified export
100
+ label_changes[] is not drift, and a name
101
+ ambiguous_names[] lists stays removed and added
102
+ --category-c <file> the output of rename-candidates: its category_c
103
+ is the Category C slice
104
+ --ccc-pairs <file> the pairs the update's CCC check added to what
105
+ rename-candidates left unpaired,
106
+ {"renamed_files": [...], "renamed_exports":
107
+ [...]}, appended to the Category C slice
108
+ --file-compare <file> build only: the output of skf-hash-content.py
109
+ compare, with --provenance-map (the map it
110
+ compared): each MODIFIED_FILE and DELETED_FILE
111
+ row goes to the {type}s_modified or
112
+ {type}s_deleted list of category_d, by the
113
+ file_type of the file_entries[] row of its
114
+ source_file (script, asset or doc; a row with
115
+ none takes the type its file_name's first
116
+ folder names, scripts/, assets/ or docs/, else
117
+ asset)
118
+ --new-files <file> build only: the output of skf-new-file-diff.py:
119
+ each new_files[] item goes to scripts_added or
120
+ assets_added by its kind
121
+ Either Category D file makes the two files the category_d slice. A given
122
+ file replaces the input's own slice. Either way, every category_c
123
+ rename then takes its pair out of the lists it was found in: a renamed
124
+ file's old_path out of category_a.deleted and new_path out of
125
+ category_a.added; a renamed export's old_name out of
126
+ category_b.deleted_exports (the one in its `old_file` when the item gives
127
+ one, else preferably the one in its `file`) and its new_name out of
128
+ category_b.new_exports (preferably the one in its `file`).
129
+
38
130
  Input JSON shape (object on stdin or in --input file):
39
131
 
40
132
  {
@@ -55,27 +147,301 @@ Input JSON shape (object on stdin or in --input file):
55
147
  },
56
148
  "category_d": {
57
149
  "scripts_modified": ["path", ...], "scripts_added": [...], "scripts_deleted": [...],
58
- "assets_modified": [...], "assets_added": [...], "assets_deleted": [...]
150
+ "assets_modified": [...], "assets_added": [...], "assets_deleted": [...],
151
+ "docs_modified": [...], "docs_deleted": [...]
59
152
  },
60
153
  "degraded_mode": false,
61
154
  "update_mode": "normal" | "gap-driven" // optional; influences deletion-ratio only
62
155
  }
63
156
 
157
+ Manifest (build's output):
158
+
159
+ {"no_changes": <every count is 0>, "degraded_mode": <bool>,
160
+ "counts": {files_changed, files_added, files_deleted, files_moved,
161
+ exports_modified, exports_new, exports_deleted,
162
+ exports_renamed, exports_moved, and one count per
163
+ category_d list},
164
+ "total_export_changes": N, "per_file": [...],
165
+ "category_d": {the eight category_d lists, each path once}}
166
+
167
+ A tracked document (a doc row of file_entries[]) has no other detector:
168
+ update-skill's Category A never lists it, so a document-only change still
169
+ counts, and the update records its new hash.
170
+
64
171
  Every category and sub-key is optional — missing keys default to
65
172
  empty lists. This lets gap-driven runs (which produce no Category D
66
173
  results) and degraded runs (which skip Category B) feed the same
67
174
  script without sentinel values.
68
175
 
176
+ Docs-only (update-skill detect-changes §1): build --doc-hashes reads the
177
+ compare-hashes output and prints {"mode": "docs-only", "no_changes":
178
+ <changed[] is empty>, "changed_urls": [each changed[].url],
179
+ "fetch_failed": [each fetch_failed[].url], "counts": {"docs_changed",
180
+ "docs_fetch_failed"}}. It takes no other input.
181
+
182
+ Rename candidates (update-skill detect-changes Category C):
183
+
184
+ Content similarity above 80% (fixed, not configurable) is a rename. A
185
+ file moved with its content unchanged is Category A's moved_files and is
186
+ left out here. For the rest:
187
+ files, Quick tier the added file's size is within 20% of the deleted
188
+ file's (the deleted size is `git cat-file -s` of
189
+ <commit>:<path> under --source-root; with no
190
+ --sizes-commit, a local source whose file is gone,
191
+ the size test is skipped) and the export names
192
+ overlap above 70% (shared names over all names of
193
+ the two files)
194
+ files, Forge+ above 80% of the exports match: an export of the
195
+ added file with the deleted file's export name and
196
+ signature (export_type, params and return_type),
197
+ over the larger file's export count
198
+ exports, Quick the names are above 80% alike (difflib ratio) and
199
+ their export types agree
200
+ exports, Forge+ the signatures are equal (export_type, params and
201
+ return_type, a parameter list or a return type
202
+ given) and the names differ
203
+ A deleted file's exports come from the provenance map, an added file's
204
+ from --extraction and --export-details. A pair is kept only when it is
205
+ each side's best match: the highest similarity wins, and a tie between
206
+ two candidates pairs neither. category_c.renamed_files holds {old_path,
207
+ new_path}; renamed_exports holds {old_name, new_name, file, old_file?},
208
+ with old_file only when it differs from file. evidence lists each pair
209
+ with its rule and similarity, and unpaired the deleted and added files
210
+ and the removed and added exports left over, for the judgment step.
211
+
212
+ Apply (update-skill write.md §3):
213
+
214
+ Reads the provenance map the update started from (--provenance-map;
215
+ without it, a degraded run's new map) and prints nothing but a summary:
216
+ the new map goes to -o. Every entry the run leaves alone keeps its exact
217
+ value; every entry it adds or rewrites carries signature_source, set
218
+ from the tool that produced the signature: ast-grep gives T1 (with
219
+ confidence T1 and the recipe's ast_node_type), a signature read by eye
220
+ (source-read) gives T1-low (confidence T1-low, ast_node_type null).
221
+ A rewritten entry keeps any key the update does not set (notes,
222
+ deprecated). New entries take source_library from the map's entries,
223
+ else --skill-name.
224
+
225
+ incremental (normal mode) reads --manifest (build's output), the renamed
226
+ exports of --category-c and --ccc-pairs (or of --input, the category
227
+ JSON's category_c) and the fresh
228
+ records: --reextract-records ({"mode": "normal", "files": [{file_path,
229
+ exports: [...]}]}, the records `records` wrote), --extraction (the
230
+ recipe runner's exports) and --export-details. A fresh record of an
231
+ export takes its line, export_type and ast labels from the runner, then
232
+ the details, then step 3's record; its params and return_type from the
233
+ details, then step 3's record (the runner's in the map's typed form).
234
+ Per file of the manifest:
235
+ MODIFIED NEW_EXPORT adds an entry, MODIFIED_EXPORT rewrites the
236
+ export's fields from its fresh record, MOVED_EXPORT sets
237
+ its line (and file), DELETED_EXPORT removes it
238
+ ADDED every fresh record of the file becomes an entry
239
+ DELETED every entry of the file is removed
240
+ MOVED the old path's entries are replaced by the new path's
241
+ fresh records, each keeping the keys of the entry of its
242
+ name it replaces
243
+ and each renamed export takes its new name and fresh fields. A docs-only
244
+ run's records ({"mode": "docs-only", "changed_urls": [...], "exports":
245
+ [{name, type, params, return_type, url}]}) replace the entries of each
246
+ changed URL (source_file is the URL; confidence and signature_source
247
+ T3).
248
+
249
+ gap-driven reads --reextract-records ({"mode": "gap-driven",
250
+ "verification": [...], "files": [...]}) and --merge-records ({"exports":
251
+ [{export_name, export_type, params, return_type}]}, merge's output for
252
+ an export no fresh record holds). Each verification record names
253
+ export_name, gap_category, severity, verification, in_map (the map holds
254
+ an entry of that name), map_entry ({source_file, source_line} of the one
255
+ entry the spot-check took, else null), source_citation, new_location,
256
+ unknown_reason, pinned_definition_lines and reachability:
257
+ verified (map_entry) unchanged
258
+ moved (map_entry) source_line (and source_file) from new_location
259
+ re-extracted the entry from its `files` record (gap-driven.md §4a);
260
+ none when its reachability is internal-unreachable
261
+ verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
262
+ reachability is not internal-unreachable
263
+ one source-read entry at the citation's line
264
+ (verified) or new_location (moved), at most one
265
+ per name and file, export_type and the
266
+ signature from --merge-records
267
+ unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
268
+ Info severity an entry from --merge-records with no
269
+ source_file or source_line
270
+ unknown, not in_map, any other severity (a missing one included)
271
+ refused: exit 3, nothing written
272
+ missing, unknown in_map, unknown on a MOVED_EXPORT
273
+ unchanged, with a warning for a person
274
+ rescoped the entry removed
275
+ Under the drift override (--drift-head and --drift-pinned) each warning
276
+ names the drift.
277
+
278
+ full (degraded mode) makes every fresh record an entry.
279
+
280
+ Every mode then applies the file changes: --file-compare (skf-hash-
281
+ content.py compare's output) sets the content_hash of a MODIFIED_FILE
282
+ row and removes a DELETED_FILE row; --new-files (skf-new-file-diff.py's
283
+ output) adds a row per new script or asset ({kind}s/<file name>, hashed
284
+ under --source-root, extraction_method file-copy); --promoted-docs (the
285
+ documents step 2 promoted, with their hashes) adds a doc row each
286
+ (docs/authoritative/<path>, promoted-authoritative). A row whose
287
+ source_file the map already holds is never added twice.
288
+
289
+ Last it sets the top-level source_commit and source_ref (when given)
290
+ and the update block, replacing an earlier update's values in place:
291
+ last_update (--generation-date), update_type, test_report_run_id,
292
+ files_changed (the manifest's files, or in gap-driven mode the files of
293
+ the entries this run changed), exports_affected (the manifest's export
294
+ changes, or the verification records), confidence_tier and
295
+ manual_sections_preserved. An older update-history key
296
+ (update_operations, update_metadata) stays as it is.
297
+
298
+ Summary on stdout: {"status": "written", "map": <-o>, "entries":
299
+ {"added", "updated", "removed"}, "file_entries": {"added", "updated",
300
+ "removed"}, "warnings": [...]}, or {"status": "refused",
301
+ "blocking_unresolved": [{export_name, severity}]} with exit 3.
302
+
303
+ Records (update-skill re-extract.md §4, and gap-driven.md §4a for the files
304
+ it scans):
305
+
306
+ Writes {"mode": "normal", "files": [{file_path, exports: [...]}]}, one
307
+ block per file --files-from names (and per other file an export names),
308
+ so the model never types a record a tool already wrote. Each export the
309
+ recipe runner found (--extraction) is seeded with its name, type,
310
+ signature, location (file:line), confidence T1, extraction_method
311
+ ast-grep, ast_node_type and ast_recipe, return_type and params, each
312
+ parameter in the provenance map's typed form (skf-extraction-
313
+ inventory.py typed_param: `name: type`, `name?: type`, ` = default`).
314
+ Where the runner left params or return_type null, --export-details
315
+ fills them. An export only --export-details holds (one read by eye,
316
+ with an export_type) is seeded from it as T1-low, source-read, with a
317
+ null ast_node_type and ast_recipe.
318
+
319
+ Then each worker patch in --patches (every *.json there, in name order:
320
+ one block {file_path, exports: [{name, ...}]} or a list of them) is
321
+ merged into the export of its name and file: signature, members,
322
+ docstring and qmd_evidence replace the seed's, and params and
323
+ return_type are set only where the seed has none. A patch for an export
324
+ no seed holds is not added, with a warning. confidence_breakdown counts
325
+ the exports by confidence, and T2 the exports with a qmd_evidence.
326
+
327
+ Gap records (update-skill gap-driven.md §3, §4 and §4a):
328
+
329
+ Reads the change manifest skf-parse-gaps.py translate wrote. Its
330
+ STRUCTURAL_FIX and metadata update entries are forwarded to merge as
331
+ they are (`forwarded`); every other entry is export-bearing. A severity
332
+ is blocking unless it is Medium, Low or Info (NON_BLOCKING), compared
333
+ case-insensitively, so a missing one is blocking. In order:
334
+
335
+ 1. Drift gate (--drift-status overridden): each export-bearing entry
336
+ but a MOVED_EXPORT needs the tree, with the first reason that fits:
337
+ a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
338
+ provenance_completeness entry "a line from the tree (rule R3)", one
339
+ whose map_match is not-found "a line and a signature from the tree",
340
+ any other "a signature from the tree". Any such entry: {"status":
341
+ "drift-blocked", "drift_blocked": [{gap_id, name, change_category,
342
+ severity, reason}]}, and no spot-check runs.
343
+ 2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
344
+ `exclude`: {"status": "blocked", "rescope_without_amendment": [names]}.
345
+ 3. Spot-checks, by skf-verify-provenance-completeness.py's definition-
346
+ lines rules, loaded from this folder, under --source-root:
347
+ DELETED_EXPORT rescoped, no check
348
+ MOVED_EXPORT checked at its map entry when map_match is
349
+ found (unknown otherwise); under the
350
+ override a `moved` becomes unknown with
351
+ unknown_reason drift-override; its
352
+ pinned_definition_lines are the numbers of
353
+ its remediation's "definition line(s) (...)"
354
+ NEW_EXPORT, MODIFIED_EXPORT
355
+ map_match found checked at the map entry (its export_type)
356
+ ambiguous unknown
357
+ not-found checked at its source_citation (no export
358
+ type) when it has one; when that pins no
359
+ line (unknown or missing), or with no
360
+ citation: targeted re-extraction (§4a) for a
361
+ provenance_completeness entry with a source
362
+ root, for a blocking one whatever its
363
+ resolved_paths, and for a missing-export or
364
+ missing-type one with resolved_paths; else
365
+ unknown
366
+ A check gives `verified` (the line is a definition line, or a module
367
+ or package entry whose file exists), `moved` (one definition line,
368
+ not the recorded one: new_location), `missing` (no such file) or
369
+ `unknown` (several definition lines, none the recorded one, or
370
+ none). A file the rules cover no language of
371
+ takes the answered `declaring_line`: the recorded line is verified,
372
+ another is moved, null is unknown. With no verifier every check is
373
+ unknown, with a `provenance: spot-checks not run` warning; a check
374
+ the verifier fails on (a file it cannot read, or any error) is
375
+ unknown, with a `provenance: spot-check failed for {name}: {error}`
376
+ warning.
377
+ --plan stops here: {"status": "planned", "reextract": {"entries":
378
+ [{gap_id, name, severity, resolved_paths}], "files": [the union of
379
+ their resolved_paths]}, "warnings"}, the files also written to
380
+ --files-out as a JSON array.
381
+ 4. Targeted re-extraction (--remediation-records, `records`' output
382
+ over those files): each routed entry with resolved_paths takes the
383
+ first export of its name in the blocks of those files, in their
384
+ order: `re-extracted`, new_location its record's location,
385
+ resolution_source remediation-paths, and the record copied into
386
+ `files` as `records` wrote it, with the answered `docstring` added.
387
+ An entry with no match, or no path to scan, is unresolved, except a
388
+ missing-export or missing-type one that is not blocking, which is
389
+ unknown. Any unresolved entry: {"status": "unresolved",
390
+ "unresolved": [{gap_id, name, severity, remediation_paths,
391
+ rejected_paths, files_scanned, exports_found_in_scan}]}.
392
+ 5. The public-reachability gate: every NEW_EXPORT that is verified,
393
+ moved or re-extracted takes the answered `reachability`, `public` or
394
+ `internal-unreachable`. An internal one keeps its outcome in its
395
+ verification record, its record is not in `files`, `counts` tallies
396
+ it under `reclassified` instead of its outcome, and it is listed in
397
+ `reclassified` (merge queues it as a metadata update; apply writes no
398
+ entry for it).
399
+
400
+ Any answer missing in step 3 or 5 (a `docstring` too, for each
401
+ re-extracted entry): {"status": "needs-judgment", "needs_judgment":
402
+ [{gap_id, name, needs: [...], file?, line?}]}. --judgments gives them,
403
+ keyed by gap id (the answers file translate reads; its keys are left).
404
+ Then it writes {"mode": "gap-driven", "files_extracted" (the blocks of
405
+ --remediation-records, 0 when it scanned none), "exports_extracted" (the
406
+ manifest's entries), "confidence_breakdown": {"T1", "T1-low",
407
+ "unlabeled", "T2": 0} (each manifest entry by the extraction_method of
408
+ its re-extracted record or, in the map, of its map entry; a pinned cited
409
+ export the map lacks and the gate passed is T1-low), "verification":
410
+ [one record per export-bearing entry: export_name, gap_category,
411
+ severity, verification, in_map, map_entry, provenance_citation,
412
+ source_citation, new_location, unknown_reason, pinned_definition_lines,
413
+ resolution_source, reachability, export_type (always null: for a cited
414
+ export the map lacks, apply takes it from merge.md's
415
+ merge-records.json)], "files": [...]}, and prints {"status": "written",
416
+ "output", "gap_count", "files_extracted", "exports_extracted",
417
+ "confidence_breakdown", "counts" (per verification outcome, and
418
+ `reclassified`), "reclassified", "forwarded", "targeted_reextraction",
419
+ "warnings"}. When targeted re-extraction ran, `targeted_reextraction`
420
+ is {resolved_count, files_scanned, exports_matched, tier}, also appended
421
+ to --evidence as one JSON line keyed `targeted_reextraction`.
422
+
69
423
  Exit codes:
70
- 0 — operation succeeded
71
- 1 — user error (malformed JSON, bad path, malformed provenance file)
424
+ 0 operation succeeded (for gap-records, whatever its `status`)
425
+ 1 user error (malformed JSON, bad path, malformed provenance file, a
426
+ helper file that is not the output it names, a gap-records answer of
427
+ the wrong type, or targeted re-extraction to match with no
428
+ --remediation-records)
429
+ 3 apply refused: a blocking gap reached the provenance write with no
430
+ source line (nothing written)
72
431
  """
73
432
 
74
433
  from __future__ import annotations
75
434
 
76
435
  import argparse
436
+ import difflib
437
+ import hashlib
438
+ import importlib.util
77
439
  import json
440
+ import os
441
+ import re
442
+ import subprocess
78
443
  import sys
444
+ import time
79
445
  from pathlib import Path
80
446
 
81
447
 
@@ -99,7 +465,10 @@ def _get_list(d: dict, *path: str) -> list:
99
465
  def _load_input(input_path: Path | None) -> dict:
100
466
  """Read JSON either from --input path or from stdin."""
101
467
  if input_path is not None:
102
- text = input_path.read_text(encoding="utf-8")
468
+ try:
469
+ text = input_path.read_text(encoding="utf-8")
470
+ except OSError as exc:
471
+ raise ValueError(f"cannot read input {input_path}: {exc}") from exc
103
472
  else:
104
473
  text = sys.stdin.read()
105
474
  try:
@@ -111,6 +480,130 @@ def _load_input(input_path: Path | None) -> dict:
111
480
  return data
112
481
 
113
482
 
483
+ def _load_helper_file(path: Path, what: str, key: str) -> dict:
484
+ """A helper's JSON output: an object holding `key`. Raises ValueError."""
485
+ try:
486
+ data = json.loads(path.read_text(encoding="utf-8"))
487
+ except OSError as exc:
488
+ raise ValueError(f"cannot read {what} {path}: {exc}") from exc
489
+ except json.JSONDecodeError as exc:
490
+ raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
491
+ if not isinstance(data, dict) or key not in data:
492
+ raise ValueError(f"{what} {path} has no `{key}`: is it the helper's output?")
493
+ return data
494
+
495
+
496
+ # --------------------------------------------------------------------------
497
+ # Helper files (update-skill detect-changes §2.1)
498
+ # --------------------------------------------------------------------------
499
+
500
+
501
+ def category_b_from_diff(diff: dict) -> dict:
502
+ """Category B's four lists from a skf-structural-diff.py diff of the
503
+ modified files (see the module docstring for the mapping)."""
504
+ lists: dict[str, list] = {
505
+ "modified_exports": [], "new_exports": [], "deleted_exports": [], "moved_exports": [],
506
+ }
507
+ for item in _get_list(diff, "removed"):
508
+ if isinstance(item, dict):
509
+ lists["deleted_exports"].append(
510
+ {"name": item.get("name"), "file": item.get("file"), "old_line": item.get("line")})
511
+ for item in _get_list(diff, "added"):
512
+ if isinstance(item, dict):
513
+ lists["new_exports"].append({"name": item.get("name"), "file": item.get("file"), "line": item.get("line")})
514
+ changed: dict[tuple, dict] = {}
515
+ for item in _get_list(diff, "changed"):
516
+ if not isinstance(item, dict):
517
+ continue
518
+ record = changed.setdefault(
519
+ (item.get("name"), item.get("file")), {"fields": set(), "line": item.get("line"), "old_line": None})
520
+ record["fields"].add(item.get("field"))
521
+ if item.get("field") == "line":
522
+ record["old_line"] = item.get("baseline_value")
523
+ unverified = list(dict.fromkeys(
524
+ (item.get("name"), item.get("file")) for item in _get_list(diff, "signature_unverified")
525
+ if isinstance(item, dict)
526
+ ))
527
+ modified: set[tuple] = set()
528
+ for key, record in changed.items():
529
+ old_line = record["old_line"] if record["old_line"] is not None else record["line"]
530
+ entry = {"name": key[0], "file": key[1], "old_line": old_line, "new_line": record["line"]}
531
+ if record["fields"] - {"line"} or key in unverified:
532
+ lists["modified_exports"].append(entry)
533
+ modified.add(key)
534
+ else:
535
+ lists["moved_exports"].append(entry)
536
+ for key in unverified:
537
+ if key not in modified:
538
+ lists["modified_exports"].append({"name": key[0], "file": key[1], "old_line": None, "new_line": None})
539
+ modified.add(key)
540
+ for item in _get_list(diff, "moved"):
541
+ if isinstance(item, dict) and (item.get("name"), item.get("current_file")) not in modified:
542
+ lists["moved_exports"].append({
543
+ "name": item.get("name"), "file": item.get("current_file"),
544
+ "old_line": item.get("previous_line"), "new_line": item.get("line"),
545
+ })
546
+ return lists
547
+
548
+
549
+ def _take_out(entries: list, name: object, files: tuple) -> None:
550
+ """Remove the first entry named `name`, preferring one in the first of
551
+ `files` that holds one (a None in `files` stands for any file)."""
552
+ for wanted in files:
553
+ for i, entry in enumerate(entries):
554
+ if isinstance(entry, dict) and entry.get("name") == name and wanted in (None, entry.get("file")):
555
+ del entries[i]
556
+ return
557
+
558
+
559
+ def apply_renames(payload: dict) -> dict:
560
+ """Take each Category C rename's pair out of the Category A and B lists
561
+ it was found in (see the module docstring). payload holds category_a
562
+ and category_b as objects, as assemble leaves them."""
563
+ cat_a, cat_b = payload["category_a"], payload["category_b"]
564
+ cat_c = payload.get("category_c", {}) or {}
565
+ for move in _get_list(cat_c, "renamed_files"):
566
+ if not isinstance(move, dict):
567
+ continue
568
+ for key, path in (("deleted", move.get("old_path")), ("added", move.get("new_path"))):
569
+ listed = _get_list(cat_a, key)
570
+ if path in listed:
571
+ cat_a[key] = [p for p in listed if p != path]
572
+ for rename in _get_list(cat_c, "renamed_exports"):
573
+ if not isinstance(rename, dict):
574
+ continue
575
+ deleted = list(_get_list(cat_b, "deleted_exports"))
576
+ new = list(_get_list(cat_b, "new_exports"))
577
+ old_files = (rename["old_file"],) if rename.get("old_file") else (rename.get("file"), None)
578
+ _take_out(deleted, rename.get("old_name"), old_files)
579
+ _take_out(new, rename.get("new_name"), (rename.get("file"), None))
580
+ cat_b["deleted_exports"], cat_b["new_exports"] = deleted, new
581
+ return payload
582
+
583
+
584
+ def assemble(payload: dict, *, category_a_doc: dict | None = None, diff: dict | None = None) -> dict:
585
+ """The category JSON build and deletion-ratio read: the input, with the
586
+ helper files' slices in place of its own, and Category C's renames taken
587
+ out of the A and B lists."""
588
+ out = dict(payload)
589
+ cat_c = dict(payload.get("category_c") or {})
590
+ if category_a_doc is not None:
591
+ out["category_a"] = {key: list(_get_list(category_a_doc, "category_a", key))
592
+ for key in ("modified", "added", "deleted")}
593
+ moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
594
+ cat_c["renamed_files"] = moves + list(_get_list(cat_c, "renamed_files"))
595
+ else:
596
+ out["category_a"] = {key: list(_get_list(payload, "category_a", key))
597
+ for key in ("modified", "added", "deleted")}
598
+ if diff is not None:
599
+ out["category_b"] = category_b_from_diff(diff)
600
+ else:
601
+ out["category_b"] = {key: list(_get_list(payload, "category_b", key))
602
+ for key in ("modified_exports", "new_exports", "deleted_exports", "moved_exports")}
603
+ out["category_c"] = cat_c
604
+ return apply_renames(out)
605
+
606
+
114
607
  # --------------------------------------------------------------------------
115
608
  # Build manifest
116
609
  # --------------------------------------------------------------------------
@@ -135,14 +628,8 @@ def build_manifest(payload: dict) -> dict:
135
628
  c_renamed_files = _get_list(cat_c, "renamed_files")
136
629
  c_renamed_exports = _get_list(cat_c, "renamed_exports")
137
630
 
138
- d = {
139
- "scripts_modified": len(_get_list(cat_d, "scripts_modified")),
140
- "scripts_added": len(_get_list(cat_d, "scripts_added")),
141
- "scripts_deleted": len(_get_list(cat_d, "scripts_deleted")),
142
- "assets_modified": len(_get_list(cat_d, "assets_modified")),
143
- "assets_added": len(_get_list(cat_d, "assets_added")),
144
- "assets_deleted": len(_get_list(cat_d, "assets_deleted")),
145
- }
631
+ d_lists = {key: list(dict.fromkeys(p for p in _get_list(cat_d, key) if isinstance(p, str))) for key in D_KEYS}
632
+ d = {key: len(paths) for key, paths in d_lists.items()}
146
633
 
147
634
  counts = {
148
635
  "files_changed": len(a_modified),
@@ -186,9 +673,51 @@ def build_manifest(payload: dict) -> dict:
186
673
  "counts": counts,
187
674
  "total_export_changes": total_export_changes,
188
675
  "per_file": per_file,
676
+ "category_d": d_lists,
189
677
  }
190
678
 
191
679
 
680
+ # Category D's lists, in the order build counts them.
681
+ D_KEYS = ("scripts_modified", "scripts_added", "scripts_deleted", "assets_modified", "assets_added",
682
+ "assets_deleted", "docs_modified", "docs_deleted")
683
+ FILE_TYPES = ("script", "asset", "doc")
684
+ _FILE_TYPE_OF_FOLDER = {"scripts": "script", "assets": "asset", "docs": "doc"}
685
+
686
+
687
+ def _file_type(row: dict) -> str:
688
+ """A file_entries[] row's file_type, else the one its file_name's first folder names, else asset."""
689
+ kind = row.get("file_type")
690
+ if kind in FILE_TYPES:
691
+ return kind
692
+ name = row.get("file_name")
693
+ folder = name.strip().replace("\\", "/").split("/", 1)[0] if isinstance(name, str) else ""
694
+ return _FILE_TYPE_OF_FOLDER.get(folder, "asset")
695
+
696
+
697
+ def category_d_from_files(compare: dict | None, new_files: dict | None, provenance: dict | None) -> dict:
698
+ """Category D's lists from skf-hash-content.py compare's rows, each typed by the file_entries[] row of its
699
+ source_file in `provenance`, and from skf-new-file-diff.py's new files, by kind. Raises ValueError."""
700
+ rows = (provenance or {}).get("file_entries") or []
701
+ if not isinstance(rows, list):
702
+ raise ValueError("provenance `file_entries` must be an array")
703
+ types: dict[str, str] = {}
704
+ for row in rows:
705
+ if isinstance(row, dict) and _norm_path(row.get("source_file")):
706
+ types.setdefault(_norm_path(row["source_file"]), _file_type(row))
707
+ lists: dict[str, list] = {key: [] for key in D_KEYS}
708
+ changes = {"MODIFIED_FILE": "modified", "DELETED_FILE": "deleted"}
709
+ for row in _get_list(compare or {}, "comparisons"):
710
+ if not isinstance(row, dict) or row.get("classification") not in changes:
711
+ continue
712
+ path = _norm_path(row.get("source_file"))
713
+ if path:
714
+ lists[f"{types.get(path, 'asset')}s_{changes[row['classification']]}"].append(path)
715
+ for item in _get_list(new_files or {}, "new_files"):
716
+ if isinstance(item, dict) and item.get("kind") in ("script", "asset") and _norm_path(item.get("source_file")):
717
+ lists[f"{item['kind']}s_added"].append(_norm_path(item["source_file"]))
718
+ return lists
719
+
720
+
192
721
  def _build_per_file(
193
722
  *,
194
723
  a_modified: list,
@@ -337,18 +866,1417 @@ def _ratio_skip(*, skip_reason: str) -> dict:
337
866
  }
338
867
 
339
868
 
869
+ # --------------------------------------------------------------------------
870
+ # Shared readers (rename-candidates and apply)
871
+ # --------------------------------------------------------------------------
872
+
873
+
874
+ def _norm_path(path: object) -> str | None:
875
+ """A file path compared as structural-diff compares it: `/` separators, no leading `./`."""
876
+ if not isinstance(path, str) or not path.strip():
877
+ return None
878
+ text = path.strip().replace("\\", "/")
879
+ while text.startswith("./"):
880
+ text = text[2:]
881
+ return text
882
+
883
+
884
+ def _load_json_file(path: Path, what: str):
885
+ try:
886
+ return json.loads(path.read_text(encoding="utf-8"))
887
+ except OSError as exc:
888
+ raise ValueError(f"cannot read {what} {path}: {exc}") from exc
889
+ except json.JSONDecodeError as exc:
890
+ raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
891
+
892
+
893
+ def _exports_of(doc, what: str) -> list[dict]:
894
+ """The export records of an extraction or export-details file ({"exports": [...]})."""
895
+ if doc is None:
896
+ return []
897
+ if not isinstance(doc, dict) or not isinstance(doc.get("exports"), list):
898
+ raise ValueError(f"{what} has no `exports` array: is it the helper's output?")
899
+ return [e for e in doc["exports"] if isinstance(e, dict)]
900
+
901
+
902
+ def _entries_of(provenance) -> list[dict]:
903
+ if provenance is None:
904
+ return []
905
+ if not isinstance(provenance, dict):
906
+ raise ValueError("the provenance map must be a JSON object")
907
+ entries = provenance.get("entries", [])
908
+ if not isinstance(entries, list):
909
+ raise ValueError("provenance `entries` must be an array")
910
+ return [e for e in entries if isinstance(e, dict)]
911
+
912
+
913
+ def _params(value) -> tuple | None:
914
+ """A parameter list compared by its text, whitespace removed."""
915
+ if value is None:
916
+ return None
917
+ if not isinstance(value, list):
918
+ return (str(value),)
919
+ out = []
920
+ for item in value:
921
+ if isinstance(item, dict):
922
+ name, kind = item.get("name"), item.get("type")
923
+ item = f"{name}: {kind}" if kind else str(name)
924
+ out.append("".join(str(item).split()))
925
+ return tuple(out)
926
+
927
+
928
+ def _signature(record: dict) -> tuple:
929
+ """(export_type, params, return_type) of an export record, the parts a signature compares."""
930
+ kind = record.get("export_type", record.get("type"))
931
+ params = record.get("params", record.get("parameters"))
932
+ ret = record.get("return_type")
933
+ return (kind if isinstance(kind, str) else None, _params(params),
934
+ "".join(ret.split()) if isinstance(ret, str) else None)
935
+
936
+
937
+ def _name_of(record: dict) -> str | None:
938
+ name = record.get("export_name", record.get("name"))
939
+ return name if isinstance(name, str) and name else None
940
+
941
+
942
+ def _file_of(record: dict) -> str | None:
943
+ return _norm_path(record.get("source_file", record.get("file", record.get("file_path"))))
944
+
945
+
946
+ # --------------------------------------------------------------------------
947
+ # Rename candidates (update-skill detect-changes Category C)
948
+ # --------------------------------------------------------------------------
949
+
950
+
951
+ RENAME_SIMILARITY = 0.80 # content similarity above this is a rename (fixed)
952
+ SIZE_TOLERANCE = 0.20 # Quick tier: the added file's size within 20% of the deleted one's
953
+ NAME_OVERLAP = 0.70 # Quick tier: shared export names above 70%
954
+ TIERS = ("Quick", "Forge", "Forge+", "Deep")
955
+
956
+
957
+ def _git_size(source_root: Path, commit: str, path: str) -> int | None:
958
+ """The size of `path` at `commit` (git cat-file -s), or None when git cannot tell."""
959
+ try:
960
+ proc = subprocess.run(["git", "-C", str(source_root), "cat-file", "-s", f"{commit}:{path}"],
961
+ capture_output=True, text=True, timeout=30, check=False)
962
+ except (OSError, subprocess.SubprocessError):
963
+ return None
964
+ if proc.returncode != 0:
965
+ return None
966
+ try:
967
+ return int(proc.stdout.strip())
968
+ except ValueError:
969
+ return None
970
+
971
+
972
+ def _file_size(source_root: Path | None, path: str) -> int | None:
973
+ if source_root is None:
974
+ return None
975
+ try:
976
+ return (source_root / path).stat().st_size
977
+ except OSError:
978
+ return None
979
+
980
+
981
+ def _best_pairs(candidates: list[tuple]) -> list[tuple]:
982
+ """Each side's best match only: (score, left, right, rule) tuples, highest score first.
983
+
984
+ A pair is kept when neither side is taken yet and no other candidate of
985
+ either side ties its score, so a tie pairs neither.
986
+ """
987
+ out, used_left, used_right = [], set(), set()
988
+ ordered = sorted(candidates, key=lambda c: (-c[0], c[1], c[2]))
989
+ for score, left, right, rule in ordered:
990
+ if left in used_left or right in used_right:
991
+ continue
992
+ rivals = [c for c in ordered if c[0] == score and (c[1] == left) != (c[2] == right)
993
+ and c[1] not in used_left and c[2] not in used_right]
994
+ if rivals:
995
+ used_left.add(left)
996
+ used_right.add(right)
997
+ for c in rivals:
998
+ used_left.add(c[1])
999
+ used_right.add(c[2])
1000
+ continue
1001
+ used_left.add(left)
1002
+ used_right.add(right)
1003
+ out.append((score, left, right, rule))
1004
+ return out
1005
+
1006
+
1007
+ def rename_candidates(category_a_doc: dict, diff: dict | None, provenance: dict, extraction: dict | None,
1008
+ details: dict | None, tier: str, *, source_root: Path | None = None,
1009
+ sizes_commit: str | None = None) -> dict:
1010
+ """Category C by the fixed rules of the module docstring."""
1011
+ if tier not in TIERS:
1012
+ raise ValueError(f"--tier must be one of {', '.join(TIERS)}, got {tier!r}")
1013
+ forge = tier != "Quick"
1014
+ moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
1015
+ moved_old = {_norm_path(m.get("old_path")) for m in moves}
1016
+ moved_new = {_norm_path(m.get("new_path")) for m in moves}
1017
+ deleted = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "deleted"))
1018
+ if p and p not in moved_old})
1019
+ added = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "added"))
1020
+ if p and p not in moved_new})
1021
+
1022
+ old_by_file: dict[str, dict[str, dict]] = {}
1023
+ for entry in _entries_of(provenance):
1024
+ name, path = _name_of(entry), _file_of(entry)
1025
+ if name and path:
1026
+ old_by_file.setdefault(path, {}).setdefault(name, entry)
1027
+ new_by_file: dict[str, dict[str, dict]] = {}
1028
+ for record in _exports_of(extraction, "--extraction file") + _exports_of(details, "--export-details file"):
1029
+ name, path = _name_of(record), _file_of(record)
1030
+ if name and path:
1031
+ merged = new_by_file.setdefault(path, {}).setdefault(name, {})
1032
+ for key, value in record.items():
1033
+ if merged.get(key) is None:
1034
+ merged[key] = value
1035
+
1036
+ file_candidates = []
1037
+ for old in deleted:
1038
+ old_exports = old_by_file.get(old, {})
1039
+ for new in added:
1040
+ new_exports = new_by_file.get(new, {})
1041
+ if not old_exports or not new_exports:
1042
+ continue
1043
+ if forge:
1044
+ matched = sum(1 for name, e in old_exports.items()
1045
+ if name in new_exports and _signature(e) == _signature(new_exports[name]))
1046
+ score = matched / max(len(old_exports), len(new_exports))
1047
+ if score > RENAME_SIMILARITY:
1048
+ file_candidates.append((round(score, 4), old, new, "signatures"))
1049
+ continue
1050
+ names_old, names_new = set(old_exports), set(new_exports)
1051
+ overlap = len(names_old & names_new) / len(names_old | names_new)
1052
+ if overlap <= NAME_OVERLAP:
1053
+ continue
1054
+ old_size = _git_size(source_root, sizes_commit, old) if (source_root and sizes_commit) else None
1055
+ new_size = _file_size(source_root, new)
1056
+ if old_size is not None:
1057
+ if new_size is None or max(old_size, new_size) == 0:
1058
+ continue
1059
+ if min(old_size, new_size) / max(old_size, new_size) < 1 - SIZE_TOLERANCE:
1060
+ continue
1061
+ rule = "size-and-names"
1062
+ else:
1063
+ rule = "names (size not checked)"
1064
+ file_candidates.append((round(overlap, 4), old, new, rule))
1065
+ file_pairs = _best_pairs(file_candidates)
1066
+ paired_old = {p[1] for p in file_pairs}
1067
+ paired_new = {p[2] for p in file_pairs}
1068
+
1069
+ removed = []
1070
+ for item in _get_list(diff or {}, "removed"):
1071
+ if isinstance(item, dict) and _name_of(item) and _file_of(item):
1072
+ name, path = _name_of(item), _file_of(item)
1073
+ removed.append({**old_by_file.get(path, {}).get(name, {}), **{k: v for k, v in item.items()
1074
+ if v is not None},
1075
+ "_key": (name, path)})
1076
+ candidates_new = []
1077
+ for item in _get_list(diff or {}, "added"):
1078
+ if isinstance(item, dict) and _name_of(item) and _file_of(item):
1079
+ name, path = _name_of(item), _file_of(item)
1080
+ candidates_new.append({**new_by_file.get(path, {}).get(name, {}),
1081
+ **{k: v for k, v in item.items() if v is not None}, "_key": (name, path)})
1082
+ seen_new = {c["_key"] for c in candidates_new}
1083
+ for path in added:
1084
+ if path in paired_new:
1085
+ continue
1086
+ for name, record in new_by_file.get(path, {}).items():
1087
+ if (name, path) not in seen_new:
1088
+ candidates_new.append({**record, "_key": (name, path)})
1089
+
1090
+ export_candidates = []
1091
+ for old in removed:
1092
+ old_sig = _signature(old)
1093
+ for new in candidates_new:
1094
+ if old["_key"][0] == new["_key"][0]:
1095
+ continue
1096
+ new_sig = _signature(new)
1097
+ if forge:
1098
+ if old_sig[1] is None and old_sig[2] is None:
1099
+ continue
1100
+ if old_sig == new_sig:
1101
+ export_candidates.append((1.0, old["_key"], new["_key"], "signature-equal"))
1102
+ continue
1103
+ if old_sig[0] and new_sig[0] and old_sig[0] != new_sig[0]:
1104
+ continue
1105
+ ratio = difflib.SequenceMatcher(None, old["_key"][0], new["_key"][0]).ratio()
1106
+ if ratio > RENAME_SIMILARITY:
1107
+ export_candidates.append((round(ratio, 4), old["_key"], new["_key"], "name-similarity"))
1108
+ export_pairs = _best_pairs(export_candidates)
1109
+
1110
+ renamed_exports = []
1111
+ for _score, (old_name, old_file), (new_name, new_file), _rule in export_pairs:
1112
+ item = {"old_name": old_name, "new_name": new_name, "file": new_file}
1113
+ if old_file != new_file:
1114
+ item["old_file"] = old_file
1115
+ renamed_exports.append(item)
1116
+ paired_removed = {p[1] for p in export_pairs}
1117
+ paired_added = {p[2] for p in export_pairs}
1118
+ return {
1119
+ "category_c": {
1120
+ "renamed_files": [{"old_path": old, "new_path": new} for _s, old, new, _r in file_pairs],
1121
+ "renamed_exports": renamed_exports,
1122
+ },
1123
+ "evidence": {
1124
+ "files": [{"old_path": old, "new_path": new, "rule": rule, "similarity": score}
1125
+ for score, old, new, rule in file_pairs],
1126
+ "exports": [{"old_name": o[0], "new_name": n[0], "file": n[1], "rule": rule, "similarity": score}
1127
+ for score, o, n, rule in export_pairs],
1128
+ },
1129
+ "unpaired": {
1130
+ "deleted_files": [p for p in deleted if p not in paired_old],
1131
+ "added_files": [p for p in added if p not in paired_new],
1132
+ "removed_exports": [{"name": r["_key"][0], "file": r["_key"][1]} for r in removed
1133
+ if r["_key"] not in paired_removed],
1134
+ "added_exports": [{"name": c["_key"][0], "file": c["_key"][1]} for c in candidates_new
1135
+ if c["_key"] not in paired_added],
1136
+ },
1137
+ "tier": tier,
1138
+ }
1139
+
1140
+
1141
+ # --------------------------------------------------------------------------
1142
+ # Apply (update-skill write.md §3)
1143
+ # --------------------------------------------------------------------------
1144
+
1145
+
1146
+ UPDATE_TYPES = ("incremental", "gap-driven", "full")
1147
+ NON_BLOCKING = ("medium", "low", "info")
1148
+ EXIT_REFUSED = 3
1149
+ REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is retried
1150
+ # The update block apply sets at the top level of the map, in this order.
1151
+ UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
1152
+ "confidence_tier", "manual_sections_preserved")
1153
+
1154
+
1155
+ def _labels(method: object, node: object) -> dict:
1156
+ """The confidence labels an extraction method implies (create-skill's entry contract)."""
1157
+ if method in ("ast-grep", "ast_bridge"):
1158
+ return {"confidence": "T1", "extraction_method": "ast-grep", "ast_node_type": node,
1159
+ "signature_source": "T1"}
1160
+ return {"confidence": "T1-low", "extraction_method": "source-read", "ast_node_type": None,
1161
+ "signature_source": "T1-low"}
1162
+
1163
+
1164
+ def _line_of(value: object) -> int | None:
1165
+ """A line number from an int, a digit string or a `file:line` / `file:start-end` location."""
1166
+ if isinstance(value, bool):
1167
+ return None
1168
+ if isinstance(value, int):
1169
+ return value
1170
+ if isinstance(value, str):
1171
+ tail = value.rsplit(":", 1)[-1].split("-", 1)[0].strip()
1172
+ if tail.isdigit():
1173
+ return int(tail)
1174
+ return None
1175
+
1176
+
1177
+ def _location(value: object) -> tuple[str | None, int | None]:
1178
+ """(file, line) of a `file:line` string or a {file, line} object."""
1179
+ if isinstance(value, dict):
1180
+ return _norm_path(value.get("file") or value.get("source_file")), _line_of(
1181
+ value.get("line", value.get("source_line")))
1182
+ if isinstance(value, str) and ":" in value:
1183
+ path, _, rest = value.rpartition(":")
1184
+ return _norm_path(path), _line_of(rest)
1185
+ return None, None
1186
+
1187
+
1188
+ class _Fresh:
1189
+ """The fresh records of one run, looked up by export name and file."""
1190
+
1191
+ def __init__(self, worker_files: list, extraction: dict | None, details: dict | None):
1192
+ self.runner = {}
1193
+ self.details = {}
1194
+ self.worker = {}
1195
+ for record in _exports_of(extraction, "--extraction file"):
1196
+ key = (_name_of(record), _file_of(record))
1197
+ if all(key):
1198
+ self.runner.setdefault(key, record)
1199
+ for record in _exports_of(details, "--export-details file"):
1200
+ key = (_name_of(record), _file_of(record))
1201
+ if all(key):
1202
+ self.details.setdefault(key, record)
1203
+ for block in worker_files:
1204
+ if not isinstance(block, dict):
1205
+ continue
1206
+ path = _norm_path(block.get("file_path"))
1207
+ for record in block.get("exports") or []:
1208
+ if isinstance(record, dict) and _name_of(record) and path:
1209
+ self.worker.setdefault((_name_of(record), path), record)
1210
+
1211
+ def keys_of(self, path: str) -> list[tuple]:
1212
+ keys = {k for source in (self.runner, self.details, self.worker) for k in source if k[1] == path}
1213
+ return sorted(keys)
1214
+
1215
+ def all_keys(self) -> list[tuple]:
1216
+ return sorted({k for source in (self.runner, self.details, self.worker) for k in source})
1217
+
1218
+ def record(self, name: str, path: str) -> dict | None:
1219
+ """The entry fields of an export's fresh record, or None when no source holds it."""
1220
+ key = (name, path)
1221
+ runner, detail, worker = self.runner.get(key), self.details.get(key), self.worker.get(key)
1222
+ if runner is None and detail is None and worker is None:
1223
+ return None
1224
+ sources = [r for r in (runner, detail, worker) if r is not None]
1225
+
1226
+ def first(*fields):
1227
+ for record in sources:
1228
+ for field in fields:
1229
+ value = record.get(field)
1230
+ if value is not None:
1231
+ return value
1232
+ return None
1233
+
1234
+ line = None
1235
+ for record in sources:
1236
+ line = _line_of(record.get("source_line"))
1237
+ if line is None and record is worker:
1238
+ line = _line_of(record.get("location"))
1239
+ if line is not None:
1240
+ break
1241
+ method = first("extraction_method")
1242
+ node = first("ast_node_type") if method in ("ast-grep", "ast_bridge") else None
1243
+ params = None
1244
+ for record in (detail, worker, runner):
1245
+ if record is None:
1246
+ continue
1247
+ value = record.get("params", record.get("parameters"))
1248
+ if value is not None:
1249
+ params = value
1250
+ break
1251
+ return {
1252
+ "export_name": name,
1253
+ "export_type": first("export_type", "type"),
1254
+ "params": params,
1255
+ "return_type": first("return_type"),
1256
+ "source_file": path,
1257
+ "source_line": line,
1258
+ **_labels(method, node),
1259
+ }
1260
+
1261
+
1262
+ def _without_none(fields: dict, keep: tuple = ()) -> dict:
1263
+ return {k: v for k, v in fields.items() if v is not None or k in keep}
1264
+
1265
+
1266
+ def _new_entry(fields: dict, library: str) -> dict:
1267
+ """An entry in create-skill's key order, from the fields a record gives."""
1268
+ order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
1269
+ "source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
1270
+ merged = {**fields, "source_library": fields.get("source_library") or library}
1271
+ out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
1272
+ return out
1273
+
1274
+
1275
+ def _rewrite(entry: dict, fields: dict) -> dict:
1276
+ """The entry with `fields` set in place; keys the update does not set keep their values."""
1277
+ out = dict(entry)
1278
+ for key, value in fields.items():
1279
+ if value is None and key not in ("ast_node_type",):
1280
+ continue
1281
+ out[key] = value
1282
+ return out
1283
+
1284
+
1285
+ def _sha256_of(path: Path) -> str:
1286
+ digest = hashlib.sha256()
1287
+ with path.open("rb") as fh:
1288
+ for chunk in iter(lambda: fh.read(65536), b""):
1289
+ digest.update(chunk)
1290
+ return "sha256:" + digest.hexdigest()
1291
+
1292
+
1293
+ class _Map:
1294
+ """The provenance map being rewritten, with what changed."""
1295
+
1296
+ def __init__(self, provenance: dict | None, skill_name: str):
1297
+ self.doc = json.loads(json.dumps(provenance)) if provenance is not None else {
1298
+ "provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
1299
+ if not isinstance(self.doc.get("entries"), list):
1300
+ raise ValueError("provenance `entries` must be an array")
1301
+ libraries = [e.get("source_library") for e in self.doc["entries"]
1302
+ if isinstance(e, dict) and isinstance(e.get("source_library"), str) and e["source_library"]]
1303
+ self.library = libraries[0] if libraries else skill_name
1304
+ self.added: list[str] = []
1305
+ self.updated: list[str] = []
1306
+ self.removed: list[str] = []
1307
+ self.touched_files: set[str] = set()
1308
+
1309
+ def find(self, name: str, path: str | None, line: int | None = None) -> int | None:
1310
+ """The index of the entry with this name (and file, and line when several share both)."""
1311
+ hits = [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _name_of(e) == name
1312
+ and (path is None or _file_of(e) == path)]
1313
+ if len(hits) > 1 and line is not None:
1314
+ hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
1315
+ return hits[0] if len(hits) == 1 else None
1316
+
1317
+ def indexes_of(self, path: str) -> list[int]:
1318
+ return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _file_of(e) == path]
1319
+
1320
+ def add(self, entry: dict) -> None:
1321
+ self.doc["entries"].append(entry)
1322
+ self.added.append(entry["export_name"])
1323
+ if entry.get("source_file"):
1324
+ self.touched_files.add(_norm_path(entry["source_file"]))
1325
+
1326
+ def update(self, index: int, fields: dict) -> None:
1327
+ before = self.doc["entries"][index]
1328
+ after = _rewrite(before, fields)
1329
+ if after != before:
1330
+ self.doc["entries"][index] = after
1331
+ self.updated.append(_name_of(after))
1332
+ for path in (_file_of(before), _file_of(after)):
1333
+ if path:
1334
+ self.touched_files.add(path)
1335
+
1336
+ def remove(self, indexes: list[int]) -> None:
1337
+ for index in sorted(set(indexes), reverse=True):
1338
+ entry = self.doc["entries"].pop(index)
1339
+ self.removed.append(_name_of(entry))
1340
+ if _file_of(entry):
1341
+ self.touched_files.add(_file_of(entry))
1342
+
1343
+
1344
+ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fresh: _Fresh,
1345
+ warnings: list[str]) -> tuple[int, int]:
1346
+ """Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block."""
1347
+ per_file = [f for f in _get_list(manifest or {}, "per_file") if isinstance(f, dict)]
1348
+
1349
+ def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
1350
+ record = fresh.record(name, path)
1351
+ if record is None:
1352
+ warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
1353
+ f"{'kept as it was' if index is not None else 'not added'}")
1354
+ return
1355
+ if index is None:
1356
+ if add:
1357
+ pmap.add(_new_entry(record, pmap.library))
1358
+ return
1359
+ pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1360
+
1361
+ for item in per_file:
1362
+ status, path = item.get("status"), _norm_path(item.get("file_path"))
1363
+ if not path:
1364
+ continue
1365
+ if status == "DELETED":
1366
+ pmap.remove(pmap.indexes_of(path))
1367
+ continue
1368
+ if status == "MOVED":
1369
+ old = _norm_path(item.get("old_path"))
1370
+ old_entries = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i]
1371
+ for i in (pmap.indexes_of(old) if old else [])}
1372
+ pmap.remove(pmap.indexes_of(old) if old else [])
1373
+ for name, key_path in fresh.keys_of(path):
1374
+ record = fresh.record(name, key_path)
1375
+ carried = old_entries.get(name)
1376
+ entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
1377
+ else _new_entry(record, pmap.library)
1378
+ pmap.add(entry)
1379
+ continue
1380
+ if status == "ADDED":
1381
+ for name, key_path in fresh.keys_of(path):
1382
+ index = pmap.find(name, key_path)
1383
+ refresh(index, name, key_path, add=True)
1384
+ continue
1385
+ for change in item.get("exports_affected") or []:
1386
+ if not isinstance(change, dict) or not _name_of(change):
1387
+ continue
1388
+ name, kind = _name_of(change), change.get("change_type")
1389
+ index = pmap.find(name, path, _line_of(change.get("old_line")))
1390
+ if kind == "DELETED_EXPORT":
1391
+ if index is not None:
1392
+ pmap.remove([index])
1393
+ elif kind == "MOVED_EXPORT":
1394
+ record = fresh.record(name, path)
1395
+ line = record["source_line"] if record and record.get("source_line") is not None \
1396
+ else _line_of(change.get("new_line"))
1397
+ if index is not None and line is not None:
1398
+ pmap.update(index, {"source_file": path, "source_line": line})
1399
+ elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1400
+ refresh(index, name, path, add=True)
1401
+ for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
1402
+ if not isinstance(rename, dict):
1403
+ continue
1404
+ old_name, new_name = rename.get("old_name"), rename.get("new_name")
1405
+ path = _norm_path(rename.get("file"))
1406
+ old_path = _norm_path(rename.get("old_file")) or path
1407
+ if not (isinstance(old_name, str) and isinstance(new_name, str) and path):
1408
+ continue
1409
+ index = pmap.find(old_name, old_path)
1410
+ record = fresh.record(new_name, path)
1411
+ if index is None:
1412
+ continue
1413
+ fields = {"export_name": new_name, "source_file": path}
1414
+ if record is not None:
1415
+ fields.update(_without_none(record, keep=("ast_node_type",)))
1416
+ pmap.update(index, fields)
1417
+ files = len(per_file)
1418
+ total = manifest.get("total_export_changes") if isinstance(manifest, dict) else None
1419
+ return files, total if isinstance(total, int) else 0
1420
+
1421
+
1422
+ def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
1423
+ changed = [u for u in records.get("changed_urls") or [] if isinstance(u, str) and u]
1424
+ exports = [e for e in records.get("exports") or [] if isinstance(e, dict)]
1425
+ for url in changed:
1426
+ old = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i] for i in pmap.indexes_of(url)}
1427
+ pmap.remove(pmap.indexes_of(url))
1428
+ for record in exports:
1429
+ if record.get("url") != url or not _name_of(record):
1430
+ continue
1431
+ fields = {"export_name": _name_of(record), "export_type": record.get("export_type", record.get("type")),
1432
+ "params": record.get("params", record.get("parameters")),
1433
+ "return_type": record.get("return_type"), "source_file": url,
1434
+ "confidence": "T3", "signature_source": "T3"}
1435
+ carried = old.get(_name_of(record))
1436
+ entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
1437
+ _without_none(fields), pmap.library)
1438
+ pmap.add(entry)
1439
+ return len(changed), len(exports)
1440
+
1441
+
1442
+ def _drift_note(drift: tuple | None) -> str:
1443
+ return f"drift override: HEAD {drift[0]} is not pinned {drift[1]}" if drift else ""
1444
+
1445
+
1446
+ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
1447
+ warnings: list[str]) -> tuple[int, int] | list[dict]:
1448
+ """Gap-driven mode: one write per verification record. A list return is the refusal."""
1449
+ verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
1450
+ reextracted = _Fresh(records.get("files") or [], None, None)
1451
+ merged = {}
1452
+ for record in _exports_of(merge_records, "--merge-records file") if merge_records is not None else []:
1453
+ if _name_of(record):
1454
+ merged.setdefault(_name_of(record), record)
1455
+
1456
+ def in_map(r: dict) -> bool:
1457
+ return bool(r.get("in_map")) or bool(r.get("map_entry"))
1458
+
1459
+ blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
1460
+ if r.get("verification") == "unknown" and not in_map(r)
1461
+ and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
1462
+ and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
1463
+ if blocking:
1464
+ return blocking
1465
+ note = _drift_note(drift)
1466
+ pinned_keys: set[tuple] = set()
1467
+ for r in verification:
1468
+ name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
1469
+ entry_file, entry_line = _location(r.get("map_entry"))
1470
+ index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
1471
+ lines = r.get("pinned_definition_lines")
1472
+ r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
1473
+ if outcome == "rescoped":
1474
+ if index is not None:
1475
+ pmap.remove([index])
1476
+ elif outcome == "verified" and index is not None:
1477
+ if drift and category == "MOVED_EXPORT":
1478
+ warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
1479
+ elif outcome == "moved" and index is not None:
1480
+ path, line = _location(r.get("new_location"))
1481
+ if path and line is not None:
1482
+ pmap.update(index, {"source_file": path, "source_line": line})
1483
+ elif outcome == "re-extracted":
1484
+ if r.get("reachability") == "internal-unreachable":
1485
+ continue # not public API: merge queued it as a metadata update
1486
+ path, line = _location(r.get("new_location"))
1487
+ record = reextracted.record(name, path) if path else None
1488
+ if record is None:
1489
+ warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
1490
+ continue
1491
+ existing = pmap.find(name, path)
1492
+ if existing is None:
1493
+ pmap.add(_new_entry(record, pmap.library))
1494
+ else:
1495
+ pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
1496
+ elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1497
+ if r.get("reachability") == "internal-unreachable":
1498
+ continue
1499
+ where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
1500
+ path, line = _location(where)
1501
+ if not path or line is None or (name, path) in pinned_keys or pmap.find(name, path) is not None:
1502
+ continue
1503
+ pinned_keys.add((name, path))
1504
+ extra = merged.get(name, {})
1505
+ pmap.add(_new_entry(_without_none({
1506
+ "export_name": name,
1507
+ "export_type": extra.get("export_type") or r.get("export_type"),
1508
+ "params": extra.get("params"), "return_type": extra.get("return_type"),
1509
+ "source_file": path, "source_line": line, **_labels("source-read", None)},
1510
+ keep=("ast_node_type",)), pmap.library))
1511
+ elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1512
+ extra = merged.get(name, {})
1513
+ pmap.add(_new_entry(_without_none({
1514
+ "export_name": name, "export_type": extra.get("export_type") or r.get("export_type"),
1515
+ "params": extra.get("params"), "return_type": extra.get("return_type"),
1516
+ **_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
1517
+ else:
1518
+ if r.get("unknown_reason") == "drift-override":
1519
+ warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
1520
+ elif drift:
1521
+ warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
1522
+ else:
1523
+ warnings.append(f"provenance: {name}: {outcome}{r5}")
1524
+ return len(pmap.touched_files), len(verification)
1525
+
1526
+
1527
+ def _apply_files(pmap: _Map, compare: dict | None, new_files: dict | None, promoted: list | None,
1528
+ source_root: Path | None) -> dict:
1529
+ """The file_entries[] changes of Category D and the promoted documents."""
1530
+ rows = pmap.doc.get("file_entries")
1531
+ if rows is None:
1532
+ rows = []
1533
+ if not isinstance(rows, list):
1534
+ raise ValueError("provenance `file_entries` must be an array")
1535
+ done = {"added": [], "updated": [], "removed": []}
1536
+ by_source = {_norm_path(r.get("source_file")): i for i, r in enumerate(rows) if isinstance(r, dict)}
1537
+ gone = set()
1538
+ for row in _get_list(compare or {}, "comparisons"):
1539
+ if not isinstance(row, dict):
1540
+ continue
1541
+ path, kind = _norm_path(row.get("source_file")), row.get("classification")
1542
+ index = by_source.get(path)
1543
+ if index is None:
1544
+ continue
1545
+ if kind == "MODIFIED_FILE" and isinstance(row.get("current_hash"), str):
1546
+ if rows[index].get("content_hash") != row["current_hash"]:
1547
+ rows[index] = {**rows[index], "content_hash": row["current_hash"]}
1548
+ done["updated"].append(path)
1549
+ elif kind == "DELETED_FILE":
1550
+ gone.add(index)
1551
+ done["removed"].append(path)
1552
+ rows = [r for i, r in enumerate(rows) if i not in gone]
1553
+ held = {_norm_path(r.get("source_file")) for r in rows if isinstance(r, dict)}
1554
+ for item in _get_list(new_files or {}, "new_files"):
1555
+ if not isinstance(item, dict):
1556
+ continue
1557
+ path, kind = _norm_path(item.get("source_file")), item.get("kind")
1558
+ if not path or path in held or kind not in ("script", "asset"):
1559
+ continue
1560
+ if source_root is None:
1561
+ raise ValueError("--new-files needs --source-root, to hash each new file")
1562
+ try:
1563
+ digest = _sha256_of(source_root / path)
1564
+ except OSError as exc:
1565
+ raise ValueError(f"cannot hash new file {path} under {source_root}: {exc}") from exc
1566
+ rows.append({"file_name": f"{kind}s/{Path(path).name}", "file_type": kind, "source_file": path,
1567
+ "confidence": "T1-low", "extraction_method": "file-copy", "content_hash": digest})
1568
+ held.add(path)
1569
+ done["added"].append(path)
1570
+ for doc in promoted or []:
1571
+ if not isinstance(doc, dict):
1572
+ continue
1573
+ path = _norm_path(doc.get("path"))
1574
+ if not path or path in held or not isinstance(doc.get("content_hash"), str):
1575
+ continue
1576
+ rows.append({"file_name": f"docs/authoritative/{path}", "file_type": "doc", "source_file": path,
1577
+ "content_hash": doc["content_hash"], "confidence": "T1-low",
1578
+ "extraction_method": "promoted-authoritative"})
1579
+ held.add(path)
1580
+ done["added"].append(path)
1581
+ if rows or "file_entries" in pmap.doc:
1582
+ pmap.doc["file_entries"] = rows
1583
+ return done
1584
+
1585
+
1586
+ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
1587
+ categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
1588
+ records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
1589
+ new_files: dict | None = None, promoted: list | None = None, source_root: Path | None = None,
1590
+ generation_date: str, test_report_run_id: str | None, confidence_tier: str,
1591
+ manual_sections_preserved: int, source_commit: str | None = None,
1592
+ source_ref: str | None = None, drift: tuple | None = None) -> tuple[dict, dict]:
1593
+ """(new map, summary) of one update (see "Apply" in the module docstring).
1594
+
1595
+ A refusal returns ({}, summary) with status "refused" and no map.
1596
+ Raises ValueError for inputs it cannot read.
1597
+ """
1598
+ if update_type not in UPDATE_TYPES:
1599
+ raise ValueError(f"--update-type must be one of {', '.join(UPDATE_TYPES)}, got {update_type!r}")
1600
+ if provenance is None and update_type != "full":
1601
+ raise ValueError("--provenance-map is required unless --update-type is full (a degraded run)")
1602
+ records = records if isinstance(records, dict) else {}
1603
+ pmap = _Map(provenance, skill_name)
1604
+ warnings: list[str] = []
1605
+ mode = records.get("mode")
1606
+ if update_type == "gap-driven":
1607
+ outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
1608
+ if isinstance(outcome, list):
1609
+ return {}, {"status": "refused", "blocking_unresolved": outcome}
1610
+ files, exports = outcome
1611
+ elif mode == "docs-only":
1612
+ files, exports = _apply_docs_only(pmap, records)
1613
+ else:
1614
+ fresh = _Fresh(records.get("files") or [], extraction, details)
1615
+ if update_type == "full":
1616
+ for name, path in fresh.all_keys():
1617
+ index = pmap.find(name, path)
1618
+ record = fresh.record(name, path)
1619
+ if index is None:
1620
+ pmap.add(_new_entry(record, pmap.library))
1621
+ else:
1622
+ pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1623
+ files = len({path for _name, path in fresh.all_keys()})
1624
+ exports = len(fresh.all_keys())
1625
+ else:
1626
+ files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings)
1627
+ file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
1628
+ doc = pmap.doc
1629
+ for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
1630
+ if value is not None:
1631
+ doc[key] = value or None
1632
+ block = {
1633
+ "last_update": generation_date,
1634
+ "update_type": update_type,
1635
+ "test_report_run_id": test_report_run_id or None,
1636
+ "files_changed": files,
1637
+ "exports_affected": exports,
1638
+ "confidence_tier": confidence_tier,
1639
+ "manual_sections_preserved": manual_sections_preserved,
1640
+ }
1641
+ for key in UPDATE_BLOCK_KEYS:
1642
+ doc[key] = block[key]
1643
+ summary = {
1644
+ "status": "written",
1645
+ "entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
1646
+ "file_entries": file_changes,
1647
+ "warnings": warnings,
1648
+ }
1649
+ return doc, summary
1650
+
1651
+
1652
+ def _write_json_atomic(path: Path, value) -> None:
1653
+ """Write `value` as indented JSON through a temporary file beside `path` and one rename, retried on Windows,
1654
+ where a scanner may hold the target open for a moment."""
1655
+ tmp = path.with_name(f".{path.name}.skf-{os.getpid()}-tmp")
1656
+ try:
1657
+ tmp.write_bytes((json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
1658
+ deadline = time.monotonic() + REPLACE_WAIT_SECONDS
1659
+ while True:
1660
+ try:
1661
+ os.replace(tmp, path)
1662
+ break
1663
+ except PermissionError:
1664
+ if os.name != "nt" or time.monotonic() >= deadline:
1665
+ raise
1666
+ time.sleep(0.05)
1667
+ except OSError:
1668
+ try:
1669
+ tmp.unlink()
1670
+ except OSError:
1671
+ pass
1672
+ raise
1673
+
1674
+
1675
+ # --------------------------------------------------------------------------
1676
+ # Records (update-skill re-extract.md §4, gap-driven.md §4a)
1677
+ # --------------------------------------------------------------------------
1678
+
1679
+
1680
+ # The fields a worker's patch may set: the ones no tool records, and the two
1681
+ # a patch sets only where the seed has none.
1682
+ PATCH_FIELDS = ("signature", "members", "docstring", "qmd_evidence")
1683
+ PATCH_IF_NULL = ("params", "return_type")
1684
+ _INVENTORY = None
1685
+
1686
+
1687
+ def _typed_param(param, language: object) -> str | None:
1688
+ """One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param, loaded once from
1689
+ this folder, so create-skill and update-skill write one form."""
1690
+ global _INVENTORY
1691
+ if _INVENTORY is None:
1692
+ path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
1693
+ spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
1694
+ if spec is None or spec.loader is None:
1695
+ raise ImportError(f"cannot load {path}")
1696
+ module = importlib.util.module_from_spec(spec)
1697
+ spec.loader.exec_module(module)
1698
+ _INVENTORY = module
1699
+ return _INVENTORY.typed_param(param, language if isinstance(language, str) else None)
1700
+
1701
+
1702
+ def _typed_params(value, language: object) -> list | None:
1703
+ if not isinstance(value, list):
1704
+ return None
1705
+ return [s for s in (_typed_param(p, language) for p in value) if s is not None]
1706
+
1707
+
1708
+ def _seed(record: dict, path: str, read_by_eye: bool) -> dict:
1709
+ """The re-extraction record of one export from the runner's (or, read by eye, step 2's) record."""
1710
+ line = _line_of(record.get("source_line"))
1711
+ out = {
1712
+ "name": _name_of(record),
1713
+ "type": record.get("export_type", record.get("type")),
1714
+ "signature": record.get("signature"),
1715
+ "location": f"{path}:{line}" if line is not None else None,
1716
+ "params": _typed_params(record.get("params", record.get("parameters")), record.get("language")),
1717
+ "return_type": record.get("return_type"),
1718
+ }
1719
+ labels = _labels("source-read" if read_by_eye else record.get("extraction_method"), record.get("ast_node_type"))
1720
+ out.update({key: labels[key] for key in ("confidence", "extraction_method", "ast_node_type")})
1721
+ out["ast_recipe"] = None if read_by_eye else record.get("ast_recipe")
1722
+ return out
1723
+
1724
+
1725
+ def _patch_blocks(folder: Path) -> list[dict]:
1726
+ """Every per-file block of the patch files in `folder` (a file holds one block or a list of them), in file
1727
+ name order. Raises ValueError."""
1728
+ if not folder.is_dir():
1729
+ raise ValueError(f"--patches {folder} is not a folder")
1730
+ blocks = []
1731
+ for path in sorted(folder.glob("*.json"), key=lambda p: p.name):
1732
+ doc = _load_json_file(path, "patch file")
1733
+ for block in doc if isinstance(doc, list) else [doc]:
1734
+ if not isinstance(block, dict) or not _norm_path(block.get("file_path")) \
1735
+ or not isinstance(block.get("exports"), list):
1736
+ raise ValueError(f"patch file {path} holds a block with no file_path or exports list")
1737
+ blocks.append(block)
1738
+ return blocks
1739
+
1740
+
1741
+ def build_records(extraction: dict | None, details: dict | None, files: list | None,
1742
+ patches: list[dict]) -> tuple[dict, dict]:
1743
+ """(reextract-records.json, summary): each export's record seeded from the runner's and step 2's records, with
1744
+ the workers' patches merged in (see "Records" in the module docstring)."""
1745
+ runner, extra = {}, {}
1746
+ for source, doc, what in ((runner, extraction, "--extraction file"), (extra, details, "--export-details file")):
1747
+ for record in _exports_of(doc, what):
1748
+ key = (_name_of(record), _file_of(record))
1749
+ if all(key):
1750
+ source.setdefault(key, record)
1751
+ seeded: dict[tuple, dict] = {}
1752
+ languages: dict[tuple, object] = {}
1753
+ warnings = []
1754
+
1755
+ def fill_if_null(key: tuple, record: dict) -> None:
1756
+ """Set params and return_type from `record` where the seed of `key` has none."""
1757
+ for field in PATCH_IF_NULL:
1758
+ value = record.get(field)
1759
+ if field == "params" and value is None:
1760
+ value = record.get("parameters")
1761
+ if seeded[key][field] is None and value is not None:
1762
+ seeded[key][field] = _typed_params(value, languages.get(key)) if field == "params" else value
1763
+
1764
+ for key, record in runner.items():
1765
+ seeded[key], languages[key] = _seed(record, key[1], read_by_eye=False), record.get("language")
1766
+ fill_if_null(key, extra.get(key, {}))
1767
+ for key, record in extra.items():
1768
+ if key in runner:
1769
+ continue
1770
+ if record.get("export_type", record.get("type")) is None:
1771
+ warnings.append(f"re-extract: {key[0]} in {key[1]}: an export-details record with no export_type the "
1772
+ f"runner does not hold; not added")
1773
+ continue
1774
+ seeded[key], languages[key] = _seed(record, key[1], read_by_eye=True), record.get("language")
1775
+ for block in patches:
1776
+ path = _norm_path(block.get("file_path"))
1777
+ for patch in block.get("exports") or []:
1778
+ key = (_name_of(patch) if isinstance(patch, dict) else None, path)
1779
+ if key not in seeded:
1780
+ warnings.append(f"re-extract: a patch names {key[0] or 'no export'} in {path}, which neither the "
1781
+ f"runner nor step 2 recorded; not added")
1782
+ continue
1783
+ for field in PATCH_FIELDS:
1784
+ if patch.get(field) is not None:
1785
+ seeded[key][field] = patch[field]
1786
+ fill_if_null(key, patch)
1787
+ paths = list(dict.fromkeys([p for p in (_norm_path(f) for f in files or []) if p]
1788
+ + sorted({key[1] for key in seeded})))
1789
+ blocks = [{"file_path": path, "exports": [seeded[key] for key in sorted(seeded) if key[1] == path]}
1790
+ for path in paths]
1791
+ exports = [record for block in blocks for record in block["exports"]]
1792
+ summary = {
1793
+ "files_extracted": len(blocks),
1794
+ "exports_extracted": len(exports),
1795
+ "confidence_breakdown": {
1796
+ "T1": sum(1 for r in exports if r["confidence"] == "T1"),
1797
+ "T1-low": sum(1 for r in exports if r["confidence"] == "T1-low"),
1798
+ "T2": sum(1 for r in exports if r.get("qmd_evidence") is not None),
1799
+ },
1800
+ "warnings": warnings,
1801
+ }
1802
+ return {"mode": "normal", "files": blocks}, summary
1803
+
1804
+
1805
+ # --------------------------------------------------------------------------
1806
+ # Gap records (update-skill gap-driven.md §3, §4 and §4a)
1807
+ # --------------------------------------------------------------------------
1808
+
1809
+
1810
+ EXPORT_CATEGORIES = ("NEW_EXPORT", "MODIFIED_EXPORT", "MOVED_EXPORT", "DELETED_EXPORT")
1811
+ DOCUMENT_CATEGORIES = ("missing-export", "missing-type") # routed to §4a by category, never halting
1812
+ DRIFT_STATUSES = ("ok", "skipped", "overridden")
1813
+ # The drift gate's reasons, gap-driven.md §3: the first that fits.
1814
+ DRIFT_RESCOPE = "a public API recount from the tree (rule R1)"
1815
+ DRIFT_COMPLETENESS = "a line from the tree (rule R3)"
1816
+ DRIFT_NOT_IN_MAP = "a line and a signature from the tree"
1817
+ DRIFT_IN_MAP = "a signature from the tree"
1818
+ REACHABILITY = ("public", "internal-unreachable")
1819
+ VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
1820
+ OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
1821
+ _DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
1822
+ _VERIFIER = None
1823
+
1824
+
1825
+ def _verifier():
1826
+ """skf-verify-provenance-completeness.py, loaded once from this folder for its definition-lines rules, or None
1827
+ when it cannot be loaded."""
1828
+ global _VERIFIER
1829
+ if _VERIFIER is None:
1830
+ path = Path(__file__).resolve().parent / "skf-verify-provenance-completeness.py"
1831
+ try:
1832
+ spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness", path)
1833
+ if spec is None or spec.loader is None:
1834
+ raise ImportError(f"cannot load {path}")
1835
+ module = importlib.util.module_from_spec(spec)
1836
+ spec.loader.exec_module(module)
1837
+ except (OSError, ImportError, SyntaxError):
1838
+ _VERIFIER = False
1839
+ else:
1840
+ _VERIFIER = module
1841
+ return _VERIFIER or None
1842
+
1843
+
1844
+ def _blocking(severity: object) -> bool:
1845
+ return str(severity or "").strip().lower() not in NON_BLOCKING
1846
+
1847
+
1848
+ def _gap_judgments(doc) -> dict:
1849
+ """The answers gap-records reads from a --judgments file, checked. Raises ValueError."""
1850
+ if not isinstance(doc, dict) or not all(isinstance(v, dict) for v in doc.values()):
1851
+ raise ValueError("--judgments must map each gap id to an object of answers")
1852
+ for gid, answers in doc.items():
1853
+ line = answers.get("declaring_line")
1854
+ if "declaring_line" in answers and line is not None and (isinstance(line, bool) or not isinstance(line, int)):
1855
+ raise ValueError(f"--judgments: {gid}'s `declaring_line` must be a line number or null")
1856
+ if "reachability" in answers and answers["reachability"] not in REACHABILITY:
1857
+ raise ValueError(f"--judgments: {gid}'s `reachability` must be one of {', '.join(REACHABILITY)}")
1858
+ if "docstring" in answers and not isinstance(answers["docstring"], (str, type(None))):
1859
+ raise ValueError(f"--judgments: {gid}'s `docstring` must be a string or null")
1860
+ return doc
1861
+
1862
+
1863
+ def _drift_reason(entry: dict) -> str | None:
1864
+ """The drift gate's reason an export-bearing entry needs the tree, or None for one that passes."""
1865
+ category = entry.get("change_category")
1866
+ if category == "DELETED_EXPORT":
1867
+ return DRIFT_RESCOPE
1868
+ if category not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1869
+ return None
1870
+ if entry.get("provenance_completeness"):
1871
+ return DRIFT_COMPLETENESS
1872
+ status = (entry.get("map_match") or {}).get("status")
1873
+ return DRIFT_IN_MAP if status in ("found", "ambiguous") else DRIFT_NOT_IN_MAP
1874
+
1875
+
1876
+ def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
1877
+ answers: dict) -> tuple[str, int | None, bool, str | None]:
1878
+ """(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
1879
+ check failed)."""
1880
+ verifier = _verifier()
1881
+ if verifier is None or root is None or not path or line is None:
1882
+ return "unknown", None, False, None
1883
+ try:
1884
+ report = verifier.definition_lines_report(path, name, root, line,
1885
+ export_type if isinstance(export_type, str) else None)
1886
+ except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
1887
+ return "unknown", None, False, f"{type(exc).__name__}: {exc}"
1888
+ check = report.get("line_check")
1889
+ if check == "file-missing":
1890
+ return "missing", None, False, None
1891
+ if check == "skipped-export-type":
1892
+ return "verified", None, False, None
1893
+ if check == "skipped-language":
1894
+ if "declaring_line" not in answers:
1895
+ return "unknown", None, True, None
1896
+ declared = answers["declaring_line"]
1897
+ if declared is None:
1898
+ return "unknown", None, False, None
1899
+ return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
1900
+ if report.get("line_is_definition"):
1901
+ return "verified", None, False, None
1902
+ defs = report.get("definition_lines") or []
1903
+ if len(defs) == 1 and defs[0] != line:
1904
+ return "moved", defs[0], False, None
1905
+ return "unknown", None, False, None
1906
+
1907
+
1908
+ def _pinned_lines(remediation: object) -> list[int] | None:
1909
+ """The definition lines a provenance-line gap's remediation lists, `... definition line (5, 7) ...`."""
1910
+ m = _DEFINITION_LINES.search(remediation) if isinstance(remediation, str) else None
1911
+ lines = [int(n) for n in re.findall(r"\d+", m.group(1))] if m else []
1912
+ return lines or None
1913
+
1914
+
1915
+ def _where(path: object, line: object) -> str | None:
1916
+ file, number = _norm_path(path), _line_of(line)
1917
+ return f"{file}:{number}" if file and number is not None else None
1918
+
1919
+
1920
+ class _GapEntry:
1921
+ """One export-bearing manifest entry as gap-records routes it."""
1922
+
1923
+ def __init__(self, entry: dict, answers: dict):
1924
+ self.entry = entry
1925
+ self.answers = answers
1926
+ self.name = _name_of(entry) or ""
1927
+ self.category = entry.get("change_category")
1928
+ match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
1929
+ self.match_status = match.get("status") or "not-found"
1930
+ found = match.get("entry") if self.match_status == "found" and isinstance(match.get("entry"), dict) else None
1931
+ self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
1932
+ if found else None
1933
+ self.map_export_type = found.get("export_type") if found else None
1934
+ self.in_map = bool(match.get("candidates"))
1935
+ citation = entry.get("source_citation")
1936
+ self.citation = citation if isinstance(citation, dict) else None
1937
+ self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
1938
+ self.outcome: str | None = None
1939
+ self.new_line: int | None = None
1940
+ self.unknown_reason: str | None = None
1941
+ self.reextract = False
1942
+ self.record: dict | None = None # the §4a record it matched
1943
+ self.record_path: str | None = None
1944
+ self.waits = False # on a declaring_line answer
1945
+ self.failure: str | None = None # why its spot-check failed
1946
+
1947
+ @property
1948
+ def blocking(self) -> bool:
1949
+ return _blocking(self.entry.get("severity"))
1950
+
1951
+ @property
1952
+ def paths(self) -> list[str]:
1953
+ return [p for p in (_norm_path(p) for p in self.entry.get("resolved_paths") or []) if p]
1954
+
1955
+ def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
1956
+ self.recorded = (_norm_path(path), _line_of(line))
1957
+ self.outcome, self.new_line, self.waits, self.failure = _spot_check(
1958
+ root, self.recorded[0], self.name, self.recorded[1], export_type, self.answers)
1959
+
1960
+ def route(self, root: Path | None, drift: bool) -> None:
1961
+ """Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
1962
+ if self.category == "DELETED_EXPORT":
1963
+ self.outcome = "rescoped"
1964
+ if self.map_entry:
1965
+ self.recorded = (_norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"]))
1966
+ return
1967
+ if self.category == "MOVED_EXPORT":
1968
+ if self.map_entry is None:
1969
+ self.outcome = "unknown"
1970
+ return
1971
+ self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
1972
+ if drift and self.outcome == "moved":
1973
+ self.outcome, self.new_line, self.unknown_reason = "unknown", None, "drift-override"
1974
+ return
1975
+ if self.match_status == "found":
1976
+ self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
1977
+ return
1978
+ if self.match_status == "ambiguous":
1979
+ self.outcome = "unknown"
1980
+ return
1981
+ if self.citation:
1982
+ self.check(root, self.citation.get("file"), self.citation.get("line"), None)
1983
+ if self.waits or self.outcome in ("verified", "moved"):
1984
+ return
1985
+ if self.entry.get("provenance_completeness") and root is not None:
1986
+ self.reextract = True
1987
+ elif self.blocking:
1988
+ self.reextract = True
1989
+ elif self.entry.get("category") in DOCUMENT_CATEGORIES and self.paths:
1990
+ self.reextract = True
1991
+ else:
1992
+ self.outcome = "unknown"
1993
+
1994
+ def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
1995
+ """Take the first export of the entry's name in the blocks of its resolved files; (files scanned, exports
1996
+ found in them)."""
1997
+ exports = 0
1998
+ for path in self.paths:
1999
+ for record in blocks.get(path, []):
2000
+ exports += 1
2001
+ if self.record is None and _name_of(record) == self.name:
2002
+ self.record, self.record_path = record, path
2003
+ if self.record is not None:
2004
+ self.outcome = "re-extracted"
2005
+ return len(self.paths), exports
2006
+
2007
+ @property
2008
+ def located(self) -> bool:
2009
+ return self.outcome in ("verified", "moved", "re-extracted")
2010
+
2011
+ @property
2012
+ def internal(self) -> bool:
2013
+ return self.category == "NEW_EXPORT" and self.answers.get("reachability") == "internal-unreachable"
2014
+
2015
+ def needs(self) -> list[str]:
2016
+ """The answers the entry still waits on once routed."""
2017
+ want = []
2018
+ if self.category == "NEW_EXPORT" and self.located and "reachability" not in self.answers:
2019
+ want.append("reachability")
2020
+ if self.outcome == "re-extracted" and "docstring" not in self.answers:
2021
+ want.append("docstring")
2022
+ return want
2023
+
2024
+ def new_location(self) -> str | None:
2025
+ if self.outcome == "moved":
2026
+ return _where(self.recorded[0], self.new_line)
2027
+ if self.outcome == "re-extracted":
2028
+ return (self.record or {}).get("location")
2029
+ return None
2030
+
2031
+ def verification(self) -> dict:
2032
+ recorded = _where(*self.recorded)
2033
+ if self.outcome == "re-extracted":
2034
+ citation = self.new_location()
2035
+ elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
2036
+ citation = recorded
2037
+ else:
2038
+ citation = "unknown"
2039
+ return {
2040
+ "export_name": self.name,
2041
+ "gap_category": self.category,
2042
+ "severity": self.entry.get("severity"),
2043
+ "verification": self.outcome,
2044
+ "in_map": self.in_map,
2045
+ "map_entry": self.map_entry,
2046
+ "provenance_citation": citation,
2047
+ "source_citation": self.citation,
2048
+ "new_location": self.new_location(),
2049
+ "unknown_reason": self.unknown_reason,
2050
+ "pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
2051
+ if self.category == "MOVED_EXPORT" else None,
2052
+ "resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
2053
+ "reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
2054
+ else None,
2055
+ "export_type": None,
2056
+ }
2057
+
2058
+ def label(self, provenance: list[dict]) -> str:
2059
+ """T1, T1-low or unlabeled: the confidence_breakdown bin of the entry."""
2060
+ if self.internal:
2061
+ return "unlabeled"
2062
+ method = None
2063
+ if self.outcome == "re-extracted":
2064
+ method = (self.record or {}).get("extraction_method")
2065
+ elif self.map_entry is not None:
2066
+ file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
2067
+ for entry in provenance:
2068
+ if (_name_of(entry) == self.name and _file_of(entry) == file
2069
+ and _line_of(entry.get("source_line")) == line):
2070
+ method = entry.get("extraction_method")
2071
+ break
2072
+ elif self.located and self.category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
2073
+ method = "source-read" # a pinned cited export the map lacks: write.md §3 writes it source-read
2074
+ if method in ("ast-grep", "ast_bridge"):
2075
+ return "T1"
2076
+ if method in ("source-read", "source_reading"):
2077
+ return "T1-low"
2078
+ return "unlabeled"
2079
+
2080
+
2081
+ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path | None, drift_status: str = "ok",
2082
+ judgments: dict | None = None, remediation: dict | None = None, plan: bool = False,
2083
+ tier: str | None = None) -> tuple[dict | None, dict]:
2084
+ """(reextract-records.json or None, the summary): see "Gap records" in the module docstring."""
2085
+ if not isinstance(manifest, dict) or not isinstance(manifest.get("entries"), list):
2086
+ raise ValueError("the change manifest has no `entries` array: is it translate's output?")
2087
+ entries = [e for e in manifest["entries"] if isinstance(e, dict)]
2088
+ judgments = judgments or {}
2089
+ root = source_root if source_root is not None and source_root.is_dir() else None
2090
+ forwarded = [{"gap_id": e.get("gap_id"), "change_category": e.get("change_category")}
2091
+ for e in entries if e.get("change_category") not in EXPORT_CATEGORIES]
2092
+ routed = [_GapEntry(e, judgments.get(e.get("gap_id")) or {}) for e in entries
2093
+ if e.get("change_category") in EXPORT_CATEGORIES]
2094
+ drift = drift_status == "overridden"
2095
+ if drift:
2096
+ blocked = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "change_category": g.category,
2097
+ "severity": g.entry.get("severity"), "reason": _drift_reason(g.entry)}
2098
+ for g in routed if _drift_reason(g.entry)]
2099
+ if blocked:
2100
+ return None, {"status": "drift-blocked", "drift_blocked": blocked}
2101
+ bare = [g.name for g in routed if g.category == "DELETED_EXPORT" and not (
2102
+ isinstance(g.entry.get("rescope"), dict) and g.entry["rescope"].get("amendment")
2103
+ and g.entry["rescope"].get("exclude"))]
2104
+ if bare:
2105
+ return None, {"status": "blocked", "rescope_without_amendment": bare}
2106
+ warnings = []
2107
+ if root is not None and _verifier() is None:
2108
+ warnings.append(VERIFIER_MISSING)
2109
+ for g in routed:
2110
+ g.route(root, drift)
2111
+ warnings += [f"provenance: spot-check failed for {g.name}: {g.failure}" for g in routed if g.failure]
2112
+ waiting = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": ["declaring_line"],
2113
+ "file": g.recorded[0], "line": g.recorded[1]} for g in routed if g.waits]
2114
+ if waiting:
2115
+ return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
2116
+ to_scan = [g for g in routed if g.reextract]
2117
+ files = list(dict.fromkeys(p for g in to_scan for p in g.paths))
2118
+ if plan:
2119
+ return None, {"status": "planned", "reextract": {
2120
+ "entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
2121
+ "resolved_paths": g.paths} for g in to_scan], "files": files}, "warnings": warnings}
2122
+ blocks: dict[str, list[dict]] = {}
2123
+ scanned = 0
2124
+ if files:
2125
+ if remediation is None:
2126
+ raise ValueError("targeted re-extraction has files to match: pass --remediation-records, the output "
2127
+ "of `records` over them")
2128
+ if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
2129
+ raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
2130
+ for block in remediation["files"]:
2131
+ if isinstance(block, dict) and _norm_path(block.get("file_path")):
2132
+ scanned += 1
2133
+ blocks.setdefault(_norm_path(block["file_path"]), []).extend(
2134
+ r for r in block.get("exports") or [] if isinstance(r, dict))
2135
+ unresolved = []
2136
+ for g in to_scan:
2137
+ files_scanned, found = g.match(blocks)
2138
+ if g.record is not None:
2139
+ continue
2140
+ if not g.blocking and g.entry.get("category") in DOCUMENT_CATEGORIES:
2141
+ g.outcome = "unknown"
2142
+ continue
2143
+ unresolved.append({"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
2144
+ "remediation_paths": g.entry.get("remediation_paths") or [],
2145
+ "rejected_paths": g.entry.get("rejected_paths") or [],
2146
+ "files_scanned": files_scanned, "exports_found_in_scan": found})
2147
+ if unresolved:
2148
+ return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
2149
+ needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
2150
+ "file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
2151
+ "line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
2152
+ for g in routed if g.needs()]
2153
+ if needs:
2154
+ return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
2155
+ kept: dict[str, list[dict]] = {}
2156
+ for g in routed:
2157
+ if g.record is not None and not g.internal:
2158
+ record = dict(g.record)
2159
+ if g.answers.get("docstring") is not None:
2160
+ record["docstring"] = g.answers["docstring"]
2161
+ if record not in kept.setdefault(g.record_path, []):
2162
+ kept[g.record_path].append(record)
2163
+ breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
2164
+ provenance_entries = _entries_of(provenance)
2165
+ for g in routed:
2166
+ breakdown[g.label(provenance_entries)] += 1
2167
+ breakdown["unlabeled"] += len(forwarded)
2168
+ records = {
2169
+ "mode": "gap-driven",
2170
+ "files_extracted": scanned,
2171
+ "exports_extracted": len(entries),
2172
+ "confidence_breakdown": breakdown,
2173
+ "verification": [g.verification() for g in routed],
2174
+ "files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
2175
+ }
2176
+ matched = [g for g in to_scan if g.record is not None]
2177
+ targeted = {"resolved_count": len(matched), "files_scanned": scanned,
2178
+ "exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
2179
+ "tier": tier} if to_scan else None
2180
+ summary = {
2181
+ "status": "written",
2182
+ "gap_count": len(entries),
2183
+ "files_extracted": scanned,
2184
+ "exports_extracted": len(entries),
2185
+ "confidence_breakdown": breakdown,
2186
+ "counts": {**{outcome: sum(1 for g in routed if g.outcome == outcome and not g.internal)
2187
+ for outcome in OUTCOMES}, "reclassified": sum(1 for g in routed if g.internal)},
2188
+ "reclassified": [g.name for g in routed if g.internal],
2189
+ "forwarded": forwarded,
2190
+ "targeted_reextraction": targeted,
2191
+ "warnings": warnings,
2192
+ }
2193
+ return records, summary
2194
+
2195
+
340
2196
  # --------------------------------------------------------------------------
341
2197
  # CLI
342
2198
  # --------------------------------------------------------------------------
343
2199
 
344
2200
 
2201
+ def _category_c(args: argparse.Namespace, own: dict | None) -> dict | None:
2202
+ """Category C from --category-c and --ccc-pairs, else the input's own
2203
+ (`own`, None when there is none). Raises ValueError."""
2204
+ if not args.category_c and not args.ccc_pairs:
2205
+ return own
2206
+ cat_c = dict(own or {})
2207
+ if args.category_c:
2208
+ doc = _load_helper_file(Path(args.category_c), "--category-c file", "category_c")
2209
+ if not isinstance(doc["category_c"], dict):
2210
+ raise ValueError(f"--category-c file {args.category_c} has no `category_c` object")
2211
+ cat_c = dict(doc["category_c"])
2212
+ if args.ccc_pairs:
2213
+ pairs = _load_json_file(Path(args.ccc_pairs), "--ccc-pairs file")
2214
+ if not isinstance(pairs, dict):
2215
+ raise ValueError(f"--ccc-pairs file {args.ccc_pairs} must hold a JSON object")
2216
+ for key in ("renamed_files", "renamed_exports"):
2217
+ cat_c[key] = list(_get_list(cat_c, key)) + [p for p in _get_list(pairs, key) if isinstance(p, dict)]
2218
+ return cat_c
2219
+
2220
+
2221
+ def _payload(args: argparse.Namespace) -> dict:
2222
+ """The category JSON of a build or deletion-ratio call: the input with
2223
+ the helper files' slices assembled in. Raises ValueError."""
2224
+ payload = _load_input(Path(args.input) if args.input else None)
2225
+ cat_c = _category_c(args, payload.get("category_c") if isinstance(payload.get("category_c"), dict) else None)
2226
+ if cat_c is not None:
2227
+ payload = {**payload, "category_c": cat_c}
2228
+ category_a_doc = (
2229
+ _load_helper_file(Path(args.category_a), "--category-a file", "category_a") if args.category_a else None
2230
+ )
2231
+ diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
2232
+ if args.category_b_diff else None
2233
+ return assemble(payload, category_a_doc=category_a_doc, diff=diff)
2234
+
2235
+
2236
+ def docs_only_manifest(doc_hashes: dict) -> dict:
2237
+ """A docs-only skill's change manifest from compare-hashes' output."""
2238
+ def urls(key: str) -> list[str]:
2239
+ return [e["url"] for e in _get_list(doc_hashes, key) if isinstance(e, dict) and isinstance(e.get("url"), str)]
2240
+
2241
+ changed, failed = urls("changed"), urls("fetch_failed")
2242
+ return {"mode": "docs-only", "no_changes": not changed, "changed_urls": changed, "fetch_failed": failed,
2243
+ "counts": {"docs_changed": len(changed), "docs_fetch_failed": len(failed)}}
2244
+
2245
+
2246
+ def _category_d(args: argparse.Namespace) -> dict | None:
2247
+ """Category D from --file-compare (typed by --provenance-map) and --new-files, else None. Raises ValueError."""
2248
+ if not args.file_compare and not args.new_files:
2249
+ if args.provenance_map:
2250
+ raise ValueError("--provenance-map goes with --file-compare")
2251
+ return None
2252
+ if args.file_compare and not args.provenance_map:
2253
+ raise ValueError("--file-compare needs --provenance-map, which types each row by its file_entries[] row")
2254
+ compare = _load_helper_file(Path(args.file_compare), "--file-compare file", "comparisons") \
2255
+ if args.file_compare else None
2256
+ new_files = _load_helper_file(Path(args.new_files), "--new-files file", "new_files") if args.new_files else None
2257
+ provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file") if args.provenance_map else None
2258
+ if provenance is not None and not isinstance(provenance, dict):
2259
+ raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
2260
+ return category_d_from_files(compare, new_files, provenance)
2261
+
2262
+
345
2263
  def _cmd_build(args: argparse.Namespace) -> int:
346
2264
  try:
347
- payload = _load_input(Path(args.input) if args.input else None)
2265
+ if args.doc_hashes:
2266
+ if (args.input or args.category_a or args.category_b_diff or args.category_c or args.ccc_pairs
2267
+ or args.file_compare or args.new_files or args.provenance_map):
2268
+ raise ValueError("--doc-hashes takes no other input")
2269
+ doc_hashes = _load_helper_file(Path(args.doc_hashes), "--doc-hashes file", "changed")
2270
+ manifest = docs_only_manifest(doc_hashes)
2271
+ else:
2272
+ payload = _payload(args)
2273
+ cat_d = _category_d(args)
2274
+ if cat_d is not None:
2275
+ payload = {**payload, "category_d": cat_d}
2276
+ manifest = build_manifest(payload)
348
2277
  except ValueError as exc:
349
2278
  print(f"error: {exc}", file=sys.stderr)
350
2279
  return 1
351
- manifest = build_manifest(payload)
352
2280
  json.dump(manifest, sys.stdout, indent=2)
353
2281
  sys.stdout.write("\n")
354
2282
  return 0
@@ -356,7 +2284,7 @@ def _cmd_build(args: argparse.Namespace) -> int:
356
2284
 
357
2285
  def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
358
2286
  try:
359
- payload = _load_input(Path(args.input) if args.input else None)
2287
+ payload = _payload(args)
360
2288
  except ValueError as exc:
361
2289
  print(f"error: {exc}", file=sys.stderr)
362
2290
  return 1
@@ -379,6 +2307,163 @@ def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
379
2307
  return 0
380
2308
 
381
2309
 
2310
+ def _optional_json(value: str | None, what: str):
2311
+ return _load_json_file(Path(value), what) if value else None
2312
+
2313
+
2314
+ def _emit(result: dict, output: str | None) -> None:
2315
+ if output:
2316
+ _write_json_atomic(Path(output), result)
2317
+ print(json.dumps({"status": "ok", "output": output}))
2318
+ else:
2319
+ json.dump(result, sys.stdout, indent=2)
2320
+ sys.stdout.write("\n")
2321
+
2322
+
2323
+ def _cmd_rename_candidates(args: argparse.Namespace) -> int:
2324
+ try:
2325
+ category_a_doc = _load_helper_file(Path(args.category_a), "--category-a file", "category_a")
2326
+ diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
2327
+ if args.category_b_diff else None
2328
+ provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
2329
+ result = rename_candidates(
2330
+ category_a_doc, diff, provenance,
2331
+ _optional_json(args.extraction, "--extraction file"),
2332
+ _optional_json(args.export_details, "--export-details file"),
2333
+ args.tier,
2334
+ source_root=Path(args.source_root) if args.source_root else None,
2335
+ sizes_commit=args.sizes_commit or None,
2336
+ )
2337
+ _emit(result, args.output)
2338
+ except ValueError as exc:
2339
+ print(f"error: {exc}", file=sys.stderr)
2340
+ return 1
2341
+ except OSError as exc:
2342
+ print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
2343
+ return 1
2344
+ return 0
2345
+
2346
+
2347
+ def _apply_categories(args: argparse.Namespace) -> dict | None:
2348
+ """The category JSON apply reads: --input, with --category-c and --ccc-pairs in place of its category_c."""
2349
+ categories = _optional_json(args.input, "--input file")
2350
+ if categories is not None and not isinstance(categories, dict):
2351
+ raise ValueError("--input file must hold a JSON object")
2352
+ own = (categories or {}).get("category_c")
2353
+ cat_c = _category_c(args, own if isinstance(own, dict) else None)
2354
+ if cat_c is None:
2355
+ return categories
2356
+ return {**(categories or {}), "category_c": cat_c}
2357
+
2358
+
2359
+ def _cmd_apply(args: argparse.Namespace) -> int:
2360
+ if bool(args.drift_head) != bool(args.drift_pinned):
2361
+ print("error: --drift-head and --drift-pinned go together", file=sys.stderr)
2362
+ return 1
2363
+ try:
2364
+ promoted = _optional_json(args.promoted_docs, "--promoted-docs file")
2365
+ if promoted is not None and not isinstance(promoted, list):
2366
+ raise ValueError("--promoted-docs file must hold a JSON array")
2367
+ doc, summary = apply_update(
2368
+ update_type=args.update_type,
2369
+ provenance=_optional_json(args.provenance_map, "--provenance-map file"),
2370
+ skill_name=args.skill_name,
2371
+ manifest=_optional_json(args.manifest, "--manifest file"),
2372
+ categories=_apply_categories(args),
2373
+ extraction=_optional_json(args.extraction, "--extraction file"),
2374
+ details=_optional_json(args.export_details, "--export-details file"),
2375
+ records=_optional_json(args.reextract_records, "--reextract-records file"),
2376
+ merge_records=_optional_json(args.merge_records, "--merge-records file"),
2377
+ compare=_optional_json(args.file_compare, "--file-compare file"),
2378
+ new_files=_optional_json(args.new_files, "--new-files file"),
2379
+ promoted=promoted,
2380
+ source_root=Path(args.source_root) if args.source_root else None,
2381
+ generation_date=args.generation_date,
2382
+ test_report_run_id=args.test_report_run_id,
2383
+ confidence_tier=args.confidence_tier,
2384
+ manual_sections_preserved=args.manual_sections_preserved,
2385
+ source_commit=args.source_commit,
2386
+ source_ref=args.source_ref,
2387
+ drift=(args.drift_head, args.drift_pinned) if args.drift_head else None,
2388
+ )
2389
+ except ValueError as exc:
2390
+ print(f"error: {exc}", file=sys.stderr)
2391
+ return 1
2392
+ if summary["status"] == "refused":
2393
+ print(json.dumps(summary))
2394
+ return EXIT_REFUSED
2395
+ try:
2396
+ _write_json_atomic(Path(args.output), doc)
2397
+ except OSError as exc:
2398
+ print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
2399
+ return 1
2400
+ print(json.dumps({**summary, "map": args.output}))
2401
+ return 0
2402
+
2403
+
2404
+ def _cmd_records(args: argparse.Namespace) -> int:
2405
+ try:
2406
+ files = _optional_json(args.files_from, "--files-from file")
2407
+ if files is not None and not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
2408
+ raise ValueError(f"--files-from file {args.files_from} must hold a JSON array of paths")
2409
+ records, summary = build_records(
2410
+ _optional_json(args.extraction, "--extraction file"),
2411
+ _optional_json(args.export_details, "--export-details file"),
2412
+ files,
2413
+ _patch_blocks(Path(args.patches)) if args.patches else [],
2414
+ )
2415
+ _write_json_atomic(Path(args.output), records)
2416
+ except ValueError as exc:
2417
+ print(f"error: {exc}", file=sys.stderr)
2418
+ return 1
2419
+ except OSError as exc:
2420
+ print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
2421
+ return 1
2422
+ print(json.dumps({"status": "written", "output": args.output, **summary}))
2423
+ return 0
2424
+
2425
+
2426
+ def _cmd_gap_records(args: argparse.Namespace) -> int:
2427
+ try:
2428
+ if args.plan and args.output:
2429
+ raise ValueError("--plan writes no records: drop -o")
2430
+ if not args.plan and not args.output:
2431
+ raise ValueError("-o is required without --plan")
2432
+ if args.files_out and not args.plan:
2433
+ raise ValueError("--files-out goes with --plan")
2434
+ if bool(args.evidence) != bool(args.tier):
2435
+ raise ValueError("--evidence and --tier go together")
2436
+ judgments = _gap_judgments(_load_json_file(Path(args.judgments), "--judgments file")) \
2437
+ if args.judgments else {}
2438
+ records, summary = gap_records(
2439
+ _load_json_file(Path(args.manifest), "--manifest file"),
2440
+ _load_json_file(Path(args.provenance_map), "--provenance-map file"),
2441
+ source_root=Path(args.source_root) if args.source_root else None,
2442
+ drift_status=args.drift_status,
2443
+ judgments=judgments,
2444
+ remediation=_optional_json(args.remediation_records, "--remediation-records file"),
2445
+ plan=args.plan,
2446
+ tier=args.tier,
2447
+ )
2448
+ if summary["status"] == "planned" and args.files_out:
2449
+ _write_json_atomic(Path(args.files_out), summary["reextract"]["files"])
2450
+ if records is not None:
2451
+ _write_json_atomic(Path(args.output), records)
2452
+ summary = {"status": summary["status"], "output": args.output,
2453
+ **{k: v for k, v in summary.items() if k != "status"}}
2454
+ if args.evidence and summary["targeted_reextraction"] is not None:
2455
+ with open(args.evidence, "a", encoding="utf-8", newline="\n") as fh:
2456
+ fh.write(json.dumps({"targeted_reextraction": summary["targeted_reextraction"]}) + "\n")
2457
+ except ValueError as exc:
2458
+ print(f"error: {exc}", file=sys.stderr)
2459
+ return 1
2460
+ except OSError as exc:
2461
+ print(f"error: cannot write: {exc}", file=sys.stderr)
2462
+ return 1
2463
+ print(json.dumps(summary, indent=2))
2464
+ return 0
2465
+
2466
+
382
2467
  def _build_parser() -> argparse.ArgumentParser:
383
2468
  parser = argparse.ArgumentParser(
384
2469
  prog="skf-build-change-manifest",
@@ -390,23 +2475,117 @@ def _build_parser() -> argparse.ArgumentParser:
390
2475
  sub = parser.add_subparsers(dest="cmd", required=True)
391
2476
 
392
2477
  p_build = sub.add_parser("build", help="aggregate categories into manifest")
393
- p_build.add_argument(
394
- "--input", default=None, help="path to JSON input (default: stdin)"
395
- )
2478
+ p_build.add_argument("--doc-hashes", metavar="FILE",
2479
+ help="a docs-only skill: skf-detect-docs.py compare-hashes output, the only input")
2480
+ p_build.add_argument("--file-compare", metavar="FILE",
2481
+ help="skf-hash-content.py compare output: Category D's modified and deleted files")
2482
+ p_build.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output: Category D's new files")
2483
+ p_build.add_argument("--provenance-map", metavar="FILE",
2484
+ help="the map --file-compare compared: each row's file_type")
396
2485
  p_build.set_defaults(func=_cmd_build)
397
2486
 
398
2487
  p_ratio = sub.add_parser(
399
2488
  "deletion-ratio",
400
2489
  help="compute §2.2 trigger; requires --provenance-map",
401
2490
  )
402
- p_ratio.add_argument(
403
- "--input", default=None, help="path to JSON input (default: stdin)"
404
- )
405
2491
  p_ratio.add_argument(
406
2492
  "--provenance-map", required=True, help="path to provenance-map.json"
407
2493
  )
408
2494
  p_ratio.set_defaults(func=_cmd_deletion_ratio)
409
2495
 
2496
+ p_rename = sub.add_parser(
2497
+ "rename-candidates",
2498
+ help="Category C by fixed rules: renamed files and renamed exports",
2499
+ )
2500
+ p_rename.add_argument("--category-a", required=True, metavar="FILE",
2501
+ help="skf-classify-changed-files.py classify output (deleted and added files)")
2502
+ p_rename.add_argument("--category-b-diff", metavar="FILE",
2503
+ help="skf-structural-diff.py output (removed and added exports)")
2504
+ p_rename.add_argument("--provenance-map", required=True, metavar="FILE",
2505
+ help="the provenance map: the deleted files' exports")
2506
+ p_rename.add_argument("--extraction", metavar="FILE", help="the recipe runner's output: the added files' exports")
2507
+ p_rename.add_argument("--export-details", metavar="FILE",
2508
+ help="the workers' export details: params, return types and exports read by eye")
2509
+ p_rename.add_argument("--tier", required=True, choices=TIERS, help="the forge tier, which picks the rules")
2510
+ p_rename.add_argument("--source-root", metavar="DIR", help="the source tree: the added files' sizes")
2511
+ p_rename.add_argument("--sizes-commit", metavar="COMMIT",
2512
+ help="the commit the deleted files' sizes are read at (git cat-file -s)")
2513
+ p_rename.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
2514
+ p_rename.set_defaults(func=_cmd_rename_candidates)
2515
+
2516
+ p_apply = sub.add_parser("apply", help="write the provenance map an update leaves")
2517
+ p_apply.add_argument("--update-type", required=True, choices=UPDATE_TYPES,
2518
+ help="incremental (normal mode), gap-driven, or full (degraded mode)")
2519
+ p_apply.add_argument("--provenance-map", metavar="FILE", help="the map the update started from")
2520
+ p_apply.add_argument("--manifest", metavar="FILE", help="the change manifest (build's output)")
2521
+ p_apply.add_argument("--input", metavar="FILE", help="the category JSON: category_c's renamed exports")
2522
+ p_apply.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
2523
+ p_apply.add_argument("--export-details", metavar="FILE", help="the workers' export details")
2524
+ p_apply.add_argument("--reextract-records", metavar="FILE", help="step 3's records")
2525
+ p_apply.add_argument("--merge-records", metavar="FILE", help="merge's records of the exports no fresh record holds")
2526
+ p_apply.add_argument("--file-compare", metavar="FILE", help="skf-hash-content.py compare output")
2527
+ p_apply.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output")
2528
+ p_apply.add_argument("--promoted-docs", metavar="FILE", help="the documents step 2 promoted, with their hashes")
2529
+ p_apply.add_argument("--source-root", metavar="DIR", help="the source tree: the new files' hashes")
2530
+ p_apply.add_argument("--skill-name", required=True, help="the skill name: source_library of a new entry")
2531
+ p_apply.add_argument("--generation-date", required=True, help="last_update: the generation_date written")
2532
+ p_apply.add_argument("--test-report-run-id", default=None, help="the run id of the test report applied")
2533
+ p_apply.add_argument("--confidence-tier", required=True, help="the forge tier the update ran at")
2534
+ p_apply.add_argument("--manual-sections-preserved", required=True, type=int,
2535
+ help="the [MANUAL] blocks the post-merge check found intact")
2536
+ p_apply.add_argument("--source-commit", default=None, help="the map's source_commit (empty: null)")
2537
+ p_apply.add_argument("--source-ref", default=None, help="the map's source_ref (empty: null)")
2538
+ p_apply.add_argument("--drift-head", default=None, help="under the drift override: HEAD's short SHA")
2539
+ p_apply.add_argument("--drift-pinned", default=None, help="under the drift override: the pinned short SHA")
2540
+ p_apply.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the new map")
2541
+ p_apply.set_defaults(func=_cmd_apply)
2542
+
2543
+ p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
2544
+ "workers' files")
2545
+ p_records.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
2546
+ p_records.add_argument("--export-details", metavar="FILE",
2547
+ help="step 2's export details: params and return types, and the exports read by eye")
2548
+ p_records.add_argument("--files-from", metavar="FILE",
2549
+ help="the files step 3 extracts (a JSON array), each a block even with no export")
2550
+ p_records.add_argument("--patches", metavar="DIR", help="the folder of the workers' per-file patches")
2551
+ p_records.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the records")
2552
+ p_records.set_defaults(func=_cmd_records)
2553
+
2554
+ p_gap = sub.add_parser("gap-records", help="spot-check, route and record a gap-driven repair's manifest entries")
2555
+ p_gap.add_argument("--manifest", required=True, metavar="FILE", help="skf-parse-gaps.py translate's manifest")
2556
+ p_gap.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the repair starts from")
2557
+ p_gap.add_argument("--source-root", metavar="DIR", help="the source tree the spot-checks read")
2558
+ p_gap.add_argument("--drift-status", choices=DRIFT_STATUSES, default="ok",
2559
+ help="the workspace drift check's status; overridden runs the drift gate")
2560
+ p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
2561
+ p_gap.add_argument("--remediation-records", metavar="FILE",
2562
+ help="`records` output over the files targeted re-extraction scanned")
2563
+ p_gap.add_argument("--plan", action="store_true",
2564
+ help="print the answers it needs and the files targeted re-extraction scans; write no records")
2565
+ p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
2566
+ p_gap.add_argument("--evidence", metavar="FILE",
2567
+ help="append the targeted_reextraction record to this JSON-lines file when it ran")
2568
+ p_gap.add_argument("--tier", help="with --evidence: the forge tier the record names")
2569
+ p_gap.add_argument("-o", "--output", metavar="FILE", help="where to write the records (without --plan)")
2570
+ p_gap.set_defaults(func=_cmd_gap_records)
2571
+
2572
+ for p in (p_build, p_ratio):
2573
+ p.add_argument(
2574
+ "--input", default=None, help="path to JSON input (default: stdin)"
2575
+ )
2576
+ p.add_argument(
2577
+ "--category-a", metavar="FILE",
2578
+ help="skf-classify-changed-files.py classify output: the Category A slice and its moves",
2579
+ )
2580
+ p.add_argument(
2581
+ "--category-b-diff", metavar="FILE",
2582
+ help="skf-structural-diff.py output over the modified files: the Category B slice",
2583
+ )
2584
+ for p in (p_build, p_ratio, p_apply):
2585
+ p.add_argument("--category-c", metavar="FILE", help="rename-candidates output: the Category C slice")
2586
+ p.add_argument("--ccc-pairs", metavar="FILE",
2587
+ help="the CCC check's pairs, {renamed_files, renamed_exports}, added to Category C")
2588
+
410
2589
  return parser
411
2590
 
412
2591