bmad-module-skill-forge 2.2.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (437) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +49 -49
  3. package/docs/404.md +10 -8
  4. package/docs/_data/pinned.yaml +22 -13
  5. package/docs/_internal/STABILITY.md +58 -35
  6. package/docs/agents.md +39 -53
  7. package/docs/architecture.md +65 -51
  8. package/docs/bmad-synergy.md +53 -51
  9. package/docs/campaign.md +127 -75
  10. package/docs/concepts.md +34 -28
  11. package/docs/examples.md +70 -68
  12. package/docs/forge-auto.md +34 -31
  13. package/docs/getting-started.md +118 -57
  14. package/docs/how-it-works.md +25 -21
  15. package/docs/index.md +15 -15
  16. package/docs/skill-model.md +70 -59
  17. package/docs/troubleshooting.md +143 -18
  18. package/docs/verifying-a-skill.md +87 -57
  19. package/docs/why-skf.md +24 -22
  20. package/docs/workflows.md +274 -110
  21. package/package.json +18 -10
  22. package/src/forger/preferences.yaml +8 -1
  23. package/src/knowledge/agentskills-spec.md +2 -0
  24. package/src/knowledge/ccc-bridge.md +40 -24
  25. package/src/knowledge/confidence-tiers.md +9 -5
  26. package/src/knowledge/overview.md +3 -3
  27. package/src/knowledge/progressive-capability.md +8 -8
  28. package/src/knowledge/provenance-tracking.md +9 -5
  29. package/src/knowledge/qmd-registry.md +3 -3
  30. package/src/knowledge/skf-knowledge-index.csv +2 -2
  31. package/src/knowledge/skill-lifecycle.md +18 -11
  32. package/src/knowledge/split-body-strategy.md +0 -3
  33. package/src/knowledge/tool-resolution.md +4 -2
  34. package/src/knowledge/version-paths.md +101 -15
  35. package/src/module-help.csv +3 -3
  36. package/src/module.yaml +10 -0
  37. package/src/shared/data/ast-grep-recipes.yaml +608 -0
  38. package/src/shared/data/ide-context-files.json +30 -0
  39. package/src/shared/health-check.md +31 -59
  40. package/src/shared/references/description-guard-protocol.md +12 -9
  41. package/src/shared/references/feasibility-report-schema.md +19 -13
  42. package/src/shared/references/headless-gate-convention.md +55 -3
  43. package/src/shared/references/output-contract-schema.md +19 -8
  44. package/src/shared/references/pipeline-contracts.md +94 -29
  45. package/src/shared/references/tessl-review.md +77 -0
  46. package/src/shared/scripts/schemas/skf-analyze-result-envelope.v1.json +194 -0
  47. package/src/shared/scripts/schemas/skf-audit-result-envelope.v1.json +192 -0
  48. package/src/shared/scripts/schemas/skf-brief-result-envelope.v1.json +32 -4
  49. package/src/shared/scripts/schemas/skf-campaign-result-envelope.v1.json +174 -0
  50. package/src/shared/scripts/schemas/skf-create-skill-result-envelope.v1.json +284 -0
  51. package/src/shared/scripts/schemas/skf-drop-skill-result-envelope.v1.json +164 -0
  52. package/src/shared/scripts/schemas/skf-export-result-envelope.v1.json +160 -0
  53. package/src/shared/scripts/schemas/skf-quick-skill-result-envelope.v1.json +261 -0
  54. package/src/shared/scripts/schemas/skf-refine-architecture-result-envelope.v1.json +192 -0
  55. package/src/shared/scripts/schemas/skf-rename-skill-result-envelope.v1.json +179 -0
  56. package/src/shared/scripts/schemas/skf-setup-result-envelope.v1.json +27 -11
  57. package/src/shared/scripts/schemas/skf-stack-result-envelope.v1.json +176 -0
  58. package/src/shared/scripts/schemas/skf-test-result-envelope.v1.json +173 -0
  59. package/src/shared/scripts/schemas/skf-update-result-envelope.v1.json +19 -5
  60. package/src/shared/scripts/schemas/skf-verify-stack-result-envelope.v1.json +180 -0
  61. package/src/shared/scripts/schemas/skill-brief.v1.json +34 -0
  62. package/src/shared/scripts/schemas/workspace-detection.v1.json +1 -1
  63. package/src/shared/scripts/skf-atomic-write.py +233 -64
  64. package/src/shared/scripts/skf-build-change-manifest.py +2753 -28
  65. package/src/shared/scripts/skf-ccc-git-hygiene.py +532 -0
  66. package/src/shared/scripts/skf-check-unit-records.py +184 -0
  67. package/src/shared/scripts/skf-check-workspace-drift.py +538 -9
  68. package/src/shared/scripts/skf-classify-changed-files.py +772 -0
  69. package/src/shared/scripts/skf-comention-pairs.py +1333 -123
  70. package/src/shared/scripts/skf-compare-file-hashes.py +267 -105
  71. package/src/shared/scripts/skf-count-imports.py +1867 -0
  72. package/src/shared/scripts/skf-count-tokens.py +1 -1
  73. package/src/shared/scripts/skf-derive-assembly-shape.py +21 -3
  74. package/src/shared/scripts/skf-description-guard.py +99 -11
  75. package/src/shared/scripts/skf-detect-docs.py +694 -2
  76. package/src/shared/scripts/skf-detect-language.py +486 -107
  77. package/src/shared/scripts/skf-detect-registry.py +887 -0
  78. package/src/shared/scripts/skf-detect-scripts-assets.py +197 -55
  79. package/src/shared/scripts/skf-detect-tools.py +209 -27
  80. package/src/shared/scripts/skf-detect-workspaces.py +506 -30
  81. package/src/shared/scripts/skf-disqualify-candidates.py +821 -182
  82. package/src/shared/scripts/skf-emit-brief-result-envelope.py +69 -191
  83. package/src/shared/scripts/skf-emit-result-envelope.py +1604 -106
  84. package/src/shared/scripts/skf-enumerate-stack-skills.py +986 -117
  85. package/src/shared/scripts/skf-extract-public-api.py +5247 -96
  86. package/src/shared/scripts/skf-extraction-inventory.py +1280 -0
  87. package/src/shared/scripts/skf-extraction-snapshot.py +664 -0
  88. package/src/shared/scripts/skf-fetch-temporal.py +674 -0
  89. package/src/shared/scripts/skf-find-cycles.py +97 -29
  90. package/src/shared/scripts/skf-find-test-report.py +476 -0
  91. package/src/shared/scripts/skf-forge-tier-rw.py +469 -205
  92. package/src/shared/scripts/skf-github-fetch.py +365 -0
  93. package/src/shared/scripts/skf-github-probe.py +700 -0
  94. package/src/shared/scripts/skf-hash-content.py +180 -20
  95. package/src/shared/scripts/skf-language-corpora.py +1 -1
  96. package/src/shared/scripts/skf-load-provenance.py +161 -27
  97. package/src/shared/scripts/skf-manifest-ops.py +1 -1
  98. package/src/shared/scripts/skf-merge-ccc-exclusions.py +2307 -223
  99. package/src/shared/scripts/skf-merge-doc-urls.py +143 -20
  100. package/src/shared/scripts/skf-names-present.py +221 -0
  101. package/src/shared/scripts/skf-pair-intersect.py +119 -28
  102. package/src/shared/scripts/skf-parse-gaps.py +1384 -0
  103. package/src/shared/scripts/skf-preapply.py +1 -1
  104. package/src/shared/scripts/skf-preflight.py +176 -28
  105. package/src/shared/scripts/skf-promote-staged.py +430 -0
  106. package/src/shared/scripts/skf-provenance-gap-dispatch.py +64 -8
  107. package/src/shared/scripts/skf-qmd-classify-collections.py +229 -45
  108. package/src/shared/scripts/skf-quick-batch.py +774 -0
  109. package/src/shared/scripts/skf-rebuild-managed-sections.py +972 -88
  110. package/src/shared/scripts/skf-recommend-scope-type.py +323 -156
  111. package/src/shared/scripts/skf-render-metadata-stats.py +156 -12
  112. package/src/shared/scripts/skf-render-quick-metadata.py +132 -28
  113. package/src/shared/scripts/skf-render-stack-metadata.py +976 -0
  114. package/src/shared/scripts/skf-resolve-authoritative-files.py +155 -14
  115. package/src/shared/scripts/skf-resolve-package.py +523 -106
  116. package/src/shared/scripts/skf-rewrite-skill-name.py +433 -47
  117. package/src/shared/scripts/skf-run-lock.py +577 -0
  118. package/src/shared/scripts/skf-scan-manifests.py +998 -57
  119. package/src/shared/scripts/skf-scan-skill-md-structure.py +1341 -37
  120. package/src/shared/scripts/skf-severity-classify.py +712 -108
  121. package/src/shared/scripts/skf-shape-detect.py +298 -204
  122. package/src/shared/scripts/skf-shard-body.py +9 -8
  123. package/src/shared/scripts/skf-skill-inventory.py +2412 -89
  124. package/src/shared/scripts/skf-skills-module.py +472 -0
  125. package/src/shared/scripts/skf-source-tree.py +1796 -0
  126. package/src/shared/scripts/skf-structural-diff.py +787 -121
  127. package/src/shared/scripts/skf-tessl-review.py +706 -0
  128. package/src/shared/scripts/skf-update-active-symlink.py +120 -67
  129. package/src/shared/scripts/skf-update-run-state.py +624 -0
  130. package/src/shared/scripts/skf-validate-brief-inputs.py +92 -29
  131. package/src/shared/scripts/skf-validate-brief-schema.py +23 -2
  132. package/src/shared/scripts/skf-validate-feasibility-report.py +577 -66
  133. package/src/shared/scripts/skf-validate-frontmatter.py +103 -26
  134. package/src/shared/scripts/skf-validate-output.py +712 -64
  135. package/src/shared/scripts/skf-validate-pins.py +1 -1
  136. package/src/shared/scripts/skf-verify-no-trace.py +116 -10
  137. package/src/shared/scripts/skf-verify-provenance-completeness.py +3945 -152
  138. package/src/shared/scripts/skf-write-skill-brief.py +406 -25
  139. package/src/shared/tool-requirements.yaml +201 -0
  140. package/src/skf-analyze-source/SKILL.md +41 -36
  141. package/src/skf-analyze-source/assets/skill-brief-schema.md +7 -8
  142. package/src/skf-analyze-source/customize.toml +36 -24
  143. package/src/skf-analyze-source/references/auto-docs-only.md +58 -20
  144. package/src/skf-analyze-source/references/continue.md +2 -13
  145. package/src/skf-analyze-source/references/discover-additional-source.md +147 -0
  146. package/src/skf-analyze-source/references/generate-briefs.md +121 -55
  147. package/src/skf-analyze-source/references/headless-contract.md +42 -15
  148. package/src/skf-analyze-source/references/identify-units.md +96 -110
  149. package/src/skf-analyze-source/references/init.md +73 -71
  150. package/src/skf-analyze-source/references/map-and-detect.md +123 -105
  151. package/src/skf-analyze-source/references/map-unit-exports.md +68 -0
  152. package/src/skf-analyze-source/references/recommend.md +51 -70
  153. package/src/skf-analyze-source/references/scan-project.md +60 -84
  154. package/src/skf-analyze-source/references/scan-root.md +39 -0
  155. package/src/skf-analyze-source/references/step-auto-scope-coexistence.md +47 -0
  156. package/src/skf-analyze-source/references/step-auto-scope-corpora.md +53 -0
  157. package/src/skf-analyze-source/references/step-auto-scope-split.md +91 -0
  158. package/src/skf-analyze-source/references/step-auto-scope.md +208 -293
  159. package/src/skf-analyze-source/references/step-shape-detect.md +14 -12
  160. package/src/skf-analyze-source/references/unit-detection-heuristics.md +37 -42
  161. package/src/skf-audit-skill/SKILL.md +43 -51
  162. package/src/skf-audit-skill/assets/drift-report-template.md +28 -3
  163. package/src/skf-audit-skill/customize.toml +41 -20
  164. package/src/skf-audit-skill/references/constituent-freshness.md +91 -0
  165. package/src/skf-audit-skill/references/doc-drift.md +141 -0
  166. package/src/skf-audit-skill/references/headless-contract.md +29 -0
  167. package/src/skf-audit-skill/references/health-check.md +2 -0
  168. package/src/skf-audit-skill/references/init.md +149 -210
  169. package/src/skf-audit-skill/references/re-index.md +103 -110
  170. package/src/skf-audit-skill/references/report.md +96 -33
  171. package/src/skf-audit-skill/references/semantic-diff.md +34 -25
  172. package/src/skf-audit-skill/references/severity-classify.md +79 -65
  173. package/src/skf-audit-skill/references/severity-rules.md +15 -6
  174. package/src/skf-audit-skill/references/structural-diff.md +80 -90
  175. package/src/skf-audit-skill/references/upstream-checkout.md +77 -0
  176. package/src/skf-audit-skill/scripts/render-drift-tables.py +712 -0
  177. package/src/skf-brief-skill/SKILL.md +37 -26
  178. package/src/skf-brief-skill/assets/description-voice-examples.md +1 -1
  179. package/src/skf-brief-skill/assets/scope-templates.md +18 -25
  180. package/src/skf-brief-skill/assets/skill-brief-schema.md +30 -23
  181. package/src/skf-brief-skill/customize.toml +48 -20
  182. package/src/skf-brief-skill/references/analyze-target.md +144 -127
  183. package/src/skf-brief-skill/references/confirm-brief.md +11 -9
  184. package/src/skf-brief-skill/references/draft-checkpoint.md +36 -17
  185. package/src/skf-brief-skill/references/gather-intent-ratify.md +68 -0
  186. package/src/skf-brief-skill/references/gather-intent.md +87 -182
  187. package/src/skf-brief-skill/references/headless-args.md +84 -16
  188. package/src/skf-brief-skill/references/health-check.md +1 -1
  189. package/src/skf-brief-skill/references/invocation-contract.md +18 -13
  190. package/src/skf-brief-skill/references/portfolio-similarity-check.md +1 -1
  191. package/src/skf-brief-skill/references/qmd-collection-registration.md +7 -10
  192. package/src/skf-brief-skill/references/scope-definition.md +123 -47
  193. package/src/skf-brief-skill/references/step-auto-brief.md +26 -100
  194. package/src/skf-brief-skill/references/step-auto-validate.md +20 -121
  195. package/src/skf-brief-skill/references/version-resolution.md +7 -10
  196. package/src/skf-brief-skill/references/write-brief.md +66 -57
  197. package/src/skf-campaign/SKILL.md +31 -34
  198. package/src/skf-campaign/assets/campaign-state-schema.json +25 -4
  199. package/src/skf-campaign/customize.toml +47 -16
  200. package/src/skf-campaign/references/campaign-contracts.md +39 -29
  201. package/src/skf-campaign/references/campaign-directive-spec.md +4 -2
  202. package/src/skf-campaign/references/health-check.md +7 -16
  203. package/src/skf-campaign/references/step-01-setup.md +46 -84
  204. package/src/skf-campaign/references/step-02-strategy.md +31 -30
  205. package/src/skf-campaign/references/step-03-pins.md +26 -18
  206. package/src/skf-campaign/references/step-04-provenance.md +24 -23
  207. package/src/skf-campaign/references/step-05-skill-loop.md +67 -48
  208. package/src/skf-campaign/references/step-06-batch.md +41 -34
  209. package/src/skf-campaign/references/step-07-capstone.md +40 -25
  210. package/src/skf-campaign/references/step-08-verify.md +28 -29
  211. package/src/skf-campaign/references/step-09-refine.md +29 -30
  212. package/src/skf-campaign/references/step-10-export.md +48 -36
  213. package/src/skf-campaign/references/step-11-maintenance.md +31 -34
  214. package/src/skf-campaign/references/step-resume.md +37 -68
  215. package/src/skf-campaign/scripts/campaign-deps.py +57 -18
  216. package/src/skf-campaign/scripts/campaign-parse-manifest.py +361 -54
  217. package/src/skf-campaign/scripts/campaign-provenance.py +96 -53
  218. package/src/skf-campaign/scripts/campaign-quality-gate.py +592 -0
  219. package/src/skf-campaign/scripts/campaign-render-batch.py +257 -49
  220. package/src/skf-campaign/scripts/campaign-render-kickoff.py +343 -50
  221. package/src/skf-campaign/scripts/campaign-report.py +204 -37
  222. package/src/skf-campaign/scripts/campaign-state.py +1128 -0
  223. package/src/skf-campaign/scripts/campaign-status.py +10 -74
  224. package/src/skf-campaign/scripts/campaign-validate-pins.py +30 -3
  225. package/src/skf-campaign/scripts/campaign-validate-state.py +34 -4
  226. package/src/skf-campaign/templates/campaign-brief-template.yaml +4 -5
  227. package/src/skf-campaign/templates/campaign-report-template.md +5 -1
  228. package/src/skf-campaign/templates/kickoff-template.md +0 -14
  229. package/src/skf-create-skill/SKILL.md +27 -20
  230. package/src/skf-create-skill/assets/compile-assembly-rules.md +11 -8
  231. package/src/skf-create-skill/assets/skill-sections.md +21 -11
  232. package/src/skf-create-skill/customize.toml +20 -8
  233. package/src/skf-create-skill/references/authoritative-files-protocol.md +36 -58
  234. package/src/skf-create-skill/references/batch-mode.md +81 -0
  235. package/src/skf-create-skill/references/ccc-index-check.md +27 -0
  236. package/src/skf-create-skill/references/compile.md +119 -70
  237. package/src/skf-create-skill/references/component-extraction.md +113 -91
  238. package/src/skf-create-skill/references/enrich.md +41 -23
  239. package/src/skf-create-skill/references/entry-points-by-hand.md +37 -0
  240. package/src/skf-create-skill/references/extract.md +204 -162
  241. package/src/skf-create-skill/references/extraction-patterns-by-hand.md +141 -0
  242. package/src/skf-create-skill/references/extraction-patterns-tracing.md +11 -11
  243. package/src/skf-create-skill/references/extraction-patterns.md +588 -201
  244. package/src/skf-create-skill/references/generate-artifacts.md +145 -93
  245. package/src/skf-create-skill/references/health-check.md +1 -1
  246. package/src/skf-create-skill/references/load-brief.md +53 -36
  247. package/src/skf-create-skill/references/relabel-rule.md +12 -0
  248. package/src/skf-create-skill/references/report.md +37 -57
  249. package/src/skf-create-skill/references/source-resolution-protocols.md +58 -169
  250. package/src/skf-create-skill/references/step-auto-shard.md +20 -93
  251. package/src/skf-create-skill/references/step-doc-rot.md +44 -60
  252. package/src/skf-create-skill/references/step-doc-sources.md +28 -19
  253. package/src/skf-create-skill/references/sub/ccc-discover.md +18 -7
  254. package/src/skf-create-skill/references/sub/fetch-docs.md +106 -59
  255. package/src/skf-create-skill/references/sub/fetch-temporal.md +53 -166
  256. package/src/skf-create-skill/references/tier-degradation-rules.md +18 -9
  257. package/src/skf-create-skill/references/validate.md +173 -99
  258. package/src/skf-create-skill/scripts/scan-doc-rot.py +232 -99
  259. package/src/skf-create-stack-skill/SKILL.md +29 -56
  260. package/src/skf-create-stack-skill/assets/metadata-contract.md +51 -0
  261. package/src/skf-create-stack-skill/assets/provenance-map-schema.md +15 -15
  262. package/src/skf-create-stack-skill/assets/stack-skill-template.md +23 -57
  263. package/src/skf-create-stack-skill/customize.toml +23 -12
  264. package/src/skf-create-stack-skill/references/compile-stack.md +57 -66
  265. package/src/skf-create-stack-skill/references/compose-mode-rules.md +24 -44
  266. package/src/skf-create-stack-skill/references/detect-integrations.md +92 -95
  267. package/src/skf-create-stack-skill/references/detect-manifests.md +46 -71
  268. package/src/skf-create-stack-skill/references/generate-output.md +138 -128
  269. package/src/skf-create-stack-skill/references/init.md +90 -32
  270. package/src/skf-create-stack-skill/references/integration-patterns.md +1 -17
  271. package/src/skf-create-stack-skill/references/invocation-contract.md +34 -0
  272. package/src/skf-create-stack-skill/references/manifest-patterns.md +32 -20
  273. package/src/skf-create-stack-skill/references/parallel-extract.md +112 -38
  274. package/src/skf-create-stack-skill/references/rank-and-confirm.md +57 -75
  275. package/src/skf-create-stack-skill/references/report.md +39 -33
  276. package/src/skf-create-stack-skill/references/validate.md +20 -39
  277. package/src/skf-drop-skill/SKILL.md +28 -51
  278. package/src/skf-drop-skill/customize.toml +44 -30
  279. package/src/skf-drop-skill/references/execute.md +83 -146
  280. package/src/skf-drop-skill/references/health-check.md +2 -0
  281. package/src/skf-drop-skill/references/invocation-contract.md +64 -0
  282. package/src/skf-drop-skill/references/report.md +46 -30
  283. package/src/skf-drop-skill/references/select.md +122 -124
  284. package/src/skf-drop-skill/scripts/dir-sizes.py +54 -26
  285. package/src/skf-drop-skill/scripts/drop-roster.py +322 -0
  286. package/src/skf-export-skill/SKILL.md +40 -62
  287. package/src/skf-export-skill/assets/managed-section-format.md +14 -50
  288. package/src/skf-export-skill/assets/snippet-format.md +3 -1
  289. package/src/skf-export-skill/customize.toml +42 -13
  290. package/src/skf-export-skill/references/generate-snippet.md +54 -46
  291. package/src/skf-export-skill/references/health-check.md +2 -0
  292. package/src/skf-export-skill/references/invocation-contract.md +15 -0
  293. package/src/skf-export-skill/references/load-skill.md +88 -62
  294. package/src/skf-export-skill/references/manifest-rebuild.md +9 -10
  295. package/src/skf-export-skill/references/multi-skill-mode.md +4 -4
  296. package/src/skf-export-skill/references/orphan-context-detection.md +13 -5
  297. package/src/skf-export-skill/references/orphan-row-detection.md +31 -22
  298. package/src/skf-export-skill/references/package.md +15 -73
  299. package/src/skf-export-skill/references/preflight-snippet-root-probe.md +51 -70
  300. package/src/skf-export-skill/references/result-envelope.md +31 -16
  301. package/src/skf-export-skill/references/summary.md +41 -17
  302. package/src/skf-export-skill/references/token-report.md +4 -12
  303. package/src/skf-export-skill/references/update-context.md +118 -237
  304. package/src/skf-forger/SKILL.md +61 -37
  305. package/src/skf-forger/bmad-skill-manifest.yaml +4 -1
  306. package/src/skf-forger/customize.toml +43 -0
  307. package/src/skf-forger/references/pipeline-mode.md +73 -18
  308. package/src/skf-forger/scripts/forge-status.py +299 -0
  309. package/src/skf-forger/scripts/parse-pipeline.py +322 -54
  310. package/src/skf-forger/scripts/pipeline-gate.py +428 -0
  311. package/src/skf-forger/scripts/pipeline-journal.py +1052 -0
  312. package/src/skf-quick-skill/SKILL.md +54 -53
  313. package/src/skf-quick-skill/assets/skill-template.md +4 -2
  314. package/src/skf-quick-skill/customize.toml +51 -20
  315. package/src/skf-quick-skill/references/batch-mode.md +74 -21
  316. package/src/skf-quick-skill/references/compile.md +30 -31
  317. package/src/skf-quick-skill/references/ecosystem-check.md +7 -57
  318. package/src/skf-quick-skill/references/finalize.md +69 -18
  319. package/src/skf-quick-skill/references/halt-contract.md +41 -29
  320. package/src/skf-quick-skill/references/health-check.md +5 -3
  321. package/src/skf-quick-skill/references/quick-extract.md +114 -45
  322. package/src/skf-quick-skill/references/registry-resolution.md +9 -64
  323. package/src/skf-quick-skill/references/resolve-target.md +116 -66
  324. package/src/skf-quick-skill/references/write-and-validate.md +79 -40
  325. package/src/skf-refine-architecture/SKILL.md +43 -53
  326. package/src/skf-refine-architecture/customize.toml +49 -20
  327. package/src/skf-refine-architecture/references/compile.md +97 -54
  328. package/src/skf-refine-architecture/references/exit-codes.md +30 -0
  329. package/src/skf-refine-architecture/references/finding-storage.md +21 -0
  330. package/src/skf-refine-architecture/references/gap-analysis.md +101 -38
  331. package/src/skf-refine-architecture/references/improvements.md +14 -20
  332. package/src/skf-refine-architecture/references/init.md +98 -57
  333. package/src/skf-refine-architecture/references/issue-detection.md +32 -39
  334. package/src/skf-refine-architecture/references/refinement-rules.md +37 -48
  335. package/src/skf-refine-architecture/references/report.md +65 -54
  336. package/src/skf-refine-architecture/scripts/skf-check-preservation.py +1619 -0
  337. package/src/skf-rename-skill/SKILL.md +19 -54
  338. package/src/skf-rename-skill/customize.toml +32 -26
  339. package/src/skf-rename-skill/references/execute.md +230 -184
  340. package/src/skf-rename-skill/references/exit-codes.md +7 -7
  341. package/src/skf-rename-skill/references/health-check.md +1 -7
  342. package/src/skf-rename-skill/references/invocation-contract.md +25 -0
  343. package/src/skf-rename-skill/references/report.md +63 -52
  344. package/src/skf-rename-skill/references/select.md +197 -115
  345. package/src/skf-rename-skill/scripts/skf-validate-rename-name.py +161 -21
  346. package/src/skf-setup/SKILL.md +26 -26
  347. package/src/skf-setup/customize.toml +14 -10
  348. package/src/skf-setup/references/auto-index.md +50 -63
  349. package/src/skf-setup/references/ccc-index.md +29 -90
  350. package/src/skf-setup/references/detect-and-tier.md +58 -67
  351. package/src/skf-setup/references/health-check.md +2 -0
  352. package/src/skf-setup/references/invocation-contract.md +14 -0
  353. package/src/skf-setup/references/report.md +43 -166
  354. package/src/skf-setup/references/tier-rules.md +5 -5
  355. package/src/skf-setup/references/write-config.md +34 -57
  356. package/src/skf-test-skill/SKILL.md +17 -57
  357. package/src/skf-test-skill/assets/output-section-formats.md +33 -34
  358. package/src/skf-test-skill/customize.toml +27 -13
  359. package/src/skf-test-skill/references/coherence-check.md +134 -62
  360. package/src/skf-test-skill/references/coverage-check-tiers.md +35 -0
  361. package/src/skf-test-skill/references/coverage-check.md +209 -264
  362. package/src/skf-test-skill/references/detect-mode.md +5 -5
  363. package/src/skf-test-skill/references/external-validators.md +94 -82
  364. package/src/skf-test-skill/references/health-check.md +6 -7
  365. package/src/skf-test-skill/references/init.md +164 -91
  366. package/src/skf-test-skill/references/invocation-contract.md +66 -0
  367. package/src/skf-test-skill/references/migration-section-rules.md +5 -2
  368. package/src/skf-test-skill/references/report.md +164 -142
  369. package/src/skf-test-skill/references/score.md +80 -107
  370. package/src/skf-test-skill/references/scoring-rules.md +42 -155
  371. package/src/skf-test-skill/references/source-access-protocol.md +18 -94
  372. package/src/skf-test-skill/references/step-hard-gate.md +51 -33
  373. package/src/skf-test-skill/scripts/aggregate-coherence.py +218 -30
  374. package/src/skf-test-skill/scripts/build-result-context.py +380 -0
  375. package/src/skf-test-skill/scripts/check-metadata-coherence.py +37 -4
  376. package/src/skf-test-skill/scripts/combine-external-scores.py +23 -6
  377. package/src/skf-test-skill/scripts/compute-score.py +223 -45
  378. package/src/skf-test-skill/scripts/gap-ledger.py +1666 -0
  379. package/src/skf-test-skill/scripts/hard-gate.py +203 -0
  380. package/src/skf-test-skill/scripts/load-coverage-inputs.py +1198 -0
  381. package/src/skf-test-skill/scripts/locate-export-segments.py +835 -0
  382. package/src/skf-test-skill/scripts/reconcile-coverage.py +547 -79
  383. package/src/skf-test-skill/scripts/score-signatures.py +479 -0
  384. package/src/skf-test-skill/scripts/stage-helper-payload.py +148 -0
  385. package/src/skf-test-skill/scripts/validate-inventory.py +311 -16
  386. package/src/skf-test-skill/scripts/verify-declared-numerator.py +133 -56
  387. package/src/skf-test-skill/templates/test-report-template.md +21 -15
  388. package/src/skf-update-skill/SKILL.md +38 -25
  389. package/src/skf-update-skill/customize.toml +27 -12
  390. package/src/skf-update-skill/references/detect-changes.md +253 -223
  391. package/src/skf-update-skill/references/gap-driven.md +336 -0
  392. package/src/skf-update-skill/references/health-check.md +31 -5
  393. package/src/skf-update-skill/references/init.md +254 -84
  394. package/src/skf-update-skill/references/invocation-contract.md +12 -0
  395. package/src/skf-update-skill/references/manual-section-rules.md +6 -23
  396. package/src/skf-update-skill/references/merge.md +182 -49
  397. package/src/skf-update-skill/references/re-extract.md +53 -262
  398. package/src/skf-update-skill/references/report.md +102 -35
  399. package/src/skf-update-skill/references/write.md +357 -155
  400. package/src/skf-update-skill/scripts/skf-new-file-diff.py +183 -46
  401. package/src/skf-verify-stack/SKILL.md +23 -42
  402. package/src/skf-verify-stack/assets/feasibility-report-template.md +11 -5
  403. package/src/skf-verify-stack/customize.toml +55 -22
  404. package/src/skf-verify-stack/references/coverage-patterns.md +17 -8
  405. package/src/skf-verify-stack/references/coverage.md +67 -64
  406. package/src/skf-verify-stack/references/exit-codes.md +21 -8
  407. package/src/skf-verify-stack/references/init.md +107 -62
  408. package/src/skf-verify-stack/references/integration-verification-rules.md +12 -13
  409. package/src/skf-verify-stack/references/integrations.md +106 -65
  410. package/src/skf-verify-stack/references/report.md +59 -42
  411. package/src/skf-verify-stack/references/requirements.md +32 -9
  412. package/src/skf-verify-stack/references/synthesize.md +53 -39
  413. package/src/skf-verify-stack/scripts/skf-coverage-tally.py +398 -52
  414. package/src/skf-verify-stack/scripts/skf-previous-report.py +330 -0
  415. package/src/skf-verify-stack/scripts/skf-report-delta.py +453 -49
  416. package/src/skf-verify-stack/scripts/skf-verdict-rollup.py +240 -27
  417. package/tools/cli/commands/install.js +11 -4
  418. package/tools/cli/commands/status.js +15 -3
  419. package/tools/cli/commands/update.js +11 -4
  420. package/tools/cli/lib/tool-check.js +320 -0
  421. package/tools/cli/lib/version-check.js +22 -9
  422. package/tools/cli/skf-cli.js +4 -1
  423. package/.gitattributes +0 -16
  424. package/.nvmrc +0 -1
  425. package/docs/_internal/RELEASING.md +0 -679
  426. package/release-audits/v1.0.0-launch-audit.md +0 -728
  427. package/src/skf-audit-skill/references/step-doc-drift.md +0 -177
  428. package/src/skf-brief-skill/references/headless-source-authority-detection.md +0 -26
  429. package/src/skf-campaign/manifest.yaml +0 -15
  430. package/src/skf-create-skill/assets/tessl-dismissal-rules.md +0 -106
  431. package/src/skf-create-skill/references/ecosystem-check.md +0 -74
  432. package/src/skf-drop-skill/references/headless-contract.md +0 -18
  433. package/src/skf-update-skill/references/merge-conflict-rules.md +0 -15
  434. package/src/skf-update-skill/references/remote-source-resolution.md +0 -98
  435. package/src/skf-update-skill/references/validate.md +0 -124
  436. package/tools/validate-docs-drift.js +0 -233
  437. package/tools/validate-docs-links.js +0 -665
@@ -1,5 +1,5 @@
1
1
  # /// script
2
- # requires-python = ">=3.10"
2
+ # requires-python = ">=3.11"
3
3
  # dependencies = []
4
4
  # ///
5
5
  """SKF Build Change Manifest — aggregate category A/B/C/D detection results.
@@ -26,15 +26,131 @@ schema, plus the provenance map for the denominator.
26
26
 
27
27
  Subcommands:
28
28
 
29
- build [--input <file>] [--no-changes-on-empty]
29
+ build [--input <file>] [--category-a <file>] [--category-b-diff <file>]
30
+ [--category-c <file>] [--ccc-pairs <file>]
31
+ [--file-compare <file> --provenance-map <file>] [--new-files <file>]
30
32
  Reads category JSON from stdin or `--input <file>`, emits the
31
- unified manifest envelope.
33
+ unified manifest envelope (see "Manifest" below).
34
+
35
+ build --doc-hashes <file>
36
+ A docs-only skill's manifest, from skf-detect-docs.py
37
+ compare-hashes' output and nothing else (see "Docs-only" below).
32
38
 
33
39
  deletion-ratio --provenance-map <file> [--input <file>]
40
+ [--category-a <file>] [--category-b-diff <file>]
41
+ [--category-c <file>] [--ccc-pairs <file>]
34
42
  Same category JSON input. Reads provenance entries[] from
35
43
  <file>, computes the §2.2 trigger envelope. Auto-skips when
36
44
  degraded_mode or update_mode==gap-driven is set in the input.
37
45
 
46
+ rename-candidates --category-a <file> --provenance-map <file>
47
+ --tier Quick|Forge|Forge+|Deep [--category-b-diff <file>]
48
+ [--extraction <file>] [--export-details <file>]
49
+ [--source-root <dir> [--sizes-commit <commit>]] [-o <file>]
50
+ Category C by fixed rules (see "Rename candidates" below): pairs
51
+ Category A's deleted files with its added ones, and the diff's
52
+ removed exports with its added ones and the added files' exports.
53
+ Prints {"category_c": {...}, "evidence": {...}, "unpaired": {...}}.
54
+
55
+ apply --update-type incremental|gap-driven|full --skill-name <name>
56
+ --generation-date <iso> --confidence-tier <tier>
57
+ --manual-sections-preserved <n> -o <file> [inputs...]
58
+ [--not-public-out <file>] [--metadata-in <file> --lists-out <file>]
59
+ The provenance map an update writes (see "Apply" below): the old
60
+ map with this run's changes, written to -o through a temporary file
61
+ and a rename, the summary's not_public list to --not-public-out and
62
+ metadata.json's exports[], scripts[] and assets[] to --lists-out
63
+ when given. Prints a summary with the warnings for a person.
64
+
65
+ metadata --metadata <metadata.json> --lists <file> --stats <file>
66
+ --version <version> --generation-date <iso>
67
+ [--source-commit <sha>] [--source-ref <ref>]
68
+ The skill's metadata.json an update leaves (see "Metadata" below),
69
+ written over --metadata through a temporary file and a rename.
70
+ Prints {"status": "written", "metadata": <path>, "version",
71
+ "exports": N, "scripts": N, "assets": N}.
72
+
73
+ records [--extraction <file>] [--export-details <file>]
74
+ [--files-from <file>] [--patches <dir>] -o <file>
75
+ Step 3's re-extraction records (see "Records" below), written to -o
76
+ through a temporary file and a rename. Prints {"status": "written",
77
+ "output", "files_extracted", "exports_extracted",
78
+ "confidence_breakdown": {"T1", "T1-low", "T2"}, "marked_not_public"
79
+ (the exports marked public: false), "warnings"}.
80
+
81
+ gap-records --manifest <file> --provenance-map <file> [--source-root <dir>]
82
+ [--drift-status ok|skipped|overridden] [--judgments <file>]
83
+ [--remediation-records <file>] [--evidence <file> --tier <tier>]
84
+ (--plan [--files-out <file>] | -o <file>)
85
+ A gap-driven repair's spot-checks, routing and verification records
86
+ (see "Gap records" below): --plan prints what to answer and which
87
+ files targeted re-extraction scans; without it, the records are
88
+ written to -o through a temporary file and a rename.
89
+
90
+ baseline-gaps --provenance-map <file> --extraction <file> --files <file>
91
+ --source-root <dir> -o <file>
92
+ Category B's baseline gaps (see "Baseline gaps" below): each map
93
+ entry of the files --files lists that the runner did not report and
94
+ its file still declares, written to -o through a temporary file and
95
+ a rename. Prints {"status": "written", "output": <-o>, "gaps": N,
96
+ "unchecked": N}.
97
+
98
+ Helper files in place of typed slices (update-skill detect-changes §2.1):
99
+
100
+ --category-a <file> the output of skf-classify-changed-files.py
101
+ classify: its category_a is the Category A
102
+ slice, and each of its moved_files (a
103
+ same-content move) is a category_c.renamed_files
104
+ item, ahead of the input's own
105
+ --category-b-diff <file> the output of skf-structural-diff.py over the
106
+ modified files: the Category B slice, mapped
107
+ removed[] -> deleted_exports {name, file, old_line}
108
+ added[] -> new_exports {name, file, line}
109
+ changed[] -> modified_exports {name, file,
110
+ old_line, new_line} for an export
111
+ with a field other than `line`, or
112
+ one signature_unverified[] lists
113
+ (its lines null when no field
114
+ changed); moved_exports for an
115
+ export whose only field is `line`
116
+ moved[] -> moved_exports {name, file:
117
+ current_file, old_file:
118
+ previous_file (only when it
119
+ differs), old_line:
120
+ previous_line, new_line: line},
121
+ unless it is a modified export,
122
+ which then takes old_file and
123
+ old_line: previous_line
124
+ label_changes[] is not drift, and a name
125
+ ambiguous_names[] lists stays removed and added
126
+ --category-c <file> the output of rename-candidates: its category_c
127
+ is the Category C slice
128
+ --ccc-pairs <file> the pairs the update's CCC check added to what
129
+ rename-candidates left unpaired,
130
+ {"renamed_files": [...], "renamed_exports":
131
+ [...]}, appended to the Category C slice
132
+ --file-compare <file> build only: the output of skf-hash-content.py
133
+ compare, with --provenance-map (the map it
134
+ compared): each MODIFIED_FILE and DELETED_FILE
135
+ row goes to the {type}s_modified or
136
+ {type}s_deleted list of category_d, by the
137
+ file_type of the file_entries[] row of its
138
+ source_file (script, asset or doc; a row with
139
+ none takes the type its file_name's first
140
+ folder names, scripts/, assets/ or docs/, else
141
+ asset)
142
+ --new-files <file> build only: the output of skf-new-file-diff.py:
143
+ each new_files[] item goes to scripts_added or
144
+ assets_added by its kind
145
+ Either Category D file makes the two files the category_d slice. A given
146
+ file replaces the input's own slice. Either way, every category_c
147
+ rename then takes its pair out of the lists it was found in: a renamed
148
+ file's old_path out of category_a.deleted and new_path out of
149
+ category_a.added; a renamed export's old_name out of
150
+ category_b.deleted_exports (the one in its `old_file` when the item gives
151
+ one, else preferably the one in its `file`) and its new_name out of
152
+ category_b.new_exports (preferably the one in its `file`).
153
+
38
154
  Input JSON shape (object on stdin or in --input file):
39
155
 
40
156
  {
@@ -44,10 +160,10 @@ Input JSON shape (object on stdin or in --input file):
44
160
  "deleted": ["path3", ...]
45
161
  },
46
162
  "category_b": {
47
- "modified_exports": [{name, file, old_line, new_line}, ...],
163
+ "modified_exports": [{name, file, old_line, new_line, old_file?}, ...],
48
164
  "new_exports": [{name, file, line}, ...],
49
165
  "deleted_exports": [{name, file, old_line}, ...],
50
- "moved_exports": [{name, file, old_line, new_line}, ...]
166
+ "moved_exports": [{name, file, old_line, new_line, old_file?}, ...]
51
167
  },
52
168
  "category_c": {
53
169
  "renamed_files": [{old_path, new_path}, ...],
@@ -55,27 +171,417 @@ Input JSON shape (object on stdin or in --input file):
55
171
  },
56
172
  "category_d": {
57
173
  "scripts_modified": ["path", ...], "scripts_added": [...], "scripts_deleted": [...],
58
- "assets_modified": [...], "assets_added": [...], "assets_deleted": [...]
174
+ "assets_modified": [...], "assets_added": [...], "assets_deleted": [...],
175
+ "docs_modified": [...], "docs_deleted": [...]
59
176
  },
60
177
  "degraded_mode": false,
61
178
  "update_mode": "normal" | "gap-driven" // optional; influences deletion-ratio only
62
179
  }
63
180
 
181
+ Manifest (build's output):
182
+
183
+ {"no_changes": <every count is 0>, "degraded_mode": <bool>,
184
+ "counts": {files_changed, files_added, files_deleted, files_moved,
185
+ exports_modified, exports_new, exports_deleted,
186
+ exports_renamed, exports_moved, and one count per
187
+ category_d list},
188
+ "total_export_changes": N, "per_file": [...],
189
+ "category_d": {the eight category_d lists, each path once}}
190
+
191
+ Each per_file item's exports_affected holds {name, change_type,
192
+ old_line, new_line}, and old_file for an export that moved across files
193
+ (a MOVED_EXPORT, or a MODIFIED_EXPORT that also changed).
194
+
195
+ A tracked document (a doc row of file_entries[]) has no other detector:
196
+ update-skill's Category A never lists it, so a document-only change still
197
+ counts, and the update records its new hash.
198
+
64
199
  Every category and sub-key is optional — missing keys default to
65
200
  empty lists. This lets gap-driven runs (which produce no Category D
66
201
  results) and degraded runs (which skip Category B) feed the same
67
202
  script without sentinel values.
68
203
 
204
+ Docs-only (update-skill detect-changes §1): build --doc-hashes reads the
205
+ compare-hashes output and prints {"mode": "docs-only", "no_changes":
206
+ <changed[] is empty>, "changed_urls": [each changed[].url],
207
+ "fetch_failed": [each fetch_failed[].url], "counts": {"docs_changed",
208
+ "docs_fetch_failed"}}. It takes no other input.
209
+
210
+ Rename candidates (update-skill detect-changes Category C):
211
+
212
+ Content similarity above 80% (fixed, not configurable) is a rename. A
213
+ file moved with its content unchanged is Category A's moved_files and is
214
+ left out here. For the rest:
215
+ files, Quick tier the added file's size is within 20% of the deleted
216
+ file's (the deleted size is `git cat-file -s` of
217
+ <commit>:<path> under --source-root; with no
218
+ --sizes-commit, a local source whose file is gone,
219
+ the size test is skipped) and the export names
220
+ overlap above 70% (shared names over all names of
221
+ the two files)
222
+ files, Forge+ above 80% of the exports match: an export of the
223
+ added file with the deleted file's export name and
224
+ signature (export_type, params and return_type),
225
+ over the larger file's export count
226
+ exports, Quick the names are above 80% alike (difflib ratio) and
227
+ their export types agree
228
+ exports, Forge+ the signatures are equal (export_type, params and
229
+ return_type, a parameter list or a return type
230
+ given) and the names differ
231
+ A deleted file's exports come from the provenance map, an added file's
232
+ from --extraction and --export-details. A pair is kept only when it is
233
+ each side's best match: the highest similarity wins, and a tie between
234
+ two candidates pairs neither. category_c.renamed_files holds {old_path,
235
+ new_path}; renamed_exports holds {old_name, new_name, file, old_file?},
236
+ with old_file only when it differs from file. evidence lists each pair
237
+ with its rule and similarity, and unpaired the deleted and added files
238
+ and the removed and added exports left over, for the judgment step.
239
+
240
+ Apply (update-skill write.md §3):
241
+
242
+ Reads the provenance map the update started from (--provenance-map;
243
+ without it, a degraded run's new map) and prints nothing but a summary:
244
+ the new map goes to -o. Every entry the run leaves alone keeps its exact
245
+ value; every entry it adds or rewrites carries signature_source, set
246
+ from the tool that produced the signature: ast-grep gives T1 (with
247
+ confidence T1 and the recipe's ast_node_type), a signature read by eye
248
+ (source-read) gives T1-low (confidence T1-low, ast_node_type null).
249
+ A rewritten entry keeps any key the update does not set (notes,
250
+ deprecated). New entries take source_library from the map's entries,
251
+ else --skill-name.
252
+
253
+ incremental (normal mode) reads --manifest (build's output), the renamed
254
+ exports of --category-c and --ccc-pairs (or of --input, the category
255
+ JSON's category_c) and the fresh
256
+ records: --reextract-records ({"mode": "normal", "files": [{file_path,
257
+ exports: [...]}]}, the records `records` wrote), --extraction (the
258
+ recipe runner's exports) and --export-details. A fresh record of an
259
+ export takes its line, export_type and ast labels from the runner, then
260
+ the details, then step 3's record; its params and return_type from the
261
+ details, then step 3's record (the runner's in the map's typed form).
262
+ Per file of the manifest:
263
+ MODIFIED NEW_EXPORT adds an entry, MODIFIED_EXPORT rewrites the
264
+ export's fields (file included) from its fresh record,
265
+ MOVED_EXPORT moves the entry to the file and the fresh
266
+ record's line, else new_line, DELETED_EXPORT removes it;
267
+ a change with an old_file finds its entry in that file
268
+ ADDED every fresh record of the file becomes an entry
269
+ DELETED every entry of the file is removed
270
+ MOVED the old path's entries are replaced by the new path's
271
+ fresh records, each keeping the keys of the entry of its
272
+ name it replaces
273
+ and each renamed export takes its new name and fresh fields. A
274
+ MOVED_EXPORT whose fresh record is the runner's (ast-grep) also takes
275
+ its confidence, extraction_method and ast_node_type (the entry's when
276
+ the record's is null), and its signature_source when the record holds
277
+ each of params and return_type the entry holds (else the entry's),
278
+ never its export_type, params or return_type; one read by eye moves
279
+ the line and file only.
280
+
281
+ Public-api skills (incremental only): a NEW_EXPORT or MODIFIED_EXPORT
282
+ the map does not hold, an ADDED file's record and a MOVED file's record
283
+ whose name no old entry carries become entries only when on the public
284
+ surface: the record's `public` mark (see "Records"), else, for a record
285
+ --reextract-records does not mark, the same rule over --extraction (its
286
+ scope.type). A name whose entry this run removes (a DELETED_EXPORT, a
287
+ DELETED file's or a MOVED file's old path's entry) is added whatever
288
+ its mark. Each one left out is listed in the summary's not_public,
289
+ {name, file}, with one warning for them all; an entry the map holds is
290
+ updated whatever its mark, and no entry is removed for it. A public-api
291
+ --extraction the rule cannot read (no entry_point_diff, an incomplete
292
+ run or one with errors, an unresolved entry point) adds every name, with
293
+ one warning. A docs-only
294
+ run's records ({"mode": "docs-only", "changed_urls": [...], "exports":
295
+ [{name, type, params, return_type, url}]}) replace the entries of each
296
+ changed URL (source_file is the URL; confidence and signature_source
297
+ T3).
298
+
299
+ gap-driven reads --reextract-records ({"mode": "gap-driven",
300
+ "verification": [...], "files": [...]}) and --merge-records ({"exports":
301
+ [{export_name, export_type, params, return_type}]}, merge's output for
302
+ an export no fresh record holds). Each verification record names
303
+ export_name, gap_category, severity, verification, in_map (the map holds
304
+ an entry of that name), map_entry ({source_file, source_line} of the one
305
+ entry the spot-check took, else null), source_citation, new_location,
306
+ unknown_reason, pinned_definition_lines and reachability:
307
+ verified (map_entry) unchanged
308
+ moved (map_entry) source_line (and source_file) from new_location
309
+ re-extracted the entry from its `files` record (gap-driven.md §4a);
310
+ none when its reachability is internal-unreachable
311
+ verified/moved, not in_map, a NEW_EXPORT or MODIFIED_EXPORT whose
312
+ reachability is not internal-unreachable
313
+ one source-read entry at the citation's line
314
+ (verified) or new_location (moved), at most one
315
+ per name and file, export_type and the
316
+ signature from --merge-records
317
+ unknown, not in_map, a NEW_EXPORT or MODIFIED_EXPORT, a Medium, Low or
318
+ Info severity an entry from --merge-records with no
319
+ source_file or source_line
320
+ unknown, not in_map, any other severity (a missing one included)
321
+ refused: exit 3, nothing written
322
+ missing, unknown in_map, unknown on a MOVED_EXPORT
323
+ unchanged, with a warning for a person
324
+ rescoped the entry removed
325
+ Under the drift override (--drift-head and --drift-pinned) each warning
326
+ names the drift.
327
+
328
+ full (degraded mode) makes every fresh record an entry.
329
+
330
+ Every mode then applies the file changes: --file-compare (skf-hash-
331
+ content.py compare's output) sets the content_hash of a MODIFIED_FILE
332
+ row and removes a DELETED_FILE row; --new-files (skf-new-file-diff.py's
333
+ output) adds a row per new script or asset ({kind}s/<file name>, hashed
334
+ under --source-root, extraction_method file-copy); --promoted-docs (the
335
+ documents step 2 promoted, with their hashes) adds a doc row each
336
+ (docs/authoritative/<path>, promoted-authoritative). A row whose
337
+ source_file the map already holds is never added twice.
338
+
339
+ Last it sets the top-level source_commit and source_ref (when given)
340
+ and the update block, replacing an earlier update's values in place:
341
+ last_update (--generation-date), update_type, test_report_run_id,
342
+ files_changed (the manifest's files, or in gap-driven mode the files of
343
+ the entries this run changed), exports_affected (the manifest's export
344
+ changes, or the verification records), confidence_tier and
345
+ manual_sections_preserved. An older update-history key
346
+ (update_operations, update_metadata) stays as it is.
347
+
348
+ Lists. With --metadata-in (the skill's metadata.json as the update
349
+ found it) and --lists-out, it also writes the lists metadata.json takes,
350
+ {"exports": [...], "scripts": [...], "assets": [...]}. exports: the
351
+ --metadata-in exports[] less each name whose entry this run removed (a
352
+ DELETED_EXPORT, a DELETED file's or a MOVED file's old path's, a
353
+ rescoped one, a changed URL's) or renamed, unless an entry of the new
354
+ map still holds it, then each name it added an entry for or renamed an
355
+ entry to, each once; a renamed name takes its old name's place. A name
356
+ left out as off the public surface (the summary's not_public) has no
357
+ entry, so it is not added. A full update, which rebuilds the map, also
358
+ drops each name the new map does not hold, and a reference app
359
+ (scope_type reference-app) gets []. scripts and assets: one row per
360
+ file_entries[] row of the new map whose file_type is script or asset
361
+ (a row with none takes the type its file_name's folder names), in the
362
+ map's order: the --metadata-in scripts[] or assets[] row of its
363
+ source_file (else of its file_name) as it stands, else a new row {file:
364
+ its file_name, purpose, source_file, confidence (default T1-low)},
365
+ whose purpose skf-detect-scripts-assets.py reads from the file under
366
+ --source-root (its file name when no --source-root is given or the
367
+ file is not there).
368
+
369
+ Metadata (update-skill write.md §2):
370
+
371
+ metadata.json as --metadata holds it, with what the update sets: version
372
+ (--version) and generation_date (--generation-date, YYYY-MM-DDTHH:MM:SSZ);
373
+ source_commit and source_ref when --source-commit or --source-ref is
374
+ given (an empty one sets null), else as they stand; exports, scripts and
375
+ assets from --lists (apply's --lists-out), scripts or assets left out
376
+ when its list is empty; confidence_distribution from --stats (the JSON
377
+ skf-render-metadata-stats.py writes with -o), and stats: the stats
378
+ helper's fields, then each other field --metadata's stats holds (a
379
+ component library's own counts, a reference app's notes, a field a
380
+ gap-driven patch added), the stats helper's fields an earlier run wrote
381
+ that this run's does not (effective_denominator,
382
+ pattern_surfaces_documented) dropped. It removes last_update and
383
+ update_type, which the provenance map's update block records, and keeps
384
+ every other field. It refuses (exit 1, nothing written) a --stats whose
385
+ scripts_count or assets_count is not the length of --lists' scripts or
386
+ assets, a --generation-date of another form, and a file it cannot read.
387
+
388
+ Summary on stdout: {"status": "written", "map": <-o>, "entries":
389
+ {"added", "updated", "removed"}, "not_public": [{name, file}],
390
+ "file_entries": {"added", "updated", "removed"}, "warnings": [...]},
391
+ or {"status": "refused",
392
+ "blocking_unresolved": [{export_name, severity}]} with exit 3.
393
+
394
+ Records (update-skill re-extract.md §4, and gap-driven.md §4a for the files
395
+ it scans):
396
+
397
+ Writes {"mode": "normal", "files": [{file_path, exports: [...]}]}, one
398
+ block per file --files-from names (and per other file an export names),
399
+ so the model never types a record a tool already wrote. Each export the
400
+ recipe runner found (--extraction) is seeded with its name, type,
401
+ signature, location (file:line), confidence T1, extraction_method
402
+ ast-grep, ast_node_type and ast_recipe, return_type and params, each
403
+ parameter in the provenance map's typed form (skf-extraction-
404
+ inventory.py typed_param: `name: type`, `name?: type`, ` = default`).
405
+ Where the runner left params or return_type null, --export-details
406
+ fills them. An export only --export-details holds (one read by eye,
407
+ with an export_type) is seeded from it as T1-low, source-read, with a
408
+ null ast_node_type and ast_recipe.
409
+
410
+ Then each worker patch in --patches (every *.json there, in name order:
411
+ one block {file_path, exports: [{name, ...}]} or a list of them) is
412
+ merged into the export of its name and file: signature, members,
413
+ docstring and qmd_evidence replace the seed's, and params and
414
+ return_type are set only where the seed has none. A patch for an export
415
+ no seed holds is not added, with a warning. confidence_breakdown counts
416
+ the exports by confidence, and T2 the exports with a qmd_evidence.
417
+
418
+ Public surface. When --extraction's scope.type is public-api (in any
419
+ case) and it has an entry_point_diff, each export of a language family
420
+ whose entry_points.by_language status is `barrel` is marked `public`:
421
+ true when its name and file are a pair of the runner's entry_point_diff
422
+ `public` (its name, or the name `local` gives it in that file) or
423
+ `extraction_gaps`, or when a `public` item of its name and family has
424
+ no file; false otherwise. A `public` item with `via` namespace and a
425
+ `members` list (what the submodule passes on, which users reach as
426
+ `pkg.sub.name`, and which counts.exports_public_api counts in its place)
427
+ gives its members' pairs (each member's name, or its `local` one, and
428
+ its file; a member with no file its name) in place of its own; one
429
+ whose `members` is null (a module the runner could not read), or that
430
+ has no `members` key (an older runner's), gives its own pair, as any
431
+ item does.
432
+ The rule is skf-extraction-inventory.py's public_surface, loaded from
433
+ this folder, by which create-skill marks its extraction inventory too.
434
+ In a normal update merge documents no export marked false that the map
435
+ does not hold, and apply adds none; gap-records drops the mark, and a
436
+ full (degraded) apply adds every record. Any other scope type, a family
437
+ with no barrel, and a run with no entry_point_diff, an incomplete one,
438
+ one with errors or one whose entry_points.unresolved is not empty mark
439
+ nothing.
440
+
441
+ Gap records (update-skill gap-driven.md §3, §4 and §4a):
442
+
443
+ Reads the change manifest skf-parse-gaps.py translate wrote. Its
444
+ STRUCTURAL_FIX and metadata update entries are forwarded to merge as
445
+ they are (`forwarded`); every other entry is export-bearing. A severity
446
+ is blocking unless it is Medium, Low or Info (NON_BLOCKING), compared
447
+ case-insensitively, so a missing one is blocking. In order:
448
+
449
+ 1. Drift gate (--drift-status overridden): each export-bearing entry
450
+ but a MOVED_EXPORT needs the tree, with the first reason that fits:
451
+ a DELETED_EXPORT "a public API recount from the tree (rule R1)", a
452
+ provenance_completeness entry "a line from the tree (rule R3)", one
453
+ whose map_match is not-found "a line and a signature from the tree",
454
+ any other "a signature from the tree". Any such entry: {"status":
455
+ "drift-blocked", "drift_blocked": [{gap_id, name, change_category,
456
+ severity, reason}]}, and no spot-check runs.
457
+ 2. A DELETED_EXPORT without a `rescope` that holds its `amendment` and
458
+ `exclude`: {"status": "blocked", "rescope_without_amendment": [names]}.
459
+ 3. Spot-checks, by skf-verify-provenance-completeness.py's definition-
460
+ lines rules, loaded from this folder, under --source-root:
461
+ DELETED_EXPORT rescoped, no check
462
+ MOVED_EXPORT checked at its map entry when map_match is
463
+ found (unknown otherwise); under the
464
+ override a `moved` becomes unknown with
465
+ unknown_reason drift-override; its
466
+ pinned_definition_lines are the numbers of
467
+ its remediation's "definition line(s) (...)"
468
+ NEW_EXPORT, MODIFIED_EXPORT
469
+ map_match found checked at the map entry (its export_type)
470
+ ambiguous unknown
471
+ not-found checked at its source_citation (no export
472
+ type) when it has one; when that pins no
473
+ line (unknown or missing), or with no
474
+ citation: targeted re-extraction (§4a) for a
475
+ provenance_completeness entry with a source
476
+ root, for a blocking one whatever its
477
+ resolved_paths, and for a missing-export or
478
+ missing-type one with resolved_paths; else
479
+ unknown
480
+ A check gives `verified` (the line is a definition line, or a module
481
+ or package entry whose file exists), `moved` (one definition line,
482
+ not the recorded one: new_location), `missing` (no such file) or
483
+ `unknown` (several definition lines, none the recorded one, or
484
+ none). A file the rules cover no language of
485
+ takes the answered `declaring_line`: the recorded line is verified,
486
+ another is moved, null is unknown. With no verifier every check is
487
+ unknown, with a `provenance: spot-checks not run` warning; a check
488
+ the verifier fails on (a file it cannot read, or any error) is
489
+ unknown, with a `provenance: spot-check failed for {name}: {error}`
490
+ warning.
491
+ --plan stops here: {"status": "planned", "reextract": {"entries":
492
+ [{gap_id, name, severity, resolved_paths}], "files": [the union of
493
+ their resolved_paths]}, "warnings"}, the files also written to
494
+ --files-out as a JSON array.
495
+ 4. Targeted re-extraction (--remediation-records, `records`' output
496
+ over those files): each routed entry with resolved_paths takes the
497
+ first export of its name in the blocks of those files, in their
498
+ order: `re-extracted`, new_location its record's location,
499
+ resolution_source remediation-paths, and the record copied into
500
+ `files` as `records` wrote it, less its `public` mark (a normal
501
+ update's), with the answered `docstring` added.
502
+ An entry with no match, or no path to scan, is unresolved, except a
503
+ missing-export or missing-type one that is not blocking, which is
504
+ unknown. Any unresolved entry: {"status": "unresolved",
505
+ "unresolved": [{gap_id, name, severity, remediation_paths,
506
+ rejected_paths, files_scanned, exports_found_in_scan}]}.
507
+ 5. The public-reachability gate: every NEW_EXPORT that is verified,
508
+ moved or re-extracted takes the answered `reachability`, `public` or
509
+ `internal-unreachable`. An internal one keeps its outcome in its
510
+ verification record, its record is not in `files`, `counts` tallies
511
+ it under `reclassified` instead of its outcome, and it is listed in
512
+ `reclassified` (merge queues it as a metadata update; apply writes no
513
+ entry for it).
514
+
515
+ Any answer missing in step 3 or 5 (a `docstring` too, for each
516
+ re-extracted entry): {"status": "needs-judgment", "needs_judgment":
517
+ [{gap_id, name, needs: [...], file?, line?}]}. --judgments gives them,
518
+ keyed by gap id (the answers file translate reads; its keys are left).
519
+ Then it writes {"mode": "gap-driven", "files_extracted" (the blocks of
520
+ --remediation-records, 0 when it scanned none), "exports_extracted" (the
521
+ manifest's entries), "confidence_breakdown": {"T1", "T1-low",
522
+ "unlabeled", "T2": 0} (each manifest entry by the extraction_method of
523
+ its re-extracted record or, in the map, of its map entry; a pinned cited
524
+ export the map lacks and the gate passed is T1-low), "verification":
525
+ [one record per export-bearing entry: export_name, gap_category,
526
+ severity, verification, in_map, map_entry, provenance_citation,
527
+ source_citation, new_location, unknown_reason, pinned_definition_lines,
528
+ resolution_source, reachability, export_type (always null: for a cited
529
+ export the map lacks, apply takes it from merge.md's
530
+ merge-records.json)], "files": [...]}, and prints {"status": "written",
531
+ "output", "gap_count", "files_extracted", "exports_extracted",
532
+ "confidence_breakdown", "counts" (per verification outcome, and
533
+ `reclassified`), "reclassified", "forwarded", "targeted_reextraction",
534
+ "warnings"}. When targeted re-extraction ran, `targeted_reextraction`
535
+ is {resolved_count, files_scanned, exports_matched, tier}, also appended
536
+ to --evidence as one JSON line keyed `targeted_reextraction`.
537
+
538
+ Baseline gaps (update-skill detect-changes Category B, step 1):
539
+
540
+ The recipe runner never reports a module, an alias that renames, a
541
+ dunder such as `__version__` or another underscore name, so the diff
542
+ would read each such map entry as a deleted export. baseline-gaps lists,
543
+ for step 2 to read by eye, each entry of a file --files lists (the
544
+ modified files, Category A's modified-files.json) whose name and file
545
+ no export of --extraction (the runner's JSON, or {"exports": []} when
546
+ it could not run) holds, under its name or its re-export target, and
547
+ whose file still declares it. The selection and the declaration rule
548
+ are skf-verify-provenance-completeness.py's `baseline_gaps` (the rule
549
+ audit-skill's extraction snapshot applies), loaded from this folder: a
550
+ verifier that cannot be loaded is an error, never an empty list. A
551
+ name the runner's own `entry_point_diff.extraction_gaps` lists at that
552
+ file is left to that list. A map whose `entries` is missing or null
553
+ has none to check. Writes {"gaps": [{name, file, line, entry: null,
554
+ export_type?}], "unchecked": [{name, file, export_type}]}: a gap's line
555
+ declares the name (line 1 of the module or package so named for a
556
+ `module` or `package` entry), and `unchecked` holds each entry no rule
557
+ can check, to read by eye: its file neither Python nor TS/JS, a file
558
+ the lookup will not read (outside the source root, a test file or a
559
+ minified bundle, a symlink on its path), or a dotted `module` or
560
+ `package` name.
561
+
69
562
  Exit codes:
70
- 0 — operation succeeded
71
- 1 — user error (malformed JSON, bad path, malformed provenance file)
563
+ 0 operation succeeded (for gap-records, whatever its `status`)
564
+ 1 user error (malformed JSON, bad path, malformed provenance file, a
565
+ helper file that is not the output it names, a gap-records answer of
566
+ the wrong type, targeted re-extraction to match with no
567
+ --remediation-records, or, for baseline-gaps, a source root that is
568
+ no folder or a verifier that cannot be loaded)
569
+ 3 apply refused: a blocking gap reached the provenance write with no
570
+ source line (nothing written)
72
571
  """
73
572
 
74
573
  from __future__ import annotations
75
574
 
76
575
  import argparse
576
+ import difflib
577
+ import hashlib
578
+ import importlib.util
77
579
  import json
580
+ import os
581
+ import re
582
+ import subprocess
78
583
  import sys
584
+ import time
79
585
  from pathlib import Path
80
586
 
81
587
 
@@ -99,7 +605,10 @@ def _get_list(d: dict, *path: str) -> list:
99
605
  def _load_input(input_path: Path | None) -> dict:
100
606
  """Read JSON either from --input path or from stdin."""
101
607
  if input_path is not None:
102
- text = input_path.read_text(encoding="utf-8")
608
+ try:
609
+ text = input_path.read_text(encoding="utf-8")
610
+ except OSError as exc:
611
+ raise ValueError(f"cannot read input {input_path}: {exc}") from exc
103
612
  else:
104
613
  text = sys.stdin.read()
105
614
  try:
@@ -111,6 +620,139 @@ def _load_input(input_path: Path | None) -> dict:
111
620
  return data
112
621
 
113
622
 
623
+ def _load_helper_file(path: Path, what: str, key: str) -> dict:
624
+ """A helper's JSON output: an object holding `key`. Raises ValueError."""
625
+ try:
626
+ data = json.loads(path.read_text(encoding="utf-8"))
627
+ except OSError as exc:
628
+ raise ValueError(f"cannot read {what} {path}: {exc}") from exc
629
+ except json.JSONDecodeError as exc:
630
+ raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
631
+ if not isinstance(data, dict) or key not in data:
632
+ raise ValueError(f"{what} {path} has no `{key}`: is it the helper's output?")
633
+ return data
634
+
635
+
636
+ # --------------------------------------------------------------------------
637
+ # Helper files (update-skill detect-changes §2.1)
638
+ # --------------------------------------------------------------------------
639
+
640
+
641
+ def category_b_from_diff(diff: dict) -> dict:
642
+ """Category B's four lists from a skf-structural-diff.py diff of the
643
+ modified files (see the module docstring for the mapping)."""
644
+ lists: dict[str, list] = {
645
+ "modified_exports": [], "new_exports": [], "deleted_exports": [], "moved_exports": [],
646
+ }
647
+ for item in _get_list(diff, "removed"):
648
+ if isinstance(item, dict):
649
+ lists["deleted_exports"].append(
650
+ {"name": item.get("name"), "file": item.get("file"), "old_line": item.get("line")})
651
+ for item in _get_list(diff, "added"):
652
+ if isinstance(item, dict):
653
+ lists["new_exports"].append({"name": item.get("name"), "file": item.get("file"), "line": item.get("line")})
654
+ changed: dict[tuple, dict] = {}
655
+ for item in _get_list(diff, "changed"):
656
+ if not isinstance(item, dict):
657
+ continue
658
+ record = changed.setdefault(
659
+ (item.get("name"), item.get("file")), {"fields": set(), "line": item.get("line"), "old_line": None})
660
+ record["fields"].add(item.get("field"))
661
+ if item.get("field") == "line":
662
+ record["old_line"] = item.get("baseline_value")
663
+ unverified = list(dict.fromkeys(
664
+ (item.get("name"), item.get("file")) for item in _get_list(diff, "signature_unverified")
665
+ if isinstance(item, dict)
666
+ ))
667
+ modified: set[tuple] = set()
668
+ for key, record in changed.items():
669
+ old_line = record["old_line"] if record["old_line"] is not None else record["line"]
670
+ entry = {"name": key[0], "file": key[1], "old_line": old_line, "new_line": record["line"]}
671
+ if record["fields"] - {"line"} or key in unverified:
672
+ lists["modified_exports"].append(entry)
673
+ modified.add(key)
674
+ else:
675
+ lists["moved_exports"].append(entry)
676
+ for key in unverified:
677
+ if key not in modified:
678
+ lists["modified_exports"].append({"name": key[0], "file": key[1], "old_line": None, "new_line": None})
679
+ modified.add(key)
680
+ for item in _get_list(diff, "moved"):
681
+ if not isinstance(item, dict):
682
+ continue
683
+ key, old_file = (item.get("name"), item.get("current_file")), _norm_path(item.get("previous_file"))
684
+ across = old_file not in (None, _norm_path(item.get("current_file"))) # apply finds the entry at old_file
685
+ if key in modified:
686
+ if across: # moved across files and changed: a MODIFIED_EXPORT that names the file it left
687
+ for entry in lists["modified_exports"]:
688
+ if (entry["name"], entry["file"]) == key:
689
+ entry.update(old_line=item.get("previous_line"), old_file=old_file)
690
+ continue
691
+ move = {"name": item.get("name"), "file": item.get("current_file")}
692
+ if across:
693
+ move["old_file"] = old_file
694
+ lists["moved_exports"].append({**move, "old_line": item.get("previous_line"), "new_line": item.get("line")})
695
+ return lists
696
+
697
+
698
+ def _take_out(entries: list, name: object, files: tuple) -> None:
699
+ """Remove the first entry named `name`, preferring one in the first of
700
+ `files` that holds one (a None in `files` stands for any file)."""
701
+ for wanted in files:
702
+ for i, entry in enumerate(entries):
703
+ if isinstance(entry, dict) and entry.get("name") == name and wanted in (None, entry.get("file")):
704
+ del entries[i]
705
+ return
706
+
707
+
708
+ def apply_renames(payload: dict) -> dict:
709
+ """Take each Category C rename's pair out of the Category A and B lists
710
+ it was found in (see the module docstring). payload holds category_a
711
+ and category_b as objects, as assemble leaves them."""
712
+ cat_a, cat_b = payload["category_a"], payload["category_b"]
713
+ cat_c = payload.get("category_c", {}) or {}
714
+ for move in _get_list(cat_c, "renamed_files"):
715
+ if not isinstance(move, dict):
716
+ continue
717
+ for key, path in (("deleted", move.get("old_path")), ("added", move.get("new_path"))):
718
+ listed = _get_list(cat_a, key)
719
+ if path in listed:
720
+ cat_a[key] = [p for p in listed if p != path]
721
+ for rename in _get_list(cat_c, "renamed_exports"):
722
+ if not isinstance(rename, dict):
723
+ continue
724
+ deleted = list(_get_list(cat_b, "deleted_exports"))
725
+ new = list(_get_list(cat_b, "new_exports"))
726
+ old_files = (rename["old_file"],) if rename.get("old_file") else (rename.get("file"), None)
727
+ _take_out(deleted, rename.get("old_name"), old_files)
728
+ _take_out(new, rename.get("new_name"), (rename.get("file"), None))
729
+ cat_b["deleted_exports"], cat_b["new_exports"] = deleted, new
730
+ return payload
731
+
732
+
733
+ def assemble(payload: dict, *, category_a_doc: dict | None = None, diff: dict | None = None) -> dict:
734
+ """The category JSON build and deletion-ratio read: the input, with the
735
+ helper files' slices in place of its own, and Category C's renames taken
736
+ out of the A and B lists."""
737
+ out = dict(payload)
738
+ cat_c = dict(payload.get("category_c") or {})
739
+ if category_a_doc is not None:
740
+ out["category_a"] = {key: list(_get_list(category_a_doc, "category_a", key))
741
+ for key in ("modified", "added", "deleted")}
742
+ moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
743
+ cat_c["renamed_files"] = moves + list(_get_list(cat_c, "renamed_files"))
744
+ else:
745
+ out["category_a"] = {key: list(_get_list(payload, "category_a", key))
746
+ for key in ("modified", "added", "deleted")}
747
+ if diff is not None:
748
+ out["category_b"] = category_b_from_diff(diff)
749
+ else:
750
+ out["category_b"] = {key: list(_get_list(payload, "category_b", key))
751
+ for key in ("modified_exports", "new_exports", "deleted_exports", "moved_exports")}
752
+ out["category_c"] = cat_c
753
+ return apply_renames(out)
754
+
755
+
114
756
  # --------------------------------------------------------------------------
115
757
  # Build manifest
116
758
  # --------------------------------------------------------------------------
@@ -135,14 +777,8 @@ def build_manifest(payload: dict) -> dict:
135
777
  c_renamed_files = _get_list(cat_c, "renamed_files")
136
778
  c_renamed_exports = _get_list(cat_c, "renamed_exports")
137
779
 
138
- d = {
139
- "scripts_modified": len(_get_list(cat_d, "scripts_modified")),
140
- "scripts_added": len(_get_list(cat_d, "scripts_added")),
141
- "scripts_deleted": len(_get_list(cat_d, "scripts_deleted")),
142
- "assets_modified": len(_get_list(cat_d, "assets_modified")),
143
- "assets_added": len(_get_list(cat_d, "assets_added")),
144
- "assets_deleted": len(_get_list(cat_d, "assets_deleted")),
145
- }
780
+ d_lists = {key: list(dict.fromkeys(p for p in _get_list(cat_d, key) if isinstance(p, str))) for key in D_KEYS}
781
+ d = {key: len(paths) for key, paths in d_lists.items()}
146
782
 
147
783
  counts = {
148
784
  "files_changed": len(a_modified),
@@ -186,9 +822,51 @@ def build_manifest(payload: dict) -> dict:
186
822
  "counts": counts,
187
823
  "total_export_changes": total_export_changes,
188
824
  "per_file": per_file,
825
+ "category_d": d_lists,
189
826
  }
190
827
 
191
828
 
829
+ # Category D's lists, in the order build counts them.
830
+ D_KEYS = ("scripts_modified", "scripts_added", "scripts_deleted", "assets_modified", "assets_added",
831
+ "assets_deleted", "docs_modified", "docs_deleted")
832
+ FILE_TYPES = ("script", "asset", "doc")
833
+ _FILE_TYPE_OF_FOLDER = {"scripts": "script", "assets": "asset", "docs": "doc"}
834
+
835
+
836
+ def _file_type(row: dict) -> str:
837
+ """A file_entries[] row's file_type, else the one its file_name's first folder names, else asset."""
838
+ kind = row.get("file_type")
839
+ if kind in FILE_TYPES:
840
+ return kind
841
+ name = row.get("file_name")
842
+ folder = name.strip().replace("\\", "/").split("/", 1)[0] if isinstance(name, str) else ""
843
+ return _FILE_TYPE_OF_FOLDER.get(folder, "asset")
844
+
845
+
846
+ def category_d_from_files(compare: dict | None, new_files: dict | None, provenance: dict | None) -> dict:
847
+ """Category D's lists from skf-hash-content.py compare's rows, each typed by the file_entries[] row of its
848
+ source_file in `provenance`, and from skf-new-file-diff.py's new files, by kind. Raises ValueError."""
849
+ rows = (provenance or {}).get("file_entries") or []
850
+ if not isinstance(rows, list):
851
+ raise ValueError("provenance `file_entries` must be an array")
852
+ types: dict[str, str] = {}
853
+ for row in rows:
854
+ if isinstance(row, dict) and _norm_path(row.get("source_file")):
855
+ types.setdefault(_norm_path(row["source_file"]), _file_type(row))
856
+ lists: dict[str, list] = {key: [] for key in D_KEYS}
857
+ changes = {"MODIFIED_FILE": "modified", "DELETED_FILE": "deleted"}
858
+ for row in _get_list(compare or {}, "comparisons"):
859
+ if not isinstance(row, dict) or row.get("classification") not in changes:
860
+ continue
861
+ path = _norm_path(row.get("source_file"))
862
+ if path:
863
+ lists[f"{types.get(path, 'asset')}s_{changes[row['classification']]}"].append(path)
864
+ for item in _get_list(new_files or {}, "new_files"):
865
+ if isinstance(item, dict) and item.get("kind") in ("script", "asset") and _norm_path(item.get("source_file")):
866
+ lists[f"{item['kind']}s_added"].append(_norm_path(item["source_file"]))
867
+ return lists
868
+
869
+
192
870
  def _build_per_file(
193
871
  *,
194
872
  a_modified: list,
@@ -209,12 +887,15 @@ def _build_per_file(
209
887
  path = entry.get("file")
210
888
  if not path:
211
889
  return
212
- exports_by_file.setdefault(path, []).append({
890
+ change = {
213
891
  "name": entry.get("name"),
214
892
  "change_type": change_type,
215
893
  "old_line": entry.get("old_line"),
216
894
  "new_line": entry.get("new_line") or entry.get("line"),
217
- })
895
+ }
896
+ if entry.get("old_file"):
897
+ change["old_file"] = entry["old_file"] # a move across files: a MOVED_EXPORT, or a MODIFIED_EXPORT
898
+ exports_by_file.setdefault(path, []).append(change)
218
899
 
219
900
  for e in b_modified:
220
901
  _record(e, "MODIFIED_EXPORT")
@@ -337,18 +1018,1665 @@ def _ratio_skip(*, skip_reason: str) -> dict:
337
1018
  }
338
1019
 
339
1020
 
1021
+ # --------------------------------------------------------------------------
1022
+ # Shared readers (rename-candidates and apply)
1023
+ # --------------------------------------------------------------------------
1024
+
1025
+
1026
+ def _norm_path(path: object) -> str | None:
1027
+ """A file path compared as structural-diff compares it: `/` separators, no leading `./`."""
1028
+ if not isinstance(path, str) or not path.strip():
1029
+ return None
1030
+ text = path.strip().replace("\\", "/")
1031
+ while text.startswith("./"):
1032
+ text = text[2:]
1033
+ return text
1034
+
1035
+
1036
+ def _load_json_file(path: Path, what: str):
1037
+ try:
1038
+ return json.loads(path.read_text(encoding="utf-8"))
1039
+ except OSError as exc:
1040
+ raise ValueError(f"cannot read {what} {path}: {exc}") from exc
1041
+ except json.JSONDecodeError as exc:
1042
+ raise ValueError(f"{what} {path} is not valid JSON: {exc}") from exc
1043
+
1044
+
1045
+ def _exports_of(doc, what: str) -> list[dict]:
1046
+ """The export records of an extraction or export-details file ({"exports": [...]})."""
1047
+ if doc is None:
1048
+ return []
1049
+ if not isinstance(doc, dict) or not isinstance(doc.get("exports"), list):
1050
+ raise ValueError(f"{what} has no `exports` array: is it the helper's output?")
1051
+ return [e for e in doc["exports"] if isinstance(e, dict)]
1052
+
1053
+
1054
+ def _entries_of(provenance) -> list[dict]:
1055
+ if provenance is None:
1056
+ return []
1057
+ if not isinstance(provenance, dict):
1058
+ raise ValueError("the provenance map must be a JSON object")
1059
+ entries = provenance.get("entries", [])
1060
+ if not isinstance(entries, list):
1061
+ raise ValueError("provenance `entries` must be an array")
1062
+ return [e for e in entries if isinstance(e, dict)]
1063
+
1064
+
1065
+ def _params(value) -> tuple | None:
1066
+ """A parameter list compared by its text, whitespace removed."""
1067
+ if value is None:
1068
+ return None
1069
+ if not isinstance(value, list):
1070
+ return (str(value),)
1071
+ out = []
1072
+ for item in value:
1073
+ if isinstance(item, dict):
1074
+ name, kind = item.get("name"), item.get("type")
1075
+ item = f"{name}: {kind}" if kind else str(name)
1076
+ out.append("".join(str(item).split()))
1077
+ return tuple(out)
1078
+
1079
+
1080
+ def _signature(record: dict) -> tuple:
1081
+ """(export_type, params, return_type) of an export record, the parts a signature compares."""
1082
+ kind = record.get("export_type", record.get("type"))
1083
+ params = record.get("params", record.get("parameters"))
1084
+ ret = record.get("return_type")
1085
+ return (kind if isinstance(kind, str) else None, _params(params),
1086
+ "".join(ret.split()) if isinstance(ret, str) else None)
1087
+
1088
+
1089
+ def _name_of(record: dict) -> str | None:
1090
+ name = record.get("export_name", record.get("name"))
1091
+ return name if isinstance(name, str) and name else None
1092
+
1093
+
1094
+ def _file_of(record: dict) -> str | None:
1095
+ return _norm_path(record.get("source_file", record.get("file", record.get("file_path"))))
1096
+
1097
+
1098
+ _INVENTORY = None
1099
+
1100
+
1101
+ def _inventory():
1102
+ """skf-extraction-inventory.py, loaded once from this folder: its typed_param and its public-surface rule, so
1103
+ create-skill and update-skill write one parameter form and keep one public surface."""
1104
+ global _INVENTORY
1105
+ if _INVENTORY is None:
1106
+ path = Path(__file__).resolve().parent / "skf-extraction-inventory.py"
1107
+ spec = importlib.util.spec_from_file_location("skf_extraction_inventory", path)
1108
+ if spec is None or spec.loader is None:
1109
+ raise ImportError(f"cannot load {path}")
1110
+ module = importlib.util.module_from_spec(spec)
1111
+ spec.loader.exec_module(module)
1112
+ _INVENTORY = module
1113
+ return _INVENTORY
1114
+
1115
+
1116
+ def public_surface(extraction: object) -> tuple[object | None, str | None]:
1117
+ """skf-extraction-inventory.py's public_surface over the recipe runner's output (see "Records" in the module
1118
+ docstring): (the public surface, None) for a public-api run, (None, why) when its surface cannot be read, and
1119
+ (None, None) for any other scope type. An input that is no object (a gap-driven or docs-only run gives none)
1120
+ is (None, None) without loading the inventory helper."""
1121
+ if not isinstance(extraction, dict):
1122
+ return None, None
1123
+ return _inventory().public_surface(extraction)
1124
+
1125
+
1126
+ # --------------------------------------------------------------------------
1127
+ # Rename candidates (update-skill detect-changes Category C)
1128
+ # --------------------------------------------------------------------------
1129
+
1130
+
1131
+ RENAME_SIMILARITY = 0.80 # content similarity above this is a rename (fixed)
1132
+ SIZE_TOLERANCE = 0.20 # Quick tier: the added file's size within 20% of the deleted one's
1133
+ NAME_OVERLAP = 0.70 # Quick tier: shared export names above 70%
1134
+ TIERS = ("Quick", "Forge", "Forge+", "Deep")
1135
+
1136
+
1137
+ def _git_size(source_root: Path, commit: str, path: str) -> int | None:
1138
+ """The size of `path` at `commit` (git cat-file -s), or None when git cannot tell."""
1139
+ try:
1140
+ proc = subprocess.run(["git", "-C", str(source_root), "cat-file", "-s", f"{commit}:{path}"],
1141
+ capture_output=True, text=True, timeout=30, check=False)
1142
+ except (OSError, subprocess.SubprocessError):
1143
+ return None
1144
+ if proc.returncode != 0:
1145
+ return None
1146
+ try:
1147
+ return int(proc.stdout.strip())
1148
+ except ValueError:
1149
+ return None
1150
+
1151
+
1152
+ def _file_size(source_root: Path | None, path: str) -> int | None:
1153
+ if source_root is None:
1154
+ return None
1155
+ try:
1156
+ return (source_root / path).stat().st_size
1157
+ except OSError:
1158
+ return None
1159
+
1160
+
1161
+ def _best_pairs(candidates: list[tuple]) -> list[tuple]:
1162
+ """Each side's best match only: (score, left, right, rule) tuples, highest score first.
1163
+
1164
+ A pair is kept when neither side is taken yet and no other candidate of
1165
+ either side ties its score, so a tie pairs neither.
1166
+ """
1167
+ out, used_left, used_right = [], set(), set()
1168
+ ordered = sorted(candidates, key=lambda c: (-c[0], c[1], c[2]))
1169
+ for score, left, right, rule in ordered:
1170
+ if left in used_left or right in used_right:
1171
+ continue
1172
+ rivals = [c for c in ordered if c[0] == score and (c[1] == left) != (c[2] == right)
1173
+ and c[1] not in used_left and c[2] not in used_right]
1174
+ if rivals:
1175
+ used_left.add(left)
1176
+ used_right.add(right)
1177
+ for c in rivals:
1178
+ used_left.add(c[1])
1179
+ used_right.add(c[2])
1180
+ continue
1181
+ used_left.add(left)
1182
+ used_right.add(right)
1183
+ out.append((score, left, right, rule))
1184
+ return out
1185
+
1186
+
1187
+ def rename_candidates(category_a_doc: dict, diff: dict | None, provenance: dict, extraction: dict | None,
1188
+ details: dict | None, tier: str, *, source_root: Path | None = None,
1189
+ sizes_commit: str | None = None) -> dict:
1190
+ """Category C by the fixed rules of the module docstring."""
1191
+ if tier not in TIERS:
1192
+ raise ValueError(f"--tier must be one of {', '.join(TIERS)}, got {tier!r}")
1193
+ forge = tier != "Quick"
1194
+ moves = [m for m in _get_list(category_a_doc, "moved_files") if isinstance(m, dict)]
1195
+ moved_old = {_norm_path(m.get("old_path")) for m in moves}
1196
+ moved_new = {_norm_path(m.get("new_path")) for m in moves}
1197
+ deleted = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "deleted"))
1198
+ if p and p not in moved_old})
1199
+ added = sorted({p for p in map(_norm_path, _get_list(category_a_doc, "category_a", "added"))
1200
+ if p and p not in moved_new})
1201
+
1202
+ old_by_file: dict[str, dict[str, dict]] = {}
1203
+ for entry in _entries_of(provenance):
1204
+ name, path = _name_of(entry), _file_of(entry)
1205
+ if name and path:
1206
+ old_by_file.setdefault(path, {}).setdefault(name, entry)
1207
+ new_by_file: dict[str, dict[str, dict]] = {}
1208
+ for record in _exports_of(extraction, "--extraction file") + _exports_of(details, "--export-details file"):
1209
+ name, path = _name_of(record), _file_of(record)
1210
+ if name and path:
1211
+ merged = new_by_file.setdefault(path, {}).setdefault(name, {})
1212
+ for key, value in record.items():
1213
+ if merged.get(key) is None:
1214
+ merged[key] = value
1215
+
1216
+ file_candidates = []
1217
+ for old in deleted:
1218
+ old_exports = old_by_file.get(old, {})
1219
+ for new in added:
1220
+ new_exports = new_by_file.get(new, {})
1221
+ if not old_exports or not new_exports:
1222
+ continue
1223
+ if forge:
1224
+ matched = sum(1 for name, e in old_exports.items()
1225
+ if name in new_exports and _signature(e) == _signature(new_exports[name]))
1226
+ score = matched / max(len(old_exports), len(new_exports))
1227
+ if score > RENAME_SIMILARITY:
1228
+ file_candidates.append((round(score, 4), old, new, "signatures"))
1229
+ continue
1230
+ names_old, names_new = set(old_exports), set(new_exports)
1231
+ overlap = len(names_old & names_new) / len(names_old | names_new)
1232
+ if overlap <= NAME_OVERLAP:
1233
+ continue
1234
+ old_size = _git_size(source_root, sizes_commit, old) if (source_root and sizes_commit) else None
1235
+ new_size = _file_size(source_root, new)
1236
+ if old_size is not None:
1237
+ if new_size is None or max(old_size, new_size) == 0:
1238
+ continue
1239
+ if min(old_size, new_size) / max(old_size, new_size) < 1 - SIZE_TOLERANCE:
1240
+ continue
1241
+ rule = "size-and-names"
1242
+ else:
1243
+ rule = "names (size not checked)"
1244
+ file_candidates.append((round(overlap, 4), old, new, rule))
1245
+ file_pairs = _best_pairs(file_candidates)
1246
+ paired_old = {p[1] for p in file_pairs}
1247
+ paired_new = {p[2] for p in file_pairs}
1248
+
1249
+ removed = []
1250
+ for item in _get_list(diff or {}, "removed"):
1251
+ if isinstance(item, dict) and _name_of(item) and _file_of(item):
1252
+ name, path = _name_of(item), _file_of(item)
1253
+ removed.append({**old_by_file.get(path, {}).get(name, {}), **{k: v for k, v in item.items()
1254
+ if v is not None},
1255
+ "_key": (name, path)})
1256
+ candidates_new = []
1257
+ for item in _get_list(diff or {}, "added"):
1258
+ if isinstance(item, dict) and _name_of(item) and _file_of(item):
1259
+ name, path = _name_of(item), _file_of(item)
1260
+ candidates_new.append({**new_by_file.get(path, {}).get(name, {}),
1261
+ **{k: v for k, v in item.items() if v is not None}, "_key": (name, path)})
1262
+ seen_new = {c["_key"] for c in candidates_new}
1263
+ for path in added:
1264
+ if path in paired_new:
1265
+ continue
1266
+ for name, record in new_by_file.get(path, {}).items():
1267
+ if (name, path) not in seen_new:
1268
+ candidates_new.append({**record, "_key": (name, path)})
1269
+
1270
+ export_candidates = []
1271
+ for old in removed:
1272
+ old_sig = _signature(old)
1273
+ for new in candidates_new:
1274
+ if old["_key"][0] == new["_key"][0]:
1275
+ continue
1276
+ new_sig = _signature(new)
1277
+ if forge:
1278
+ if old_sig[1] is None and old_sig[2] is None:
1279
+ continue
1280
+ if old_sig == new_sig:
1281
+ export_candidates.append((1.0, old["_key"], new["_key"], "signature-equal"))
1282
+ continue
1283
+ if old_sig[0] and new_sig[0] and old_sig[0] != new_sig[0]:
1284
+ continue
1285
+ ratio = difflib.SequenceMatcher(None, old["_key"][0], new["_key"][0]).ratio()
1286
+ if ratio > RENAME_SIMILARITY:
1287
+ export_candidates.append((round(ratio, 4), old["_key"], new["_key"], "name-similarity"))
1288
+ export_pairs = _best_pairs(export_candidates)
1289
+
1290
+ renamed_exports = []
1291
+ for _score, (old_name, old_file), (new_name, new_file), _rule in export_pairs:
1292
+ item = {"old_name": old_name, "new_name": new_name, "file": new_file}
1293
+ if old_file != new_file:
1294
+ item["old_file"] = old_file
1295
+ renamed_exports.append(item)
1296
+ paired_removed = {p[1] for p in export_pairs}
1297
+ paired_added = {p[2] for p in export_pairs}
1298
+ return {
1299
+ "category_c": {
1300
+ "renamed_files": [{"old_path": old, "new_path": new} for _s, old, new, _r in file_pairs],
1301
+ "renamed_exports": renamed_exports,
1302
+ },
1303
+ "evidence": {
1304
+ "files": [{"old_path": old, "new_path": new, "rule": rule, "similarity": score}
1305
+ for score, old, new, rule in file_pairs],
1306
+ "exports": [{"old_name": o[0], "new_name": n[0], "file": n[1], "rule": rule, "similarity": score}
1307
+ for score, o, n, rule in export_pairs],
1308
+ },
1309
+ "unpaired": {
1310
+ "deleted_files": [p for p in deleted if p not in paired_old],
1311
+ "added_files": [p for p in added if p not in paired_new],
1312
+ "removed_exports": [{"name": r["_key"][0], "file": r["_key"][1]} for r in removed
1313
+ if r["_key"] not in paired_removed],
1314
+ "added_exports": [{"name": c["_key"][0], "file": c["_key"][1]} for c in candidates_new
1315
+ if c["_key"] not in paired_added],
1316
+ },
1317
+ "tier": tier,
1318
+ }
1319
+
1320
+
1321
+ # --------------------------------------------------------------------------
1322
+ # Apply (update-skill write.md §3)
1323
+ # --------------------------------------------------------------------------
1324
+
1325
+
1326
+ UPDATE_TYPES = ("incremental", "gap-driven", "full")
1327
+ NON_BLOCKING = ("medium", "low", "info")
1328
+ EXIT_REFUSED = 3
1329
+ REPLACE_WAIT_SECONDS = 5.0 # how long a Windows rename onto a file held open is retried
1330
+ # The update block apply sets at the top level of the map, in this order.
1331
+ UPDATE_BLOCK_KEYS = ("last_update", "update_type", "test_report_run_id", "files_changed", "exports_affected",
1332
+ "confidence_tier", "manual_sections_preserved")
1333
+
1334
+
1335
+ def _labels(method: object, node: object) -> dict:
1336
+ """The confidence labels an extraction method implies (create-skill's entry contract)."""
1337
+ if method in ("ast-grep", "ast_bridge"):
1338
+ return {"confidence": "T1", "extraction_method": "ast-grep", "ast_node_type": node,
1339
+ "signature_source": "T1"}
1340
+ return {"confidence": "T1-low", "extraction_method": "source-read", "ast_node_type": None,
1341
+ "signature_source": "T1-low"}
1342
+
1343
+
1344
+ def _line_of(value: object) -> int | None:
1345
+ """A line number from an int, a digit string or a `file:line` / `file:start-end` location."""
1346
+ if isinstance(value, bool):
1347
+ return None
1348
+ if isinstance(value, int):
1349
+ return value
1350
+ if isinstance(value, str):
1351
+ tail = value.rsplit(":", 1)[-1].split("-", 1)[0].strip()
1352
+ if tail.isdigit():
1353
+ return int(tail)
1354
+ return None
1355
+
1356
+
1357
+ def _location(value: object) -> tuple[str | None, int | None]:
1358
+ """(file, line) of a `file:line` string or a {file, line} object."""
1359
+ if isinstance(value, dict):
1360
+ return _norm_path(value.get("file") or value.get("source_file")), _line_of(
1361
+ value.get("line", value.get("source_line")))
1362
+ if isinstance(value, str) and ":" in value:
1363
+ path, _, rest = value.rpartition(":")
1364
+ return _norm_path(path), _line_of(rest)
1365
+ return None, None
1366
+
1367
+
1368
+ class _Fresh:
1369
+ """The fresh records of one run, looked up by export name and file."""
1370
+
1371
+ def __init__(self, worker_files: list, extraction: dict | None, details: dict | None):
1372
+ self.runner = {}
1373
+ self.details = {}
1374
+ self.worker = {}
1375
+ self.surface, self.surface_problem = public_surface(extraction)
1376
+ for record in _exports_of(extraction, "--extraction file"):
1377
+ key = (_name_of(record), _file_of(record))
1378
+ if all(key):
1379
+ self.runner.setdefault(key, record)
1380
+ for record in _exports_of(details, "--export-details file"):
1381
+ key = (_name_of(record), _file_of(record))
1382
+ if all(key):
1383
+ self.details.setdefault(key, record)
1384
+ for block in worker_files:
1385
+ if not isinstance(block, dict):
1386
+ continue
1387
+ path = _norm_path(block.get("file_path"))
1388
+ for record in block.get("exports") or []:
1389
+ if isinstance(record, dict) and _name_of(record) and path:
1390
+ self.worker.setdefault((_name_of(record), path), record)
1391
+
1392
+ def keys_of(self, path: str) -> list[tuple]:
1393
+ keys = {k for source in (self.runner, self.details, self.worker) for k in source if k[1] == path}
1394
+ return sorted(keys)
1395
+
1396
+ def all_keys(self) -> list[tuple]:
1397
+ return sorted({k for source in (self.runner, self.details, self.worker) for k in source})
1398
+
1399
+ def public(self, name: str, path: str) -> bool | None:
1400
+ """The `public` mark `records` gave the export, else the public-surface rule over --extraction; None when
1401
+ neither marks it."""
1402
+ mark = (self.worker.get((name, path)) or {}).get("public")
1403
+ if isinstance(mark, bool):
1404
+ return mark
1405
+ if self.surface is None:
1406
+ return None
1407
+ record = self.runner.get((name, path)) or self.details.get((name, path)) or {}
1408
+ return self.surface.public(name, path, record.get("language"))
1409
+
1410
+ def record(self, name: str, path: str) -> dict | None:
1411
+ """The entry fields of an export's fresh record, or None when no source holds it."""
1412
+ key = (name, path)
1413
+ runner, detail, worker = self.runner.get(key), self.details.get(key), self.worker.get(key)
1414
+ if runner is None and detail is None and worker is None:
1415
+ return None
1416
+ sources = [r for r in (runner, detail, worker) if r is not None]
1417
+
1418
+ def first(*fields):
1419
+ for record in sources:
1420
+ for field in fields:
1421
+ value = record.get(field)
1422
+ if value is not None:
1423
+ return value
1424
+ return None
1425
+
1426
+ line = None
1427
+ for record in sources:
1428
+ line = _line_of(record.get("source_line"))
1429
+ if line is None and record is worker:
1430
+ line = _line_of(record.get("location"))
1431
+ if line is not None:
1432
+ break
1433
+ method = first("extraction_method")
1434
+ node = first("ast_node_type") if method in ("ast-grep", "ast_bridge") else None
1435
+ params = None
1436
+ for record in (detail, worker, runner):
1437
+ if record is None:
1438
+ continue
1439
+ value = record.get("params", record.get("parameters"))
1440
+ if value is not None:
1441
+ params = value
1442
+ break
1443
+ return {
1444
+ "export_name": name,
1445
+ "export_type": first("export_type", "type"),
1446
+ "params": params,
1447
+ "return_type": first("return_type"),
1448
+ "source_file": path,
1449
+ "source_line": line,
1450
+ **_labels(method, node),
1451
+ }
1452
+
1453
+
1454
+ def _without_none(fields: dict, keep: tuple = ()) -> dict:
1455
+ return {k: v for k, v in fields.items() if v is not None or k in keep}
1456
+
1457
+
1458
+ def _new_entry(fields: dict, library: str) -> dict:
1459
+ """An entry in create-skill's key order, from the fields a record gives."""
1460
+ order = ("export_name", "export_type", "source_library", "params", "return_type", "source_file",
1461
+ "source_line", "confidence", "extraction_method", "ast_node_type", "signature_source")
1462
+ merged = {**fields, "source_library": fields.get("source_library") or library}
1463
+ out = {k: merged[k] for k in order if k in merged and (merged[k] is not None or k == "ast_node_type")}
1464
+ return out
1465
+
1466
+
1467
+ def _rewrite(entry: dict, fields: dict) -> dict:
1468
+ """The entry with `fields` set in place; keys the update does not set keep their values."""
1469
+ out = dict(entry)
1470
+ for key, value in fields.items():
1471
+ if value is None and key not in ("ast_node_type",):
1472
+ continue
1473
+ out[key] = value
1474
+ return out
1475
+
1476
+
1477
+ def _sha256_of(path: Path) -> str:
1478
+ digest = hashlib.sha256()
1479
+ with path.open("rb") as fh:
1480
+ for chunk in iter(lambda: fh.read(65536), b""):
1481
+ digest.update(chunk)
1482
+ return "sha256:" + digest.hexdigest()
1483
+
1484
+
1485
+ class _Map:
1486
+ """The provenance map being rewritten, with what changed."""
1487
+
1488
+ def __init__(self, provenance: dict | None, skill_name: str):
1489
+ self.doc = json.loads(json.dumps(provenance)) if provenance is not None else {
1490
+ "provenance_version": "2.0", "skill_name": skill_name, "skill_type": "single", "entries": []}
1491
+ if not isinstance(self.doc.get("entries"), list):
1492
+ raise ValueError("provenance `entries` must be an array")
1493
+ libraries = [e.get("source_library") for e in self.doc["entries"]
1494
+ if isinstance(e, dict) and isinstance(e.get("source_library"), str) and e["source_library"]]
1495
+ self.library = libraries[0] if libraries else skill_name
1496
+ self.added: list[str] = []
1497
+ self.updated: list[str] = []
1498
+ self.removed: list[str] = []
1499
+ self.renamed: list[tuple[str, str]] = [] # (old name, new name)
1500
+ self.touched_files: set[str] = set()
1501
+
1502
+ def find(self, name: str, path: str | None, line: int | None = None) -> int | None:
1503
+ """The index of the entry with this name (and file, and line when several share both)."""
1504
+ hits = [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _name_of(e) == name
1505
+ and (path is None or _file_of(e) == path)]
1506
+ if len(hits) > 1 and line is not None:
1507
+ hits = [i for i in hits if _line_of(self.doc["entries"][i].get("source_line")) == line]
1508
+ return hits[0] if len(hits) == 1 else None
1509
+
1510
+ def indexes_of(self, path: str) -> list[int]:
1511
+ return [i for i, e in enumerate(self.doc["entries"]) if isinstance(e, dict) and _file_of(e) == path]
1512
+
1513
+ def add(self, entry: dict) -> None:
1514
+ self.doc["entries"].append(entry)
1515
+ self.added.append(entry["export_name"])
1516
+ if entry.get("source_file"):
1517
+ self.touched_files.add(_norm_path(entry["source_file"]))
1518
+
1519
+ def update(self, index: int, fields: dict) -> None:
1520
+ before = self.doc["entries"][index]
1521
+ after = _rewrite(before, fields)
1522
+ if after != before:
1523
+ self.doc["entries"][index] = after
1524
+ self.updated.append(_name_of(after))
1525
+ if _name_of(before) and _name_of(after) and _name_of(before) != _name_of(after):
1526
+ self.renamed.append((_name_of(before), _name_of(after)))
1527
+ for path in (_file_of(before), _file_of(after)):
1528
+ if path:
1529
+ self.touched_files.add(path)
1530
+
1531
+ def remove(self, indexes: list[int]) -> None:
1532
+ for index in sorted(set(indexes), reverse=True):
1533
+ entry = self.doc["entries"].pop(index)
1534
+ self.removed.append(_name_of(entry))
1535
+ if _file_of(entry):
1536
+ self.touched_files.add(_file_of(entry))
1537
+
1538
+
1539
+ def _apply_normal(pmap: _Map, manifest: dict | None, categories: dict | None, fresh: _Fresh,
1540
+ warnings: list[str], not_public: list[dict]) -> tuple[int, int]:
1541
+ """Normal mode: the manifest's file and export changes. Returns (files, exports) for the update block; each
1542
+ name it leaves out as off a public-api skill's public surface goes to `not_public`."""
1543
+ per_file = [f for f in _get_list(manifest or {}, "per_file") if isinstance(f, dict)]
1544
+ # the names of the entries this run removes (a DELETED_EXPORT, a DELETED file's, a MOVED file's old path's):
1545
+ # one that comes back in another file is re-added whatever its mark, so no held export is lost
1546
+ removed: set[str] = set()
1547
+ for item in per_file:
1548
+ path = _norm_path(item.get("old_path") if item.get("status") == "MOVED" else item.get("file_path"))
1549
+ if item.get("status") in ("DELETED", "MOVED") and path:
1550
+ removed.update(_name_of(pmap.doc["entries"][i]) for i in pmap.indexes_of(path))
1551
+ removed.update(_name_of(c) for c in item.get("exports_affected") or []
1552
+ if isinstance(c, dict) and c.get("change_type") == "DELETED_EXPORT")
1553
+ listed: set[tuple[str, str]] = set()
1554
+
1555
+ def off_surface(name: str, path: str) -> bool:
1556
+ """True for a name the run would add that is off the public surface, which it lists."""
1557
+ if name in removed or fresh.public(name, path) is not False:
1558
+ return False
1559
+ if (name, path) not in listed:
1560
+ listed.add((name, path))
1561
+ not_public.append({"name": name, "file": path})
1562
+ return True
1563
+
1564
+ def refresh(index: int | None, name: str, path: str, *, add: bool) -> None:
1565
+ record = fresh.record(name, path)
1566
+ if record is None:
1567
+ warnings.append(f"provenance: {name} in {path}: no extraction record, entry "
1568
+ f"{'kept as it was' if index is not None else 'not added'}")
1569
+ return
1570
+ if index is None:
1571
+ if add and not off_surface(name, path):
1572
+ pmap.add(_new_entry(record, pmap.library))
1573
+ return
1574
+ pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1575
+
1576
+ for item in per_file:
1577
+ status, path = item.get("status"), _norm_path(item.get("file_path"))
1578
+ if not path:
1579
+ continue
1580
+ if status == "DELETED":
1581
+ pmap.remove(pmap.indexes_of(path))
1582
+ continue
1583
+ if status == "MOVED":
1584
+ old = _norm_path(item.get("old_path"))
1585
+ old_entries = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i]
1586
+ for i in (pmap.indexes_of(old) if old else [])}
1587
+ pmap.remove(pmap.indexes_of(old) if old else [])
1588
+ for name, key_path in fresh.keys_of(path):
1589
+ record = fresh.record(name, key_path)
1590
+ carried = old_entries.get(name)
1591
+ if carried is None and off_surface(name, key_path):
1592
+ continue
1593
+ entry = _rewrite(carried, _without_none(record, keep=("ast_node_type",))) if carried \
1594
+ else _new_entry(record, pmap.library)
1595
+ pmap.add(entry)
1596
+ continue
1597
+ if status == "ADDED":
1598
+ for name, key_path in fresh.keys_of(path):
1599
+ index = pmap.find(name, key_path)
1600
+ refresh(index, name, key_path, add=True)
1601
+ continue
1602
+ for change in item.get("exports_affected") or []:
1603
+ if not isinstance(change, dict) or not _name_of(change):
1604
+ continue
1605
+ name, kind = _name_of(change), change.get("change_type")
1606
+ # a move across files (a MOVED_EXPORT, or a MODIFIED_EXPORT that also moved) names the file its entry is at
1607
+ index = pmap.find(name, _norm_path(change.get("old_file")) or path, _line_of(change.get("old_line")))
1608
+ if kind == "DELETED_EXPORT":
1609
+ if index is not None:
1610
+ pmap.remove([index])
1611
+ elif kind == "MOVED_EXPORT":
1612
+ record = fresh.record(name, path)
1613
+ line = record["source_line"] if record and record.get("source_line") is not None \
1614
+ else _line_of(change.get("new_line"))
1615
+ if index is not None and line is not None:
1616
+ fields = {"source_file": path, "source_line": line}
1617
+ if record is not None and record["extraction_method"] == "ast-grep":
1618
+ # the runner matched it: its labels, never its export_type, params or return_type
1619
+ fields.update(confidence=record["confidence"], extraction_method=record["extraction_method"])
1620
+ if record["ast_node_type"] is not None:
1621
+ fields["ast_node_type"] = record["ast_node_type"]
1622
+ entry = pmap.doc["entries"][index]
1623
+ # its signature_source only when the record holds each signature part the entry does
1624
+ if all(record.get(part) is not None for part in ("params", "return_type")
1625
+ if entry.get(part) is not None):
1626
+ fields["signature_source"] = record["signature_source"]
1627
+ pmap.update(index, fields)
1628
+ elif kind in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1629
+ refresh(index, name, path, add=True)
1630
+ for rename in _get_list(categories or {}, "category_c", "renamed_exports"):
1631
+ if not isinstance(rename, dict):
1632
+ continue
1633
+ old_name, new_name = rename.get("old_name"), rename.get("new_name")
1634
+ path = _norm_path(rename.get("file"))
1635
+ old_path = _norm_path(rename.get("old_file")) or path
1636
+ if not (isinstance(old_name, str) and isinstance(new_name, str) and path):
1637
+ continue
1638
+ index = pmap.find(old_name, old_path)
1639
+ record = fresh.record(new_name, path)
1640
+ if index is None:
1641
+ continue
1642
+ fields = {"export_name": new_name, "source_file": path}
1643
+ if record is not None:
1644
+ fields.update(_without_none(record, keep=("ast_node_type",)))
1645
+ pmap.update(index, fields)
1646
+ files = len(per_file)
1647
+ total = manifest.get("total_export_changes") if isinstance(manifest, dict) else None
1648
+ return files, total if isinstance(total, int) else 0
1649
+
1650
+
1651
+ def _apply_docs_only(pmap: _Map, records: dict) -> tuple[int, int]:
1652
+ changed = [u for u in records.get("changed_urls") or [] if isinstance(u, str) and u]
1653
+ exports = [e for e in records.get("exports") or [] if isinstance(e, dict)]
1654
+ for url in changed:
1655
+ old = {_name_of(pmap.doc["entries"][i]): pmap.doc["entries"][i] for i in pmap.indexes_of(url)}
1656
+ pmap.remove(pmap.indexes_of(url))
1657
+ for record in exports:
1658
+ if record.get("url") != url or not _name_of(record):
1659
+ continue
1660
+ fields = {"export_name": _name_of(record), "export_type": record.get("export_type", record.get("type")),
1661
+ "params": record.get("params", record.get("parameters")),
1662
+ "return_type": record.get("return_type"), "source_file": url,
1663
+ "confidence": "T3", "signature_source": "T3"}
1664
+ carried = old.get(_name_of(record))
1665
+ entry = _rewrite(carried, _without_none(fields)) if carried else _new_entry(
1666
+ _without_none(fields), pmap.library)
1667
+ pmap.add(entry)
1668
+ return len(changed), len(exports)
1669
+
1670
+
1671
+ def _drift_note(drift: tuple | None) -> str:
1672
+ return f"drift override: HEAD {drift[0]} is not pinned {drift[1]}" if drift else ""
1673
+
1674
+
1675
+ def _apply_gap_driven(pmap: _Map, records: dict, merge_records: dict | None, drift: tuple | None,
1676
+ warnings: list[str]) -> tuple[int, int] | list[dict]:
1677
+ """Gap-driven mode: one write per verification record. A list return is the refusal."""
1678
+ verification = [r for r in records.get("verification") or [] if isinstance(r, dict) and _name_of(r)]
1679
+ reextracted = _Fresh(records.get("files") or [], None, None)
1680
+ merged = {}
1681
+ for record in _exports_of(merge_records, "--merge-records file") if merge_records is not None else []:
1682
+ if _name_of(record):
1683
+ merged.setdefault(_name_of(record), record)
1684
+
1685
+ def in_map(r: dict) -> bool:
1686
+ return bool(r.get("in_map")) or bool(r.get("map_entry"))
1687
+
1688
+ blocking = [{"export_name": _name_of(r), "severity": r.get("severity")} for r in verification
1689
+ if r.get("verification") == "unknown" and not in_map(r)
1690
+ and r.get("gap_category") in ("NEW_EXPORT", "MODIFIED_EXPORT")
1691
+ and str(r.get("severity") or "").strip().lower() not in NON_BLOCKING]
1692
+ if blocking:
1693
+ return blocking
1694
+ note = _drift_note(drift)
1695
+ pinned_keys: set[tuple] = set()
1696
+ for r in verification:
1697
+ name, outcome, category = _name_of(r), r.get("verification"), r.get("gap_category")
1698
+ entry_file, entry_line = _location(r.get("map_entry"))
1699
+ index = pmap.find(name, entry_file, entry_line) if r.get("map_entry") else None
1700
+ lines = r.get("pinned_definition_lines")
1701
+ r5 = f"; definition lines per the test report: {lines}" if category == "MOVED_EXPORT" else ""
1702
+ if outcome == "rescoped":
1703
+ if index is not None:
1704
+ pmap.remove([index])
1705
+ elif outcome == "verified" and index is not None:
1706
+ if drift and category == "MOVED_EXPORT":
1707
+ warnings.append(f"provenance: {name}: verified at HEAD only ({note}{r5})")
1708
+ elif outcome == "moved" and index is not None:
1709
+ path, line = _location(r.get("new_location"))
1710
+ if path and line is not None:
1711
+ pmap.update(index, {"source_file": path, "source_line": line})
1712
+ elif outcome == "re-extracted":
1713
+ if r.get("reachability") == "internal-unreachable":
1714
+ continue # not public API: merge queued it as a metadata update
1715
+ path, line = _location(r.get("new_location"))
1716
+ record = reextracted.record(name, path) if path else None
1717
+ if record is None:
1718
+ warnings.append(f"provenance: {name}: re-extracted, but no record of it in the run's files")
1719
+ continue
1720
+ existing = pmap.find(name, path)
1721
+ if existing is None:
1722
+ pmap.add(_new_entry(record, pmap.library))
1723
+ else:
1724
+ pmap.update(existing, _without_none(record, keep=("ast_node_type",)))
1725
+ elif outcome in ("verified", "moved") and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1726
+ if r.get("reachability") == "internal-unreachable":
1727
+ continue
1728
+ where = r.get("source_citation") if outcome == "verified" else r.get("new_location")
1729
+ path, line = _location(where)
1730
+ if not path or line is None or (name, path) in pinned_keys or pmap.find(name, path) is not None:
1731
+ continue
1732
+ pinned_keys.add((name, path))
1733
+ extra = merged.get(name, {})
1734
+ pmap.add(_new_entry(_without_none({
1735
+ "export_name": name,
1736
+ "export_type": extra.get("export_type") or r.get("export_type"),
1737
+ "params": extra.get("params"), "return_type": extra.get("return_type"),
1738
+ "source_file": path, "source_line": line, **_labels("source-read", None)},
1739
+ keep=("ast_node_type",)), pmap.library))
1740
+ elif outcome == "unknown" and not in_map(r) and category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
1741
+ extra = merged.get(name, {})
1742
+ pmap.add(_new_entry(_without_none({
1743
+ "export_name": name, "export_type": extra.get("export_type") or r.get("export_type"),
1744
+ "params": extra.get("params"), "return_type": extra.get("return_type"),
1745
+ **_labels("source-read", None)}, keep=("ast_node_type",)), pmap.library))
1746
+ else:
1747
+ if r.get("unknown_reason") == "drift-override":
1748
+ warnings.append(f"provenance: {name}: unknown ({note}; no line taken from HEAD{r5})")
1749
+ elif drift:
1750
+ warnings.append(f"provenance: {name}: {outcome} ({note}{r5})")
1751
+ else:
1752
+ warnings.append(f"provenance: {name}: {outcome}{r5}")
1753
+ return len(pmap.touched_files), len(verification)
1754
+
1755
+
1756
+ def _apply_files(pmap: _Map, compare: dict | None, new_files: dict | None, promoted: list | None,
1757
+ source_root: Path | None) -> dict:
1758
+ """The file_entries[] changes of Category D and the promoted documents."""
1759
+ rows = pmap.doc.get("file_entries")
1760
+ if rows is None:
1761
+ rows = []
1762
+ if not isinstance(rows, list):
1763
+ raise ValueError("provenance `file_entries` must be an array")
1764
+ done = {"added": [], "updated": [], "removed": []}
1765
+ by_source = {_norm_path(r.get("source_file")): i for i, r in enumerate(rows) if isinstance(r, dict)}
1766
+ gone = set()
1767
+ for row in _get_list(compare or {}, "comparisons"):
1768
+ if not isinstance(row, dict):
1769
+ continue
1770
+ path, kind = _norm_path(row.get("source_file")), row.get("classification")
1771
+ index = by_source.get(path)
1772
+ if index is None:
1773
+ continue
1774
+ if kind == "MODIFIED_FILE" and isinstance(row.get("current_hash"), str):
1775
+ if rows[index].get("content_hash") != row["current_hash"]:
1776
+ rows[index] = {**rows[index], "content_hash": row["current_hash"]}
1777
+ done["updated"].append(path)
1778
+ elif kind == "DELETED_FILE":
1779
+ gone.add(index)
1780
+ done["removed"].append(path)
1781
+ rows = [r for i, r in enumerate(rows) if i not in gone]
1782
+ held = {_norm_path(r.get("source_file")) for r in rows if isinstance(r, dict)}
1783
+ for item in _get_list(new_files or {}, "new_files"):
1784
+ if not isinstance(item, dict):
1785
+ continue
1786
+ path, kind = _norm_path(item.get("source_file")), item.get("kind")
1787
+ if not path or path in held or kind not in ("script", "asset"):
1788
+ continue
1789
+ if source_root is None:
1790
+ raise ValueError("--new-files needs --source-root, to hash each new file")
1791
+ try:
1792
+ digest = _sha256_of(source_root / path)
1793
+ except OSError as exc:
1794
+ raise ValueError(f"cannot hash new file {path} under {source_root}: {exc}") from exc
1795
+ rows.append({"file_name": f"{kind}s/{Path(path).name}", "file_type": kind, "source_file": path,
1796
+ "confidence": "T1-low", "extraction_method": "file-copy", "content_hash": digest})
1797
+ held.add(path)
1798
+ done["added"].append(path)
1799
+ for doc in promoted or []:
1800
+ if not isinstance(doc, dict):
1801
+ continue
1802
+ path = _norm_path(doc.get("path"))
1803
+ if not path or path in held or not isinstance(doc.get("content_hash"), str):
1804
+ continue
1805
+ rows.append({"file_name": f"docs/authoritative/{path}", "file_type": "doc", "source_file": path,
1806
+ "content_hash": doc["content_hash"], "confidence": "T1-low",
1807
+ "extraction_method": "promoted-authoritative"})
1808
+ held.add(path)
1809
+ done["added"].append(path)
1810
+ if rows or "file_entries" in pmap.doc:
1811
+ pmap.doc["file_entries"] = rows
1812
+ return done
1813
+
1814
+
1815
+ REFERENCE_APP = "reference-app"
1816
+
1817
+
1818
+ def _export_name(item: object) -> str | None:
1819
+ """The name of a metadata.json exports[] item: a string, or an object's name or export_name."""
1820
+ if isinstance(item, str):
1821
+ return item or None
1822
+ return _name_of(item) if isinstance(item, dict) else None
1823
+
1824
+
1825
+ def exports_after(metadata: object, pmap: _Map, update_type: str) -> list:
1826
+ """metadata.json's exports[] after the update (see "Exports" in the module docstring)."""
1827
+ if not isinstance(metadata, dict):
1828
+ raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
1829
+ if metadata.get("scope_type") == REFERENCE_APP:
1830
+ return []
1831
+ current = metadata.get("exports") if isinstance(metadata.get("exports"), list) else []
1832
+ held = {_name_of(e) for e in pmap.doc["entries"] if isinstance(e, dict)} - {None}
1833
+ renamed = dict(pmap.renamed)
1834
+ gone = ({n for n in pmap.removed if n} | set(renamed)) - held
1835
+ if update_type == "full":
1836
+ gone |= {n for n in map(_export_name, current) if n and n not in held}
1837
+ out: list = []
1838
+ seen: set[str] = set()
1839
+
1840
+ def put(item: object, name: str | None) -> None:
1841
+ if name and name not in seen:
1842
+ seen.add(name)
1843
+ out.append(item)
1844
+
1845
+ for item in current:
1846
+ name = _export_name(item)
1847
+ if name in gone:
1848
+ new = renamed.get(name)
1849
+ put(new, new if new in held else None)
1850
+ else:
1851
+ put(item, name)
1852
+ for name in pmap.added + [new for _old, new in pmap.renamed]:
1853
+ put(name, name if name in held else None)
1854
+ return out
1855
+
1856
+
1857
+ _DETECTOR = None
1858
+
1859
+
1860
+ def _detector():
1861
+ """skf-detect-scripts-assets.py, loaded once from this folder: the purpose a new script's or asset's row
1862
+ takes, read from the file as create-skill's detector reads it."""
1863
+ global _DETECTOR
1864
+ if _DETECTOR is None:
1865
+ path = Path(__file__).resolve().parent / "skf-detect-scripts-assets.py"
1866
+ spec = importlib.util.spec_from_file_location("skf_detect_scripts_assets", path)
1867
+ if spec is None or spec.loader is None or not path.is_file():
1868
+ raise ValueError(f"cannot load {path.name} beside {Path(__file__).name}: re-install SKF")
1869
+ module = importlib.util.module_from_spec(spec)
1870
+ spec.loader.exec_module(module)
1871
+ _DETECTOR = module
1872
+ return _DETECTOR
1873
+
1874
+
1875
+ def _purpose(source_file: str, kind: str, source_root: Path | None) -> str:
1876
+ """A new script's or asset's purpose: the detector's, read from the file under `source_root`, else its name."""
1877
+ path = source_root / source_file if source_root is not None else None
1878
+ if path is None or not path.is_file():
1879
+ return Path(source_file).name
1880
+ detector = _detector()
1881
+ hint = detector.asset_type(path, source_root) if kind == "asset" else None
1882
+ return detector.extract_purpose(path, asset_type_hint=hint)
1883
+
1884
+
1885
+ def file_rows_after(metadata: dict, pmap: _Map, source_root: Path | None) -> dict[str, list]:
1886
+ """metadata.json's scripts[] and assets[] after the update (see "Lists" in the module docstring)."""
1887
+ by_source: dict[str, dict] = {}
1888
+ by_file: dict[str, dict] = {}
1889
+ for key in ("scripts", "assets"):
1890
+ rows = metadata.get(key)
1891
+ for row in rows if isinstance(rows, list) else []:
1892
+ if not isinstance(row, dict):
1893
+ continue
1894
+ source = _norm_path(row.get("source_file"))
1895
+ if source:
1896
+ by_source.setdefault(source, row)
1897
+ if isinstance(row.get("file"), str):
1898
+ by_file.setdefault(row["file"], row)
1899
+ lists: dict[str, list] = {"scripts": [], "assets": []}
1900
+ entries = pmap.doc.get("file_entries")
1901
+ for entry in entries if isinstance(entries, list) else []:
1902
+ if not isinstance(entry, dict) or _file_type(entry) not in ("script", "asset"):
1903
+ continue
1904
+ kind = _file_type(entry)
1905
+ source = _norm_path(entry.get("source_file"))
1906
+ held = by_source.get(source) if source else None
1907
+ if held is None and isinstance(entry.get("file_name"), str):
1908
+ held = by_file.get(entry["file_name"])
1909
+ if held is None:
1910
+ held = {"file": entry.get("file_name") or f"{kind}s/{Path(source or '').name}",
1911
+ "purpose": _purpose(source, kind, source_root) if source else "",
1912
+ "source_file": source, "confidence": entry.get("confidence") or "T1-low"}
1913
+ lists[f"{kind}s"].append(held)
1914
+ return lists
1915
+
1916
+
1917
+ def apply_update(*, update_type: str, provenance: dict | None, skill_name: str, manifest: dict | None = None,
1918
+ categories: dict | None = None, extraction: dict | None = None, details: dict | None = None,
1919
+ records: dict | None = None, merge_records: dict | None = None, compare: dict | None = None,
1920
+ new_files: dict | None = None, promoted: list | None = None, source_root: Path | None = None,
1921
+ generation_date: str, test_report_run_id: str | None, confidence_tier: str,
1922
+ manual_sections_preserved: int, source_commit: str | None = None,
1923
+ source_ref: str | None = None, drift: tuple | None = None,
1924
+ metadata: dict | None = None) -> tuple[dict, dict]:
1925
+ """(new map, summary) of one update (see "Apply" in the module docstring). With `metadata` (the skill's
1926
+ metadata.json), the summary's `lists` holds the exports[], scripts[] and assets[] it takes after the update.
1927
+
1928
+ A refusal returns ({}, summary) with status "refused" and no map.
1929
+ Raises ValueError for inputs it cannot read.
1930
+ """
1931
+ if update_type not in UPDATE_TYPES:
1932
+ raise ValueError(f"--update-type must be one of {', '.join(UPDATE_TYPES)}, got {update_type!r}")
1933
+ if provenance is None and update_type != "full":
1934
+ raise ValueError("--provenance-map is required unless --update-type is full (a degraded run)")
1935
+ records = records if isinstance(records, dict) else {}
1936
+ pmap = _Map(provenance, skill_name)
1937
+ warnings: list[str] = []
1938
+ not_public: list[dict] = []
1939
+ mode = records.get("mode")
1940
+ if update_type == "gap-driven":
1941
+ outcome = _apply_gap_driven(pmap, records, merge_records, drift, warnings)
1942
+ if isinstance(outcome, list):
1943
+ return {}, {"status": "refused", "blocking_unresolved": outcome}
1944
+ files, exports = outcome
1945
+ elif mode == "docs-only":
1946
+ files, exports = _apply_docs_only(pmap, records)
1947
+ else:
1948
+ fresh = _Fresh(records.get("files") or [], extraction, details)
1949
+ if update_type == "full":
1950
+ for name, path in fresh.all_keys():
1951
+ index = pmap.find(name, path)
1952
+ record = fresh.record(name, path)
1953
+ if index is None:
1954
+ pmap.add(_new_entry(record, pmap.library))
1955
+ else:
1956
+ pmap.update(index, _without_none(record, keep=("ast_node_type",)))
1957
+ files = len({path for _name, path in fresh.all_keys()})
1958
+ exports = len(fresh.all_keys())
1959
+ else:
1960
+ files, exports = _apply_normal(pmap, manifest, categories, fresh, warnings, not_public)
1961
+ if fresh.surface_problem:
1962
+ warnings.append(f"provenance: the public surface of this public-api skill could not be applied "
1963
+ f"({fresh.surface_problem}): every new export was added")
1964
+ if not_public:
1965
+ warnings.append(_not_public_warning(not_public))
1966
+ file_changes = _apply_files(pmap, compare, new_files, promoted, source_root)
1967
+ doc = pmap.doc
1968
+ for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
1969
+ if value is not None:
1970
+ doc[key] = value or None
1971
+ block = {
1972
+ "last_update": generation_date,
1973
+ "update_type": update_type,
1974
+ "test_report_run_id": test_report_run_id or None,
1975
+ "files_changed": files,
1976
+ "exports_affected": exports,
1977
+ "confidence_tier": confidence_tier,
1978
+ "manual_sections_preserved": manual_sections_preserved,
1979
+ }
1980
+ for key in UPDATE_BLOCK_KEYS:
1981
+ doc[key] = block[key]
1982
+ summary = {
1983
+ "status": "written",
1984
+ "entries": {"added": pmap.added, "updated": pmap.updated, "removed": pmap.removed},
1985
+ "not_public": not_public,
1986
+ "file_entries": file_changes,
1987
+ "warnings": warnings,
1988
+ }
1989
+ if metadata is not None:
1990
+ summary["lists"] = {"exports": exports_after(metadata, pmap, update_type),
1991
+ **file_rows_after(metadata, pmap, source_root)}
1992
+ return doc, summary
1993
+
1994
+
1995
+ GENERATION_DATE_RE = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ")
1996
+ # the stats fields skf-render-metadata-stats.py writes, some only for a shape or a judgment
1997
+ STATS_HELPER_FIELDS = ("exports_documented", "exports_public_api", "exports_internal", "exports_total",
1998
+ "public_api_coverage", "total_coverage", "scripts_count", "assets_count",
1999
+ "pattern_surfaces_documented", "effective_denominator")
2000
+
2001
+
2002
+ def update_metadata(metadata: object, lists: object, stats: object, *, version: str, generation_date: str,
2003
+ source_commit: str | None = None, source_ref: str | None = None) -> dict:
2004
+ """The metadata.json an update leaves (see "Metadata" in the module docstring). Raises ValueError."""
2005
+ if not isinstance(metadata, dict):
2006
+ raise ValueError("--metadata file must hold a JSON object (the skill's metadata.json)")
2007
+ if not isinstance(lists, dict) or not all(isinstance(lists.get(k), list) for k in ("exports", "scripts",
2008
+ "assets")):
2009
+ raise ValueError("--lists file must hold the exports, scripts and assets lists apply wrote (--lists-out)")
2010
+ if not isinstance(stats, dict) or not isinstance(stats.get("stats"), dict) \
2011
+ or not isinstance(stats.get("confidence_distribution"), dict):
2012
+ raise ValueError("--stats file must hold the stats helper's JSON: a stats and a confidence_distribution "
2013
+ "object")
2014
+ if not version.strip():
2015
+ raise ValueError("--version must not be empty")
2016
+ if not GENERATION_DATE_RE.fullmatch(generation_date):
2017
+ raise ValueError(f"--generation-date must be YYYY-MM-DDTHH:MM:SSZ, read from the clock; got "
2018
+ f"{generation_date!r}")
2019
+ for kind in ("scripts", "assets"):
2020
+ counted = stats["stats"].get(f"{kind}_count")
2021
+ if counted != len(lists[kind]):
2022
+ raise ValueError(f"the stats count {counted} {kind} but apply's lists hold {len(lists[kind])}: run the "
2023
+ f"stats helper again with the lists' {kind}")
2024
+ out = {k: v for k, v in metadata.items() if k not in ("last_update", "update_type")}
2025
+ out["version"] = version
2026
+ out["generation_date"] = generation_date
2027
+ for key, value in (("source_commit", source_commit), ("source_ref", source_ref)):
2028
+ if value is not None:
2029
+ out[key] = value or None
2030
+ out["exports"] = lists["exports"]
2031
+ for kind in ("scripts", "assets"):
2032
+ if lists[kind]:
2033
+ out[kind] = lists[kind]
2034
+ else:
2035
+ out.pop(kind, None)
2036
+ old_stats = metadata.get("stats") if isinstance(metadata.get("stats"), dict) else {}
2037
+ kept = {k: v for k, v in old_stats.items() if k not in STATS_HELPER_FIELDS and k not in stats["stats"]}
2038
+ out["stats"] = {**stats["stats"], **kept}
2039
+ out["confidence_distribution"] = stats["confidence_distribution"]
2040
+ return out
2041
+
2042
+
2043
+ NOT_PUBLIC_NAMED = 10 # the names the not_public warning spells out; the summary lists them all
2044
+
2045
+
2046
+ def _not_public_warning(not_public: list[dict]) -> str:
2047
+ """The one warning for every name apply left out as off a public-api skill's public surface."""
2048
+ names = [item["name"] for item in not_public]
2049
+ shown = ", ".join(names[:NOT_PUBLIC_NAMED])
2050
+ more = f" and {len(names) - NOT_PUBLIC_NAMED} more" if len(names) > NOT_PUBLIC_NAMED else ""
2051
+ return (f"provenance: {len(names)} new export(s) off the public surface of this public-api skill not added "
2052
+ f"(the summary's not_public lists each with its file): {shown}{more}")
2053
+
2054
+
2055
+ def _write_json_atomic(path: Path, value) -> None:
2056
+ """Write `value` as indented JSON through a temporary file beside `path` and one rename, retried on Windows,
2057
+ where a scanner may hold the target open for a moment."""
2058
+ tmp = path.with_name(f".{path.name}.skf-{os.getpid()}-tmp")
2059
+ try:
2060
+ tmp.write_bytes((json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
2061
+ deadline = time.monotonic() + REPLACE_WAIT_SECONDS
2062
+ while True:
2063
+ try:
2064
+ os.replace(tmp, path)
2065
+ break
2066
+ except PermissionError:
2067
+ if os.name != "nt" or time.monotonic() >= deadline:
2068
+ raise
2069
+ time.sleep(0.05)
2070
+ except OSError:
2071
+ try:
2072
+ tmp.unlink()
2073
+ except OSError:
2074
+ pass
2075
+ raise
2076
+
2077
+
2078
+ # --------------------------------------------------------------------------
2079
+ # Records (update-skill re-extract.md §4, gap-driven.md §4a)
2080
+ # --------------------------------------------------------------------------
2081
+
2082
+
2083
+ # The fields a worker's patch may set: the ones no tool records, and the two
2084
+ # a patch sets only where the seed has none.
2085
+ PATCH_FIELDS = ("signature", "members", "docstring", "qmd_evidence")
2086
+ PATCH_IF_NULL = ("params", "return_type")
2087
+
2088
+
2089
+ def _typed_param(param, language: object) -> str | None:
2090
+ """One parameter as the provenance map writes it: skf-extraction-inventory.py's typed_param, so create-skill
2091
+ and update-skill write one form."""
2092
+ return _inventory().typed_param(param, language if isinstance(language, str) else None)
2093
+
2094
+
2095
+ def _typed_params(value, language: object) -> list | None:
2096
+ if not isinstance(value, list):
2097
+ return None
2098
+ return [s for s in (_typed_param(p, language) for p in value) if s is not None]
2099
+
2100
+
2101
+ def _seed(record: dict, path: str, read_by_eye: bool) -> dict:
2102
+ """The re-extraction record of one export from the runner's (or, read by eye, step 2's) record."""
2103
+ line = _line_of(record.get("source_line"))
2104
+ out = {
2105
+ "name": _name_of(record),
2106
+ "type": record.get("export_type", record.get("type")),
2107
+ "signature": record.get("signature"),
2108
+ "location": f"{path}:{line}" if line is not None else None,
2109
+ "params": _typed_params(record.get("params", record.get("parameters")), record.get("language")),
2110
+ "return_type": record.get("return_type"),
2111
+ }
2112
+ labels = _labels("source-read" if read_by_eye else record.get("extraction_method"), record.get("ast_node_type"))
2113
+ out.update({key: labels[key] for key in ("confidence", "extraction_method", "ast_node_type")})
2114
+ out["ast_recipe"] = None if read_by_eye else record.get("ast_recipe")
2115
+ return out
2116
+
2117
+
2118
+ def _patch_blocks(folder: Path) -> list[dict]:
2119
+ """Every per-file block of the patch files in `folder` (a file holds one block or a list of them), in file
2120
+ name order. Raises ValueError."""
2121
+ if not folder.is_dir():
2122
+ raise ValueError(f"--patches {folder} is not a folder")
2123
+ blocks = []
2124
+ for path in sorted(folder.glob("*.json"), key=lambda p: p.name):
2125
+ doc = _load_json_file(path, "patch file")
2126
+ for block in doc if isinstance(doc, list) else [doc]:
2127
+ if not isinstance(block, dict) or not _norm_path(block.get("file_path")) \
2128
+ or not isinstance(block.get("exports"), list):
2129
+ raise ValueError(f"patch file {path} holds a block with no file_path or exports list")
2130
+ blocks.append(block)
2131
+ return blocks
2132
+
2133
+
2134
+ def build_records(extraction: dict | None, details: dict | None, files: list | None,
2135
+ patches: list[dict]) -> tuple[dict, dict]:
2136
+ """(reextract-records.json, summary): each export's record seeded from the runner's and step 2's records, with
2137
+ the workers' patches merged in (see "Records" in the module docstring)."""
2138
+ runner, extra = {}, {}
2139
+ for source, doc, what in ((runner, extraction, "--extraction file"), (extra, details, "--export-details file")):
2140
+ for record in _exports_of(doc, what):
2141
+ key = (_name_of(record), _file_of(record))
2142
+ if all(key):
2143
+ source.setdefault(key, record)
2144
+ seeded: dict[tuple, dict] = {}
2145
+ languages: dict[tuple, object] = {}
2146
+ warnings = []
2147
+
2148
+ def fill_if_null(key: tuple, record: dict) -> None:
2149
+ """Set params and return_type from `record` where the seed of `key` has none."""
2150
+ for field in PATCH_IF_NULL:
2151
+ value = record.get(field)
2152
+ if field == "params" and value is None:
2153
+ value = record.get("parameters")
2154
+ if seeded[key][field] is None and value is not None:
2155
+ seeded[key][field] = _typed_params(value, languages.get(key)) if field == "params" else value
2156
+
2157
+ for key, record in runner.items():
2158
+ seeded[key], languages[key] = _seed(record, key[1], read_by_eye=False), record.get("language")
2159
+ fill_if_null(key, extra.get(key, {}))
2160
+ for key, record in extra.items():
2161
+ if key in runner:
2162
+ continue
2163
+ if record.get("export_type", record.get("type")) is None:
2164
+ warnings.append(f"re-extract: {key[0]} in {key[1]}: an export-details record with no export_type the "
2165
+ f"runner does not hold; not added")
2166
+ continue
2167
+ seeded[key], languages[key] = _seed(record, key[1], read_by_eye=True), record.get("language")
2168
+ for block in patches:
2169
+ path = _norm_path(block.get("file_path"))
2170
+ for patch in block.get("exports") or []:
2171
+ key = (_name_of(patch) if isinstance(patch, dict) else None, path)
2172
+ if key not in seeded:
2173
+ warnings.append(f"re-extract: a patch names {key[0] or 'no export'} in {path}, which neither the "
2174
+ f"runner nor step 2 recorded; not added")
2175
+ continue
2176
+ for field in PATCH_FIELDS:
2177
+ if patch.get(field) is not None:
2178
+ seeded[key][field] = patch[field]
2179
+ fill_if_null(key, patch)
2180
+ surface, _problem = public_surface(extraction) # apply warns when a public-api run gives no surface
2181
+ for key in seeded if surface is not None else ():
2182
+ mark = surface.public(key[0], key[1], languages.get(key))
2183
+ if mark is not None:
2184
+ seeded[key]["public"] = mark
2185
+ paths = list(dict.fromkeys([p for p in (_norm_path(f) for f in files or []) if p]
2186
+ + sorted({key[1] for key in seeded})))
2187
+ blocks = [{"file_path": path, "exports": [seeded[key] for key in sorted(seeded) if key[1] == path]}
2188
+ for path in paths]
2189
+ exports = [record for block in blocks for record in block["exports"]]
2190
+ summary = {
2191
+ "files_extracted": len(blocks),
2192
+ "exports_extracted": len(exports),
2193
+ "confidence_breakdown": {
2194
+ "T1": sum(1 for r in exports if r["confidence"] == "T1"),
2195
+ "T1-low": sum(1 for r in exports if r["confidence"] == "T1-low"),
2196
+ "T2": sum(1 for r in exports if r.get("qmd_evidence") is not None),
2197
+ },
2198
+ "marked_not_public": sum(1 for r in exports if r.get("public") is False),
2199
+ "warnings": warnings,
2200
+ }
2201
+ return {"mode": "normal", "files": blocks}, summary
2202
+
2203
+
2204
+ # --------------------------------------------------------------------------
2205
+ # Gap records (update-skill gap-driven.md §3, §4 and §4a)
2206
+ # --------------------------------------------------------------------------
2207
+
2208
+
2209
+ EXPORT_CATEGORIES = ("NEW_EXPORT", "MODIFIED_EXPORT", "MOVED_EXPORT", "DELETED_EXPORT")
2210
+ DOCUMENT_CATEGORIES = ("missing-export", "missing-type") # routed to §4a by category, never halting
2211
+ DRIFT_STATUSES = ("ok", "skipped", "overridden")
2212
+ # The drift gate's reasons, gap-driven.md §3: the first that fits.
2213
+ DRIFT_RESCOPE = "a public API recount from the tree (rule R1)"
2214
+ DRIFT_COMPLETENESS = "a line from the tree (rule R3)"
2215
+ DRIFT_NOT_IN_MAP = "a line and a signature from the tree"
2216
+ DRIFT_IN_MAP = "a signature from the tree"
2217
+ REACHABILITY = ("public", "internal-unreachable")
2218
+ VERIFIER_MISSING = "provenance: spot-checks not run: skf-verify-provenance-completeness.py is missing; re-install SKF"
2219
+ OUTCOMES = ("verified", "moved", "missing", "re-extracted", "rescoped", "unknown")
2220
+ _DEFINITION_LINES = re.compile(r"definition lines?\s*\(([\d,\s]+)\)", re.IGNORECASE)
2221
+ _VERIFIER = None
2222
+
2223
+
2224
+ def _verifier():
2225
+ """skf-verify-provenance-completeness.py, loaded once from this folder for its definition-lines rules, or None
2226
+ when it cannot be loaded."""
2227
+ global _VERIFIER
2228
+ if _VERIFIER is None:
2229
+ path = Path(__file__).resolve().parent / "skf-verify-provenance-completeness.py"
2230
+ try:
2231
+ spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness", path)
2232
+ if spec is None or spec.loader is None:
2233
+ raise ImportError(f"cannot load {path}")
2234
+ module = importlib.util.module_from_spec(spec)
2235
+ spec.loader.exec_module(module)
2236
+ except (OSError, ImportError, SyntaxError):
2237
+ _VERIFIER = False
2238
+ else:
2239
+ _VERIFIER = module
2240
+ return _VERIFIER or None
2241
+
2242
+
2243
+ def _blocking(severity: object) -> bool:
2244
+ return str(severity or "").strip().lower() not in NON_BLOCKING
2245
+
2246
+
2247
+ def _gap_judgments(doc) -> dict:
2248
+ """The answers gap-records reads from a --judgments file, checked. Raises ValueError."""
2249
+ if not isinstance(doc, dict) or not all(isinstance(v, dict) for v in doc.values()):
2250
+ raise ValueError("--judgments must map each gap id to an object of answers")
2251
+ for gid, answers in doc.items():
2252
+ line = answers.get("declaring_line")
2253
+ if "declaring_line" in answers and line is not None and (isinstance(line, bool) or not isinstance(line, int)):
2254
+ raise ValueError(f"--judgments: {gid}'s `declaring_line` must be a line number or null")
2255
+ if "reachability" in answers and answers["reachability"] not in REACHABILITY:
2256
+ raise ValueError(f"--judgments: {gid}'s `reachability` must be one of {', '.join(REACHABILITY)}")
2257
+ if "docstring" in answers and not isinstance(answers["docstring"], (str, type(None))):
2258
+ raise ValueError(f"--judgments: {gid}'s `docstring` must be a string or null")
2259
+ return doc
2260
+
2261
+
2262
+ def _drift_reason(entry: dict) -> str | None:
2263
+ """The drift gate's reason an export-bearing entry needs the tree, or None for one that passes."""
2264
+ category = entry.get("change_category")
2265
+ if category == "DELETED_EXPORT":
2266
+ return DRIFT_RESCOPE
2267
+ if category not in ("NEW_EXPORT", "MODIFIED_EXPORT"):
2268
+ return None
2269
+ if entry.get("provenance_completeness"):
2270
+ return DRIFT_COMPLETENESS
2271
+ status = (entry.get("map_match") or {}).get("status")
2272
+ return DRIFT_IN_MAP if status in ("found", "ambiguous") else DRIFT_NOT_IN_MAP
2273
+
2274
+
2275
+ def _spot_check(root: Path | None, path: str | None, name: str, line: int | None, export_type: object,
2276
+ answers: dict) -> tuple[str, int | None, bool, str | None]:
2277
+ """(outcome, the line a `moved` export moves to, whether it waits on a `declaring_line` answer, why the
2278
+ check failed)."""
2279
+ verifier = _verifier()
2280
+ if verifier is None or root is None or not path or line is None:
2281
+ return "unknown", None, False, None
2282
+ try:
2283
+ report = verifier.definition_lines_report(path, name, root, line,
2284
+ export_type if isinstance(export_type, str) else None)
2285
+ except Exception as exc: # noqa: BLE001 - a file it cannot read, or any verifier failure: this entry only
2286
+ return "unknown", None, False, f"{type(exc).__name__}: {exc}"
2287
+ check = report.get("line_check")
2288
+ if check == "file-missing":
2289
+ return "missing", None, False, None
2290
+ if check == "skipped-export-type":
2291
+ return "verified", None, False, None
2292
+ if check == "skipped-language":
2293
+ if "declaring_line" not in answers:
2294
+ return "unknown", None, True, None
2295
+ declared = answers["declaring_line"]
2296
+ if declared is None:
2297
+ return "unknown", None, False, None
2298
+ return ("verified", None, False, None) if declared == line else ("moved", declared, False, None)
2299
+ if report.get("line_is_definition"):
2300
+ return "verified", None, False, None
2301
+ defs = report.get("definition_lines") or []
2302
+ if len(defs) == 1 and defs[0] != line:
2303
+ return "moved", defs[0], False, None
2304
+ return "unknown", None, False, None
2305
+
2306
+
2307
+ def _pinned_lines(remediation: object) -> list[int] | None:
2308
+ """The definition lines a provenance-line gap's remediation lists, `... definition line (5, 7) ...`."""
2309
+ m = _DEFINITION_LINES.search(remediation) if isinstance(remediation, str) else None
2310
+ lines = [int(n) for n in re.findall(r"\d+", m.group(1))] if m else []
2311
+ return lines or None
2312
+
2313
+
2314
+ def _where(path: object, line: object) -> str | None:
2315
+ file, number = _norm_path(path), _line_of(line)
2316
+ return f"{file}:{number}" if file and number is not None else None
2317
+
2318
+
2319
+ class _GapEntry:
2320
+ """One export-bearing manifest entry as gap-records routes it."""
2321
+
2322
+ def __init__(self, entry: dict, answers: dict):
2323
+ self.entry = entry
2324
+ self.answers = answers
2325
+ self.name = _name_of(entry) or ""
2326
+ self.category = entry.get("change_category")
2327
+ match = entry.get("map_match") if isinstance(entry.get("map_match"), dict) else {}
2328
+ self.match_status = match.get("status") or "not-found"
2329
+ found = match.get("entry") if self.match_status == "found" and isinstance(match.get("entry"), dict) else None
2330
+ self.map_entry = {"source_file": found.get("source_file"), "source_line": found.get("source_line")} \
2331
+ if found else None
2332
+ self.map_export_type = found.get("export_type") if found else None
2333
+ self.in_map = bool(match.get("candidates"))
2334
+ citation = entry.get("source_citation")
2335
+ self.citation = citation if isinstance(citation, dict) else None
2336
+ self.recorded: tuple[str | None, int | None] = (None, None) # the file and line a check read
2337
+ self.outcome: str | None = None
2338
+ self.new_line: int | None = None
2339
+ self.unknown_reason: str | None = None
2340
+ self.reextract = False
2341
+ self.record: dict | None = None # the §4a record it matched
2342
+ self.record_path: str | None = None
2343
+ self.waits = False # on a declaring_line answer
2344
+ self.failure: str | None = None # why its spot-check failed
2345
+
2346
+ @property
2347
+ def blocking(self) -> bool:
2348
+ return _blocking(self.entry.get("severity"))
2349
+
2350
+ @property
2351
+ def paths(self) -> list[str]:
2352
+ return [p for p in (_norm_path(p) for p in self.entry.get("resolved_paths") or []) if p]
2353
+
2354
+ def check(self, root: Path | None, path: object, line: object, export_type: object) -> None:
2355
+ self.recorded = (_norm_path(path), _line_of(line))
2356
+ self.outcome, self.new_line, self.waits, self.failure = _spot_check(
2357
+ root, self.recorded[0], self.name, self.recorded[1], export_type, self.answers)
2358
+
2359
+ def route(self, root: Path | None, drift: bool) -> None:
2360
+ """Spot-check the entry, or send it to targeted re-extraction (see "Gap records" in the module docstring)."""
2361
+ if self.category == "DELETED_EXPORT":
2362
+ self.outcome = "rescoped"
2363
+ if self.map_entry:
2364
+ self.recorded = (_norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"]))
2365
+ return
2366
+ if self.category == "MOVED_EXPORT":
2367
+ if self.map_entry is None:
2368
+ self.outcome = "unknown"
2369
+ return
2370
+ self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
2371
+ if drift and self.outcome == "moved":
2372
+ self.outcome, self.new_line, self.unknown_reason = "unknown", None, "drift-override"
2373
+ return
2374
+ if self.match_status == "found":
2375
+ self.check(root, self.map_entry["source_file"], self.map_entry["source_line"], self.map_export_type)
2376
+ return
2377
+ if self.match_status == "ambiguous":
2378
+ self.outcome = "unknown"
2379
+ return
2380
+ if self.citation:
2381
+ self.check(root, self.citation.get("file"), self.citation.get("line"), None)
2382
+ if self.waits or self.outcome in ("verified", "moved"):
2383
+ return
2384
+ if self.entry.get("provenance_completeness") and root is not None:
2385
+ self.reextract = True
2386
+ elif self.blocking:
2387
+ self.reextract = True
2388
+ elif self.entry.get("category") in DOCUMENT_CATEGORIES and self.paths:
2389
+ self.reextract = True
2390
+ else:
2391
+ self.outcome = "unknown"
2392
+
2393
+ def match(self, blocks: dict[str, list[dict]]) -> tuple[int, int]:
2394
+ """Take the first export of the entry's name in the blocks of its resolved files; (files scanned, exports
2395
+ found in them)."""
2396
+ exports = 0
2397
+ for path in self.paths:
2398
+ for record in blocks.get(path, []):
2399
+ exports += 1
2400
+ if self.record is None and _name_of(record) == self.name:
2401
+ self.record, self.record_path = record, path
2402
+ if self.record is not None:
2403
+ self.outcome = "re-extracted"
2404
+ return len(self.paths), exports
2405
+
2406
+ @property
2407
+ def located(self) -> bool:
2408
+ return self.outcome in ("verified", "moved", "re-extracted")
2409
+
2410
+ @property
2411
+ def internal(self) -> bool:
2412
+ return self.category == "NEW_EXPORT" and self.answers.get("reachability") == "internal-unreachable"
2413
+
2414
+ def needs(self) -> list[str]:
2415
+ """The answers the entry still waits on once routed."""
2416
+ want = []
2417
+ if self.category == "NEW_EXPORT" and self.located and "reachability" not in self.answers:
2418
+ want.append("reachability")
2419
+ if self.outcome == "re-extracted" and "docstring" not in self.answers:
2420
+ want.append("docstring")
2421
+ return want
2422
+
2423
+ def new_location(self) -> str | None:
2424
+ if self.outcome == "moved":
2425
+ return _where(self.recorded[0], self.new_line)
2426
+ if self.outcome == "re-extracted":
2427
+ return (self.record or {}).get("location")
2428
+ return None
2429
+
2430
+ def verification(self) -> dict:
2431
+ recorded = _where(*self.recorded)
2432
+ if self.outcome == "re-extracted":
2433
+ citation = self.new_location()
2434
+ elif self.outcome in ("verified", "moved", "missing", "rescoped") and recorded:
2435
+ citation = recorded
2436
+ else:
2437
+ citation = "unknown"
2438
+ return {
2439
+ "export_name": self.name,
2440
+ "gap_category": self.category,
2441
+ "severity": self.entry.get("severity"),
2442
+ "verification": self.outcome,
2443
+ "in_map": self.in_map,
2444
+ "map_entry": self.map_entry,
2445
+ "provenance_citation": citation,
2446
+ "source_citation": self.citation,
2447
+ "new_location": self.new_location(),
2448
+ "unknown_reason": self.unknown_reason,
2449
+ "pinned_definition_lines": _pinned_lines(self.entry.get("remediation"))
2450
+ if self.category == "MOVED_EXPORT" else None,
2451
+ "resolution_source": "remediation-paths" if self.outcome == "re-extracted" else None,
2452
+ "reachability": self.answers.get("reachability") if self.category == "NEW_EXPORT" and self.located
2453
+ else None,
2454
+ "export_type": None,
2455
+ }
2456
+
2457
+ def label(self, provenance: list[dict]) -> str:
2458
+ """T1, T1-low or unlabeled: the confidence_breakdown bin of the entry."""
2459
+ if self.internal:
2460
+ return "unlabeled"
2461
+ method = None
2462
+ if self.outcome == "re-extracted":
2463
+ method = (self.record or {}).get("extraction_method")
2464
+ elif self.map_entry is not None:
2465
+ file, line = _norm_path(self.map_entry["source_file"]), _line_of(self.map_entry["source_line"])
2466
+ for entry in provenance:
2467
+ if (_name_of(entry) == self.name and _file_of(entry) == file
2468
+ and _line_of(entry.get("source_line")) == line):
2469
+ method = entry.get("extraction_method")
2470
+ break
2471
+ elif self.located and self.category in ("NEW_EXPORT", "MODIFIED_EXPORT"):
2472
+ method = "source-read" # a pinned cited export the map lacks: write.md §3 writes it source-read
2473
+ if method in ("ast-grep", "ast_bridge"):
2474
+ return "T1"
2475
+ if method in ("source-read", "source_reading"):
2476
+ return "T1-low"
2477
+ return "unlabeled"
2478
+
2479
+
2480
+ def gap_records(manifest: dict, provenance: dict | None, *, source_root: Path | None, drift_status: str = "ok",
2481
+ judgments: dict | None = None, remediation: dict | None = None, plan: bool = False,
2482
+ tier: str | None = None) -> tuple[dict | None, dict]:
2483
+ """(reextract-records.json or None, the summary): see "Gap records" in the module docstring."""
2484
+ if not isinstance(manifest, dict) or not isinstance(manifest.get("entries"), list):
2485
+ raise ValueError("the change manifest has no `entries` array: is it translate's output?")
2486
+ entries = [e for e in manifest["entries"] if isinstance(e, dict)]
2487
+ judgments = judgments or {}
2488
+ root = source_root if source_root is not None and source_root.is_dir() else None
2489
+ forwarded = [{"gap_id": e.get("gap_id"), "change_category": e.get("change_category")}
2490
+ for e in entries if e.get("change_category") not in EXPORT_CATEGORIES]
2491
+ routed = [_GapEntry(e, judgments.get(e.get("gap_id")) or {}) for e in entries
2492
+ if e.get("change_category") in EXPORT_CATEGORIES]
2493
+ drift = drift_status == "overridden"
2494
+ if drift:
2495
+ blocked = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "change_category": g.category,
2496
+ "severity": g.entry.get("severity"), "reason": _drift_reason(g.entry)}
2497
+ for g in routed if _drift_reason(g.entry)]
2498
+ if blocked:
2499
+ return None, {"status": "drift-blocked", "drift_blocked": blocked}
2500
+ bare = [g.name for g in routed if g.category == "DELETED_EXPORT" and not (
2501
+ isinstance(g.entry.get("rescope"), dict) and g.entry["rescope"].get("amendment")
2502
+ and g.entry["rescope"].get("exclude"))]
2503
+ if bare:
2504
+ return None, {"status": "blocked", "rescope_without_amendment": bare}
2505
+ warnings = []
2506
+ if root is not None and _verifier() is None:
2507
+ warnings.append(VERIFIER_MISSING)
2508
+ for g in routed:
2509
+ g.route(root, drift)
2510
+ warnings += [f"provenance: spot-check failed for {g.name}: {g.failure}" for g in routed if g.failure]
2511
+ waiting = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": ["declaring_line"],
2512
+ "file": g.recorded[0], "line": g.recorded[1]} for g in routed if g.waits]
2513
+ if waiting:
2514
+ return None, {"status": "needs-judgment", "needs_judgment": waiting, "warnings": warnings}
2515
+ to_scan = [g for g in routed if g.reextract]
2516
+ files = list(dict.fromkeys(p for g in to_scan for p in g.paths))
2517
+ if plan:
2518
+ return None, {"status": "planned", "reextract": {
2519
+ "entries": [{"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
2520
+ "resolved_paths": g.paths} for g in to_scan], "files": files}, "warnings": warnings}
2521
+ blocks: dict[str, list[dict]] = {}
2522
+ scanned = 0
2523
+ if files:
2524
+ if remediation is None:
2525
+ raise ValueError("targeted re-extraction has files to match: pass --remediation-records, the output "
2526
+ "of `records` over them")
2527
+ if not isinstance(remediation, dict) or not isinstance(remediation.get("files"), list):
2528
+ raise ValueError("--remediation-records has no `files` array: is it the output of `records`?")
2529
+ for block in remediation["files"]:
2530
+ if isinstance(block, dict) and _norm_path(block.get("file_path")):
2531
+ scanned += 1
2532
+ blocks.setdefault(_norm_path(block["file_path"]), []).extend(
2533
+ r for r in block.get("exports") or [] if isinstance(r, dict))
2534
+ unresolved = []
2535
+ for g in to_scan:
2536
+ files_scanned, found = g.match(blocks)
2537
+ if g.record is not None:
2538
+ continue
2539
+ if not g.blocking and g.entry.get("category") in DOCUMENT_CATEGORIES:
2540
+ g.outcome = "unknown"
2541
+ continue
2542
+ unresolved.append({"gap_id": g.entry.get("gap_id"), "name": g.name, "severity": g.entry.get("severity"),
2543
+ "remediation_paths": g.entry.get("remediation_paths") or [],
2544
+ "rejected_paths": g.entry.get("rejected_paths") or [],
2545
+ "files_scanned": files_scanned, "exports_found_in_scan": found})
2546
+ if unresolved:
2547
+ return None, {"status": "unresolved", "unresolved": unresolved, "warnings": warnings}
2548
+ needs = [{"gap_id": g.entry.get("gap_id"), "name": g.name, "needs": g.needs(),
2549
+ "file": _location(g.new_location())[0] if g.outcome != "verified" else g.recorded[0],
2550
+ "line": _location(g.new_location())[1] if g.outcome != "verified" else g.recorded[1]}
2551
+ for g in routed if g.needs()]
2552
+ if needs:
2553
+ return None, {"status": "needs-judgment", "needs_judgment": needs, "warnings": warnings}
2554
+ kept: dict[str, list[dict]] = {}
2555
+ for g in routed:
2556
+ if g.record is not None and not g.internal:
2557
+ record = dict(g.record)
2558
+ record.pop("public", None) # a repair documents and maps every record: the mark is a normal update's
2559
+ if g.answers.get("docstring") is not None:
2560
+ record["docstring"] = g.answers["docstring"]
2561
+ if record not in kept.setdefault(g.record_path, []):
2562
+ kept[g.record_path].append(record)
2563
+ breakdown = {"T1": 0, "T1-low": 0, "unlabeled": 0, "T2": 0}
2564
+ provenance_entries = _entries_of(provenance)
2565
+ for g in routed:
2566
+ breakdown[g.label(provenance_entries)] += 1
2567
+ breakdown["unlabeled"] += len(forwarded)
2568
+ records = {
2569
+ "mode": "gap-driven",
2570
+ "files_extracted": scanned,
2571
+ "exports_extracted": len(entries),
2572
+ "confidence_breakdown": breakdown,
2573
+ "verification": [g.verification() for g in routed],
2574
+ "files": [{"file_path": path, "exports": exports} for path, exports in kept.items()],
2575
+ }
2576
+ matched = [g for g in to_scan if g.record is not None]
2577
+ targeted = {"resolved_count": len(matched), "files_scanned": scanned,
2578
+ "exports_matched": len({(g.record_path, _name_of(g.record)) for g in matched}),
2579
+ "tier": tier} if to_scan else None
2580
+ summary = {
2581
+ "status": "written",
2582
+ "gap_count": len(entries),
2583
+ "files_extracted": scanned,
2584
+ "exports_extracted": len(entries),
2585
+ "confidence_breakdown": breakdown,
2586
+ "counts": {**{outcome: sum(1 for g in routed if g.outcome == outcome and not g.internal)
2587
+ for outcome in OUTCOMES}, "reclassified": sum(1 for g in routed if g.internal)},
2588
+ "reclassified": [g.name for g in routed if g.internal],
2589
+ "forwarded": forwarded,
2590
+ "targeted_reextraction": targeted,
2591
+ "warnings": warnings,
2592
+ }
2593
+ return records, summary
2594
+
2595
+
340
2596
  # --------------------------------------------------------------------------
341
2597
  # CLI
342
2598
  # --------------------------------------------------------------------------
343
2599
 
344
2600
 
2601
+ def _category_c(args: argparse.Namespace, own: dict | None) -> dict | None:
2602
+ """Category C from --category-c and --ccc-pairs, else the input's own
2603
+ (`own`, None when there is none). Raises ValueError."""
2604
+ if not args.category_c and not args.ccc_pairs:
2605
+ return own
2606
+ cat_c = dict(own or {})
2607
+ if args.category_c:
2608
+ doc = _load_helper_file(Path(args.category_c), "--category-c file", "category_c")
2609
+ if not isinstance(doc["category_c"], dict):
2610
+ raise ValueError(f"--category-c file {args.category_c} has no `category_c` object")
2611
+ cat_c = dict(doc["category_c"])
2612
+ if args.ccc_pairs:
2613
+ pairs = _load_json_file(Path(args.ccc_pairs), "--ccc-pairs file")
2614
+ if not isinstance(pairs, dict):
2615
+ raise ValueError(f"--ccc-pairs file {args.ccc_pairs} must hold a JSON object")
2616
+ for key in ("renamed_files", "renamed_exports"):
2617
+ cat_c[key] = list(_get_list(cat_c, key)) + [p for p in _get_list(pairs, key) if isinstance(p, dict)]
2618
+ return cat_c
2619
+
2620
+
2621
+ def _payload(args: argparse.Namespace) -> dict:
2622
+ """The category JSON of a build or deletion-ratio call: the input with
2623
+ the helper files' slices assembled in. Raises ValueError."""
2624
+ payload = _load_input(Path(args.input) if args.input else None)
2625
+ cat_c = _category_c(args, payload.get("category_c") if isinstance(payload.get("category_c"), dict) else None)
2626
+ if cat_c is not None:
2627
+ payload = {**payload, "category_c": cat_c}
2628
+ category_a_doc = (
2629
+ _load_helper_file(Path(args.category_a), "--category-a file", "category_a") if args.category_a else None
2630
+ )
2631
+ diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
2632
+ if args.category_b_diff else None
2633
+ return assemble(payload, category_a_doc=category_a_doc, diff=diff)
2634
+
2635
+
2636
+ def docs_only_manifest(doc_hashes: dict) -> dict:
2637
+ """A docs-only skill's change manifest from compare-hashes' output."""
2638
+ def urls(key: str) -> list[str]:
2639
+ return [e["url"] for e in _get_list(doc_hashes, key) if isinstance(e, dict) and isinstance(e.get("url"), str)]
2640
+
2641
+ changed, failed = urls("changed"), urls("fetch_failed")
2642
+ return {"mode": "docs-only", "no_changes": not changed, "changed_urls": changed, "fetch_failed": failed,
2643
+ "counts": {"docs_changed": len(changed), "docs_fetch_failed": len(failed)}}
2644
+
2645
+
2646
+ def _category_d(args: argparse.Namespace) -> dict | None:
2647
+ """Category D from --file-compare (typed by --provenance-map) and --new-files, else None. Raises ValueError."""
2648
+ if not args.file_compare and not args.new_files:
2649
+ if args.provenance_map:
2650
+ raise ValueError("--provenance-map goes with --file-compare")
2651
+ return None
2652
+ if args.file_compare and not args.provenance_map:
2653
+ raise ValueError("--file-compare needs --provenance-map, which types each row by its file_entries[] row")
2654
+ compare = _load_helper_file(Path(args.file_compare), "--file-compare file", "comparisons") \
2655
+ if args.file_compare else None
2656
+ new_files = _load_helper_file(Path(args.new_files), "--new-files file", "new_files") if args.new_files else None
2657
+ provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file") if args.provenance_map else None
2658
+ if provenance is not None and not isinstance(provenance, dict):
2659
+ raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
2660
+ return category_d_from_files(compare, new_files, provenance)
2661
+
2662
+
345
2663
  def _cmd_build(args: argparse.Namespace) -> int:
346
2664
  try:
347
- payload = _load_input(Path(args.input) if args.input else None)
2665
+ if args.doc_hashes:
2666
+ if (args.input or args.category_a or args.category_b_diff or args.category_c or args.ccc_pairs
2667
+ or args.file_compare or args.new_files or args.provenance_map):
2668
+ raise ValueError("--doc-hashes takes no other input")
2669
+ doc_hashes = _load_helper_file(Path(args.doc_hashes), "--doc-hashes file", "changed")
2670
+ manifest = docs_only_manifest(doc_hashes)
2671
+ else:
2672
+ payload = _payload(args)
2673
+ cat_d = _category_d(args)
2674
+ if cat_d is not None:
2675
+ payload = {**payload, "category_d": cat_d}
2676
+ manifest = build_manifest(payload)
348
2677
  except ValueError as exc:
349
2678
  print(f"error: {exc}", file=sys.stderr)
350
2679
  return 1
351
- manifest = build_manifest(payload)
352
2680
  json.dump(manifest, sys.stdout, indent=2)
353
2681
  sys.stdout.write("\n")
354
2682
  return 0
@@ -356,7 +2684,7 @@ def _cmd_build(args: argparse.Namespace) -> int:
356
2684
 
357
2685
  def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
358
2686
  try:
359
- payload = _load_input(Path(args.input) if args.input else None)
2687
+ payload = _payload(args)
360
2688
  except ValueError as exc:
361
2689
  print(f"error: {exc}", file=sys.stderr)
362
2690
  return 1
@@ -379,6 +2707,271 @@ def _cmd_deletion_ratio(args: argparse.Namespace) -> int:
379
2707
  return 0
380
2708
 
381
2709
 
2710
+ def _optional_json(value: str | None, what: str):
2711
+ return _load_json_file(Path(value), what) if value else None
2712
+
2713
+
2714
+ def _emit(result: dict, output: str | None) -> None:
2715
+ if output:
2716
+ _write_json_atomic(Path(output), result)
2717
+ print(json.dumps({"status": "ok", "output": output}))
2718
+ else:
2719
+ json.dump(result, sys.stdout, indent=2)
2720
+ sys.stdout.write("\n")
2721
+
2722
+
2723
+ def _cmd_rename_candidates(args: argparse.Namespace) -> int:
2724
+ try:
2725
+ category_a_doc = _load_helper_file(Path(args.category_a), "--category-a file", "category_a")
2726
+ diff = _load_helper_file(Path(args.category_b_diff), "--category-b-diff file", "removed") \
2727
+ if args.category_b_diff else None
2728
+ provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
2729
+ result = rename_candidates(
2730
+ category_a_doc, diff, provenance,
2731
+ _optional_json(args.extraction, "--extraction file"),
2732
+ _optional_json(args.export_details, "--export-details file"),
2733
+ args.tier,
2734
+ source_root=Path(args.source_root) if args.source_root else None,
2735
+ sizes_commit=args.sizes_commit or None,
2736
+ )
2737
+ _emit(result, args.output)
2738
+ except ValueError as exc:
2739
+ print(f"error: {exc}", file=sys.stderr)
2740
+ return 1
2741
+ except OSError as exc:
2742
+ print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
2743
+ return 1
2744
+ return 0
2745
+
2746
+
2747
+ def _apply_categories(args: argparse.Namespace) -> dict | None:
2748
+ """The category JSON apply reads: --input, with --category-c and --ccc-pairs in place of its category_c."""
2749
+ categories = _optional_json(args.input, "--input file")
2750
+ if categories is not None and not isinstance(categories, dict):
2751
+ raise ValueError("--input file must hold a JSON object")
2752
+ own = (categories or {}).get("category_c")
2753
+ cat_c = _category_c(args, own if isinstance(own, dict) else None)
2754
+ if cat_c is None:
2755
+ return categories
2756
+ return {**(categories or {}), "category_c": cat_c}
2757
+
2758
+
2759
+ def _metadata_in(path: str | None) -> dict | None:
2760
+ """apply's --metadata-in: the skill's metadata.json, an object."""
2761
+ if not path:
2762
+ return None
2763
+ metadata = _load_json_file(Path(path), "--metadata-in file")
2764
+ if not isinstance(metadata, dict):
2765
+ raise ValueError("--metadata-in file must hold a JSON object (the skill's metadata.json)")
2766
+ return metadata
2767
+
2768
+
2769
+ def _cmd_metadata(args: argparse.Namespace) -> int:
2770
+ target = Path(args.metadata)
2771
+ try:
2772
+ metadata = update_metadata(
2773
+ _load_json_file(target, "--metadata file"), _load_json_file(Path(args.lists), "--lists file"),
2774
+ _load_json_file(Path(args.stats), "--stats file"), version=args.version,
2775
+ generation_date=args.generation_date, source_commit=args.source_commit, source_ref=args.source_ref)
2776
+ _write_json_atomic(target, metadata)
2777
+ except ValueError as exc:
2778
+ print(f"error: {exc}", file=sys.stderr)
2779
+ return 1
2780
+ except OSError as exc:
2781
+ print(f"error: cannot write {args.metadata}: {exc}", file=sys.stderr)
2782
+ return 1
2783
+ print(json.dumps({"status": "written", "metadata": target.as_posix(), "version": metadata["version"],
2784
+ "exports": len(metadata["exports"]), "scripts": len(metadata.get("scripts") or []),
2785
+ "assets": len(metadata.get("assets") or [])}))
2786
+ return 0
2787
+
2788
+
2789
+ def _cmd_apply(args: argparse.Namespace) -> int:
2790
+ if bool(args.drift_head) != bool(args.drift_pinned):
2791
+ print("error: --drift-head and --drift-pinned go together", file=sys.stderr)
2792
+ return 1
2793
+ if bool(args.metadata_in) != bool(args.lists_out):
2794
+ print("error: --metadata-in and --lists-out go together", file=sys.stderr)
2795
+ return 1
2796
+ try:
2797
+ promoted = _optional_json(args.promoted_docs, "--promoted-docs file")
2798
+ if promoted is not None and not isinstance(promoted, list):
2799
+ raise ValueError("--promoted-docs file must hold a JSON array")
2800
+ doc, summary = apply_update(
2801
+ update_type=args.update_type,
2802
+ provenance=_optional_json(args.provenance_map, "--provenance-map file"),
2803
+ skill_name=args.skill_name,
2804
+ manifest=_optional_json(args.manifest, "--manifest file"),
2805
+ categories=_apply_categories(args),
2806
+ extraction=_optional_json(args.extraction, "--extraction file"),
2807
+ details=_optional_json(args.export_details, "--export-details file"),
2808
+ records=_optional_json(args.reextract_records, "--reextract-records file"),
2809
+ merge_records=_optional_json(args.merge_records, "--merge-records file"),
2810
+ compare=_optional_json(args.file_compare, "--file-compare file"),
2811
+ new_files=_optional_json(args.new_files, "--new-files file"),
2812
+ promoted=promoted,
2813
+ source_root=Path(args.source_root) if args.source_root else None,
2814
+ generation_date=args.generation_date,
2815
+ test_report_run_id=args.test_report_run_id,
2816
+ confidence_tier=args.confidence_tier,
2817
+ manual_sections_preserved=args.manual_sections_preserved,
2818
+ source_commit=args.source_commit,
2819
+ source_ref=args.source_ref,
2820
+ drift=(args.drift_head, args.drift_pinned) if args.drift_head else None,
2821
+ metadata=_metadata_in(args.metadata_in),
2822
+ )
2823
+ except ValueError as exc:
2824
+ print(f"error: {exc}", file=sys.stderr)
2825
+ return 1
2826
+ if summary["status"] == "refused":
2827
+ print(json.dumps(summary))
2828
+ return EXIT_REFUSED
2829
+ lists = summary.pop("lists", None)
2830
+ for path, value in ((args.not_public_out, summary["not_public"]), (args.lists_out, lists),
2831
+ (args.output, doc)):
2832
+ try:
2833
+ if path:
2834
+ _write_json_atomic(Path(path), value)
2835
+ except OSError as exc:
2836
+ print(f"error: cannot write {path}: {exc}", file=sys.stderr)
2837
+ return 1
2838
+ print(json.dumps({**summary, "map": args.output}))
2839
+ return 0
2840
+
2841
+
2842
+ def _cmd_records(args: argparse.Namespace) -> int:
2843
+ try:
2844
+ files = _optional_json(args.files_from, "--files-from file")
2845
+ if files is not None and not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
2846
+ raise ValueError(f"--files-from file {args.files_from} must hold a JSON array of paths")
2847
+ records, summary = build_records(
2848
+ _optional_json(args.extraction, "--extraction file"),
2849
+ _optional_json(args.export_details, "--export-details file"),
2850
+ files,
2851
+ _patch_blocks(Path(args.patches)) if args.patches else [],
2852
+ )
2853
+ _write_json_atomic(Path(args.output), records)
2854
+ except ValueError as exc:
2855
+ print(f"error: {exc}", file=sys.stderr)
2856
+ return 1
2857
+ except OSError as exc:
2858
+ print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
2859
+ return 1
2860
+ print(json.dumps({"status": "written", "output": args.output, **summary}))
2861
+ return 0
2862
+
2863
+
2864
+ def _cmd_gap_records(args: argparse.Namespace) -> int:
2865
+ try:
2866
+ if args.plan and args.output:
2867
+ raise ValueError("--plan writes no records: drop -o")
2868
+ if not args.plan and not args.output:
2869
+ raise ValueError("-o is required without --plan")
2870
+ if args.files_out and not args.plan:
2871
+ raise ValueError("--files-out goes with --plan")
2872
+ if bool(args.evidence) != bool(args.tier):
2873
+ raise ValueError("--evidence and --tier go together")
2874
+ judgments = _gap_judgments(_load_json_file(Path(args.judgments), "--judgments file")) \
2875
+ if args.judgments else {}
2876
+ records, summary = gap_records(
2877
+ _load_json_file(Path(args.manifest), "--manifest file"),
2878
+ _load_json_file(Path(args.provenance_map), "--provenance-map file"),
2879
+ source_root=Path(args.source_root) if args.source_root else None,
2880
+ drift_status=args.drift_status,
2881
+ judgments=judgments,
2882
+ remediation=_optional_json(args.remediation_records, "--remediation-records file"),
2883
+ plan=args.plan,
2884
+ tier=args.tier,
2885
+ )
2886
+ if summary["status"] == "planned" and args.files_out:
2887
+ _write_json_atomic(Path(args.files_out), summary["reextract"]["files"])
2888
+ if records is not None:
2889
+ _write_json_atomic(Path(args.output), records)
2890
+ summary = {"status": summary["status"], "output": args.output,
2891
+ **{k: v for k, v in summary.items() if k != "status"}}
2892
+ if args.evidence and summary["targeted_reextraction"] is not None:
2893
+ with open(args.evidence, "a", encoding="utf-8", newline="\n") as fh:
2894
+ fh.write(json.dumps({"targeted_reextraction": summary["targeted_reextraction"]}) + "\n")
2895
+ except ValueError as exc:
2896
+ print(f"error: {exc}", file=sys.stderr)
2897
+ return 1
2898
+ except OSError as exc:
2899
+ print(f"error: cannot write: {exc}", file=sys.stderr)
2900
+ return 1
2901
+ print(json.dumps(summary, indent=2))
2902
+ return 0
2903
+
2904
+
2905
+ # --------------------------------------------------------------------------
2906
+ # Baseline gaps (update-skill detect-changes Category B)
2907
+ # --------------------------------------------------------------------------
2908
+
2909
+
2910
+ VERIFIER_FILE = "skf-verify-provenance-completeness.py"
2911
+
2912
+
2913
+ def _baseline_verifier():
2914
+ """The verifier beside this script, for its `baseline_gaps`. Unlike `_verifier()`, which lets a gap-driven
2915
+ spot-check go on as unknown, one that cannot be loaded (or has no `baseline_gaps`) is a ValueError: with no
2916
+ verifier no gap is listed, and every name the runner leaves out would read as a deleted export."""
2917
+ path = Path(__file__).resolve().parent / VERIFIER_FILE
2918
+ where = f"cannot load {VERIFIER_FILE} beside {Path(__file__).name}"
2919
+ try:
2920
+ spec = importlib.util.spec_from_file_location("skf_verify_provenance_completeness_baseline", path)
2921
+ if spec is None or spec.loader is None or not path.is_file():
2922
+ raise ImportError(f"no file at {path}")
2923
+ module = importlib.util.module_from_spec(spec)
2924
+ spec.loader.exec_module(module)
2925
+ except Exception as exc: # whatever the script raises, the call reports it
2926
+ raise ValueError(f"{where}: {exc}; re-install SKF") from exc
2927
+ if not callable(getattr(module, "baseline_gaps", None)):
2928
+ raise ValueError(f"{where}: it has no baseline_gaps; re-install SKF")
2929
+ return module
2930
+
2931
+
2932
+ def baseline_gaps(provenance: dict, extraction: dict, files: list, source_root: Path, verifier) -> dict:
2933
+ """{"gaps", "unchecked"}: the map entries of `files` the runner's `extraction` lacks whose file still declares
2934
+ them (see "Baseline gaps" in the module docstring)."""
2935
+ if not isinstance(extraction, dict):
2936
+ raise ValueError("--extraction file has no `exports` array: is it the helper's output?")
2937
+ held = {(name, file) for e in _exports_of(extraction, "--extraction file")
2938
+ if (name := _name_of(e)) is not None and (file := _file_of(e)) is not None}
2939
+ diff = extraction.get("entry_point_diff")
2940
+ listed = {(item["name"], _norm_path(item["file"]))
2941
+ for item in (diff.get("extraction_gaps") or [] if isinstance(diff, dict) else [])
2942
+ if isinstance(item, dict) and isinstance(item.get("name"), str) and _norm_path(item.get("file"))}
2943
+ if provenance.get("entries") is not None and not isinstance(provenance["entries"], list):
2944
+ raise ValueError("provenance `entries` must be an array") # missing or null: no entry (#684)
2945
+ wanted = {f for f in (_norm_path(p) for p in files) if f is not None}
2946
+ gaps, unchecked = verifier.baseline_gaps(provenance, source_root, held, wanted, listed)
2947
+ return {"gaps": gaps, "unchecked": unchecked}
2948
+
2949
+
2950
+ def _cmd_baseline_gaps(args: argparse.Namespace) -> int:
2951
+ try:
2952
+ source_root = Path(args.source_root)
2953
+ if not source_root.is_dir():
2954
+ raise ValueError(f"--source-root {args.source_root} is not a folder")
2955
+ files = _load_json_file(Path(args.files), "--files file")
2956
+ if not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
2957
+ raise ValueError(f"--files file {args.files} must hold a JSON array of paths")
2958
+ provenance = _load_json_file(Path(args.provenance_map), "--provenance-map file")
2959
+ if not isinstance(provenance, dict):
2960
+ raise ValueError(f"--provenance-map file {args.provenance_map} must hold a JSON object")
2961
+ extraction = _load_json_file(Path(args.extraction), "--extraction file")
2962
+ result = baseline_gaps(provenance, extraction, files, source_root, _baseline_verifier())
2963
+ _write_json_atomic(Path(args.output), result)
2964
+ except ValueError as exc:
2965
+ print(f"error: {exc}", file=sys.stderr)
2966
+ return 1
2967
+ except OSError as exc:
2968
+ print(f"error: cannot write {args.output}: {exc}", file=sys.stderr)
2969
+ return 1
2970
+ print(json.dumps({"status": "written", "output": args.output, "gaps": len(result["gaps"]),
2971
+ "unchecked": len(result["unchecked"])}))
2972
+ return 0
2973
+
2974
+
382
2975
  def _build_parser() -> argparse.ArgumentParser:
383
2976
  parser = argparse.ArgumentParser(
384
2977
  prog="skf-build-change-manifest",
@@ -390,23 +2983,155 @@ def _build_parser() -> argparse.ArgumentParser:
390
2983
  sub = parser.add_subparsers(dest="cmd", required=True)
391
2984
 
392
2985
  p_build = sub.add_parser("build", help="aggregate categories into manifest")
393
- p_build.add_argument(
394
- "--input", default=None, help="path to JSON input (default: stdin)"
395
- )
2986
+ p_build.add_argument("--doc-hashes", metavar="FILE",
2987
+ help="a docs-only skill: skf-detect-docs.py compare-hashes output, the only input")
2988
+ p_build.add_argument("--file-compare", metavar="FILE",
2989
+ help="skf-hash-content.py compare output: Category D's modified and deleted files")
2990
+ p_build.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output: Category D's new files")
2991
+ p_build.add_argument("--provenance-map", metavar="FILE",
2992
+ help="the map --file-compare compared: each row's file_type")
396
2993
  p_build.set_defaults(func=_cmd_build)
397
2994
 
398
2995
  p_ratio = sub.add_parser(
399
2996
  "deletion-ratio",
400
2997
  help="compute §2.2 trigger; requires --provenance-map",
401
2998
  )
402
- p_ratio.add_argument(
403
- "--input", default=None, help="path to JSON input (default: stdin)"
404
- )
405
2999
  p_ratio.add_argument(
406
3000
  "--provenance-map", required=True, help="path to provenance-map.json"
407
3001
  )
408
3002
  p_ratio.set_defaults(func=_cmd_deletion_ratio)
409
3003
 
3004
+ p_rename = sub.add_parser(
3005
+ "rename-candidates",
3006
+ help="Category C by fixed rules: renamed files and renamed exports",
3007
+ )
3008
+ p_rename.add_argument("--category-a", required=True, metavar="FILE",
3009
+ help="skf-classify-changed-files.py classify output (deleted and added files)")
3010
+ p_rename.add_argument("--category-b-diff", metavar="FILE",
3011
+ help="skf-structural-diff.py output (removed and added exports)")
3012
+ p_rename.add_argument("--provenance-map", required=True, metavar="FILE",
3013
+ help="the provenance map: the deleted files' exports")
3014
+ p_rename.add_argument("--extraction", metavar="FILE", help="the recipe runner's output: the added files' exports")
3015
+ p_rename.add_argument("--export-details", metavar="FILE",
3016
+ help="the workers' export details: params, return types and exports read by eye")
3017
+ p_rename.add_argument("--tier", required=True, choices=TIERS, help="the forge tier, which picks the rules")
3018
+ p_rename.add_argument("--source-root", metavar="DIR", help="the source tree: the added files' sizes")
3019
+ p_rename.add_argument("--sizes-commit", metavar="COMMIT",
3020
+ help="the commit the deleted files' sizes are read at (git cat-file -s)")
3021
+ p_rename.add_argument("-o", "--output", metavar="FILE", help="write the JSON here instead of stdout")
3022
+ p_rename.set_defaults(func=_cmd_rename_candidates)
3023
+
3024
+ p_apply = sub.add_parser("apply", help="write the provenance map an update leaves")
3025
+ p_apply.add_argument("--update-type", required=True, choices=UPDATE_TYPES,
3026
+ help="incremental (normal mode), gap-driven, or full (degraded mode)")
3027
+ p_apply.add_argument("--provenance-map", metavar="FILE", help="the map the update started from")
3028
+ p_apply.add_argument("--manifest", metavar="FILE", help="the change manifest (build's output)")
3029
+ p_apply.add_argument("--input", metavar="FILE", help="the category JSON: category_c's renamed exports")
3030
+ p_apply.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
3031
+ p_apply.add_argument("--export-details", metavar="FILE", help="the workers' export details")
3032
+ p_apply.add_argument("--reextract-records", metavar="FILE", help="step 3's records")
3033
+ p_apply.add_argument("--merge-records", metavar="FILE", help="merge's records of the exports no fresh record holds")
3034
+ p_apply.add_argument("--file-compare", metavar="FILE", help="skf-hash-content.py compare output")
3035
+ p_apply.add_argument("--new-files", metavar="FILE", help="skf-new-file-diff.py output")
3036
+ p_apply.add_argument("--promoted-docs", metavar="FILE", help="the documents step 2 promoted, with their hashes")
3037
+ p_apply.add_argument("--source-root", metavar="DIR", help="the source tree: the new files' hashes")
3038
+ p_apply.add_argument("--skill-name", required=True, help="the skill name: source_library of a new entry")
3039
+ p_apply.add_argument("--generation-date", required=True, help="last_update: the generation_date written")
3040
+ p_apply.add_argument("--test-report-run-id", default=None, help="the run id of the test report applied")
3041
+ p_apply.add_argument("--confidence-tier", required=True, help="the forge tier the update ran at")
3042
+ p_apply.add_argument("--manual-sections-preserved", required=True, type=int,
3043
+ help="the [MANUAL] blocks the post-merge check found intact")
3044
+ p_apply.add_argument("--source-commit", default=None, help="the map's source_commit (empty: null)")
3045
+ p_apply.add_argument("--source-ref", default=None, help="the map's source_ref (empty: null)")
3046
+ p_apply.add_argument("--drift-head", default=None, help="under the drift override: HEAD's short SHA")
3047
+ p_apply.add_argument("--drift-pinned", default=None, help="under the drift override: the pinned short SHA")
3048
+ p_apply.add_argument("--not-public-out", metavar="FILE",
3049
+ help="also write the summary's not_public list here (a JSON array)")
3050
+ p_apply.add_argument("--metadata-in", metavar="FILE",
3051
+ help="the skill's metadata.json, whose exports[], scripts[] and assets[] the update "
3052
+ "starts from")
3053
+ p_apply.add_argument("--lists-out", metavar="FILE",
3054
+ help="write the exports[], scripts[] and assets[] metadata.json takes after the update "
3055
+ "here (a JSON object)")
3056
+ p_apply.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the new map")
3057
+ p_apply.set_defaults(func=_cmd_apply)
3058
+
3059
+ p_meta = sub.add_parser(
3060
+ "metadata", help="write the metadata.json an update leaves",
3061
+ description="Rewrite the skill's metadata.json in place. It sets version and generation_date, "
3062
+ "source_commit and source_ref when given, exports, scripts and assets from --lists (a script "
3063
+ "or asset list left out when empty), confidence_distribution from --stats and the stats "
3064
+ "helper's stats fields, dropping one an earlier run wrote that --stats lacks; it removes "
3065
+ "last_update and update_type and keeps every other field, a patched one included. It refuses "
3066
+ "a --stats whose scripts_count or assets_count is not the length of --lists' scripts or assets.")
3067
+ p_meta.add_argument("--metadata", required=True, metavar="FILE",
3068
+ help="the skill's metadata.json, rewritten in place")
3069
+ p_meta.add_argument("--lists", required=True, metavar="FILE", help="the lists apply wrote (--lists-out)")
3070
+ p_meta.add_argument("--stats", required=True, metavar="FILE",
3071
+ help="the JSON skf-render-metadata-stats.py wrote (its -o file)")
3072
+ p_meta.add_argument("--version", required=True, help="the version this update writes")
3073
+ p_meta.add_argument("--generation-date", required=True, help="the clock's UTC time, YYYY-MM-DDTHH:MM:SSZ")
3074
+ p_meta.add_argument("--source-commit", default=None, help="the commit this update read (an empty one: null)")
3075
+ p_meta.add_argument("--source-ref", default=None, help="the ref this update read (an empty one: null)")
3076
+ p_meta.set_defaults(func=_cmd_metadata)
3077
+
3078
+ p_records = sub.add_parser("records", help="write step 3's re-extraction records from the runner's and the "
3079
+ "workers' files")
3080
+ p_records.add_argument("--extraction", metavar="FILE", help="the recipe runner's output")
3081
+ p_records.add_argument("--export-details", metavar="FILE",
3082
+ help="step 2's export details: params and return types, and the exports read by eye")
3083
+ p_records.add_argument("--files-from", metavar="FILE",
3084
+ help="the files step 3 extracts (a JSON array), each a block even with no export")
3085
+ p_records.add_argument("--patches", metavar="DIR", help="the folder of the workers' per-file patches")
3086
+ p_records.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the records")
3087
+ p_records.set_defaults(func=_cmd_records)
3088
+
3089
+ p_gap = sub.add_parser("gap-records", help="spot-check, route and record a gap-driven repair's manifest entries")
3090
+ p_gap.add_argument("--manifest", required=True, metavar="FILE", help="skf-parse-gaps.py translate's manifest")
3091
+ p_gap.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the repair starts from")
3092
+ p_gap.add_argument("--source-root", metavar="DIR", help="the source tree the spot-checks read")
3093
+ p_gap.add_argument("--drift-status", choices=DRIFT_STATUSES, default="ok",
3094
+ help="the workspace drift check's status; overridden runs the drift gate")
3095
+ p_gap.add_argument("--judgments", metavar="FILE", help="the answers to its needs_judgment[], keyed by gap id")
3096
+ p_gap.add_argument("--remediation-records", metavar="FILE",
3097
+ help="`records` output over the files targeted re-extraction scanned")
3098
+ p_gap.add_argument("--plan", action="store_true",
3099
+ help="print the answers it needs and the files targeted re-extraction scans; write no records")
3100
+ p_gap.add_argument("--files-out", metavar="FILE", help="with --plan: write those files here, a JSON array")
3101
+ p_gap.add_argument("--evidence", metavar="FILE",
3102
+ help="append the targeted_reextraction record to this JSON-lines file when it ran")
3103
+ p_gap.add_argument("--tier", help="with --evidence: the forge tier the record names")
3104
+ p_gap.add_argument("-o", "--output", metavar="FILE", help="where to write the records (without --plan)")
3105
+ p_gap.set_defaults(func=_cmd_gap_records)
3106
+
3107
+ p_base = sub.add_parser("baseline-gaps", help="list the map entries the runner left out that their modified "
3108
+ "file still declares, for Category B's step 2 to read")
3109
+ p_base.add_argument("--provenance-map", required=True, metavar="FILE", help="the map the update starts from")
3110
+ p_base.add_argument("--extraction", required=True, metavar="FILE",
3111
+ help="the recipe runner's output ({\"exports\": []} when it could not run)")
3112
+ p_base.add_argument("--files", required=True, metavar="FILE",
3113
+ help="the files whose entries are checked: Category A's modified-files.json")
3114
+ p_base.add_argument("--source-root", required=True, metavar="DIR", help="the source tree the files are read in")
3115
+ p_base.add_argument("-o", "--output", required=True, metavar="FILE", help="where to write the gaps")
3116
+ p_base.set_defaults(func=_cmd_baseline_gaps)
3117
+
3118
+ for p in (p_build, p_ratio):
3119
+ p.add_argument(
3120
+ "--input", default=None, help="path to JSON input (default: stdin)"
3121
+ )
3122
+ p.add_argument(
3123
+ "--category-a", metavar="FILE",
3124
+ help="skf-classify-changed-files.py classify output: the Category A slice and its moves",
3125
+ )
3126
+ p.add_argument(
3127
+ "--category-b-diff", metavar="FILE",
3128
+ help="skf-structural-diff.py output over the modified files: the Category B slice",
3129
+ )
3130
+ for p in (p_build, p_ratio, p_apply):
3131
+ p.add_argument("--category-c", metavar="FILE", help="rename-candidates output: the Category C slice")
3132
+ p.add_argument("--ccc-pairs", metavar="FILE",
3133
+ help="the CCC check's pairs, {renamed_files, renamed_exports}, added to Category C")
3134
+
410
3135
  return parser
411
3136
 
412
3137