akm-cli 0.9.16 → 0.9.17-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/CHANGELOG.md +2101 -0
  2. package/STABILITY.md +11 -10
  3. package/dist/akm +124 -193
  4. package/dist/akm-migrate +38 -19
  5. package/dist/assets/hints/cli-hints-full.md +6 -7
  6. package/dist/assets/improve-strategies/catchup.json +0 -3
  7. package/dist/assets/improve-strategies/consolidate.json +0 -1
  8. package/dist/assets/improve-strategies/default.json +1 -2
  9. package/dist/assets/improve-strategies/proactive-maintenance.json +1 -2
  10. package/dist/assets/improve-strategies/quick.json +1 -2
  11. package/dist/assets/improve-strategies/reflect-distill.json +1 -2
  12. package/dist/assets/improve-strategies/thorough.json +0 -3
  13. package/dist/assets/prompts/consolidate-pair.md +20 -0
  14. package/dist/assets/prompts/consolidate-system.md +4 -11
  15. package/dist/assets/prompts/retrieval-relevance-judge.md +6 -0
  16. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +20 -20
  17. package/dist/assets/stash-skeleton/facts/conventions/domains.md +2 -2
  18. package/dist/assets/templates/html/health.html +3 -5
  19. package/dist/cli/retired-commands.js +1 -1
  20. package/dist/cli/shared.js +6 -2
  21. package/dist/cli/unknown-flags.js +24 -1
  22. package/dist/cli.js +68 -10
  23. package/dist/commands/agent/agent-dispatch.js +1 -1
  24. package/dist/commands/command/command-execution.js +24 -62
  25. package/dist/commands/feedback-cli.js +0 -1
  26. package/dist/commands/health/accept-rate.js +6 -0
  27. package/dist/commands/health/archive-usage.js +92 -0
  28. package/dist/commands/health/checks.js +83 -74
  29. package/dist/commands/health/config-skew.js +38 -0
  30. package/dist/commands/health/data-dir-usage.js +25 -13
  31. package/dist/commands/health/egress.js +54 -0
  32. package/dist/commands/health/html-report.js +1 -42
  33. package/dist/commands/health/improve-metrics.js +136 -591
  34. package/dist/commands/health/md-report.js +1 -6
  35. package/dist/commands/health/plugin-staleness.js +53 -3
  36. package/dist/commands/health/renderers.js +12 -4
  37. package/dist/commands/health/report-view-model.js +14 -120
  38. package/dist/commands/health/types-improve.js +4 -19
  39. package/dist/commands/health/windows.js +64 -74
  40. package/dist/commands/health.js +145 -143
  41. package/dist/commands/improve/consolidate/chunking.js +26 -117
  42. package/dist/commands/improve/consolidate/continuity-check.js +137 -0
  43. package/dist/commands/improve/consolidate/pair-pass.js +791 -0
  44. package/dist/commands/improve/consolidate/sanitize.js +54 -149
  45. package/dist/commands/improve/consolidate.js +589 -1127
  46. package/dist/commands/improve/content-hash.js +16 -24
  47. package/dist/commands/improve/distill/content-repair.js +18 -100
  48. package/dist/commands/improve/distill-guards.js +20 -81
  49. package/dist/commands/improve/distill-promotion-policy.js +23 -243
  50. package/dist/commands/improve/distill.js +608 -1041
  51. package/dist/commands/improve/eligibility.js +126 -390
  52. package/dist/commands/improve/execution.js +8 -10
  53. package/dist/commands/improve/extract-prompt.js +1 -2
  54. package/dist/commands/improve/extract.js +487 -1046
  55. package/dist/commands/improve/feedback-valence.js +0 -25
  56. package/dist/commands/improve/improve-cli.js +75 -169
  57. package/dist/commands/improve/improve-result-file.js +10 -66
  58. package/dist/commands/improve/improve-strategies.js +52 -4
  59. package/dist/commands/improve/improve-usage-report.js +18 -64
  60. package/dist/commands/improve/improve.js +480 -1074
  61. package/dist/commands/improve/ledger.js +119 -0
  62. package/dist/commands/improve/locks.js +2 -8
  63. package/dist/commands/improve/loop-stages.js +415 -1073
  64. package/dist/commands/improve/memory/derived-ref.js +12 -77
  65. package/dist/commands/improve/memory/memory-belief.js +16 -118
  66. package/dist/commands/improve/memory/memory-improve.js +266 -14
  67. package/dist/commands/improve/outcome-loop.js +28 -156
  68. package/dist/commands/improve/planner.js +5 -15
  69. package/dist/commands/improve/preparation.js +779 -2319
  70. package/dist/commands/improve/proactive-maintenance.js +34 -101
  71. package/dist/commands/improve/reflect-noise.js +104 -280
  72. package/dist/commands/improve/reflect.js +642 -1353
  73. package/dist/commands/improve/retrieval-gate.js +127 -0
  74. package/dist/commands/improve/retrieval-scope.js +92 -0
  75. package/dist/commands/improve/salience.js +41 -240
  76. package/dist/commands/improve/session-asset.js +19 -100
  77. package/dist/commands/improve/stage.js +322 -0
  78. package/dist/commands/lint/base-linter.js +37 -15
  79. package/dist/commands/proposal/drain.js +261 -578
  80. package/dist/commands/proposal/proposal-cli.js +19 -20
  81. package/dist/commands/proposal/proposal-types.js +31 -24
  82. package/dist/commands/proposal/proposal.js +38 -8
  83. package/dist/commands/proposal/propose.js +134 -160
  84. package/dist/commands/proposal/repository.js +1097 -1394
  85. package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
  86. package/dist/commands/proposal/validators/proposal-validators.js +1 -1
  87. package/dist/commands/proposal/validators/proposals.js +22 -89
  88. package/dist/commands/read/curate.js +105 -462
  89. package/dist/commands/read/knowledge.js +3 -2
  90. package/dist/commands/read/search-cli.js +16 -33
  91. package/dist/commands/read/search.js +17 -23
  92. package/dist/commands/read/show.js +57 -108
  93. package/dist/commands/sources/bundle-cli.js +25 -2
  94. package/dist/commands/sources/bundle-config-ops.js +4 -0
  95. package/dist/commands/sources/dangerous-env-audit.js +1 -2
  96. package/dist/commands/sources/info.js +127 -29
  97. package/dist/commands/sources/installed-stashes.js +197 -746
  98. package/dist/commands/sources/schema-repair.js +98 -129
  99. package/dist/commands/sources/source-add.js +62 -12
  100. package/dist/commands/sources/source-manage.js +9 -2
  101. package/dist/commands/sources/stash-cli.js +24 -4
  102. package/dist/commands/tasks/explain.js +10 -13
  103. package/dist/commands/tasks/tasks-cli.js +12 -13
  104. package/dist/commands/tasks/tasks.js +350 -936
  105. package/dist/commands/tasks/validate.js +26 -24
  106. package/dist/commands/workflow/plan.js +22 -29
  107. package/dist/commands/workflow-cli.js +4 -4
  108. package/dist/core/adapter/adapters/akm-adapter.js +2 -1
  109. package/dist/core/adapter/adapters/akm-lint.js +2 -3
  110. package/dist/core/adapter/adapters/akm-metadata.js +42 -12
  111. package/dist/core/adapter/adapters/akm-task-adapter.js +29 -8
  112. package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
  113. package/dist/core/adapter/execution-source.js +17 -29
  114. package/dist/core/asset/asset-placement.js +4 -13
  115. package/dist/core/asset/frontmatter.js +106 -1
  116. package/dist/core/asset/resolve-ref.js +1 -1
  117. package/dist/core/bundle-id.js +42 -5
  118. package/dist/core/bundle-rename.js +285 -0
  119. package/dist/core/config/config-io.js +1 -2
  120. package/dist/core/config/config-schema.js +9 -34
  121. package/dist/core/config/config-walker.js +1 -1
  122. package/dist/core/config/config.js +184 -111
  123. package/dist/core/config/engine-semantics.js +0 -2
  124. package/dist/core/config/legacy-source-shape-shim.js +38 -9
  125. package/dist/core/config/schema/embedding.js +20 -5
  126. package/dist/core/config/schema/engines.js +5 -0
  127. package/dist/core/config/schema/execution.js +1 -1
  128. package/dist/core/config/schema/experimental.js +1 -1
  129. package/dist/core/config/schema/improve-processes.js +54 -125
  130. package/dist/core/config/schema/improve.js +4 -42
  131. package/dist/core/config/schema/index-config.js +9 -48
  132. package/dist/core/config/schema/scheduler.js +12 -12
  133. package/dist/core/config/schema/search.js +6 -22
  134. package/dist/core/env-secret-ref.js +0 -1
  135. package/dist/core/errors.js +8 -9
  136. package/dist/core/file-change.js +13 -5
  137. package/dist/core/file-lock.js +76 -173
  138. package/dist/core/improve-result.js +35 -7
  139. package/dist/core/improve-types.js +0 -1
  140. package/dist/core/logs-db.js +2 -2
  141. package/dist/core/loopback.js +7 -12
  142. package/dist/core/non-task-input.js +20 -0
  143. package/dist/core/parse.js +13 -16
  144. package/dist/core/paths.js +0 -24
  145. package/dist/core/redaction.js +109 -2
  146. package/dist/core/run-lock.js +2 -5
  147. package/dist/core/spawn-env.js +1 -1
  148. package/dist/core/state/migrations.js +123 -61
  149. package/dist/core/state-db-scope.js +2 -4
  150. package/dist/core/state-db.js +126 -692
  151. package/dist/core/time.js +0 -20
  152. package/dist/core/type-presentation.js +1 -9
  153. package/dist/core/write-source.js +294 -1005
  154. package/dist/execution/input-contract.js +1 -1
  155. package/dist/execution/resolved-request.js +135 -689
  156. package/dist/execution/source.js +63 -257
  157. package/dist/execution/target-ref.js +1 -1
  158. package/dist/indexer/bundle-identity-guard.js +2 -2
  159. package/dist/indexer/db/llm-cache.js +2 -2
  160. package/dist/indexer/ensure-index.js +77 -73
  161. package/dist/indexer/index-rebuild-lock.js +3 -11
  162. package/dist/indexer/index-writer-lock.js +8 -17
  163. package/dist/indexer/index-written-assets.js +141 -154
  164. package/dist/indexer/indexer.js +400 -1124
  165. package/dist/indexer/links/declared-links.js +90 -0
  166. package/dist/indexer/materialize-embeddings.js +60 -397
  167. package/dist/indexer/passes/memory-inference.js +96 -90
  168. package/dist/indexer/passes/metadata.js +132 -219
  169. package/dist/indexer/read-preflight.js +0 -7
  170. package/dist/indexer/scan/doc-to-entry.js +2 -3
  171. package/dist/indexer/scan/drain-dir.js +1 -1
  172. package/dist/indexer/search/db-search.js +190 -590
  173. package/dist/indexer/search/fts-query.js +30 -41
  174. package/dist/indexer/search/ranking.js +28 -154
  175. package/dist/indexer/search/search-attribution.js +12 -32
  176. package/dist/indexer/search/search-fields.js +11 -15
  177. package/dist/indexer/search/search-hit-enrichers.js +54 -85
  178. package/dist/indexer/search/search-source.js +1 -4
  179. package/dist/indexer/usage/usage-events.js +36 -7
  180. package/dist/indexer/walk/walker.js +3 -4
  181. package/dist/integrations/agent/engine-fallback.js +23 -40
  182. package/dist/integrations/agent/engine-resolution.js +93 -183
  183. package/dist/integrations/agent/execution.js +507 -0
  184. package/dist/integrations/agent/model-map.js +28 -156
  185. package/dist/integrations/agent/request-lowering.js +66 -141
  186. package/dist/integrations/agent/runner-dispatch.js +143 -321
  187. package/dist/integrations/agent/runner.js +54 -14
  188. package/dist/integrations/lockfile.js +53 -101
  189. package/dist/llm/client.js +18 -6
  190. package/dist/llm/embedders/deterministic.js +2 -3
  191. package/dist/llm/embedders/profile.js +71 -0
  192. package/dist/llm/embedders/remote.js +11 -17
  193. package/dist/llm/feature-gate.js +0 -8
  194. package/dist/llm/index-passes.js +3 -5
  195. package/dist/llm/memory-infer.js +1 -2
  196. package/dist/llm/structured-call.js +5 -24
  197. package/dist/output/generic-render.js +23 -11
  198. package/dist/output/html-render.js +13 -10
  199. package/dist/output/render-registry.js +3 -32
  200. package/dist/output/shapes/helpers.js +25 -38
  201. package/dist/output/shapes/passthrough.js +1 -9
  202. package/dist/{indexer/graph/graph-types.js → output/text/bundle-rename.js} +4 -1
  203. package/dist/output/text/command-format.js +69 -31
  204. package/dist/output/text/helpers.js +1 -1
  205. package/dist/output/text/migrate.js +5 -14
  206. package/dist/output/text/proposal-format.js +48 -3
  207. package/dist/output/text/show-format.js +13 -17
  208. package/dist/output/text/workflow-format.js +0 -32
  209. package/dist/output/text.js +2 -0
  210. package/dist/registry/factory.js +4 -19
  211. package/dist/registry/network.js +66 -220
  212. package/dist/registry/providers/index.js +0 -2
  213. package/dist/registry/providers/skills-sh.js +3 -14
  214. package/dist/registry/providers/static-index.js +24 -26
  215. package/dist/registry/resolve.js +55 -131
  216. package/dist/scripts/akm-migrate-node.js +42948 -92369
  217. package/dist/scripts/akm-migrate.js +42935 -92354
  218. package/dist/setup/registry-stash-loader.js +4 -13
  219. package/dist/setup/semantic-assets.js +3 -44
  220. package/dist/setup/setup.js +1 -1
  221. package/dist/setup/steps/connection.js +5 -6
  222. package/dist/setup/steps/platforms.js +2 -2
  223. package/dist/setup/steps/tasks.js +25 -15
  224. package/dist/sources/provider-factory.js +17 -18
  225. package/dist/sources/providers/filesystem.js +2 -3
  226. package/dist/sources/providers/git-install.js +7 -1
  227. package/dist/sources/providers/git-provider.js +0 -3
  228. package/dist/sources/providers/git-stash.js +83 -21
  229. package/dist/sources/providers/npm.js +2 -4
  230. package/dist/sources/providers/provider-utils.js +5 -10
  231. package/dist/sources/providers/website.js +0 -2
  232. package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
  233. package/dist/sources/website-url.js +2 -2
  234. package/dist/storage/database.js +9 -35
  235. package/dist/storage/repositories/improve-ledger-repository.js +209 -0
  236. package/dist/storage/repositories/index-connection.js +39 -72
  237. package/dist/storage/repositories/index-entries-repository.js +131 -129
  238. package/dist/storage/repositories/index-entry-mapper.js +1 -2
  239. package/dist/storage/repositories/index-entry-schema.js +101 -268
  240. package/dist/storage/repositories/index-fts-repository.js +86 -256
  241. package/dist/storage/repositories/index-links-repository.js +143 -0
  242. package/dist/storage/repositories/index-llm-cache-repository.js +7 -9
  243. package/dist/storage/repositories/index-meta-repository.js +6 -4
  244. package/dist/storage/repositories/index-schema.js +257 -325
  245. package/dist/storage/repositories/index-utility-repository.js +8 -29
  246. package/dist/storage/repositories/index-vec-repository.js +133 -414
  247. package/dist/storage/repositories/outcome-repository.js +2 -1
  248. package/dist/storage/repositories/proposals-repository.js +104 -1
  249. package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
  250. package/dist/storage/repositories/salience-repository.js +1 -19
  251. package/dist/storage/repositories/task-history-repository.js +26 -4
  252. package/dist/storage/repositories/workflow-runs-repository.js +53 -244
  253. package/dist/storage/sqlite-migrations.js +136 -0
  254. package/dist/storage/sqlite-pragmas.js +11 -9
  255. package/dist/storage/sqlite-transaction.js +170 -0
  256. package/dist/storage/state-db-integrity.js +130 -0
  257. package/dist/tasks/activation-config.js +134 -62
  258. package/dist/tasks/backends/cron.js +191 -302
  259. package/dist/tasks/backends/exec-utils.js +2 -5
  260. package/dist/tasks/backends/launchd.js +141 -748
  261. package/dist/tasks/backends/schtasks.js +119 -623
  262. package/dist/tasks/prepare/prepare-support.js +5 -15
  263. package/dist/tasks/prepare/prepare.js +0 -2
  264. package/dist/tasks/resolve-akm-bin.js +20 -79
  265. package/dist/tasks/run/attempt-lifecycle.js +0 -1
  266. package/dist/tasks/run/load-task.js +1 -1
  267. package/dist/tasks/scheduler-binding.js +20 -238
  268. package/dist/tasks/scheduler-invocation.js +136 -244
  269. package/dist/tasks/scheduler-lock.js +53 -0
  270. package/dist/tasks/scheduler-sync.js +368 -679
  271. package/dist/tasks/source/parse-task-source.js +55 -9
  272. package/dist/tasks/source/task-source-v3-frozen.js +3 -4
  273. package/dist/tasks/source/task-to-v4.js +464 -88
  274. package/dist/workflows/authoring/authoring.js +3 -12
  275. package/dist/workflows/compile.js +211 -0
  276. package/dist/workflows/concurrency-policy.js +13 -74
  277. package/dist/workflows/exec/child-invocation.js +3 -17
  278. package/dist/workflows/exec/child-workflow.js +32 -141
  279. package/dist/workflows/exec/dispatch-redaction.js +13 -53
  280. package/dist/workflows/exec/environment.js +98 -0
  281. package/dist/workflows/exec/exec-unit.js +33 -140
  282. package/dist/workflows/exec/frozen-judge.js +7 -59
  283. package/dist/workflows/exec/native-executor.js +82 -341
  284. package/dist/workflows/exec/param-secrets.js +29 -47
  285. package/dist/workflows/exec/run-workflow.js +154 -387
  286. package/dist/workflows/exec/scheduler.js +9 -36
  287. package/dist/workflows/exec/step-work.js +127 -430
  288. package/dist/workflows/exec/unit-dispatch.js +11 -63
  289. package/dist/workflows/exec/unit-writer.js +8 -52
  290. package/dist/workflows/exec/worktree.js +39 -273
  291. package/dist/workflows/freeze/child-output-references.js +4 -15
  292. package/dist/workflows/freeze/environment.js +99 -92
  293. package/dist/workflows/freeze/freeze.js +172 -0
  294. package/dist/workflows/freeze/step-values.js +19 -21
  295. package/dist/workflows/freeze/targets/child-workflow.js +23 -92
  296. package/dist/workflows/freeze/targets/command.js +10 -33
  297. package/dist/workflows/freeze/targets/script.js +5 -12
  298. package/dist/workflows/freeze/targets/shell.js +3 -6
  299. package/dist/workflows/freeze/targets/task.js +25 -80
  300. package/dist/workflows/freeze/task-bindings.js +20 -67
  301. package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
  302. package/dist/workflows/ir/params.js +6 -51
  303. package/dist/workflows/ir/plan-hash.js +2 -34
  304. package/dist/workflows/parser.js +140 -43
  305. package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
  306. package/dist/workflows/renderer.js +36 -69
  307. package/dist/workflows/resource-limits.js +12 -120
  308. package/dist/workflows/runtime/agent-identity.js +8 -40
  309. package/dist/workflows/runtime/run-outputs.js +3 -6
  310. package/dist/workflows/runtime/run-plan.js +316 -0
  311. package/dist/workflows/runtime/runs.js +48 -200
  312. package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
  313. package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
  314. package/dist/workflows/validate-summary.js +2 -7
  315. package/docs/integration/bundling-akm.md +49 -42
  316. package/docs/migration/README.md +1 -0
  317. package/docs/migration/release-notes/0.9.17.md +43 -0
  318. package/docs/migration/v0.9.1-to-v0.9.2.md +23 -7
  319. package/docs/reference/cli.md +232 -135
  320. package/docs/reference/configuration.md +71 -57
  321. package/docs/reference/data-and-telemetry.md +20 -21
  322. package/docs/reference/tasks.md +105 -39
  323. package/docs/reference/workflow-schema.md +14 -18
  324. package/docs/reference/workflows.md +6 -9
  325. package/package.json +1 -1
  326. package/schemas/akm-config.json +115 -738
  327. package/schemas/akm-workflow.json +1 -0
  328. package/dist/assets/improve-strategies/graph-refresh.json +0 -15
  329. package/dist/assets/prompts/contradiction-judge.md +0 -33
  330. package/dist/assets/prompts/graph-extract-system.md +0 -1
  331. package/dist/assets/prompts/graph-extract-user-prompt.md +0 -35
  332. package/dist/assets/prompts/metadata-enhance-system.md +0 -1
  333. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +0 -4
  334. package/dist/commands/health/advisories.js +0 -150
  335. package/dist/commands/health/metrics.js +0 -329
  336. package/dist/commands/health/surfaces.js +0 -102
  337. package/dist/commands/improve/anti-collapse.js +0 -83
  338. package/dist/commands/improve/collapse-detector.js +0 -432
  339. package/dist/commands/improve/consolidate/eligibility.js +0 -48
  340. package/dist/commands/improve/consolidate/merge.js +0 -149
  341. package/dist/commands/improve/distill/promote-memory.js +0 -291
  342. package/dist/commands/improve/distill/quality-gate.js +0 -337
  343. package/dist/commands/improve/eval-cases.js +0 -52
  344. package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
  345. package/dist/commands/improve/proposal-envelope.js +0 -31
  346. package/dist/commands/improve/run-context.js +0 -123
  347. package/dist/commands/improve/shared.js +0 -31
  348. package/dist/commands/improve/source-identity.js +0 -28
  349. package/dist/commands/improve/triage.js +0 -96
  350. package/dist/commands/proposal/drain-policies.js +0 -151
  351. package/dist/commands/sources/update-transaction.js +0 -220
  352. package/dist/core/action-contributors.js +0 -28
  353. package/dist/core/config/config-version-shim.js +0 -101
  354. package/dist/core/fs-txn.js +0 -405
  355. package/dist/core/lexical-score.js +0 -25
  356. package/dist/core/maintenance-barrier.js +0 -167
  357. package/dist/execution/executable-identity.js +0 -105
  358. package/dist/execution/guarded-source.js +0 -427
  359. package/dist/indexer/db/graph-db.js +0 -444
  360. package/dist/indexer/graph/graph-boost.js +0 -427
  361. package/dist/indexer/graph/graph-dedup.js +0 -95
  362. package/dist/indexer/graph/graph-extraction.js +0 -1108
  363. package/dist/indexer/search/name-match.js +0 -35
  364. package/dist/indexer/search/ranking-contributors.js +0 -515
  365. package/dist/indexer/search/ranking-types.js +0 -4
  366. package/dist/indexer/walk/project-context.js +0 -192
  367. package/dist/integrations/agent/execution-cascade.js +0 -566
  368. package/dist/integrations/agent/execution-definitions.js +0 -202
  369. package/dist/integrations/agent/execution-lowering.js +0 -841
  370. package/dist/integrations/agent/execution-preparation.js +0 -98
  371. package/dist/integrations/agent/inline-execution.js +0 -74
  372. package/dist/llm/graph-extract.js +0 -728
  373. package/dist/llm/metadata-enhance.js +0 -96
  374. package/dist/registry/create-provider-registry.js +0 -29
  375. package/dist/registry/pinned-request-helper.js +0 -247
  376. package/dist/registry/pinned-transport.js +0 -717
  377. package/dist/sources/providers/index.js +0 -14
  378. package/dist/storage/engines/sqlite-migrations.js +0 -271
  379. package/dist/storage/repositories/canaries-repository.js +0 -107
  380. package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
  381. package/dist/storage/repositories/registry-cache.js +0 -113
  382. package/dist/tasks/scheduler-sync-preview.js +0 -52
  383. package/dist/tasks/source/task-to-v3.js +0 -507
  384. package/dist/workflows/freeze/resolve-steps.js +0 -86
  385. package/dist/workflows/freeze/source-freeze.js +0 -64
  386. package/dist/workflows/ir/compile.js +0 -321
  387. package/dist/workflows/ir/environment-v4.js +0 -330
  388. package/dist/workflows/ir/freeze-v4.js +0 -153
  389. package/dist/workflows/ir/schema-v4.js +0 -745
  390. package/dist/workflows/ir/schema.js +0 -354
  391. package/dist/workflows/program/schema.js +0 -77
  392. package/dist/workflows/runtime/checkin.js +0 -57
  393. package/dist/workflows/runtime/plan-classifier.js +0 -196
  394. package/dist/workflows/runtime/unit-checkin.js +0 -45
  395. package/dist/workflows/runtime/unit-phases.js +0 -20
  396. package/dist/workflows/schema.js +0 -4
  397. package/dist/workflows/source-ir/compile.js +0 -200
  398. package/dist/workflows/source-ir/program.js +0 -50
  399. package/dist/workflows/source-ir/result.js +0 -26
  400. package/dist/workflows/source-ir/schema.js +0 -786
  401. package/dist/workflows/source-ir/triggers.js +0 -79
  402. package/dist/workflows/source-ir/uses.js +0 -40
  403. package/dist/workflows/validator.js +0 -60
@@ -2,49 +2,15 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  /**
5
- * `akm distill <ref>` — feedback distillation into lesson proposals (#228).
5
+ * `akm distill <ref>` — distil an asset and its feedback into a lesson (or,
6
+ * for a reinforced memory, a knowledge) proposal. One bounded LLM call, then
7
+ * the shared judge → mint path in `./stage`; the proposal queue is the only
8
+ * way to a live asset. Every invocation emits one `distill_invoked` event
9
+ * carrying its `outcome` (config-disabled runs emit none).
6
10
  *
7
- * The command reads a target asset and any recent feedback events about it,
8
- * asks an LLM to distil a *lesson* (per v1 spec §13) the agent should
9
- * remember next time, and queues the result as a {@link Proposal} (source
10
- * `"distill"`). The proposal queue is the *only* path to a live asset — this
11
- * command never mutates source files directly. Acceptance is a human (or
12
- * automated) decision via `akm proposal accept`.
13
- *
14
- * # Architectural seams
15
- *
16
- * - **Single bounded in-tree LLM call.** Routed through `callStructured`
17
- * under the `distill` gate (v1 spec §14; 0.8.0 unified the orchestration
18
- * and LLM-call gates under `processes.distill.enabled`). The wrapper
19
- * enforces a hard timeout (default 600s / 10 min — overridable via
20
- * `opts.timeoutMs`) and converts disable / throw / timeout
21
- * into a `null` return from `fn`, which we treat as a graceful
22
- * "skipped" outcome (exit 0, no proposal, `distill_invoked` event with
23
- * `outcome: "skipped"`).
24
- * - **Stateless.** No module-level state — every callable is a pure
25
- * function of its arguments and an injectable `chat` seam. The
26
- * architecture seam test (`tests/architecture/llm-stateless-seam.test.ts`)
27
- * applies.
28
- * - **Output substrate.** Proposal creation goes through the `proposals`
29
- * module so distill shares its persistence + validation pipeline with
30
- * `akm reflect` / `akm propose`. Validation failures (LLM returned a
31
- * lesson without required `description` / `when_to_use` frontmatter) are
32
- * a *different* graceful path: no proposal is created, the structured
33
- * error is surfaced, and the command exits non-zero.
34
- *
35
- * # Lesson-name derivation rule
36
- *
37
- * A nested input preserves its first legitimate scope segment
38
- * (`memories/project-a/deploy` → `lessons/project-a/memory-deploy-lesson`). An
39
- * unscoped input stays flat; asset types are not project scopes. Origin prefixes
40
- * remain durable provenance but are not embedded in the output path.
41
- *
42
- * # Why we do not call `runAgent`
43
- *
44
- * Distillation is in-tree per the v1 spec ("bounded in-tree LLM call"). The
45
- * agent dispatch path is a heavier shell-out used by the curator/agent
46
- * surfaces — distill must be cheap, deterministic-ish, and bounded so it can
47
- * be invoked from CI / automation without spinning up an agent harness.
11
+ * Lesson refs: a nested input keeps its first scope segment
12
+ * (`memories/project-a/deploy` → `lessons/project-a/memory-deploy-lesson`); an
13
+ * unscoped input stays flat.
48
14
  */
49
15
  import fs from "node:fs";
50
16
  import distillKnowledgeSystemPrompt from "../../assets/prompts/distill-knowledge-system.md" with { type: "text" };
@@ -54,6 +20,7 @@ import { parseFrontmatter, writeSalienceToFrontmatter } from "../../core/asset/f
54
20
  import { stripMarkdownFences } from "../../core/asset/markdown.js";
55
21
  import { conceptIdFromTypeName, parseRefInput } from "../../core/asset/resolve-ref.js";
56
22
  import { authoringRulesForType } from "../../core/authoring-rules.js";
23
+ import { resolveStashDir } from "../../core/common.js";
57
24
  import { getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
58
25
  import { UsageError } from "../../core/errors.js";
59
26
  import { appendEvent, readEvents } from "../../core/events.js";
@@ -62,110 +29,47 @@ import { parseEmbeddedJsonResponse } from "../../core/parse.js";
62
29
  import { getDbPath } from "../../core/paths.js";
63
30
  import { resolveStandardsContext } from "../../core/standards/resolve-standards-context.js";
64
31
  import { withStateDb } from "../../core/state-db.js";
65
- import { warnVerbose } from "../../core/warn.js";
32
+ import { warn, warnVerbose } from "../../core/warn.js";
33
+ import { recordWrittenPath } from "../../core/write-provenance.js";
66
34
  import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
67
- import { disposeLoweredExecutionDispatchLease, } from "../../integrations/agent/execution-lowering.js";
68
- import { callStructured, preflightStructuredLlmRunner } from "../../llm/structured-call.js";
35
+ import { assertRunnerCredentials } from "../../integrations/agent/runner-dispatch.js";
69
36
  import { closeDatabase, openReadonlyExistingDatabase } from "../../storage/repositories/index-connection.js";
70
37
  import { getAllEntries } from "../../storage/repositories/index-entries-repository.js";
71
- import { isProposalSkipped, listProposals, listProposalsReadOnly, proposalContent, } from "../proposal/repository.js";
72
- import { stripFrontmatterBody as stripBodyForFidelity } from "./content-hash.js";
38
+ import { listProposals } from "../proposal/repository.js";
39
+ import { detectDoubleFrontmatter, isValidDescription } from "../proposal/validators/proposal-quality-validators.js";
40
+ import { akmSearch } from "../read/search.js";
41
+ import { stripFrontmatterBody } from "./content-hash.js";
73
42
  import { autoRepairLessonFrontmatter, autoSwapDescriptionWhenToUse, collectLessonQualityFindings, repairLessonDescriptionTruncation, } from "./distill/content-repair.js";
74
- import { memoryKnowledgePromotionRequiresDispatch, planMemoryKnowledgePromotion, promoteMemoryToKnowledge, } from "./distill/promote-memory.js";
75
- import { fetchTopSimilarLessons, persistOutputEncodingSalience, runLessonQualityJudge, writeQualityRejection, } from "./distill/quality-gate.js";
76
43
  import { buildClsContext, checkDistillFidelity, DEFAULT_CLS_ADJACENT_COUNT } from "./distill-guards.js";
77
- import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
44
+ import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./distill-promotion-policy.js";
78
45
  import { buildRefVocabulary, scoreEncodingSalience } from "./encoding-salience.js";
79
- import { resolveImproveLlmExecution } from "./execution.js";
80
46
  import { resolveImproveStrategy, resolveProcessEnabled } from "./improve-strategies.js";
81
- import { emitProposal } from "./proposal-envelope.js";
82
- import { createRunContext, resolveRunStashDir } from "./run-context.js";
47
+ import { recordLedgerAttempt } from "./ledger.js";
83
48
  import { computeSalience, upsertAssetSalience } from "./salience.js";
84
- import { MAX_REJECTED_PROPOSALS } from "./shared.js";
85
- import { durableImproveRef } from "./source-identity.js";
49
+ import { callStage, mintProposal, noticeSet, rejectedProposalContext, runLessonQualityJudge, stageRunner, } from "./stage.js";
86
50
  /**
87
- * Asset-ref types that `akm distill` structurally refuses as inputs.
88
- *
89
- * Distill *produces* lessons from non-lesson sources (memory, skill, knowledge,
90
- * etc.). Calling distill on an existing `lessons/*` ref would derive
91
- * `lessons/lesson-<name>-lesson-lesson` (double `-lesson` suffix) — the
92
- * recursive-ref defect observed across 323 archived rejected proposals.
93
- *
94
- * 08-F2: `env` and `secret` are refused as a STRUCTURAL floor — distill reads
95
- * the input asset's bytes via `readFileSync` and hands them to the LLM, so
96
- * secret material must never be a distill input. This gate is code, not config:
97
- * it holds even when `allowedTypes` config is mis-set in unattended cron.
98
- *
99
- * The runtime gate inside {@link akmDistill} still refuses these inputs
100
- * defensively (returning an `outcome: "skipped"` envelope with `skipReason:
101
- * "recursive_lesson_input"`). This exported set is the planner-side companion:
102
- * callers that schedule distill attempts (e.g. `akm improve`'s distill queue)
103
- * import it so refs of these types never enter the queue in the first place.
104
- *
105
- * Source of truth: this set drives the gate in `akmDistill` and is consumed
106
- * directly by the improve planner. Adding a new structurally-refused input
107
- * type means updating this constant — the planner picks the change up for
108
- * free.
51
+ * Input types distill structurally refuses: a lesson is the distilled form
52
+ * (distilling one would mint `lessons/lesson-…-lesson`), and env/secret bytes
53
+ * must never reach the model. The improve planner skips these before queuing.
109
54
  */
110
55
  export const DISTILL_REFUSED_INPUT_TYPES = new Set(["lesson", "env", "secret"]);
111
- /**
112
- * Returns true when `type` is structurally refused as an input by
113
- * {@link akmDistill}. See {@link DISTILL_REFUSED_INPUT_TYPES}.
114
- */
115
56
  export function isDistillRefusedInputType(type) {
116
57
  return DISTILL_REFUSED_INPUT_TYPES.has(type);
117
58
  }
118
- // ── Lesson-ref derivation ───────────────────────────────────────────────────
119
- /** Derive the proposed lesson ref from the input ref. See module docblock. */
59
+ /** Derive the proposed lesson ref from the input ref. */
120
60
  export function deriveLessonRef(inputRef) {
121
61
  const parsed = parseRefInput(inputRef);
122
- // Strip the bundle: a feedback signal recorded against `team//skills/deploy`
123
- // distils into the same lesson namespace as `skills/deploy`. The proposal
124
- // id (a UUID) keeps the queue entries distinct, so collisions are not a
125
- // problem — and reviewers want to see them next to each other anyway.
126
62
  const parts = parsed.name.split("/");
127
63
  const scope = parts.length > 1 ? parts.shift() : undefined;
128
- const slug = `${parsed.type}-${parts.join("-")}`.toLowerCase();
129
- // Replace anything outside the canonical asset-name charset with `-`. Keep
130
- // it deterministic so re-runs produce the same ref.
131
- const safe = slug
64
+ const clean = (value) => value
65
+ .toLowerCase()
132
66
  .replace(/[^a-z0-9-]+/g, "-")
133
67
  .replace(/-+/g, "-")
134
68
  .replace(/^-|-$/g, "");
135
- const safeScope = scope
136
- ?.toLowerCase()
137
- .replace(/[^a-z0-9-]+/g, "-")
138
- .replace(/-+/g, "-")
139
- .replace(/^-|-$/g, "");
140
- return `lessons/${safeScope ? `${safeScope}/` : ""}${safe}-lesson`;
69
+ const safeScope = scope ? clean(scope) : "";
70
+ return `lessons/${safeScope ? `${safeScope}/` : ""}${clean(`${parsed.type}-${parts.join("-")}`)}-lesson`;
141
71
  }
142
- // ── Content quality validators ──────────────────────────────────────────────
143
- //
144
- // The actual implementations now live in `core/proposal-quality-validators.ts`
145
- // so the same checks run inside `runProposalValidators` on `proposal accept`.
146
- import { detectDoubleFrontmatter, isValidDescription } from "../proposal/validators/proposal-quality-validators.js";
147
- // ── Prompt assembly ─────────────────────────────────────────────────────────
148
- const LESSON_SYSTEM_PROMPT = distillLessonSystemPrompt;
149
- const KNOWLEDGE_SYSTEM_PROMPT = distillKnowledgeSystemPrompt;
150
- // ── Structured-output schemas (responseSchema lift) ─────────────────────────
151
- //
152
- // PR 1 of the asset-writers decision (see knowledge/projects/akm/
153
- // asset-writers-investigation/00-synthesis): on providers that honour
154
- // `response_format: json_schema`, ask the LLM for a typed JSON object and
155
- // re-assemble the markdown locally. The "emit raw markdown with embedded
156
- // frontmatter" path supports providers that ignore
157
- // the schema (and for the `chat` test seam, which is wired to return strings
158
- // today). Shape-level rejection codes — MALFORMED_FRONTMATTER_BLOCK,
159
- // FRONTMATTER_NOT_OBJECT, INVALID_YAML, UNBALANCED_CODE_FENCE — become
160
- // unreachable on the structured path. Content-quality validators
161
- // (isValidDescription / isValidWhenToUse) keep firing post-assembly because
162
- // the LLM still controls the string contents of typed fields.
163
- /**
164
- * JSON Schema for structured lesson distillation. Mirrors the LESSON_SYSTEM_PROMPT
165
- * frontmatter contract. Required: description, when_to_use, body. Optional:
166
- * tags (string array) so providers that volunteer categorisation hints survive
167
- * the round-trip without being rejected as additionalProperties.
168
- */
72
+ // ── Output contract ──────────────────────────────────────────────────────────
169
73
  export const DISTILL_LESSON_JSON_SCHEMA = {
170
74
  type: "object",
171
75
  required: ["description", "when_to_use", "body"],
@@ -193,21 +97,12 @@ export const DISTILL_LESSON_JSON_SCHEMA = {
193
97
  },
194
98
  },
195
99
  };
196
- /**
197
- * JSON Schema for structured knowledge distillation. Mirrors the
198
- * KNOWLEDGE_SYSTEM_PROMPT contract. Required: description, body. Optional:
199
- * tags, sources.
200
- */
201
100
  export const DISTILL_KNOWLEDGE_JSON_SCHEMA = {
202
101
  type: "object",
203
102
  required: ["description", "body"],
204
103
  additionalProperties: false,
205
104
  properties: {
206
- description: {
207
- type: "string",
208
- minLength: 1,
209
- description: "One-line summary of the knowledge asset.",
210
- },
105
+ description: { type: "string", minLength: 1, description: "One-line summary of the knowledge asset." },
211
106
  body: {
212
107
  type: "string",
213
108
  minLength: 1,
@@ -226,38 +121,31 @@ export const DISTILL_KNOWLEDGE_JSON_SCHEMA = {
226
121
  },
227
122
  };
228
123
  /**
229
- * Assemble a markdown asset from a structured-output payload. Returns `null`
230
- * when the payload is missing required fields — the caller then falls through
231
- * to the prompt-contract markdown path. We deliberately do NOT validate
232
- * content quality here (isValidDescription / isValidWhenToUse run downstream
233
- * on the assembled content); this helper only catches shape-level emptiness
234
- * that the schema may not have rejected (e.g. a provider that ignored
235
- * `minLength` but still returned the field).
124
+ * Assemble markdown from a structured-output payload, or `null` when a
125
+ * required field is empty (the caller then treats the response as markdown).
236
126
  */
237
127
  export function assembleStructuredDistillMarkdown(payload, kind) {
238
128
  if (payload === null || typeof payload !== "object")
239
129
  return null;
240
- const description = typeof payload.description === "string" ? payload.description.trim() : "";
241
- const body = typeof payload.body === "string" ? payload.body.trim() : "";
242
- if (description.length === 0 || body.length === 0)
130
+ const text = (value) => (typeof value === "string" ? value.trim() : "");
131
+ const list = (value) => Array.isArray(value) ? value.filter((v) => typeof v === "string" && v.trim().length > 0) : [];
132
+ const description = text(payload.description);
133
+ const body = text(payload.body);
134
+ if (!description || !body)
243
135
  return null;
244
136
  const fm = { description };
245
137
  if (kind === "lesson") {
246
- const whenToUse = typeof payload.when_to_use === "string" ? payload.when_to_use.trim() : "";
247
- if (whenToUse.length === 0)
138
+ const whenToUse = text(payload.when_to_use);
139
+ if (!whenToUse)
248
140
  return null;
249
141
  fm.when_to_use = whenToUse;
250
142
  }
251
- if (Array.isArray(payload.tags)) {
252
- const tags = payload.tags.filter((t) => typeof t === "string" && t.trim().length > 0);
253
- if (tags.length > 0)
254
- fm.tags = tags;
255
- }
256
- if (kind === "knowledge" && Array.isArray(payload.sources)) {
257
- const sources = payload.sources.filter((s) => typeof s === "string" && s.trim().length > 0);
258
- if (sources.length > 0)
259
- fm.xrefs = sources;
260
- }
143
+ const tags = list(payload.tags);
144
+ if (tags.length > 0)
145
+ fm.tags = tags;
146
+ const sources = kind === "knowledge" ? list(payload.sources) : [];
147
+ if (sources.length > 0)
148
+ fm.xrefs = sources;
261
149
  return assembleAssetFromString(serializeFrontmatterQuoted(fm), body);
262
150
  }
263
151
  function validateKnowledgeContent(content, inputRef) {
@@ -270,1017 +158,673 @@ function validateKnowledgeContent(content, inputRef) {
270
158
  message: `Distilled knowledge for ${inputRef} must include a non-empty markdown body.`,
271
159
  });
272
160
  }
273
- // Knowledge proposals don't strictly require a description, but if one is
274
- // present it must be a real summary — not a placeholder like `---` or a
275
- // truncated heading. Without this check, distill can land knowledge assets
276
- // with `description: ---` (observed in the wild when the LLM has nothing
277
- // meaningful to say about a session-checkpoint memory).
278
- const fm = (parsed.data ?? {});
279
- if (fm.description !== undefined) {
280
- // Knowledge can legitimately mention the topic name in its description, so
281
- // suppress the ref-restatement heuristic that's tuned for lesson assets.
282
- const descCheck = isValidDescription(fm.description, inputRef, { skipRefTailCheck: true });
283
- if (!descCheck.ok) {
161
+ // A present description must be a real summary (not `---` or a heading fragment).
162
+ const description = parsed.data?.description;
163
+ if (description !== undefined) {
164
+ const check = isValidDescription(description, inputRef, { skipRefTailCheck: true });
165
+ if (!check.ok) {
284
166
  findings.push({
285
167
  kind: "invalid-description",
286
168
  field: "description",
287
- message: `Distilled knowledge for ${inputRef} has an invalid description: ${descCheck.reason}.`,
169
+ message: `Distilled knowledge for ${inputRef} has an invalid description: ${check.reason}.`,
288
170
  });
289
171
  }
290
172
  }
291
- // Double-frontmatter pollution shows up in knowledge too — the LLM sometimes
292
- // re-emits the source asset's frontmatter inside its own response, leaving
293
- // two `---`-delimited blocks back-to-back.
294
- const dfm = detectDoubleFrontmatter(content);
295
- if (dfm) {
173
+ const doubled = detectDoubleFrontmatter(content);
174
+ if (doubled) {
296
175
  findings.push({
297
- kind: dfm.kind,
176
+ kind: doubled.kind,
298
177
  field: "body",
299
- message: `Distilled knowledge for ${inputRef}: ${dfm.message}`,
178
+ message: `Distilled knowledge for ${inputRef}: ${doubled.message}`,
300
179
  });
301
180
  }
302
181
  return findings;
303
182
  }
304
183
  /**
305
- * Pure: build the user-prompt body. Exported for tests.
306
- *
307
- * D-3 (#371): restructures the feedback section from raw JSON event lines into
308
- * a Reflexion-style verbal contrast (`## What worked` / `## What failed`).
309
- * The verbal format allows LLMs to use feedback as gradient signal rather than
310
- * just metadata — capturing the +8% AlfWorld lift from arXiv:2303.11366 and
311
- * the contrast-based rule-learning gain from ExpeL arXiv:2308.10144.
184
+ * The distill user prompt. Feedback is rendered as "What worked" / "What
185
+ * failed" contrast when it carries signals, else as a flat event list.
312
186
  */
313
187
  export function buildDistillPrompt(input) {
314
- const lines = [];
315
- lines.push(`Asset ref: ${input.inputRef}`);
316
- lines.push("");
188
+ const lines = [`Asset ref: ${input.inputRef}`, ""];
317
189
  if (input.standardsContext?.trim()) {
318
- lines.push("Standards to follow (the rulebook for this target):");
319
- lines.push(input.standardsContext.trim());
320
- lines.push("");
321
- }
322
- {
323
- const authoringRules = authoringRulesForType(input.proposalKind ?? "lesson");
324
- if (authoringRules) {
325
- lines.push(authoringRules);
326
- lines.push("");
327
- }
190
+ lines.push("Standards to follow (the rulebook for this target):", input.standardsContext.trim(), "");
328
191
  }
192
+ const authoringRules = authoringRulesForType(input.proposalKind ?? "lesson");
193
+ if (authoringRules)
194
+ lines.push(authoringRules, "");
329
195
  lines.push("Asset content:");
330
196
  if (input.assetContent) {
331
- // The output contract also uses YAML fences. Feeding source frontmatter
332
- // verbatim caused local models to copy it into the lesson body, producing
333
- // deterministic double-frontmatter rejection. Distillation needs the
334
- // source body; its metadata is not evidence to reproduce.
335
- const body = parseFrontmatter(input.assetContent).content.trim().slice(0, 3000);
336
- lines.push("```");
337
- lines.push(body);
338
- lines.push("```");
197
+ // Source frontmatter is not evidence; fed verbatim, models copied it into the body.
198
+ lines.push("```", parseFrontmatter(input.assetContent).content.trim().slice(0, 3000), "```");
339
199
  }
340
200
  else {
341
201
  lines.push("(asset is not currently indexed; distil from feedback signal alone)");
342
202
  }
343
203
  lines.push("");
204
+ const flat = (event) => `- ${event.ts} ${event.eventType}${event.metadata ? ` ${JSON.stringify(event.metadata)}` : ""}`;
344
205
  if (input.feedback.length === 0) {
345
206
  lines.push("Recent feedback: (no feedback events recorded — distil from the asset itself)");
346
207
  }
347
208
  else {
348
- // D-3 (#371): verbal contrast format for Reflexion verbal-gradient lift.
349
- // Partition events into positive ("what worked") and negative ("what failed").
350
- const positive = [];
351
- const negative = [];
352
- const neutral = [];
209
+ const worked = [];
210
+ const failed = [];
211
+ const other = [];
353
212
  for (const event of input.feedback) {
354
- const meta = (event.metadata ?? {});
355
- const signal = typeof meta.signal === "string" ? meta.signal : undefined;
356
- const reason = typeof meta.reason === "string" ? meta.reason : "";
357
- const note = typeof meta.note === "string" ? meta.note : "";
358
- const detail = reason || note;
359
- const line = detail ? `- ${event.ts}: ${detail}` : `- ${event.ts}: feedback received`;
360
- if (signal === "positive")
361
- positive.push(line);
362
- else if (signal === "negative")
363
- negative.push(line);
213
+ const meta = event.metadata ?? {};
214
+ const detail = (typeof meta.reason === "string" ? meta.reason : "") || (typeof meta.note === "string" ? meta.note : "");
215
+ const line = `- ${event.ts}: ${detail || "feedback received"}`;
216
+ if (meta.signal === "positive")
217
+ worked.push(line);
218
+ else if (meta.signal === "negative")
219
+ failed.push(line);
364
220
  else
365
- neutral.push(`- ${event.ts} ${event.eventType}${event.metadata ? ` ${JSON.stringify(event.metadata)}` : ""}`);
221
+ other.push(flat(event));
366
222
  }
367
- if (positive.length > 0 || negative.length > 0) {
368
- if (positive.length > 0) {
369
- lines.push("## What worked");
370
- for (const l of positive)
371
- lines.push(l);
372
- lines.push("");
373
- }
374
- if (negative.length > 0) {
375
- lines.push("## What failed");
376
- for (const l of negative)
377
- lines.push(l);
378
- lines.push("");
379
- }
380
- if (neutral.length > 0) {
381
- lines.push("## Other signals");
382
- for (const l of neutral)
383
- lines.push(l);
384
- lines.push("");
223
+ if (worked.length > 0 || failed.length > 0) {
224
+ for (const [heading, section] of [
225
+ ["## What worked", worked],
226
+ ["## What failed", failed],
227
+ ["## Other signals", other],
228
+ ]) {
229
+ if (section.length > 0)
230
+ lines.push(heading, ...section, "");
385
231
  }
386
232
  }
387
233
  else {
388
- // No positive/negative signals — fall back to the pre-D3 flat format for
389
- // non-feedback event types (e.g. reflect_invoked, distill_invoked).
390
- lines.push("Recent feedback events (most recent last):");
391
- for (const event of input.feedback) {
392
- const meta = event.metadata ? ` ${JSON.stringify(event.metadata)}` : "";
393
- lines.push(`- ${event.ts} ${event.eventType}${meta}`);
394
- }
395
- lines.push("");
234
+ lines.push("Recent feedback events (most recent last):", ...input.feedback.map(flat), "");
396
235
  }
397
236
  }
398
237
  if (input.rejectedProposals && input.rejectedProposals.length > 0) {
399
- lines.push("");
400
- lines.push("Previously rejected proposals for this ref (Reflexion context):");
401
- lines.push("The following proposals were already reviewed and rejected. " +
238
+ lines.push("", "Previously rejected proposals for this ref (Reflexion context):", "The following proposals were already reviewed and rejected. " +
402
239
  "Your new proposal MUST differ meaningfully in approach, framing, or evidence.");
403
240
  for (const rp of input.rejectedProposals) {
404
241
  lines.push(`- Rejection reason: ${rp.reason}`);
405
- if (rp.contentPreview) {
242
+ if (rp.contentPreview)
406
243
  lines.push(` Content preview: ${rp.contentPreview.slice(0, 200).replace(/\n/g, " ")}`);
407
- }
408
244
  }
409
245
  }
410
- if (input.proposalKind === "knowledge") {
411
- lines.push("Produce the knowledge markdown file now. Start your response with `---` on the first line, followed by a `description:` field whose value is a 1-sentence summary (20–400 chars). Never use placeholder values like `---`, `tbd`, `n/a`, or a single dash. If the source has nothing meaningful to summarize, do NOT produce a proposal — return an empty response instead. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body.");
412
- }
413
- else {
414
- lines.push("Produce the lesson markdown file now. Start your response with `---` on the first line, followed by `description:` and `when_to_use:` fields. Both must be real one-sentence summaries (20–400 chars) — never placeholder values like `---`, `tbd`, or `n/a`. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body.");
415
- }
246
+ lines.push(input.proposalKind === "knowledge"
247
+ ? "Produce the knowledge markdown file now. Start your response with `---` on the first line, followed by a `description:` field whose value is a 1-sentence summary (20–400 chars). Never use placeholder values like `---`, `tbd`, `n/a`, or a single dash. If the source has nothing meaningful to summarize, do NOT produce a proposal — return an empty response instead. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body."
248
+ : "Produce the lesson markdown file now. Start your response with `---` on the first line, followed by `description:` and `when_to_use:` fields. Both must be real one-sentence summaries (20–400 chars) — never placeholder values like `---`, `tbd`, or `n/a`. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body.");
416
249
  return lines.join("\n");
417
250
  }
418
- // ── Main entry point ────────────────────────────────────────────────────────
419
- /**
420
- * Run a single bounded distillation pass for `ref`. Always emits exactly one
421
- * `distill_invoked` event (with `outcome` in the metadata) regardless of the
422
- * branch taken — so observers can count invocations cheaply.
423
- */
424
- /**
425
- * Best-effort load of the distill INPUT asset plus the #608 encoding-time
426
- * salience scoring: read the source, build the once-per-invocation bigram ref
427
- * vocabulary, then score the asset (novelty×0.40 + magnitude×0.35 +
428
- * predictionError×0.25) and mirror the result to both the asset frontmatter and
429
- * `state.db :: asset_salience`. Every write is best-effort. Extracted verbatim
430
- * from `akmDistill`; returns the (possibly salience-stamped) content plus the
431
- * ref vocabulary the caller reuses when scoring the distilled OUTPUT (G4).
432
- */
433
- async function loadAndScoreInputSalience(args) {
434
- const { inputRef, durableInputRef, salienceWriteKey, stash, outcomeWeightEnabled, lookup, ctx } = args;
435
- // Best-effort load: when the asset is not yet indexed we still proceed —
436
- // the LLM is asked to distil from "available signal" (feedback alone).
437
- let assetContent = null;
438
- let assetFilePath = null;
439
- try {
440
- const filePath = await lookup(durableInputRef);
441
- if (filePath && fs.existsSync(filePath)) {
442
- assetFilePath = filePath;
443
- assetContent = ctx.readAsset(filePath);
444
- }
445
- }
446
- catch {
447
- assetContent = null;
448
- }
449
- // ── #608: Encoding-time salience scoring ────────────────────────────────
450
- // Score the source asset with the three-signal model (novelty × 0.40 +
451
- // magnitude × 0.35 + predictionError × 0.25) and persist the result to:
452
- // 1. The asset's frontmatter (human-readable mirror; idempotent delta gate).
453
- // 2. state.db :: asset_salience (canonical; feeds improve's high-salience gate).
454
- // Both writes are best-effort — a DB error never blocks distillation.
455
- //
456
- // The bigram ref vocabulary is built ONCE per invocation — the novelty signal
457
- // reuses it when scoring the distilled OUTPUT at proposal creation (G4).
458
- let existingRefVocabulary = new Set();
459
- try {
460
- const indexDb = openReadonlyExistingDatabase(getDbPath(), { isolatedSnapshot: true });
461
- if (indexDb) {
462
- try {
463
- const allRefs = getAllEntries(indexDb).map((e) => e.itemRef);
464
- existingRefVocabulary = buildRefVocabulary(allRefs);
465
- }
466
- finally {
467
- closeDatabase(indexDb);
468
- }
469
- }
470
- }
471
- catch {
472
- // Index not available — novelty defaults to type-floor.
473
- }
474
- if (args.persistSalience !== false && assetContent && assetFilePath) {
475
- try {
476
- const parsedRef = parseRefInput(inputRef);
477
- // G4: predictionError decays with revision count — the prior hardcoded
478
- // `revisionCount: 0` made it a dead constant 1.0. Use the number of
479
- // proposals ever raised against this ref as the revision proxy.
480
- let revisionCount = 0;
481
- try {
482
- revisionCount = listProposals(stash, { ref: inputRef, includeArchive: true }).length;
483
- }
484
- catch {
485
- // best-effort: unknown history scores as a first encounter
486
- }
487
- const salienceResult = scoreEncodingSalience({
488
- body: assetContent,
489
- type: parsedRef.type,
490
- existingRefVocabulary,
491
- revisionCount,
492
- });
493
- // 1. Write salience to the source asset frontmatter (idempotent).
494
- const updatedContent = writeSalienceToFrontmatter(assetContent, salienceResult.score, salienceResult);
495
- if (updatedContent !== assetContent) {
496
- ctx.writeAsset(assetFilePath, updatedContent);
497
- assetContent = updatedContent;
498
- }
499
- // 2. Persist encoding_salience to state.db.
500
- try {
501
- withStateDb((stateDb) => {
502
- const vector = computeSalience({
503
- ref: inputRef,
504
- type: parsedRef.type,
505
- retrievalFreq: 0,
506
- encodingSalience: salienceResult.score,
507
- outcomeWeightEnabled,
508
- });
509
- upsertAssetSalience(stateDb, salienceWriteKey, vector);
510
- });
511
- }
512
- catch {
513
- // State DB unavailable — frontmatter mirror is the only persistence.
514
- }
515
- }
516
- catch {
517
- // Scoring errors never block distillation.
518
- }
519
- }
520
- return { assetContent, existingRefVocabulary };
251
+ // ── Invocation ───────────────────────────────────────────────────────────────
252
+ const DISABLED_MESSAGE = "distill is disabled in config; enable processes.distill.enabled to activate.";
253
+ function emitDistill(run, meta) {
254
+ appendEvent({ eventType: "distill_invoked", ref: run.ledgerRef, metadata: { ...meta, ...run.eligMeta } }, run.options.eventsCtx);
521
255
  }
522
- /**
523
- * Recursive-distillation + secret-material input guard. Distill produces
524
- * *lessons* from non-lesson sources; a lesson input would derive a recursive
525
- * `lessons/lesson-<name>` ref (the 323-archived-proposals defect) and
526
- * env/secret inputs must never be read or sent to the LLM. Emits the
527
- * `distill_invoked(skipped)` event and returns the terminal skipped result,
528
- * or `null` when the input type is allowed. Extracted verbatim from
529
- * `akmDistill` (R25/R31 — the events-ctx threading pushed it over the bar).
530
- */
531
- function refuseDisallowedDistillInput(args) {
532
- const { options, parsedInputRef, inputRef, durableInputRef, eligMeta } = args;
533
- if (!isDistillRefusedInputType(parsedInputRef.type))
534
- return null;
535
- // 08-F2: env/secret are a secret-material refusal (never read the bytes);
536
- // lesson is the recursive-form refusal. Both skip BEFORE any readFileSync.
537
- const isSecretInput = parsedInputRef.type === "env" || parsedInputRef.type === "secret";
538
- const skippedRef = isSecretInput ? inputRef : conceptIdFromTypeName("lesson", parsedInputRef.name);
539
- const message = isSecretInput
540
- ? `Distill refuses ${parsedInputRef.type} inputs — secret material must never be sent to the LLM.`
541
- : "Distill refuses lesson inputs — lessons are the distilled form, not a source.";
542
- appendEvent({
543
- eventType: "distill_invoked",
544
- // Key on item_ref when the planner resolved one, otherwise the conceptId.
545
- ref: options.itemRef ?? durableInputRef,
546
- metadata: {
547
- outcome: "skipped",
548
- proposalRef: skippedRef,
549
- message,
550
- skipReason: isSecretInput ? "refused_secret_input" : "recursive_lesson_input",
551
- ...eligMeta,
552
- },
553
- }, options.eventsCtx);
554
- return {
555
- schemaVersion: 1,
556
- ok: true,
557
- outcome: "skipped",
558
- inputRef,
559
- proposalRef: skippedRef,
560
- skipReason: isSecretInput ? "refused_secret_input" : "recursive_lesson_input",
561
- message,
562
- };
563
- }
564
- async function prepareDistillExecution(args) {
565
- const { options, config, inputRef, durableInputRef, salienceWriteKey } = args;
566
- const stash = resolveRunStashDir(options.stashDir);
567
- const chat = options.chat;
568
- const executionNotices = new Map();
569
- const collectNotices = (notices) => {
570
- for (const notice of notices)
571
- executionNotices.set(JSON.stringify(notice), notice);
572
- };
573
- const resolvedExecution = !Object.hasOwn(options, "llmRunner")
574
- ? resolveImproveLlmExecution({
575
- config,
576
- profile: options.improveProfile,
577
- process: getImproveProcessConfig("distill", options.improveProfile),
578
- processName: "distill",
579
- })
580
- : null;
581
- if (resolvedExecution)
582
- collectNotices(resolvedExecution.notices);
583
- const distillRunner = Object.hasOwn(options, "llmRunner")
584
- ? (options.llmRunner ?? undefined)
585
- : resolvedExecution?.runner;
586
- const withNotices = (result) => executionNotices.size > 0 ? { ...result, notices: Object.freeze([...executionNotices.values()]) } : result;
587
- const lookup = options.lookupFn ?? ((ref) => defaultLookup(ref, stash));
588
- const readEventsImpl = options.readEventsFn ??
589
- ((readOptions) => readEvents(readOptions, { readOnly: true }));
590
- const outcomeWeightEnabled = config.improve?.salience?.outcomeWeightEnabled !== false;
591
- const fetchSimilarLessonsFn = options.fetchSimilarLessonsFn ?? ((query, n) => fetchTopSimilarLessons(query, n, options.stashDir));
592
- const assetCtx = createRunContext({
593
- stashDir: stash,
594
- config,
595
- eventsCtx: options.eventsCtx ?? {},
596
- proposalsCtx: options.ctx ?? {},
597
- chat,
598
- getLlmRunner: () => distillRunner ?? null,
599
- sourceRun: options.sourceRun ?? `distill-${Date.now()}`,
600
- dryRun: false,
601
- signal: options.signal,
602
- }).withFreshAssetMemo();
603
- const initialSalience = await loadAndScoreInputSalience({
604
- inputRef,
605
- durableInputRef,
606
- salienceWriteKey,
607
- stash,
608
- config,
609
- outcomeWeightEnabled,
610
- lookup,
611
- ctx: assetCtx,
612
- persistSalience: false,
613
- });
614
- const assetState = { ...initialSalience };
615
- const persistInputSalience = async () => {
616
- const scored = await loadAndScoreInputSalience({
617
- inputRef,
618
- durableInputRef,
619
- salienceWriteKey,
620
- stash,
621
- config,
622
- outcomeWeightEnabled,
623
- lookup,
624
- ctx: assetCtx,
625
- });
626
- assetState.assetContent = scored.assetContent ?? assetState.assetContent;
627
- assetState.existingRefVocabulary = scored.existingRefVocabulary;
628
- };
629
- const feedbackState = readDistillFeedback({ readEventsImpl, options, durableInputRef });
630
- const feedback = feedbackState.filteredEvents.slice(-20).map((event) => ({
631
- ts: event.ts,
632
- eventType: event.eventType,
633
- ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
634
- }));
635
- return {
636
- stash,
637
- chat,
638
- collectNotices,
639
- distillRunner,
640
- withNotices,
641
- lookup,
642
- outcomeWeightEnabled,
643
- fetchSimilarLessonsFn,
644
- assetState,
645
- persistInputSalience,
646
- feedback,
647
- ...feedbackState,
648
- };
256
+ /** The exclusion diagnostics for an event (count only) or a result (count + fully-filtered). */
257
+ function exclusionMeta(run, forResult) {
258
+ if (!run.exclusion)
259
+ return {};
260
+ return forResult ? { ...run.exclusion } : { filteredFeedbackCount: run.exclusion.filteredFeedbackCount };
649
261
  }
650
262
  export async function akmDistill(options) {
651
263
  const inputRef = options.ref.trim();
652
264
  if (!inputRef) {
653
265
  throw new UsageError("Asset ref is required. Usage: akm distill <ref>", "MISSING_REQUIRED_ARGUMENT");
654
266
  }
655
- // Validate the ref shape up front so a typo never reaches the LLM.
656
267
  const parsedInputRef = parseRefInput(inputRef);
657
- const durableInputRef = durableImproveRef(inputRef);
658
- // The input asset's durable salience write key is item_ref when resolved,
659
- // otherwise the input conceptId.
660
- const salienceWriteKey = options.itemRef ?? durableInputRef;
661
- const targetKind = options.proposalKind ?? "lesson";
662
268
  const config = options.config ?? loadConfig();
663
- const improveProfile = options.improveProfile ?? resolveImproveStrategy(undefined, config).config;
664
- options = { ...options, improveProfile };
665
- if (!resolveProcessEnabled("distill", improveProfile)) {
666
- const proposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
269
+ const profile = options.improveProfile ?? resolveImproveStrategy(undefined, config).config;
270
+ options = { ...options, improveProfile: profile };
271
+ const targetKind = options.proposalKind ?? "lesson";
272
+ const kind = targetKind === "knowledge" ? "knowledge" : "lesson";
273
+ const outputRef = kind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
274
+ if (!resolveProcessEnabled("distill", profile)) {
667
275
  return {
668
276
  schemaVersion: 1,
669
277
  ok: true,
670
278
  outcome: "config_disabled",
671
279
  inputRef,
672
- proposalRef: proposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef),
673
- proposalKind,
674
- message: "distill is disabled in config; enable processes.distill.enabled to activate.",
280
+ proposalRef: outputRef,
281
+ proposalKind: kind,
282
+ message: DISABLED_MESSAGE,
675
283
  };
676
284
  }
677
- // Attribution tagging: spread into every distill_invoked event's metadata so
678
- // the lane that selected this asset is recorded uniformly across all outcome
679
- // branches. Empty object when no lane was supplied (direct `akm distill`).
680
- const eligMeta = options.eligibilitySource
681
- ? { eligibilitySource: options.eligibilitySource }
682
- : {};
683
- // Recursive-distillation guard (see refuseDisallowedDistillInput). The
684
- // refused-type set is exported as {@link DISTILL_REFUSED_INPUT_TYPES} so
685
- // the improve planner can skip these refs before queuing distill attempts;
686
- // this runtime check stays as a defensive backstop for direct callers.
687
- const refused = refuseDisallowedDistillInput({ options, parsedInputRef, inputRef, durableInputRef, eligMeta });
688
- if (refused)
689
- return refused;
690
- const prepared = await prepareDistillExecution({
285
+ const ledgerRef = options.itemRef ?? inputRef;
286
+ const eligMeta = options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {};
287
+ if (isDistillRefusedInputType(parsedInputRef.type)) {
288
+ const secret = parsedInputRef.type === "env" || parsedInputRef.type === "secret";
289
+ const proposalRef = secret ? inputRef : conceptIdFromTypeName("lesson", parsedInputRef.name);
290
+ const skipReason = secret ? "refused_secret_input" : "recursive_lesson_input";
291
+ const message = secret
292
+ ? `Distill refuses ${parsedInputRef.type} inputs — secret material must never be sent to the LLM.`
293
+ : "Distill refuses lesson inputs — lessons are the distilled form, not a source.";
294
+ emitDistill({ ledgerRef, eligMeta, options }, { outcome: "skipped", proposalRef, message, skipReason });
295
+ return { schemaVersion: 1, ok: true, outcome: "skipped", inputRef, proposalRef, skipReason, message };
296
+ }
297
+ const stash = options.stashDir ?? resolveStashDir();
298
+ const notices = noticeSet();
299
+ const lookup = options.lookupFn ?? ((ref) => defaultLookup(ref, stash));
300
+ const asset = await loadInput(lookup, inputRef);
301
+ const run = {
691
302
  options,
692
- config,
693
303
  inputRef,
694
- durableInputRef,
695
- salienceWriteKey,
696
- });
697
- const { stash, chat, collectNotices, distillRunner, withNotices, lookup, outcomeWeightEnabled, fetchSimilarLessonsFn, assetState, persistInputSalience, feedback, filteredEvents, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, } = prepared;
698
- const promotionContext = {
699
- targetKind,
700
- inputRef,
701
- durableInputRef,
702
- ...(options.itemRef ? { itemRef: options.itemRef } : {}),
703
- assetContent: assetState.assetContent,
704
- filteredEvents,
705
- config,
706
- strategy: options.improveProfile,
707
- llmRunner: distillRunner,
708
- signal: options.signal,
709
- chat,
304
+ ledgerRef,
710
305
  stash,
711
- lookup,
712
- fetchSimilarLessonsFn,
713
- existingRefVocabulary: assetState.existingRefVocabulary,
714
- outcomeWeightEnabled,
306
+ config,
307
+ profile,
308
+ runner: stageRunner(options, config, profile, "distill", notices.add),
309
+ notices,
715
310
  eligMeta,
716
- eligibilitySource: options.eligibilitySource,
717
- sourceRun: options.sourceRun,
718
- proposalsCtx: options.ctx,
719
- eventsCtx: options.eventsCtx,
720
- exclusionSetSize: exclusionSet.size,
721
- filteredFeedbackCount,
722
- feedbackFullyFiltered,
723
- onNotices: collectNotices,
311
+ asset,
312
+ vocabulary: loadRefVocabulary(),
313
+ outcomeWeightEnabled: config.improve?.salience?.outcomeWeightEnabled !== false,
314
+ similar: options.fetchSimilarLessonsFn ?? fetchTopSimilarLessons,
315
+ lookup,
724
316
  };
725
- const promotionPlan = await planMemoryKnowledgePromotion(promotionContext);
726
- let dispatchLease;
727
- try {
728
- // Memory→knowledge promotion branch (D-1/#369). When the target ref is a
729
- // reinforced memory, distill graduates it into a knowledge proposal instead
730
- // of a lesson — the whole branch (LLM contradiction-merge, quality gate,
731
- // proposal creation, event emit) lives in `promoteMemoryToKnowledge` and is
732
- // terminal when it fires. A `null` return means "not a promotion candidate";
733
- // fall through to the ordinary lesson/knowledge distillation path.
734
- if (promotionPlan && distillRunner && memoryKnowledgePromotionRequiresDispatch(promotionContext, promotionPlan)) {
735
- dispatchLease = await preflightStructuredLlmRunner(distillRunner);
736
- }
737
- const promotionResult = await promoteMemoryToKnowledge({ ...promotionContext, ...(dispatchLease ? { lease: dispatchLease } : {}) }, promotionPlan);
738
- if (promotionResult) {
739
- await persistInputSalience();
740
- return withNotices(promotionResult);
741
- }
742
- const effectiveProposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
743
- const effectiveLessonRef = effectiveProposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
744
- const messages = await buildDistillMessages({
745
- options,
746
- stash,
747
- inputRef,
748
- assetContent: assetState.assetContent,
749
- feedback,
750
- effectiveProposalKind,
751
- effectiveLessonRef,
752
- fetchSimilarLessonsFn,
753
- });
754
- if (!dispatchLease && distillRunner)
755
- dispatchLease = await preflightStructuredLlmRunner(distillRunner);
756
- const { raw, fallbackReason } = await runDistillLlmCall({
757
- config,
758
- options,
759
- distillRunner,
760
- lease: dispatchLease,
761
- messages,
762
- effectiveProposalKind,
763
- onNotices: collectNotices,
764
- });
765
- await persistInputSalience();
766
- if (raw === null || raw.trim() === "") {
767
- return withNotices(distillEmptyResponseResult({
768
- fallbackReason,
769
- inputRef,
770
- durableInputRef,
771
- ...(options.itemRef ? { itemRef: options.itemRef } : {}),
772
- effectiveLessonRef,
773
- effectiveProposalKind,
774
- exclusionSet,
775
- filteredFeedbackCount,
776
- feedbackFullyFiltered,
777
- eligMeta,
778
- eventsCtx: options.eventsCtx,
779
- }));
780
- }
781
- const assembled = assembleAndValidateDistillContent({
782
- raw,
783
- effectiveProposalKind,
784
- inputRef,
785
- durableInputRef,
786
- ...(options.itemRef ? { itemRef: options.itemRef } : {}),
787
- effectiveLessonRef,
788
- exclusionSet,
789
- filteredFeedbackCount,
790
- eligMeta,
791
- eventsCtx: options.eventsCtx,
792
- stash,
793
- });
794
- if ("rejection" in assembled)
795
- return withNotices(assembled.rejection);
796
- const { content, descriptionSwapped } = assembled;
797
- const gate = await applyDistillQualityGate({
798
- config,
799
- options,
800
- content,
801
- assetContent: assetState.assetContent,
802
- chat,
803
- distillRunner,
804
- lease: dispatchLease,
805
- fetchSimilarLessonsFn,
806
- stash,
807
- inputRef,
808
- effectiveLessonRef,
809
- exclusionSet,
810
- filteredFeedbackCount,
811
- feedbackFullyFiltered,
812
- onNotices: collectNotices,
813
- });
814
- if ("rejection" in gate)
815
- return withNotices(gate.rejection);
816
- const lessonJudgeConfidence = gate.confidence;
817
- return withNotices(await emitDistillLessonProposal({
818
- content,
819
- options,
820
- distillRunner,
821
- assetContent: assetState.assetContent,
822
- inputRef,
823
- durableInputRef,
824
- effectiveLessonRef,
825
- effectiveProposalKind,
826
- stash,
827
- exclusionSet,
828
- filteredFeedbackCount,
829
- feedbackFullyFiltered,
830
- lessonJudgeConfidence,
831
- existingRefVocabulary: assetState.existingRefVocabulary,
832
- outcomeWeightEnabled,
833
- descriptionSwapped,
834
- eligMeta,
835
- }));
836
- }
837
- finally {
838
- if (dispatchLease)
839
- disposeLoweredExecutionDispatchLease(dispatchLease);
840
- }
317
+ const feedbackEvents = readDistillFeedback(run);
318
+ const result = await distill(run, targetKind, kind, outputRef, feedbackEvents);
319
+ return { ...result, ...notices.fields() };
841
320
  }
842
- // ── Helpers ─────────────────────────────────────────────────────────────────
843
- /**
844
- * The distill-propose pass: the optional WS-3b distill→source fidelity check
845
- * (routes contradictions to human review), provenance xref round-trip, proposal
846
- * creation, and the queued/skipped `distill_invoked` emit + output salience
847
- * scoring (G4). Extracted verbatim from `akmDistill`; every outcome shape and
848
- * event is byte-identical.
849
- */
850
- async function emitDistillLessonProposal(args) {
851
- const { options, distillRunner, assetContent, inputRef, durableInputRef, effectiveLessonRef, effectiveProposalKind, stash, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, lessonJudgeConfidence, existingRefVocabulary, outcomeWeightEnabled, descriptionSwapped, eligMeta, } = args;
852
- let content = args.content;
853
- // WS-3b: Distill→source fidelity check (step 10).
854
- // When fidelityCheck.enabled, check the distill proposal against its cited
855
- // source memories. A contradiction flag routes to human review (not auto-accept).
856
- // DEFAULT OFF. Fail-open: any error is treated as no-contradiction.
857
- const fidelityConfig = getImproveProcessConfig("distill", options.improveProfile)?.fidelityCheck ??
858
- {};
859
- if (fidelityConfig.enabled && assetContent) {
860
- try {
861
- const proposalBody = stripBodyForFidelity(content);
862
- const sourceBodies = [stripBodyForFidelity(assetContent)];
863
- const fidelityResult = checkDistillFidelity(proposalBody, sourceBodies, fidelityConfig);
864
- if (fidelityResult.contradictionDetected) {
865
- // Route to human review by writing a quality rejection with reviewNeeded=true.
866
- return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed
867
- fidelityResult.reason ?? "Proposal may contradict cited source memories.", {
868
- reviewNeeded: true,
869
- fidelityContradiction: true,
870
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
871
- }, options.eligibilitySource, options.eventsCtx);
872
- }
873
- }
874
- catch {
875
- // Fail open — fidelity check is supplemental.
876
- }
877
- }
878
- // Round-trip the parsed frontmatter so the proposal carries it as a
879
- // structured payload alongside the raw content (matches the shape used by
880
- // other proposal sources).
881
- //
882
- // Serialize canonical provenance into the content that promotion writes.
883
- const parsed = parseFrontmatter(content);
884
- const existingXrefs = Array.isArray(parsed.data.xrefs) ? parsed.data.xrefs.map(String) : [];
885
- const frontmatterWithXrefs = {
886
- ...parsed.data,
887
- xrefs: [...new Set([...existingXrefs, durableInputRef])],
888
- };
889
- delete frontmatterWithXrefs.sources;
890
- content = assembleAsset(frontmatterWithXrefs, parsed.content);
891
- const proposalResult2 = emitProposal({ stashDir: stash, proposalsCtx: options.ctx }, {
892
- ref: effectiveLessonRef,
893
- // §23.6 fingerprint model-id term (WI-6.4). Uses the RESOLVED connection
894
- // (profile/config fallback included), not the raw option — a standalone
895
- // `akm distill` run must fingerprint under the model that actually
896
- // generated the content, matching the promote-memory branch.
897
- ...(distillRunner?.connection.model ? { modelId: distillRunner.connection.model } : {}),
898
- source: "distill",
899
- ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
900
- payload: {
901
- content,
902
- frontmatter: frontmatterWithXrefs,
903
- },
904
- ...(lessonJudgeConfidence !== undefined ? { confidence: lessonJudgeConfidence } : {}),
905
- // Attribution tagging: persist the eligibility lane on the proposal.
906
- ...(options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {}),
907
- });
908
- if (isProposalSkipped(proposalResult2)) {
909
- appendEvent({
910
- eventType: "distill_invoked",
911
- // Use item_ref when resolved, otherwise the input conceptId.
912
- ref: options.itemRef ?? durableInputRef,
913
- metadata: {
914
- outcome: "skipped",
915
- proposalRef: effectiveLessonRef,
916
- message: proposalResult2.message,
917
- skipReason: proposalResult2.reason,
918
- ...eligMeta,
321
+ async function distill(run, targetKind, kind, outputRef, feedbackEvents) {
322
+ // A reinforced memory graduates to knowledge without a generation call.
323
+ const promotion = targetKind === "lesson" ? null : await planPromotion(run, feedbackEvents);
324
+ if (promotion) {
325
+ if (run.runner && (promotion.existing || qualityGateEnabled(run)))
326
+ assertRunnerCredentials(run.runner);
327
+ const promoted = await promoteToKnowledge(run, promotion);
328
+ stampInputSalience(run);
329
+ return promoted;
330
+ }
331
+ const feedback = feedbackEvents.slice(-20).map((event) => ({
332
+ ts: event.ts,
333
+ eventType: event.eventType,
334
+ ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
335
+ }));
336
+ const { system, prompt } = await buildDistillMessages(run, feedback, kind, outputRef);
337
+ const call = run.runner
338
+ ? await callStage({
339
+ feature: "distill",
340
+ runner: run.runner,
341
+ system,
342
+ prompt,
343
+ gate: { config: run.config, enabled: true },
344
+ // The injected test transport never sees the schema.
345
+ request: {
346
+ ...(run.options.chat === undefined
347
+ ? { responseSchema: kind === "knowledge" ? DISTILL_KNOWLEDGE_JSON_SCHEMA : DISTILL_LESSON_JSON_SCHEMA }
348
+ : { chat: run.options.chat }),
349
+ ...(run.options.signal ? { signal: run.options.signal } : {}),
919
350
  },
920
- }, options.eventsCtx);
351
+ onNotices: run.notices.add,
352
+ })
353
+ : { ok: false, reason: "error" };
354
+ // Durable input salience waits until the credential-bearing dispatch returned.
355
+ stampInputSalience(run);
356
+ if (!call.ok || call.raw.trim() === "") {
357
+ if (!call.ok)
358
+ warnVerbose(`[akm] LLM fallback for distill: ${call.reason}`);
359
+ emitDistill(run, {
360
+ outcome: "llm_failed",
361
+ proposalRef: outputRef,
362
+ proposalKind: kind,
363
+ ...exclusionMeta(run, false),
364
+ });
921
365
  return {
922
366
  schemaVersion: 1,
923
367
  ok: true,
924
- outcome: "skipped",
925
- inputRef,
926
- proposalRef: effectiveLessonRef,
927
- skipReason: proposalResult2.reason,
928
- message: proposalResult2.message,
368
+ outcome: "llm_failed",
369
+ inputRef: run.inputRef,
370
+ proposalRef: outputRef,
371
+ proposalKind: kind,
372
+ message: "LLM call returned no usable output (timeout, empty, or error).",
373
+ ...exclusionMeta(run, true),
929
374
  };
930
375
  }
931
- const proposal2 = proposalResult2;
932
- // G4: content-score the distilled OUTPUT so it carries a real encoding
933
- // salience (encoding_source='content') from creation — lessons never get
934
- // another chance (they are refused as distill inputs).
935
- persistOutputEncodingSalience(durableImproveRef(effectiveLessonRef), content, existingRefVocabulary, outcomeWeightEnabled);
936
- appendEvent({
937
- eventType: "distill_invoked",
938
- // Use item_ref when resolved, otherwise the input conceptId.
939
- ref: options.itemRef ?? durableInputRef,
940
- metadata: {
941
- outcome: "queued",
942
- proposalRef: effectiveLessonRef,
943
- proposalKind: effectiveProposalKind,
944
- proposalId: proposal2.id,
945
- // R3: judge verdicts are longitudinally queryable, not just a one-shot
946
- // proposal.confidence write (normalized 1–5 score / 5).
947
- ...(lessonJudgeConfidence !== undefined ? { judgeConfidence: lessonJudgeConfidence } : {}),
948
- ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
949
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
950
- ...(descriptionSwapped > 0 ? { descriptionSwapped } : {}),
951
- ...eligMeta,
952
- },
953
- }, options.eventsCtx);
954
- return {
955
- schemaVersion: 1,
956
- ok: true,
957
- outcome: "queued",
958
- inputRef,
959
- proposalRef: effectiveLessonRef,
960
- proposalKind: effectiveProposalKind,
961
- proposalId: proposal2.id,
962
- proposal: proposal2,
963
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
964
- ...(descriptionSwapped > 0 ? { descriptionSwapped } : {}),
965
- };
376
+ const assembled = assembleDistilledContent(run, call.raw, kind, outputRef);
377
+ if ("rejection" in assembled)
378
+ return assembled.rejection;
379
+ return judgeAndQueue(run, {
380
+ ref: outputRef,
381
+ kind,
382
+ content: assembled.content,
383
+ source: run.asset.content,
384
+ descriptionSwapped: assembled.descriptionSwapped,
385
+ });
966
386
  }
967
- /**
968
- * Turn the raw LLM response into validated proposal content: prefer the
969
- * structured-JSON assembly, else strip markdown fences; run the lesson-only
970
- * auto-repair chain (frontmatter repair, description↔when_to_use swap, truncation
971
- * repair); then lint/validate, emitting `distill_invoked(validation_failed)` and
972
- * throwing a `UsageError` on any finding. Extracted verbatim from `akmDistill`.
973
- */
974
- function assembleAndValidateDistillContent(args) {
975
- const { raw, effectiveProposalKind, inputRef, durableInputRef, itemRef, effectiveLessonRef, exclusionSet, filteredFeedbackCount, eligMeta, eventsCtx, stash, } = args;
976
- // Structured-output path: when the provider honoured the JSON schema, `raw`
977
- // is a JSON object string (not a markdown blob). Try to parse it and assemble
978
- // the canonical `---\nfm\n---\n\nbody` form before using the markdown
979
- // response path. Failure here (non-JSON response, missing
980
- // required field, unexpected types) is non-fatal — we drop down to the
981
- // markdown path which has its own auto-repair + lint pass.
982
- let content;
983
- const structuredCandidate = parseEmbeddedJsonResponse(raw);
984
- const structuredAssembled = structuredCandidate && !Array.isArray(structuredCandidate)
985
- ? assembleStructuredDistillMarkdown(structuredCandidate, effectiveProposalKind)
986
- : null;
987
- if (structuredAssembled !== null) {
988
- content = structuredAssembled;
989
- }
990
- else {
991
- // Strip any stray fence the LLM might have added around the markdown.
992
- content = stripMarkdownFences(raw);
993
- }
994
- // Lesson-path content normalization (see distill/content-repair): auto-repair
995
- // missing frontmatter, description↔when_to_use auto-swap, and truncation
996
- // repair. Knowledge output skips all three (no lesson frontmatter contract).
997
- if (effectiveProposalKind !== "knowledge") {
998
- content = autoRepairLessonFrontmatter(content, inputRef);
999
- }
387
+ /** Turn the response into validated content: structured JSON or markdown, then lesson repairs and lint. */
388
+ function assembleDistilledContent(run, raw, kind, outputRef) {
389
+ const structured = parseEmbeddedJsonResponse(raw);
390
+ let content = (structured && !Array.isArray(structured) ? assembleStructuredDistillMarkdown(structured, kind) : null) ??
391
+ stripMarkdownFences(raw);
1000
392
  let descriptionSwapped = 0;
1001
- if (effectiveProposalKind !== "knowledge") {
1002
- const swapResult = autoSwapDescriptionWhenToUse(content, inputRef);
1003
- content = swapResult.content;
1004
- descriptionSwapped = swapResult.swapped;
1005
- }
1006
- if (effectiveProposalKind !== "knowledge") {
393
+ if (kind === "lesson") {
394
+ content = autoRepairLessonFrontmatter(content, run.inputRef);
395
+ ({ content, swapped: descriptionSwapped } = autoSwapDescriptionWhenToUse(content, run.inputRef));
1007
396
  content = repairLessonDescriptionTruncation(content);
1008
397
  }
1009
- // Parse + lint the lesson before creating the proposal. The lint is the
1010
- // canonical gate for required frontmatter (v1 spec §13): a field that is
1011
- // genuinely missing or empty means there is no valid asset to write, so
1012
- // that failure stays a hard reject — but still emit `distill_invoked` so
1013
- // the failure is observable.
1014
- const structuralFindings = effectiveProposalKind === "knowledge"
1015
- ? validateKnowledgeContent(content, inputRef)
1016
- : lintLessonContent(content, `distill:${inputRef}`).findings;
1017
- // Additional lesson-only quality validators — reject the systematic failure
1018
- // modes seen across 323 archived rejected proposals (see distill/content-repair).
1019
- const qualityFindings = effectiveProposalKind !== "knowledge" && structuralFindings.length === 0
1020
- ? collectLessonQualityFindings(content, inputRef)
1021
- : [];
1022
- if (structuralFindings.length > 0) {
1023
- appendEvent({
1024
- eventType: "distill_invoked",
1025
- // Use item_ref when resolved, otherwise the input conceptId.
1026
- ref: itemRef ?? durableInputRef,
1027
- metadata: {
1028
- outcome: "validation_failed",
1029
- proposalRef: effectiveLessonRef,
1030
- proposalKind: effectiveProposalKind,
1031
- findingKinds: structuralFindings.map((f) => f.kind),
1032
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
1033
- ...eligMeta,
1034
- },
1035
- }, eventsCtx);
1036
- const message = structuralFindings.map((f) => f.message).join("\n");
1037
- throw new UsageError(`Distilled ${effectiveProposalKind} failed validation:\n${message}`, "MISSING_REQUIRED_ARGUMENT", effectiveProposalKind === "knowledge"
398
+ // Required structure missing means there is no asset to write: a hard reject.
399
+ const structural = kind === "knowledge"
400
+ ? validateKnowledgeContent(content, run.inputRef)
401
+ : lintLessonContent(content, `distill:${run.inputRef}`).findings;
402
+ if (structural.length > 0) {
403
+ emitDistill(run, {
404
+ outcome: "validation_failed",
405
+ proposalRef: outputRef,
406
+ proposalKind: kind,
407
+ findingKinds: structural.map((f) => f.kind),
408
+ ...exclusionMeta(run, false),
409
+ });
410
+ throw new UsageError(`Distilled ${kind} failed validation:\n${structural.map((f) => f.message).join("\n")}`, "MISSING_REQUIRED_ARGUMENT", kind === "knowledge"
1038
411
  ? "Knowledge proposals require a non-empty markdown body."
1039
412
  : "Lessons require non-empty `description` and `when_to_use` frontmatter fields. See v1 spec §13.");
1040
413
  }
1041
- if (qualityFindings.length > 0) {
414
+ // Heuristic quality findings go to a human, not the bin.
415
+ const quality = kind === "lesson" ? collectLessonQualityFindings(content, run.inputRef) : [];
416
+ if (quality.length > 0) {
1042
417
  return {
1043
- rejection: writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed — no judge score exists for a structural/heuristic finding
1044
- qualityFindings.map((f) => f.message).join("\n"), {
418
+ rejection: rejectDistilled(run, outputRef, content, 2.0, quality.map((f) => f.message).join("\n"), {
1045
419
  reviewNeeded: true,
1046
- proposalKind: effectiveProposalKind,
1047
- findingKinds: qualityFindings.map((f) => f.kind),
1048
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
1049
- }, eligMeta.eligibilitySource, eventsCtx),
420
+ proposalKind: kind,
421
+ findingKinds: quality.map((f) => f.kind),
422
+ }),
1050
423
  };
1051
424
  }
1052
425
  return { content, descriptionSwapped };
1053
426
  }
427
+ function qualityGateEnabled(run) {
428
+ return run.profile.processes?.distill?.qualityGate?.enabled ?? true;
429
+ }
1054
430
  /**
1055
- * The single bounded distill LLM call: gate-checked via `callStructured`
1056
- * (R26 migration off the raw `chatCompletion` scaffold), passing the
1057
- * lesson/knowledge JSON schema on the production path and keeping the
1058
- * injected test fake schema-blind. Returns the raw response (or `null`) and
1059
- * the fallback reason.
431
+ * Judge the distilled content, then queue it. A rejected, uncertain or
432
+ * source-contradicting result is recorded instead (see {@link writeQualityRejection}).
1060
433
  */
1061
- async function runDistillLlmCall(args) {
1062
- const { config, options, distillRunner, lease, messages, effectiveProposalKind, onNotices } = args;
1063
- const distillSchema = effectiveProposalKind === "knowledge" ? DISTILL_KNOWLEDGE_JSON_SCHEMA : DISTILL_LESSON_JSON_SCHEMA;
1064
- let fallbackReason;
1065
- const enabled = resolveProcessEnabled("distill", options.improveProfile ?? resolveImproveStrategy(undefined, config).config);
1066
- const recordFallback = (feature, reason) => {
1067
- fallbackReason = reason;
1068
- // Log the fallback reason; the caller (raw === null path) handles
1069
- // emitting the distill_invoked event so we don't double-emit here.
1070
- warnVerbose(`[akm] LLM fallback for ${feature}: ${reason}`);
1071
- };
1072
- if (enabled && !distillRunner) {
1073
- // No LLM connection configured. At HEAD this threw a ConfigError inside
1074
- // the gated fn and tryLlmFeature routed it through the "error" fallback;
1075
- // reproduce that terminal state directly (the gate-disabled case above
1076
- // still dominates: when disabled, callStructured takes the "disabled"
1077
- // fallback before any LLM lookup, exactly as before).
1078
- recordFallback("distill", "error");
1079
- return { raw: null, fallbackReason };
434
+ async function judgeAndQueue(run, out) {
435
+ let content = out.content;
436
+ let confidence;
437
+ if (qualityGateEnabled(run)) {
438
+ const similarLessons = await run.similar(content.slice(0, 500), 3);
439
+ const verdict = await runLessonQualityJudge(run.config, content, out.source ?? "", run.options.chat, {
440
+ ...(similarLessons.length > 0 ? { similarLessons } : {}),
441
+ ...(run.runner ? { llmRunner: run.runner } : {}),
442
+ ...(run.options.signal ? { signal: run.options.signal } : {}),
443
+ onNotices: run.notices.add,
444
+ });
445
+ if (!verdict.pass) {
446
+ return rejectDistilled(run, out.ref, content, verdict.score, verdict.reason, {
447
+ ...(verdict.reviewNeeded ? { reviewNeeded: true } : {}),
448
+ ...(verdict.criteria ? { criteria: verdict.criteria } : {}),
449
+ });
450
+ }
451
+ if (verdict.score > 0)
452
+ confidence = verdict.score / 5;
453
+ }
454
+ let frontmatter;
455
+ if (out.promotion) {
456
+ const data = parseFrontmatter(content).data;
457
+ if (Object.keys(data).length > 0)
458
+ frontmatter = data;
1080
459
  }
1081
- const raw = await callStructured({
1082
- feature: "distill",
1083
- akmConfig: config,
1084
- enabled,
1085
- // Safe: when the gate is open, distillRunner is defined (guard above); when
1086
- // it is closed, the transport never runs and the runner is never read.
1087
- runner: distillRunner,
1088
- ...(lease ? { lease } : {}),
1089
- messages,
1090
- request: options.chat === undefined
1091
- ? // Production path: pass the JSON schema so providers that honour
1092
- // `response_format: json_schema` enforce shape upstream. Providers
1093
- // that ignore the option fall through to the prompt-contract
1094
- // markdown path.
1095
- {
1096
- responseSchema: distillSchema,
1097
- ...(options.signal ? { signal: options.signal } : {}),
460
+ else {
461
+ // Optional check against the cited source; a contradiction goes to a human.
462
+ const fidelity = getImproveProcessConfig("distill", run.profile)?.fidelityCheck ?? {};
463
+ if (fidelity.enabled && out.source) {
464
+ try {
465
+ const verdict = checkDistillFidelity(stripFrontmatterBody(content), [stripFrontmatterBody(out.source)], fidelity);
466
+ if (verdict.contradictionDetected) {
467
+ return rejectDistilled(run, out.ref, content, 2.0, verdict.reason ?? "Proposal may contradict cited source memories.", { reviewNeeded: true, fidelityContradiction: true });
1098
468
  }
1099
- : // Test seam: keep the injected fake as the transport; fakes never
1100
- // see the schema (they return markdown strings).
1101
- { chat: options.chat, ...(options.signal ? { signal: options.signal } : {}) },
1102
- parse: (raw) => raw ?? null,
1103
- onError: (_cls, err) => {
1104
- // At HEAD a transport throw escaped to tryLlmFeature's catch, which
1105
- // fired onFallback("error"); reproduce that observable state.
1106
- void err;
1107
- recordFallback("distill", "error");
1108
- return null;
1109
- },
1110
- fallback: null,
1111
- onFallback: (evt) => recordFallback(evt.feature, evt.reason),
1112
- onNotices,
469
+ }
470
+ catch {
471
+ // The fidelity check is supplemental.
472
+ }
473
+ }
474
+ // Canonical provenance goes into the content promotion writes.
475
+ const parsed = parseFrontmatter(content);
476
+ const xrefs = Array.isArray(parsed.data.xrefs) ? parsed.data.xrefs.map(String) : [];
477
+ frontmatter = { ...parsed.data, xrefs: [...new Set([...xrefs, run.inputRef])] };
478
+ delete frontmatter.sources;
479
+ content = assembleAsset(frontmatter, parsed.content);
480
+ }
481
+ const proposal = mintProposal(run.stash, run.options.ctx, {
482
+ ref: out.ref,
483
+ source: "distill",
484
+ ...(run.options.sourceRun !== undefined ? { sourceRun: run.options.sourceRun } : {}),
485
+ payload: { content, ...(frontmatter ? { frontmatter } : {}) },
486
+ ...(confidence !== undefined ? { confidence } : {}),
487
+ ...(run.options.eligibilitySource ? { eligibilitySource: run.options.eligibilitySource } : {}),
488
+ // The ledger keys the attempt by the input, not the output.
489
+ attemptedRefs: [run.ledgerRef],
490
+ }, { judged: confidence !== undefined });
491
+ persistOutputEncodingSalience(run, out.ref, content);
492
+ const swapped = out.descriptionSwapped ? { descriptionSwapped: out.descriptionSwapped } : {};
493
+ emitDistill(run, {
494
+ outcome: "queued",
495
+ proposalRef: out.ref,
496
+ proposalKind: out.kind,
497
+ proposalId: proposal.id,
498
+ ...(confidence !== undefined ? { judgeConfidence: confidence } : {}),
499
+ ...(run.options.sourceRun !== undefined ? { sourceRun: run.options.sourceRun } : {}),
500
+ ...exclusionMeta(run, false),
501
+ ...swapped,
502
+ });
503
+ return {
504
+ schemaVersion: 1,
505
+ ok: true,
506
+ outcome: "queued",
507
+ inputRef: run.inputRef,
508
+ proposalRef: out.ref,
509
+ proposalKind: out.kind,
510
+ proposalId: proposal.id,
511
+ proposal,
512
+ ...exclusionMeta(run, true),
513
+ ...swapped,
514
+ };
515
+ }
516
+ function rejectDistilled(run, proposalRef, content, score, reason, meta) {
517
+ return writeQualityRejection({
518
+ stash: run.stash,
519
+ inputRef: run.inputRef,
520
+ proposalRef,
521
+ content,
522
+ score,
523
+ reason,
524
+ meta: { ...meta, ...exclusionMeta(run, true) },
525
+ eligibilitySource: run.options.eligibilitySource,
526
+ eventsCtx: run.options.eventsCtx,
527
+ proposalsCtx: run.options.ctx,
528
+ sourceRun: run.options.sourceRun,
529
+ ledgerRef: run.ledgerRef,
1113
530
  });
1114
- return { raw, fallbackReason };
1115
531
  }
1116
532
  /**
1117
- * Build the terminal result for an empty/failed distill LLM response,
1118
- * distinguishing the config-gate-off branch (event suppressed) from a real
1119
- * transport/timeout/empty failure (emits `distill_invoked(llm_failed)`).
1120
- * Extracted verbatim from `akmDistill`.
533
+ * Record a distill quality-gate outcome and return its envelope.
534
+ * `quality_rejected` lands in the improve ledger under the input's key (its
535
+ * rejection window keeps selection from regenerating it); `review_needed`
536
+ * mints a pending proposal for a human, stamped `deferred`/`quality-gate` so
537
+ * the triage drain leaves it alone. Content the mint refuses still records
538
+ * the attempt.
1121
539
  */
1122
- function distillEmptyResponseResult(args) {
1123
- const { fallbackReason, inputRef, durableInputRef, itemRef, effectiveLessonRef, effectiveProposalKind, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, eligMeta, eventsCtx, } = args;
1124
- // Distinguish "config gate disabled" from "LLM call failed". For the
1125
- // config-disabled branch, we ALSO suppress the `distill_invoked` event
1126
- // because no LLM work was actually invoked — emitting the event causes
1127
- // the planner to accumulate phantom invocations that drown out real
1128
- // signal.
1129
- if (fallbackReason === "disabled") {
1130
- return {
1131
- schemaVersion: 1,
1132
- ok: true,
1133
- outcome: "config_disabled",
1134
- inputRef,
1135
- proposalRef: effectiveLessonRef,
1136
- proposalKind: effectiveProposalKind,
1137
- message: "distill is disabled in config; enable processes.distill.enabled to activate.",
1138
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
1139
- };
540
+ export function writeQualityRejection(args) {
541
+ const meta = args.meta ?? {};
542
+ const outcome = meta.reviewNeeded ? "review_needed" : "quality_rejected";
543
+ const ledgerRef = args.ledgerRef ?? args.inputRef;
544
+ const access = { proposalsCtx: args.proposalsCtx, eventsCtx: args.eventsCtx };
545
+ const attempt = { stashDir: args.stash, ref: ledgerRef, source: "distill", detail: args.reason };
546
+ let proposal;
547
+ if (outcome === "quality_rejected") {
548
+ recordLedgerAttempt(access, { ...attempt, outcome: "quality_rejected" });
549
+ }
550
+ else {
551
+ try {
552
+ proposal = mintProposal(args.stash, args.proposalsCtx, {
553
+ ref: args.proposalRef,
554
+ source: "distill",
555
+ ...(args.sourceRun !== undefined ? { sourceRun: args.sourceRun } : {}),
556
+ payload: { content: args.content },
557
+ attemptedRefs: [ledgerRef],
558
+ ...(args.eligibilitySource ? { eligibilitySource: args.eligibilitySource } : {}),
559
+ }, { review: { reason: "quality-review", gate: "quality-gate" } });
560
+ }
561
+ catch (error) {
562
+ warn(`[akm] writeQualityRejection: failed to queue ${args.proposalRef} for review: ${error instanceof Error ? error.message : String(error)}`);
563
+ recordLedgerAttempt(access, { ...attempt, outcome: "review_needed" });
564
+ }
1140
565
  }
1141
- // LLM was actually invoked but produced nothing usable (transport error,
1142
- // timeout, or empty/whitespace response). Emit the event so the failure
1143
- // is observable.
566
+ const eligMeta = args.eligibilitySource ? { eligibilitySource: args.eligibilitySource } : {};
1144
567
  appendEvent({
1145
568
  eventType: "distill_invoked",
1146
- // Use item_ref when resolved, otherwise the input conceptId.
1147
- ref: itemRef ?? durableInputRef,
569
+ ref: ledgerRef,
1148
570
  metadata: {
1149
- outcome: "llm_failed",
1150
- proposalRef: effectiveLessonRef,
1151
- proposalKind: effectiveProposalKind,
1152
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
571
+ outcome,
572
+ proposalRef: args.proposalRef,
573
+ score: args.score,
574
+ reason: args.reason,
575
+ ...meta,
1153
576
  ...eligMeta,
1154
577
  },
1155
- }, eventsCtx);
578
+ }, args.eventsCtx);
1156
579
  return {
1157
580
  schemaVersion: 1,
1158
581
  ok: true,
1159
- outcome: "llm_failed",
1160
- inputRef,
1161
- proposalRef: effectiveLessonRef,
1162
- proposalKind: effectiveProposalKind,
1163
- message: "LLM call returned no usable output (timeout, empty, or error).",
1164
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
582
+ outcome,
583
+ inputRef: args.inputRef,
584
+ proposalRef: args.proposalRef,
585
+ score: args.score,
586
+ reason: args.reason,
587
+ ...(proposal ? { proposalId: proposal.id, proposal } : {}),
588
+ ...meta,
1165
589
  };
1166
590
  }
591
+ async function planPromotion(run, feedbackEvents) {
592
+ const assessment = assessMemoryKnowledgePromotionCandidate({
593
+ inputRef: run.inputRef,
594
+ assetContent: run.asset.content,
595
+ feedbackEvents,
596
+ });
597
+ if (!assessment.promote || !assessment.content)
598
+ return null;
599
+ const existingPath = await run.lookup(assessment.knowledgeRef);
600
+ let existing = null;
601
+ try {
602
+ if (existingPath && fs.existsSync(existingPath))
603
+ existing = fs.readFileSync(existingPath, "utf8");
604
+ }
605
+ catch {
606
+ existing = null;
607
+ }
608
+ return { knowledgeRef: assessment.knowledgeRef, content: assessment.content, existing };
609
+ }
1167
610
  /**
1168
- * The P2-B LLM-as-judge quality gate (fail-CLOSED; D-5/#388 three-band). Returns
1169
- * a terminal rejection result when the judge rejects (or routes to review), or
1170
- * the normalized [0,1] confidence to carry onto the proposal. Extracted verbatim
1171
- * from `akmDistill`.
611
+ * Promote a reinforced memory to knowledge. An existing destination is
612
+ * reconciled by the model (ADD/UPDATE swap content in, NOOP keeps what is
613
+ * there); without a model the existing content is appended for the reviewer.
1172
614
  */
1173
- async function applyDistillQualityGate(args) {
1174
- const { config, options, content, assetContent, chat, distillRunner, lease, fetchSimilarLessonsFn, stash, inputRef, effectiveLessonRef, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, onNotices, } = args;
1175
- if (!(options.improveProfile?.processes?.distill?.qualityGate?.enabled ?? true)) {
1176
- return { confidence: undefined };
1177
- }
1178
- // D-4 / #390: retrieve top-3 similar lessons for dedup check in judge.
1179
- const similarLessons = await fetchSimilarLessonsFn(content.slice(0, 500), 3);
1180
- const judgeResult = await runLessonQualityJudge(config, content, assetContent ?? "", chat, {
1181
- ...(similarLessons.length > 0 ? { similarLessons } : {}),
1182
- ...(distillRunner ? { llmRunner: distillRunner } : {}),
1183
- ...(lease ? { lease } : {}),
1184
- ...(options.signal ? { signal: options.signal } : {}),
1185
- onNotices,
1186
- });
1187
- if (!judgeResult.pass) {
1188
- if (judgeResult.reviewNeeded) {
615
+ async function promoteToKnowledge(run, plan) {
616
+ let content = plan.content;
617
+ if (plan.existing && run.runner) {
618
+ const merged = await callStage({
619
+ feature: "distill",
620
+ runner: run.runner,
621
+ system: "Return only valid JSON. No prose.",
622
+ prompt: [
623
+ "You are merging two versions of a knowledge document.",
624
+ "Existing content is already committed; new content comes from a memory distillation run.",
625
+ "Choose one of: ADD (combine both), UPDATE (replace existing with new), NOOP (keep existing unchanged).",
626
+ 'Return ONLY valid JSON: {"action": "ADD"|"UPDATE"|"NOOP", "content": "<merged markdown if ADD/UPDATE, empty string if NOOP>"}',
627
+ "",
628
+ "## Existing knowledge content",
629
+ "```",
630
+ plan.existing.slice(0, 3000),
631
+ "```",
632
+ "",
633
+ "## New content from distillation",
634
+ "```",
635
+ plan.content.slice(0, 3000),
636
+ "```",
637
+ ].join("\n"),
638
+ request: {
639
+ ...(run.options.signal ? { signal: run.options.signal } : {}),
640
+ ...(run.options.chat ? { chat: run.options.chat } : {}),
641
+ },
642
+ onNotices: run.notices.add,
643
+ });
644
+ const decision = merged.ok
645
+ ? parseEmbeddedJsonResponse(merged.raw)
646
+ : undefined;
647
+ if (decision?.action === "NOOP") {
648
+ emitDistill(run, {
649
+ outcome: "skipped",
650
+ proposalRef: plan.knowledgeRef,
651
+ message: "D-1: LLM resolved destination conflict as NOOP — existing content kept",
652
+ });
1189
653
  return {
1190
- rejection: writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, {
1191
- reviewNeeded: true,
1192
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
1193
- }, options.eligibilitySource, options.eventsCtx),
654
+ schemaVersion: 1,
655
+ ok: true,
656
+ outcome: "skipped",
657
+ inputRef: run.inputRef,
658
+ proposalRef: plan.knowledgeRef,
659
+ skipReason: "conflict_noop",
660
+ message: "Existing knowledge content unchanged (contradiction resolution: NOOP)",
1194
661
  };
1195
662
  }
1196
- return {
1197
- rejection: writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}, options.eligibilitySource, options.eventsCtx),
1198
- };
663
+ if ((decision?.action === "ADD" || decision?.action === "UPDATE") && decision.content?.trim()) {
664
+ content = decision.content;
665
+ }
666
+ }
667
+ else if (plan.existing) {
668
+ content = [
669
+ plan.content,
670
+ "",
671
+ "---",
672
+ "<!-- D-1 / #369: Existing knowledge content is shown below for reviewer reference. -->",
673
+ "<!-- Review: decide whether to ADD (merge), UPDATE (replace), or NOOP (keep existing). -->",
674
+ "",
675
+ "## Existing content (for reviewer reference)",
676
+ "",
677
+ plan.existing,
678
+ ].join("\n");
679
+ }
680
+ return judgeAndQueue(run, {
681
+ ref: plan.knowledgeRef,
682
+ kind: "knowledge",
683
+ content,
684
+ source: run.asset.content,
685
+ promotion: true,
686
+ });
687
+ }
688
+ // ── Inputs ───────────────────────────────────────────────────────────────────
689
+ /** Read the input asset (best-effort: an unindexed asset distils from feedback alone). */
690
+ async function loadInput(lookup, inputRef) {
691
+ try {
692
+ const filePath = await lookup(inputRef);
693
+ if (filePath && fs.existsSync(filePath))
694
+ return { path: filePath, content: fs.readFileSync(filePath, "utf8") };
695
+ }
696
+ catch {
697
+ // An index miss is not fatal.
698
+ }
699
+ return { path: null, content: null };
700
+ }
701
+ /** The index's ref bigram vocabulary, for the novelty term of encoding salience. */
702
+ function loadRefVocabulary() {
703
+ try {
704
+ const db = openReadonlyExistingDatabase(getDbPath(), { isolatedSnapshot: true });
705
+ if (!db)
706
+ return new Set();
707
+ try {
708
+ return buildRefVocabulary(getAllEntries(db).map((e) => e.itemRef));
709
+ }
710
+ finally {
711
+ closeDatabase(db);
712
+ }
713
+ }
714
+ catch {
715
+ return new Set();
1199
716
  }
1200
- // Normalize 1-5 judge score to [0, 1]. Only a real passing verdict
1201
- // reaches here (07 P0-2: the judge now fails CLOSED on no-LLM / timeout /
1202
- // parse failure, so those return pass:false and never fall through to
1203
- // this line). A defensive score>0 guard keeps confidence undefined for any
1204
- // non-positive score the auto-accept gate should treat as unscored.
1205
- return { confidence: judgeResult.score > 0 ? judgeResult.score / 5 : undefined };
1206
717
  }
1207
718
  /**
1208
- * Read the target ref's `feedback` events and apply the #267 exclusion filter.
1209
- * Returns the filtered events plus the exclusion tallies the outcome branches
1210
- * carry. Extracted verbatim from `akmDistill`.
719
+ * Score the input's encoding salience and mirror it to the asset frontmatter
720
+ * and `asset_salience` (keyed by the ledger ref). Best-effort throughout.
1211
721
  */
1212
- function readDistillFeedback(args) {
1213
- const { readEventsImpl, options, durableInputRef } = args;
1214
- const { events: unfilteredEvents } = readEventsImpl({
1215
- ref: options.itemRef ?? durableInputRef,
1216
- type: "feedback",
1217
- excludeTags: options.excludeTags,
1218
- includeTags: options.includeTags,
1219
- });
1220
- const events = unfilteredEvents;
1221
- // #267 — feedback exclusion. Filter events whose `ref` matches the
1222
- // exclusion list BEFORE the prompt is built. The original event stream
1223
- // is never mutated; only the `feedback` slice that reaches the LLM is
1224
- // affected. Exclusion refs are compared with event refs exactly.
1225
- const exclusionList = options.excludeFeedbackFromRefs ?? [];
1226
- const exclusionSet = new Set(exclusionList.map((ref) => ref.trim()).filter((ref) => ref.length > 0));
1227
- const originalEventCount = events.length;
1228
- const filteredEvents = exclusionSet.size > 0 ? events.filter((e) => !(e.ref !== undefined && exclusionSet.has(e.ref))) : events;
1229
- const filteredFeedbackCount = originalEventCount - filteredEvents.length;
1230
- const feedbackFullyFiltered = exclusionSet.size > 0 && originalEventCount > 0 && filteredEvents.length === 0;
1231
- return { filteredEvents, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered };
722
+ function stampInputSalience(run) {
723
+ const { content, path: filePath } = run.asset;
724
+ if (!content || !filePath)
725
+ return;
726
+ try {
727
+ const type = parseRefInput(run.inputRef).type;
728
+ let revisionCount = 0;
729
+ try {
730
+ // Revisions so far: every proposal raised against this ref.
731
+ revisionCount = listProposals(run.stash, { ref: run.inputRef, includeArchive: true }).length;
732
+ }
733
+ catch {
734
+ // Unknown history scores as a first encounter.
735
+ }
736
+ const scored = scoreEncodingSalience({ body: content, type, existingRefVocabulary: run.vocabulary, revisionCount });
737
+ const updated = writeSalienceToFrontmatter(content, scored.score, scored);
738
+ if (updated !== content) {
739
+ fs.writeFileSync(filePath, updated, "utf8");
740
+ recordWrittenPath(filePath);
741
+ run.asset.content = updated;
742
+ }
743
+ try {
744
+ withStateDb((stateDb) => upsertAssetSalience(stateDb, run.ledgerRef, computeSalience({
745
+ ref: run.inputRef,
746
+ type,
747
+ retrievalFreq: 0,
748
+ encodingSalience: scored.score,
749
+ outcomeWeightEnabled: run.outcomeWeightEnabled,
750
+ })));
751
+ }
752
+ catch {
753
+ // The frontmatter mirror is the only persistence then.
754
+ }
755
+ }
756
+ catch {
757
+ // Scoring never blocks distillation.
758
+ }
1232
759
  }
1233
760
  /**
1234
- * Build the distill chat messages: inject the last 1–3 rejected proposals
1235
- * (Reflexion verbal-RL), the optional WS-3b CLS adjacent-context, and the stash
1236
- * authoring standards, then assemble the system+user prompt. Extracted verbatim
1237
- * from `akmDistill`.
761
+ * Content-score a distilled output so it carries a real encoding salience from
762
+ * creation — lessons are refused as inputs, so this is their only chance.
1238
763
  */
1239
- async function buildDistillMessages(args) {
1240
- const { options, stash, inputRef, assetContent, feedback, effectiveProposalKind, effectiveLessonRef, fetchSimilarLessonsFn, } = args;
1241
- // Inject last 1–3 rejected proposals for this ref as Reflexion-style
1242
- // verbal-RL context so the LLM avoids regenerating refused proposals.
1243
- const rejectedForRef = listProposalsReadOnly(stash, { ref: inputRef, status: "rejected", includeArchive: true }, options.ctx)
1244
- .sort((a, b) => new Date(b.updatedAt ?? 0).getTime() - new Date(a.updatedAt ?? 0).getTime())
1245
- .slice(0, MAX_REJECTED_PROPOSALS)
1246
- .map((p) => ({
1247
- reason: p.review?.reason ?? "no reason given",
1248
- contentPreview: proposalContent(p).slice(0, 500),
1249
- }));
1250
- // WS-3b CLS interleaving (step 9).
1251
- // When cls.enabled, inject embedding-retrieved adjacent lessons/knowledge
1252
- // into the distill prompt so the LLM avoids overwriting prior generalizations
1253
- // (catastrophic interference). DEFAULT OFF.
1254
- const clsConfig = getImproveProcessConfig("distill", options.improveProfile)?.cls ?? {};
764
+ function persistOutputEncodingSalience(run, ref, body) {
765
+ try {
766
+ const type = parseRefInput(ref).type;
767
+ const scored = scoreEncodingSalience({ body, type, existingRefVocabulary: run.vocabulary, revisionCount: 0 });
768
+ withStateDb((stateDb) => upsertAssetSalience(stateDb, ref, computeSalience({
769
+ ref,
770
+ type,
771
+ retrievalFreq: 0,
772
+ encodingSalience: scored.score,
773
+ outcomeWeightEnabled: run.outcomeWeightEnabled,
774
+ })));
775
+ }
776
+ catch {
777
+ // Scoring never blocks proposal creation.
778
+ }
779
+ }
780
+ /** The ref's feedback events, minus any `excludeFeedbackFromRefs` matches. */
781
+ function readDistillFeedback(run) {
782
+ const read = run.options.readEventsFn ??
783
+ ((readOptions) => readEvents(readOptions, { readOnly: true }));
784
+ const { events } = read({
785
+ ref: run.ledgerRef,
786
+ type: "feedback",
787
+ excludeTags: run.options.excludeTags,
788
+ includeTags: run.options.includeTags,
789
+ });
790
+ const excluded = new Set((run.options.excludeFeedbackFromRefs ?? []).map((ref) => ref.trim()).filter((ref) => ref.length > 0));
791
+ if (excluded.size === 0)
792
+ return events;
793
+ const kept = events.filter((e) => !(e.ref !== undefined && excluded.has(e.ref)));
794
+ run.exclusion = {
795
+ filteredFeedbackCount: events.length - kept.length,
796
+ feedbackFullyFiltered: events.length > 0 && kept.length === 0,
797
+ };
798
+ return kept;
799
+ }
800
+ /** System + user prompt: rejected-proposal context, optional CLS neighbours, stash standards. */
801
+ async function buildDistillMessages(run, feedback, kind, outputRef) {
802
+ const rejectedProposals = rejectedProposalContext(run.stash, run.inputRef, run.options.ctx);
803
+ // CLS interleaving (default off): show related lessons so the model does not overwrite them.
804
+ const cls = getImproveProcessConfig("distill", run.profile)?.cls ?? {};
1255
805
  let clsContext = "";
1256
- if (clsConfig.enabled) {
806
+ if (cls.enabled) {
1257
807
  try {
1258
- const adjacentCount = clsConfig.adjacentCount ?? DEFAULT_CLS_ADJACENT_COUNT;
1259
- // Use the asset content or input ref as the query for adjacent retrieval.
1260
- const clsQuery = assetContent ? assetContent.slice(0, 500) : inputRef;
1261
- const adjacentItems = await fetchSimilarLessonsFn(clsQuery, adjacentCount);
1262
- clsContext = buildClsContext(adjacentItems, clsConfig);
808
+ const query = run.asset.content ? run.asset.content.slice(0, 500) : run.inputRef;
809
+ clsContext = buildClsContext(await run.similar(query, cls.adjacentCount ?? DEFAULT_CLS_ADJACENT_COUNT), cls);
1263
810
  }
1264
811
  catch {
1265
- // Fail open — CLS is supplemental, never required.
812
+ // CLS context is supplemental.
1266
813
  }
1267
814
  }
1268
- // Distill output is a lesson/knowledge (non-wiki) → stash authoring
1269
- // standards. Resolved once for this single call.
1270
- const standardsContext = resolveStandardsContext(effectiveLessonRef, stash);
1271
- const baseUserPrompt = buildDistillPrompt({
1272
- inputRef,
1273
- assetContent,
815
+ const standardsContext = resolveStandardsContext(outputRef, run.stash);
816
+ const prompt = buildDistillPrompt({
817
+ inputRef: run.inputRef,
818
+ assetContent: run.asset.content,
1274
819
  feedback,
1275
- proposalKind: effectiveProposalKind,
1276
- ...(rejectedForRef.length > 0 ? { rejectedProposals: rejectedForRef } : {}),
820
+ proposalKind: kind,
821
+ ...(rejectedProposals.length > 0 ? { rejectedProposals } : {}),
1277
822
  ...(standardsContext.trim() ? { standardsContext } : {}),
1278
823
  });
1279
- const userPrompt = clsContext ? `${baseUserPrompt}${clsContext}` : baseUserPrompt;
1280
- return [
1281
- { role: "system", content: effectiveProposalKind === "knowledge" ? KNOWLEDGE_SYSTEM_PROMPT : LESSON_SYSTEM_PROMPT },
1282
- { role: "user", content: userPrompt },
1283
- ];
824
+ return {
825
+ system: kind === "knowledge" ? distillKnowledgeSystemPrompt : distillLessonSystemPrompt,
826
+ prompt: `${prompt}${clsContext}`,
827
+ };
1284
828
  }
1285
829
  async function defaultLookup(ref, stashDir) {
1286
830
  return resolveAssetPath(ref, {
@@ -1291,3 +835,26 @@ async function defaultLookup(ref, stashDir) {
1291
835
  honorOrigin: false,
1292
836
  });
1293
837
  }
838
+ /** Top-N existing lessons similar to `query` (empty when search is unavailable). */
839
+ async function fetchTopSimilarLessons(query, n) {
840
+ try {
841
+ const result = await akmSearch({ query, type: "lesson", limit: n, skipLogging: true, eventSource: "improve" });
842
+ return (result?.hits ?? [])
843
+ .filter((h) => "path" in h && typeof h.path === "string")
844
+ .slice(0, n)
845
+ .map((h) => {
846
+ let content = "";
847
+ try {
848
+ if (h.path && fs.existsSync(h.path))
849
+ content = fs.readFileSync(h.path, "utf8");
850
+ }
851
+ catch {
852
+ // best-effort
853
+ }
854
+ return { ref: h.ref, content };
855
+ });
856
+ }
857
+ catch {
858
+ return [];
859
+ }
860
+ }