akm-cli 0.9.17-alpha.2 → 0.9.17-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. package/CHANGELOG.md +756 -0
  2. package/dist/akm +94 -196
  3. package/dist/cli/shared.js +6 -2
  4. package/dist/cli.js +22 -9
  5. package/dist/commands/agent/agent-dispatch.js +1 -1
  6. package/dist/commands/command/command-execution.js +24 -62
  7. package/dist/commands/feedback-cli.js +0 -1
  8. package/dist/commands/health/accept-rate.js +2 -2
  9. package/dist/commands/health/checks.js +30 -75
  10. package/dist/commands/health/config-skew.js +38 -0
  11. package/dist/commands/health/egress.js +54 -0
  12. package/dist/commands/health/html-report.js +0 -38
  13. package/dist/commands/health/improve-metrics.js +123 -562
  14. package/dist/commands/health/plugin-staleness.js +53 -3
  15. package/dist/commands/health/renderers.js +12 -4
  16. package/dist/commands/health/report-view-model.js +11 -106
  17. package/dist/commands/health/types-improve.js +4 -19
  18. package/dist/commands/health/windows.js +64 -73
  19. package/dist/commands/health.js +122 -143
  20. package/dist/commands/improve/consolidate/chunking.js +25 -100
  21. package/dist/commands/improve/consolidate/sanitize.js +54 -149
  22. package/dist/commands/improve/consolidate.js +538 -1075
  23. package/dist/commands/improve/content-hash.js +16 -24
  24. package/dist/commands/improve/distill/content-repair.js +18 -100
  25. package/dist/commands/improve/distill-guards.js +20 -81
  26. package/dist/commands/improve/distill-promotion-policy.js +23 -243
  27. package/dist/commands/improve/distill.js +608 -1075
  28. package/dist/commands/improve/eligibility.js +126 -400
  29. package/dist/commands/improve/execution.js +3 -5
  30. package/dist/commands/improve/extract.js +487 -1046
  31. package/dist/commands/improve/feedback-valence.js +0 -25
  32. package/dist/commands/improve/improve-cli.js +29 -166
  33. package/dist/commands/improve/improve-result-file.js +10 -66
  34. package/dist/commands/improve/improve-strategies.js +12 -7
  35. package/dist/commands/improve/improve-usage-report.js +18 -64
  36. package/dist/commands/improve/improve.js +443 -1063
  37. package/dist/commands/improve/ledger.js +114 -0
  38. package/dist/commands/improve/locks.js +2 -8
  39. package/dist/commands/improve/loop-stages.js +459 -1172
  40. package/dist/commands/improve/memory/derived-ref.js +12 -77
  41. package/dist/commands/improve/memory/memory-belief.js +14 -118
  42. package/dist/commands/improve/memory/memory-improve.js +4 -3
  43. package/dist/commands/improve/outcome-loop.js +28 -156
  44. package/dist/commands/improve/planner.js +5 -10
  45. package/dist/commands/improve/preparation.js +851 -2339
  46. package/dist/commands/improve/proactive-maintenance.js +34 -101
  47. package/dist/commands/improve/reflect-noise.js +104 -280
  48. package/dist/commands/improve/reflect.js +621 -1367
  49. package/dist/commands/improve/salience.js +46 -232
  50. package/dist/commands/improve/session-asset.js +19 -100
  51. package/dist/commands/improve/stage.js +323 -0
  52. package/dist/commands/proposal/drain.js +251 -644
  53. package/dist/commands/proposal/proposal-cli.js +3 -18
  54. package/dist/commands/proposal/proposal-types.js +20 -41
  55. package/dist/commands/proposal/proposal.js +1 -2
  56. package/dist/commands/proposal/propose.js +134 -160
  57. package/dist/commands/proposal/repository.js +502 -1487
  58. package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
  59. package/dist/commands/proposal/validators/proposal-validators.js +1 -1
  60. package/dist/commands/proposal/validators/proposals.js +13 -89
  61. package/dist/commands/read/curate.js +63 -413
  62. package/dist/commands/read/search-cli.js +16 -33
  63. package/dist/commands/read/search.js +17 -23
  64. package/dist/commands/read/show.js +2 -13
  65. package/dist/commands/sources/bundle-cli.js +25 -2
  66. package/dist/commands/sources/bundle-config-ops.js +7 -0
  67. package/dist/commands/sources/dangerous-env-audit.js +1 -2
  68. package/dist/commands/sources/info.js +2 -11
  69. package/dist/commands/sources/installed-stashes.js +197 -746
  70. package/dist/commands/sources/schema-repair.js +98 -129
  71. package/dist/commands/sources/source-add.js +62 -12
  72. package/dist/commands/sources/stash-cli.js +1 -1
  73. package/dist/commands/tasks/explain.js +10 -13
  74. package/dist/commands/tasks/tasks-cli.js +9 -8
  75. package/dist/commands/tasks/tasks.js +326 -930
  76. package/dist/commands/tasks/validate.js +42 -21
  77. package/dist/commands/workflow/plan.js +22 -29
  78. package/dist/commands/workflow-cli.js +4 -4
  79. package/dist/core/adapter/adapters/akm-adapter.js +0 -1
  80. package/dist/core/adapter/adapters/akm-lint.js +2 -3
  81. package/dist/core/adapter/adapters/akm-metadata.js +11 -12
  82. package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
  83. package/dist/core/adapter/execution-source.js +17 -29
  84. package/dist/core/asset/resolve-ref.js +1 -1
  85. package/dist/core/bundle-id.js +42 -5
  86. package/dist/core/bundle-rename.js +291 -0
  87. package/dist/core/config/config-io.js +1 -2
  88. package/dist/core/config/config-schema.js +1 -33
  89. package/dist/core/config/config-walker.js +1 -1
  90. package/dist/core/config/config.js +163 -68
  91. package/dist/core/config/legacy-source-shape-shim.js +38 -9
  92. package/dist/core/config/schema/embedding.js +20 -5
  93. package/dist/core/config/schema/engines.js +5 -0
  94. package/dist/core/config/schema/execution.js +1 -1
  95. package/dist/core/config/schema/experimental.js +1 -1
  96. package/dist/core/config/schema/improve-processes.js +21 -95
  97. package/dist/core/config/schema/improve.js +4 -42
  98. package/dist/core/config/schema/scheduler.js +12 -12
  99. package/dist/core/config/schema/search.js +6 -22
  100. package/dist/core/env-secret-ref.js +0 -1
  101. package/dist/core/errors.js +8 -9
  102. package/dist/core/file-lock.js +76 -173
  103. package/dist/core/logs-db.js +2 -2
  104. package/dist/core/paths.js +0 -27
  105. package/dist/core/redaction.js +109 -2
  106. package/dist/core/run-lock.js +2 -5
  107. package/dist/core/spawn-env.js +1 -1
  108. package/dist/core/state/migrations.js +108 -61
  109. package/dist/core/state-db-scope.js +2 -4
  110. package/dist/core/state-db.js +126 -692
  111. package/dist/core/type-presentation.js +1 -9
  112. package/dist/core/write-source.js +293 -1012
  113. package/dist/execution/input-contract.js +1 -1
  114. package/dist/execution/resolved-request.js +135 -689
  115. package/dist/execution/source.js +63 -257
  116. package/dist/execution/target-ref.js +1 -1
  117. package/dist/indexer/bundle-identity-guard.js +2 -2
  118. package/dist/indexer/db/graph-db.js +106 -46
  119. package/dist/indexer/ensure-index.js +44 -85
  120. package/dist/indexer/graph/graph-extraction.js +340 -562
  121. package/dist/indexer/graph/graph-related.js +130 -0
  122. package/dist/indexer/index-rebuild-lock.js +3 -11
  123. package/dist/indexer/index-writer-lock.js +8 -17
  124. package/dist/indexer/index-written-assets.js +139 -151
  125. package/dist/indexer/indexer.js +524 -846
  126. package/dist/indexer/materialize-embeddings.js +60 -397
  127. package/dist/indexer/passes/memory-inference.js +81 -90
  128. package/dist/indexer/passes/metadata.js +132 -200
  129. package/dist/indexer/read-preflight.js +0 -7
  130. package/dist/indexer/scan/doc-to-entry.js +1 -3
  131. package/dist/indexer/scan/drain-dir.js +1 -1
  132. package/dist/indexer/search/db-search.js +181 -590
  133. package/dist/indexer/search/fts-query.js +30 -41
  134. package/dist/indexer/search/ranking.js +28 -154
  135. package/dist/indexer/search/search-attribution.js +12 -32
  136. package/dist/indexer/search/search-fields.js +11 -15
  137. package/dist/indexer/search/search-hit-enrichers.js +54 -85
  138. package/dist/indexer/search/search-source.js +1 -4
  139. package/dist/indexer/usage/usage-events.js +2 -7
  140. package/dist/integrations/agent/engine-fallback.js +23 -40
  141. package/dist/integrations/agent/engine-resolution.js +93 -183
  142. package/dist/integrations/agent/execution.js +507 -0
  143. package/dist/integrations/agent/model-map.js +28 -156
  144. package/dist/integrations/agent/request-lowering.js +66 -141
  145. package/dist/integrations/agent/runner-dispatch.js +143 -321
  146. package/dist/integrations/agent/runner.js +54 -14
  147. package/dist/integrations/lockfile.js +53 -101
  148. package/dist/llm/embedders/deterministic.js +2 -3
  149. package/dist/llm/embedders/profile.js +71 -0
  150. package/dist/llm/embedders/remote.js +10 -15
  151. package/dist/llm/graph-extract.js +3 -12
  152. package/dist/llm/index-passes.js +3 -5
  153. package/dist/llm/memory-infer.js +1 -2
  154. package/dist/llm/metadata-enhance.js +1 -2
  155. package/dist/llm/structured-call.js +5 -24
  156. package/dist/output/generic-render.js +23 -11
  157. package/dist/output/html-render.js +13 -10
  158. package/dist/output/render-registry.js +3 -32
  159. package/dist/output/shapes/helpers.js +2 -34
  160. package/dist/output/shapes/passthrough.js +1 -9
  161. package/dist/{indexer/search/ranking-types.js → output/text/bundle-rename.js} +4 -1
  162. package/dist/output/text/command-format.js +60 -23
  163. package/dist/output/text/helpers.js +1 -1
  164. package/dist/output/text/migrate.js +5 -14
  165. package/dist/output/text/proposal-format.js +1 -2
  166. package/dist/output/text/workflow-format.js +0 -32
  167. package/dist/output/text.js +2 -0
  168. package/dist/registry/factory.js +4 -19
  169. package/dist/registry/network.js +66 -220
  170. package/dist/registry/providers/index.js +0 -2
  171. package/dist/registry/providers/skills-sh.js +3 -14
  172. package/dist/registry/providers/static-index.js +24 -26
  173. package/dist/registry/resolve.js +55 -131
  174. package/dist/scripts/akm-migrate-node.js +43937 -93313
  175. package/dist/scripts/akm-migrate.js +43697 -93071
  176. package/dist/setup/registry-stash-loader.js +4 -13
  177. package/dist/setup/semantic-assets.js +3 -44
  178. package/dist/setup/setup.js +1 -1
  179. package/dist/setup/steps/tasks.js +25 -15
  180. package/dist/sources/provider-factory.js +17 -18
  181. package/dist/sources/providers/filesystem.js +2 -3
  182. package/dist/sources/providers/git-install.js +7 -1
  183. package/dist/sources/providers/git-provider.js +0 -3
  184. package/dist/sources/providers/git-stash.js +0 -17
  185. package/dist/sources/providers/npm.js +2 -4
  186. package/dist/sources/providers/provider-utils.js +5 -10
  187. package/dist/sources/providers/website.js +0 -2
  188. package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
  189. package/dist/sources/website-url.js +2 -2
  190. package/dist/storage/database.js +9 -35
  191. package/dist/storage/repositories/improve-ledger-repository.js +168 -0
  192. package/dist/storage/repositories/index-connection.js +34 -70
  193. package/dist/storage/repositories/index-entries-repository.js +69 -111
  194. package/dist/storage/repositories/index-entry-mapper.js +1 -2
  195. package/dist/storage/repositories/index-entry-schema.js +83 -269
  196. package/dist/storage/repositories/index-fts-repository.js +86 -256
  197. package/dist/storage/repositories/index-llm-cache-repository.js +17 -0
  198. package/dist/storage/repositories/index-meta-repository.js +6 -4
  199. package/dist/storage/repositories/index-schema.js +192 -220
  200. package/dist/storage/repositories/index-utility-repository.js +8 -29
  201. package/dist/storage/repositories/index-vec-repository.js +133 -414
  202. package/dist/storage/repositories/outcome-repository.js +2 -1
  203. package/dist/storage/repositories/proposals-repository.js +35 -0
  204. package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
  205. package/dist/storage/repositories/task-history-repository.js +26 -4
  206. package/dist/storage/repositories/workflow-runs-repository.js +53 -244
  207. package/dist/storage/sqlite-migrations.js +136 -0
  208. package/dist/storage/sqlite-pragmas.js +11 -9
  209. package/dist/storage/sqlite-transaction.js +170 -0
  210. package/dist/storage/state-db-integrity.js +34 -27
  211. package/dist/tasks/activation-config.js +134 -62
  212. package/dist/tasks/backends/cron.js +129 -277
  213. package/dist/tasks/backends/exec-utils.js +2 -5
  214. package/dist/tasks/backends/launchd.js +125 -745
  215. package/dist/tasks/backends/schtasks.js +101 -620
  216. package/dist/tasks/prepare/prepare-support.js +5 -15
  217. package/dist/tasks/prepare/prepare.js +0 -2
  218. package/dist/tasks/resolve-akm-bin.js +20 -79
  219. package/dist/tasks/run/attempt-lifecycle.js +0 -1
  220. package/dist/tasks/scheduler-binding.js +18 -238
  221. package/dist/tasks/scheduler-invocation.js +52 -52
  222. package/dist/tasks/scheduler-lock.js +53 -0
  223. package/dist/tasks/scheduler-sync.js +363 -679
  224. package/dist/tasks/source/parse-task-source.js +160 -10
  225. package/dist/tasks/source/task-source-v3-frozen.js +3 -4
  226. package/dist/tasks/source/task-to-v4.js +2 -2
  227. package/dist/workflows/authoring/authoring.js +3 -12
  228. package/dist/workflows/compile.js +211 -0
  229. package/dist/workflows/concurrency-policy.js +13 -74
  230. package/dist/workflows/exec/child-invocation.js +3 -17
  231. package/dist/workflows/exec/child-workflow.js +32 -141
  232. package/dist/workflows/exec/dispatch-redaction.js +13 -53
  233. package/dist/workflows/exec/environment.js +98 -0
  234. package/dist/workflows/exec/exec-unit.js +33 -140
  235. package/dist/workflows/exec/frozen-judge.js +7 -59
  236. package/dist/workflows/exec/native-executor.js +82 -341
  237. package/dist/workflows/exec/param-secrets.js +29 -47
  238. package/dist/workflows/exec/run-workflow.js +154 -387
  239. package/dist/workflows/exec/scheduler.js +9 -36
  240. package/dist/workflows/exec/step-work.js +127 -430
  241. package/dist/workflows/exec/unit-dispatch.js +11 -63
  242. package/dist/workflows/exec/unit-writer.js +8 -52
  243. package/dist/workflows/exec/worktree.js +39 -273
  244. package/dist/workflows/freeze/child-output-references.js +4 -15
  245. package/dist/workflows/freeze/environment.js +99 -92
  246. package/dist/workflows/freeze/freeze.js +172 -0
  247. package/dist/workflows/freeze/step-values.js +19 -21
  248. package/dist/workflows/freeze/targets/child-workflow.js +23 -92
  249. package/dist/workflows/freeze/targets/command.js +10 -33
  250. package/dist/workflows/freeze/targets/script.js +5 -12
  251. package/dist/workflows/freeze/targets/shell.js +3 -6
  252. package/dist/workflows/freeze/targets/task.js +25 -80
  253. package/dist/workflows/freeze/task-bindings.js +20 -67
  254. package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
  255. package/dist/workflows/ir/params.js +6 -51
  256. package/dist/workflows/ir/plan-hash.js +2 -34
  257. package/dist/workflows/parser.js +140 -43
  258. package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
  259. package/dist/workflows/renderer.js +36 -69
  260. package/dist/workflows/resource-limits.js +12 -120
  261. package/dist/workflows/runtime/agent-identity.js +8 -40
  262. package/dist/workflows/runtime/run-outputs.js +3 -6
  263. package/dist/workflows/runtime/run-plan.js +316 -0
  264. package/dist/workflows/runtime/runs.js +48 -200
  265. package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
  266. package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
  267. package/dist/workflows/validate-summary.js +2 -7
  268. package/docs/integration/bundling-akm.md +49 -42
  269. package/docs/migration/README.md +1 -0
  270. package/docs/migration/release-notes/0.9.17.md +41 -0
  271. package/docs/migration/v0.9.1-to-v0.9.2.md +19 -7
  272. package/docs/reference/cli.md +182 -125
  273. package/docs/reference/configuration.md +49 -56
  274. package/docs/reference/data-and-telemetry.md +19 -20
  275. package/docs/reference/tasks.md +86 -38
  276. package/docs/reference/workflow-schema.md +14 -18
  277. package/docs/reference/workflows.md +6 -9
  278. package/package.json +1 -1
  279. package/schemas/akm-config.json +87 -406
  280. package/dist/commands/health/advisories.js +0 -150
  281. package/dist/commands/health/metrics.js +0 -329
  282. package/dist/commands/health/surfaces.js +0 -102
  283. package/dist/commands/improve/anti-collapse.js +0 -83
  284. package/dist/commands/improve/collapse-detector.js +0 -432
  285. package/dist/commands/improve/consolidate/eligibility.js +0 -48
  286. package/dist/commands/improve/consolidate/merge.js +0 -146
  287. package/dist/commands/improve/distill/promote-memory.js +0 -329
  288. package/dist/commands/improve/distill/quality-gate.js +0 -500
  289. package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
  290. package/dist/commands/improve/proposal-envelope.js +0 -31
  291. package/dist/commands/improve/run-context.js +0 -123
  292. package/dist/commands/improve/shared.js +0 -21
  293. package/dist/commands/improve/source-identity.js +0 -28
  294. package/dist/commands/improve/triage.js +0 -96
  295. package/dist/commands/proposal/drain-policies.js +0 -151
  296. package/dist/commands/sources/update-transaction.js +0 -220
  297. package/dist/core/action-contributors.js +0 -28
  298. package/dist/core/config/config-version-shim.js +0 -101
  299. package/dist/core/config/retired-experimental-keys-shim.js +0 -62
  300. package/dist/core/fs-txn.js +0 -405
  301. package/dist/core/lexical-score.js +0 -25
  302. package/dist/core/maintenance-barrier.js +0 -167
  303. package/dist/execution/executable-identity.js +0 -105
  304. package/dist/execution/guarded-source.js +0 -427
  305. package/dist/indexer/graph/graph-boost.js +0 -427
  306. package/dist/indexer/graph/graph-dedup.js +0 -95
  307. package/dist/indexer/search/name-match.js +0 -35
  308. package/dist/indexer/search/ranking-contributors.js +0 -515
  309. package/dist/indexer/walk/project-context.js +0 -192
  310. package/dist/integrations/agent/execution-cascade.js +0 -566
  311. package/dist/integrations/agent/execution-definitions.js +0 -202
  312. package/dist/integrations/agent/execution-lowering.js +0 -841
  313. package/dist/integrations/agent/execution-preparation.js +0 -98
  314. package/dist/integrations/agent/inline-execution.js +0 -74
  315. package/dist/registry/create-provider-registry.js +0 -29
  316. package/dist/registry/pinned-request-helper.js +0 -247
  317. package/dist/registry/pinned-transport.js +0 -717
  318. package/dist/sources/providers/index.js +0 -14
  319. package/dist/storage/engines/sqlite-migrations.js +0 -271
  320. package/dist/storage/repositories/canaries-repository.js +0 -107
  321. package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
  322. package/dist/storage/repositories/registry-cache.js +0 -113
  323. package/dist/tasks/scheduler-sync-preview.js +0 -52
  324. package/dist/workflows/freeze/resolve-steps.js +0 -86
  325. package/dist/workflows/freeze/source-freeze.js +0 -64
  326. package/dist/workflows/ir/compile.js +0 -321
  327. package/dist/workflows/ir/environment-v4.js +0 -330
  328. package/dist/workflows/ir/freeze-v4.js +0 -153
  329. package/dist/workflows/ir/schema-v4.js +0 -745
  330. package/dist/workflows/ir/schema.js +0 -354
  331. package/dist/workflows/program/schema.js +0 -78
  332. package/dist/workflows/runtime/checkin.js +0 -57
  333. package/dist/workflows/runtime/plan-classifier.js +0 -196
  334. package/dist/workflows/runtime/unit-checkin.js +0 -45
  335. package/dist/workflows/runtime/unit-phases.js +0 -20
  336. package/dist/workflows/schema.js +0 -4
  337. package/dist/workflows/source-ir/compile.js +0 -200
  338. package/dist/workflows/source-ir/program.js +0 -50
  339. package/dist/workflows/source-ir/result.js +0 -26
  340. package/dist/workflows/source-ir/schema.js +0 -786
  341. package/dist/workflows/source-ir/triggers.js +0 -79
  342. package/dist/workflows/source-ir/uses.js +0 -40
  343. package/dist/workflows/validator.js +0 -60
@@ -2,49 +2,15 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  /**
5
- * `akm distill <ref>` — feedback distillation into lesson proposals (#228).
5
+ * `akm distill <ref>` — distil an asset and its feedback into a lesson (or,
6
+ * for a reinforced memory, a knowledge) proposal. One bounded LLM call, then
7
+ * the shared judge → mint path in `./stage`; the proposal queue is the only
8
+ * way to a live asset. Every invocation emits one `distill_invoked` event
9
+ * carrying its `outcome` (config-disabled runs emit none).
6
10
  *
7
- * The command reads a target asset and any recent feedback events about it,
8
- * asks an LLM to distil a *lesson* (per v1 spec §13) the agent should
9
- * remember next time, and queues the result as a {@link Proposal} (source
10
- * `"distill"`). The proposal queue is the *only* path to a live asset — this
11
- * command never mutates source files directly. Acceptance is a human (or
12
- * automated) decision via `akm proposal accept`.
13
- *
14
- * # Architectural seams
15
- *
16
- * - **Single bounded in-tree LLM call.** Routed through `callStructured`
17
- * under the `distill` gate (v1 spec §14; 0.8.0 unified the orchestration
18
- * and LLM-call gates under `processes.distill.enabled`). The wrapper
19
- * enforces a hard timeout (default 600s / 10 min — overridable via
20
- * `opts.timeoutMs`) and converts disable / throw / timeout
21
- * into a `null` return from `fn`, which we treat as a graceful
22
- * "skipped" outcome (exit 0, no proposal, `distill_invoked` event with
23
- * `outcome: "skipped"`).
24
- * - **Stateless.** No module-level state — every callable is a pure
25
- * function of its arguments and an injectable `chat` seam. The
26
- * architecture seam test (`tests/architecture/llm-stateless-seam.test.ts`)
27
- * applies.
28
- * - **Output substrate.** Proposal creation goes through the `proposals`
29
- * module so distill shares its persistence + validation pipeline with
30
- * `akm reflect` / `akm propose`. Validation failures (LLM returned a
31
- * lesson without required `description` / `when_to_use` frontmatter) are
32
- * a *different* graceful path: no proposal is created, the structured
33
- * error is surfaced, and the command exits non-zero.
34
- *
35
- * # Lesson-name derivation rule
36
- *
37
- * A nested input preserves its first legitimate scope segment
38
- * (`memories/project-a/deploy` → `lessons/project-a/memory-deploy-lesson`). An
39
- * unscoped input stays flat; asset types are not project scopes. Origin prefixes
40
- * remain durable provenance but are not embedded in the output path.
41
- *
42
- * # Why we do not call `runAgent`
43
- *
44
- * Distillation is in-tree per the v1 spec ("bounded in-tree LLM call"). The
45
- * agent dispatch path is a heavier shell-out used by the curator/agent
46
- * surfaces — distill must be cheap, deterministic-ish, and bounded so it can
47
- * be invoked from CI / automation without spinning up an agent harness.
11
+ * Lesson refs: a nested input keeps its first scope segment
12
+ * (`memories/project-a/deploy` → `lessons/project-a/memory-deploy-lesson`); an
13
+ * unscoped input stays flat.
48
14
  */
49
15
  import fs from "node:fs";
50
16
  import distillKnowledgeSystemPrompt from "../../assets/prompts/distill-knowledge-system.md" with { type: "text" };
@@ -54,6 +20,7 @@ import { parseFrontmatter, writeSalienceToFrontmatter } from "../../core/asset/f
54
20
  import { stripMarkdownFences } from "../../core/asset/markdown.js";
55
21
  import { conceptIdFromTypeName, parseRefInput } from "../../core/asset/resolve-ref.js";
56
22
  import { authoringRulesForType } from "../../core/authoring-rules.js";
23
+ import { resolveStashDir } from "../../core/common.js";
57
24
  import { getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
58
25
  import { UsageError } from "../../core/errors.js";
59
26
  import { appendEvent, readEvents } from "../../core/events.js";
@@ -62,111 +29,47 @@ import { parseEmbeddedJsonResponse } from "../../core/parse.js";
62
29
  import { getDbPath } from "../../core/paths.js";
63
30
  import { resolveStandardsContext } from "../../core/standards/resolve-standards-context.js";
64
31
  import { withStateDb } from "../../core/state-db.js";
65
- import { warnVerbose } from "../../core/warn.js";
32
+ import { warn, warnVerbose } from "../../core/warn.js";
33
+ import { recordWrittenPath } from "../../core/write-provenance.js";
66
34
  import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
67
- import { disposeLoweredExecutionDispatchLease, } from "../../integrations/agent/execution-lowering.js";
68
- import { callStructured, preflightStructuredLlmRunner } from "../../llm/structured-call.js";
35
+ import { assertRunnerCredentials } from "../../integrations/agent/runner-dispatch.js";
69
36
  import { closeDatabase, openReadonlyExistingDatabase } from "../../storage/repositories/index-connection.js";
70
37
  import { getAllEntries } from "../../storage/repositories/index-entries-repository.js";
71
- import { isStaleTargetRejection } from "../proposal/proposal-types.js";
72
- import { isProposalSkipped, listProposals, listProposalsReadOnly, } from "../proposal/repository.js";
73
- import { stripFrontmatterBody as stripBodyForFidelity } from "./content-hash.js";
38
+ import { listProposals } from "../proposal/repository.js";
39
+ import { detectDoubleFrontmatter, isValidDescription } from "../proposal/validators/proposal-quality-validators.js";
40
+ import { akmSearch } from "../read/search.js";
41
+ import { stripFrontmatterBody } from "./content-hash.js";
74
42
  import { autoRepairLessonFrontmatter, autoSwapDescriptionWhenToUse, collectLessonQualityFindings, repairLessonDescriptionTruncation, } from "./distill/content-repair.js";
75
- import { memoryKnowledgePromotionRequiresDispatch, planMemoryKnowledgePromotion, promoteMemoryToKnowledge, } from "./distill/promote-memory.js";
76
- import { fetchTopSimilarLessons, persistOutputEncodingSalience, runLessonQualityJudge, writeQualityRejection, } from "./distill/quality-gate.js";
77
43
  import { buildClsContext, checkDistillFidelity, DEFAULT_CLS_ADJACENT_COUNT } from "./distill-guards.js";
78
- import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
44
+ import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./distill-promotion-policy.js";
79
45
  import { buildRefVocabulary, scoreEncodingSalience } from "./encoding-salience.js";
80
- import { resolveImproveLlmExecution } from "./execution.js";
81
46
  import { resolveImproveStrategy, resolveProcessEnabled } from "./improve-strategies.js";
82
- import { emitProposal } from "./proposal-envelope.js";
83
- import { createRunContext, resolveRunStashDir } from "./run-context.js";
47
+ import { recordLedgerAttempt } from "./ledger.js";
84
48
  import { computeSalience, upsertAssetSalience } from "./salience.js";
85
- import { MAX_REJECTED_PROPOSALS } from "./shared.js";
86
- import { durableImproveRef } from "./source-identity.js";
49
+ import { callStage, mintProposal, noticeSet, rejectedProposalContext, runLessonQualityJudge, stageRunner, } from "./stage.js";
87
50
  /**
88
- * Asset-ref types that `akm distill` structurally refuses as inputs.
89
- *
90
- * Distill *produces* lessons from non-lesson sources (memory, skill, knowledge,
91
- * etc.). Calling distill on an existing `lessons/*` ref would derive
92
- * `lessons/lesson-<name>-lesson-lesson` (double `-lesson` suffix) — the
93
- * recursive-ref defect observed across 323 archived rejected proposals.
94
- *
95
- * 08-F2: `env` and `secret` are refused as a STRUCTURAL floor — distill reads
96
- * the input asset's bytes via `readFileSync` and hands them to the LLM, so
97
- * secret material must never be a distill input. This gate is code, not config:
98
- * it holds even when `allowedTypes` config is mis-set in unattended cron.
99
- *
100
- * The runtime gate inside {@link akmDistill} still refuses these inputs
101
- * defensively (returning an `outcome: "skipped"` envelope with `skipReason:
102
- * "recursive_lesson_input"`). This exported set is the planner-side companion:
103
- * callers that schedule distill attempts (e.g. `akm improve`'s distill queue)
104
- * import it so refs of these types never enter the queue in the first place.
105
- *
106
- * Source of truth: this set drives the gate in `akmDistill` and is consumed
107
- * directly by the improve planner. Adding a new structurally-refused input
108
- * type means updating this constant — the planner picks the change up for
109
- * free.
51
+ * Input types distill structurally refuses: a lesson is the distilled form
52
+ * (distilling one would mint `lessons/lesson-…-lesson`), and env/secret bytes
53
+ * must never reach the model. The improve planner skips these before queuing.
110
54
  */
111
55
  export const DISTILL_REFUSED_INPUT_TYPES = new Set(["lesson", "env", "secret"]);
112
- /**
113
- * Returns true when `type` is structurally refused as an input by
114
- * {@link akmDistill}. See {@link DISTILL_REFUSED_INPUT_TYPES}.
115
- */
116
56
  export function isDistillRefusedInputType(type) {
117
57
  return DISTILL_REFUSED_INPUT_TYPES.has(type);
118
58
  }
119
- // ── Lesson-ref derivation ───────────────────────────────────────────────────
120
- /** Derive the proposed lesson ref from the input ref. See module docblock. */
59
+ /** Derive the proposed lesson ref from the input ref. */
121
60
  export function deriveLessonRef(inputRef) {
122
61
  const parsed = parseRefInput(inputRef);
123
- // Strip the bundle: a feedback signal recorded against `team//skills/deploy`
124
- // distils into the same lesson namespace as `skills/deploy`. The proposal
125
- // id (a UUID) keeps the queue entries distinct, so collisions are not a
126
- // problem — and reviewers want to see them next to each other anyway.
127
62
  const parts = parsed.name.split("/");
128
63
  const scope = parts.length > 1 ? parts.shift() : undefined;
129
- const slug = `${parsed.type}-${parts.join("-")}`.toLowerCase();
130
- // Replace anything outside the canonical asset-name charset with `-`. Keep
131
- // it deterministic so re-runs produce the same ref.
132
- const safe = slug
64
+ const clean = (value) => value
65
+ .toLowerCase()
133
66
  .replace(/[^a-z0-9-]+/g, "-")
134
67
  .replace(/-+/g, "-")
135
68
  .replace(/^-|-$/g, "");
136
- const safeScope = scope
137
- ?.toLowerCase()
138
- .replace(/[^a-z0-9-]+/g, "-")
139
- .replace(/-+/g, "-")
140
- .replace(/^-|-$/g, "");
141
- return `lessons/${safeScope ? `${safeScope}/` : ""}${safe}-lesson`;
69
+ const safeScope = scope ? clean(scope) : "";
70
+ return `lessons/${safeScope ? `${safeScope}/` : ""}${clean(`${parsed.type}-${parts.join("-")}`)}-lesson`;
142
71
  }
143
- // ── Content quality validators ──────────────────────────────────────────────
144
- //
145
- // The actual implementations now live in `core/proposal-quality-validators.ts`
146
- // so the same checks run inside `runProposalValidators` on `proposal accept`.
147
- import { detectDoubleFrontmatter, isValidDescription } from "../proposal/validators/proposal-quality-validators.js";
148
- // ── Prompt assembly ─────────────────────────────────────────────────────────
149
- const LESSON_SYSTEM_PROMPT = distillLessonSystemPrompt;
150
- const KNOWLEDGE_SYSTEM_PROMPT = distillKnowledgeSystemPrompt;
151
- // ── Structured-output schemas (responseSchema lift) ─────────────────────────
152
- //
153
- // PR 1 of the asset-writers decision (see knowledge/projects/akm/
154
- // asset-writers-investigation/00-synthesis): on providers that honour
155
- // `response_format: json_schema`, ask the LLM for a typed JSON object and
156
- // re-assemble the markdown locally. The "emit raw markdown with embedded
157
- // frontmatter" path supports providers that ignore
158
- // the schema (and for the `chat` test seam, which is wired to return strings
159
- // today). Shape-level rejection codes — MALFORMED_FRONTMATTER_BLOCK,
160
- // FRONTMATTER_NOT_OBJECT, INVALID_YAML, UNBALANCED_CODE_FENCE — become
161
- // unreachable on the structured path. Content-quality validators
162
- // (isValidDescription / isValidWhenToUse) keep firing post-assembly because
163
- // the LLM still controls the string contents of typed fields.
164
- /**
165
- * JSON Schema for structured lesson distillation. Mirrors the LESSON_SYSTEM_PROMPT
166
- * frontmatter contract. Required: description, when_to_use, body. Optional:
167
- * tags (string array) so providers that volunteer categorisation hints survive
168
- * the round-trip without being rejected as additionalProperties.
169
- */
72
+ // ── Output contract ──────────────────────────────────────────────────────────
170
73
  export const DISTILL_LESSON_JSON_SCHEMA = {
171
74
  type: "object",
172
75
  required: ["description", "when_to_use", "body"],
@@ -194,21 +97,12 @@ export const DISTILL_LESSON_JSON_SCHEMA = {
194
97
  },
195
98
  },
196
99
  };
197
- /**
198
- * JSON Schema for structured knowledge distillation. Mirrors the
199
- * KNOWLEDGE_SYSTEM_PROMPT contract. Required: description, body. Optional:
200
- * tags, sources.
201
- */
202
100
  export const DISTILL_KNOWLEDGE_JSON_SCHEMA = {
203
101
  type: "object",
204
102
  required: ["description", "body"],
205
103
  additionalProperties: false,
206
104
  properties: {
207
- description: {
208
- type: "string",
209
- minLength: 1,
210
- description: "One-line summary of the knowledge asset.",
211
- },
105
+ description: { type: "string", minLength: 1, description: "One-line summary of the knowledge asset." },
212
106
  body: {
213
107
  type: "string",
214
108
  minLength: 1,
@@ -227,38 +121,31 @@ export const DISTILL_KNOWLEDGE_JSON_SCHEMA = {
227
121
  },
228
122
  };
229
123
  /**
230
- * Assemble a markdown asset from a structured-output payload. Returns `null`
231
- * when the payload is missing required fields — the caller then falls through
232
- * to the prompt-contract markdown path. We deliberately do NOT validate
233
- * content quality here (isValidDescription / isValidWhenToUse run downstream
234
- * on the assembled content); this helper only catches shape-level emptiness
235
- * that the schema may not have rejected (e.g. a provider that ignored
236
- * `minLength` but still returned the field).
124
+ * Assemble markdown from a structured-output payload, or `null` when a
125
+ * required field is empty (the caller then treats the response as markdown).
237
126
  */
238
127
  export function assembleStructuredDistillMarkdown(payload, kind) {
239
128
  if (payload === null || typeof payload !== "object")
240
129
  return null;
241
- const description = typeof payload.description === "string" ? payload.description.trim() : "";
242
- const body = typeof payload.body === "string" ? payload.body.trim() : "";
243
- if (description.length === 0 || body.length === 0)
130
+ const text = (value) => (typeof value === "string" ? value.trim() : "");
131
+ const list = (value) => Array.isArray(value) ? value.filter((v) => typeof v === "string" && v.trim().length > 0) : [];
132
+ const description = text(payload.description);
133
+ const body = text(payload.body);
134
+ if (!description || !body)
244
135
  return null;
245
136
  const fm = { description };
246
137
  if (kind === "lesson") {
247
- const whenToUse = typeof payload.when_to_use === "string" ? payload.when_to_use.trim() : "";
248
- if (whenToUse.length === 0)
138
+ const whenToUse = text(payload.when_to_use);
139
+ if (!whenToUse)
249
140
  return null;
250
141
  fm.when_to_use = whenToUse;
251
142
  }
252
- if (Array.isArray(payload.tags)) {
253
- const tags = payload.tags.filter((t) => typeof t === "string" && t.trim().length > 0);
254
- if (tags.length > 0)
255
- fm.tags = tags;
256
- }
257
- if (kind === "knowledge" && Array.isArray(payload.sources)) {
258
- const sources = payload.sources.filter((s) => typeof s === "string" && s.trim().length > 0);
259
- if (sources.length > 0)
260
- fm.xrefs = sources;
261
- }
143
+ const tags = list(payload.tags);
144
+ if (tags.length > 0)
145
+ fm.tags = tags;
146
+ const sources = kind === "knowledge" ? list(payload.sources) : [];
147
+ if (sources.length > 0)
148
+ fm.xrefs = sources;
262
149
  return assembleAssetFromString(serializeFrontmatterQuoted(fm), body);
263
150
  }
264
151
  function validateKnowledgeContent(content, inputRef) {
@@ -271,1052 +158,675 @@ function validateKnowledgeContent(content, inputRef) {
271
158
  message: `Distilled knowledge for ${inputRef} must include a non-empty markdown body.`,
272
159
  });
273
160
  }
274
- // Knowledge proposals don't strictly require a description, but if one is
275
- // present it must be a real summary — not a placeholder like `---` or a
276
- // truncated heading. Without this check, distill can land knowledge assets
277
- // with `description: ---` (observed in the wild when the LLM has nothing
278
- // meaningful to say about a session-checkpoint memory).
279
- const fm = (parsed.data ?? {});
280
- if (fm.description !== undefined) {
281
- // Knowledge can legitimately mention the topic name in its description, so
282
- // suppress the ref-restatement heuristic that's tuned for lesson assets.
283
- const descCheck = isValidDescription(fm.description, inputRef, { skipRefTailCheck: true });
284
- if (!descCheck.ok) {
161
+ // A present description must be a real summary (not `---` or a heading fragment).
162
+ const description = parsed.data?.description;
163
+ if (description !== undefined) {
164
+ const check = isValidDescription(description, inputRef, { skipRefTailCheck: true });
165
+ if (!check.ok) {
285
166
  findings.push({
286
167
  kind: "invalid-description",
287
168
  field: "description",
288
- message: `Distilled knowledge for ${inputRef} has an invalid description: ${descCheck.reason}.`,
169
+ message: `Distilled knowledge for ${inputRef} has an invalid description: ${check.reason}.`,
289
170
  });
290
171
  }
291
172
  }
292
- // Double-frontmatter pollution shows up in knowledge too — the LLM sometimes
293
- // re-emits the source asset's frontmatter inside its own response, leaving
294
- // two `---`-delimited blocks back-to-back.
295
- const dfm = detectDoubleFrontmatter(content);
296
- if (dfm) {
173
+ const doubled = detectDoubleFrontmatter(content);
174
+ if (doubled) {
297
175
  findings.push({
298
- kind: dfm.kind,
176
+ kind: doubled.kind,
299
177
  field: "body",
300
- message: `Distilled knowledge for ${inputRef}: ${dfm.message}`,
178
+ message: `Distilled knowledge for ${inputRef}: ${doubled.message}`,
301
179
  });
302
180
  }
303
181
  return findings;
304
182
  }
305
183
  /**
306
- * Pure: build the user-prompt body. Exported for tests.
307
- *
308
- * D-3 (#371): restructures the feedback section from raw JSON event lines into
309
- * a Reflexion-style verbal contrast (`## What worked` / `## What failed`).
310
- * The verbal format allows LLMs to use feedback as gradient signal rather than
311
- * just metadata — capturing the +8% AlfWorld lift from arXiv:2303.11366 and
312
- * the contrast-based rule-learning gain from ExpeL arXiv:2308.10144.
184
+ * The distill user prompt. Feedback is rendered as "What worked" / "What
185
+ * failed" contrast when it carries signals, else as a flat event list.
313
186
  */
314
187
  export function buildDistillPrompt(input) {
315
- const lines = [];
316
- lines.push(`Asset ref: ${input.inputRef}`);
317
- lines.push("");
188
+ const lines = [`Asset ref: ${input.inputRef}`, ""];
318
189
  if (input.standardsContext?.trim()) {
319
- lines.push("Standards to follow (the rulebook for this target):");
320
- lines.push(input.standardsContext.trim());
321
- lines.push("");
322
- }
323
- {
324
- const authoringRules = authoringRulesForType(input.proposalKind ?? "lesson");
325
- if (authoringRules) {
326
- lines.push(authoringRules);
327
- lines.push("");
328
- }
190
+ lines.push("Standards to follow (the rulebook for this target):", input.standardsContext.trim(), "");
329
191
  }
192
+ const authoringRules = authoringRulesForType(input.proposalKind ?? "lesson");
193
+ if (authoringRules)
194
+ lines.push(authoringRules, "");
330
195
  lines.push("Asset content:");
331
196
  if (input.assetContent) {
332
- // The output contract also uses YAML fences. Feeding source frontmatter
333
- // verbatim caused local models to copy it into the lesson body, producing
334
- // deterministic double-frontmatter rejection. Distillation needs the
335
- // source body; its metadata is not evidence to reproduce.
336
- const body = parseFrontmatter(input.assetContent).content.trim().slice(0, 3000);
337
- lines.push("```");
338
- lines.push(body);
339
- lines.push("```");
197
+ // Source frontmatter is not evidence; fed verbatim, models copied it into the body.
198
+ lines.push("```", parseFrontmatter(input.assetContent).content.trim().slice(0, 3000), "```");
340
199
  }
341
200
  else {
342
201
  lines.push("(asset is not currently indexed; distil from feedback signal alone)");
343
202
  }
344
203
  lines.push("");
204
+ const flat = (event) => `- ${event.ts} ${event.eventType}${event.metadata ? ` ${JSON.stringify(event.metadata)}` : ""}`;
345
205
  if (input.feedback.length === 0) {
346
206
  lines.push("Recent feedback: (no feedback events recorded — distil from the asset itself)");
347
207
  }
348
208
  else {
349
- // D-3 (#371): verbal contrast format for Reflexion verbal-gradient lift.
350
- // Partition events into positive ("what worked") and negative ("what failed").
351
- const positive = [];
352
- const negative = [];
353
- const neutral = [];
209
+ const worked = [];
210
+ const failed = [];
211
+ const other = [];
354
212
  for (const event of input.feedback) {
355
- const meta = (event.metadata ?? {});
356
- const signal = typeof meta.signal === "string" ? meta.signal : undefined;
357
- const reason = typeof meta.reason === "string" ? meta.reason : "";
358
- const note = typeof meta.note === "string" ? meta.note : "";
359
- const detail = reason || note;
360
- const line = detail ? `- ${event.ts}: ${detail}` : `- ${event.ts}: feedback received`;
361
- if (signal === "positive")
362
- positive.push(line);
363
- else if (signal === "negative")
364
- negative.push(line);
213
+ const meta = event.metadata ?? {};
214
+ const detail = (typeof meta.reason === "string" ? meta.reason : "") || (typeof meta.note === "string" ? meta.note : "");
215
+ const line = `- ${event.ts}: ${detail || "feedback received"}`;
216
+ if (meta.signal === "positive")
217
+ worked.push(line);
218
+ else if (meta.signal === "negative")
219
+ failed.push(line);
365
220
  else
366
- neutral.push(`- ${event.ts} ${event.eventType}${event.metadata ? ` ${JSON.stringify(event.metadata)}` : ""}`);
221
+ other.push(flat(event));
367
222
  }
368
- if (positive.length > 0 || negative.length > 0) {
369
- if (positive.length > 0) {
370
- lines.push("## What worked");
371
- for (const l of positive)
372
- lines.push(l);
373
- lines.push("");
374
- }
375
- if (negative.length > 0) {
376
- lines.push("## What failed");
377
- for (const l of negative)
378
- lines.push(l);
379
- lines.push("");
380
- }
381
- if (neutral.length > 0) {
382
- lines.push("## Other signals");
383
- for (const l of neutral)
384
- lines.push(l);
385
- lines.push("");
223
+ if (worked.length > 0 || failed.length > 0) {
224
+ for (const [heading, section] of [
225
+ ["## What worked", worked],
226
+ ["## What failed", failed],
227
+ ["## Other signals", other],
228
+ ]) {
229
+ if (section.length > 0)
230
+ lines.push(heading, ...section, "");
386
231
  }
387
232
  }
388
233
  else {
389
- // No positive/negative signals — fall back to the pre-D3 flat format for
390
- // non-feedback event types (e.g. reflect_invoked, distill_invoked).
391
- lines.push("Recent feedback events (most recent last):");
392
- for (const event of input.feedback) {
393
- const meta = event.metadata ? ` ${JSON.stringify(event.metadata)}` : "";
394
- lines.push(`- ${event.ts} ${event.eventType}${meta}`);
395
- }
396
- lines.push("");
234
+ lines.push("Recent feedback events (most recent last):", ...input.feedback.map(flat), "");
397
235
  }
398
236
  }
399
237
  if (input.rejectedProposals && input.rejectedProposals.length > 0) {
400
- lines.push("");
401
- lines.push("Previously rejected proposals for this ref (Reflexion context):");
402
- lines.push("The following proposals were already reviewed and rejected. " +
238
+ lines.push("", "Previously rejected proposals for this ref (Reflexion context):", "The following proposals were already reviewed and rejected. " +
403
239
  "Your new proposal MUST differ meaningfully in approach, framing, or evidence.");
404
240
  for (const rp of input.rejectedProposals) {
405
241
  lines.push(`- Rejection reason: ${rp.reason}`);
406
- if (rp.contentPreview) {
242
+ if (rp.contentPreview)
407
243
  lines.push(` Content preview: ${rp.contentPreview.slice(0, 200).replace(/\n/g, " ")}`);
408
- }
409
244
  }
410
245
  }
411
- if (input.proposalKind === "knowledge") {
412
- lines.push("Produce the knowledge markdown file now. Start your response with `---` on the first line, followed by a `description:` field whose value is a 1-sentence summary (20–400 chars). Never use placeholder values like `---`, `tbd`, `n/a`, or a single dash. If the source has nothing meaningful to summarize, do NOT produce a proposal — return an empty response instead. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body.");
413
- }
414
- else {
415
- lines.push("Produce the lesson markdown file now. Start your response with `---` on the first line, followed by `description:` and `when_to_use:` fields. Both must be real one-sentence summaries (20–400 chars) — never placeholder values like `---`, `tbd`, or `n/a`. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body.");
416
- }
246
+ lines.push(input.proposalKind === "knowledge"
247
+ ? "Produce the knowledge markdown file now. Start your response with `---` on the first line, followed by a `description:` field whose value is a 1-sentence summary (20–400 chars). Never use placeholder values like `---`, `tbd`, `n/a`, or a single dash. If the source has nothing meaningful to summarize, do NOT produce a proposal — return an empty response instead. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body."
248
+ : "Produce the lesson markdown file now. Start your response with `---` on the first line, followed by `description:` and `when_to_use:` fields. Both must be real one-sentence summaries (20–400 chars) — never placeholder values like `---`, `tbd`, or `n/a`. The frontmatter block ends with a second `---` line; do not emit any additional `---` fences in the body.");
417
249
  return lines.join("\n");
418
250
  }
419
- // ── Main entry point ────────────────────────────────────────────────────────
420
- /**
421
- * Run a single bounded distillation pass for `ref`. Always emits exactly one
422
- * `distill_invoked` event (with `outcome` in the metadata) regardless of the
423
- * branch taken — so observers can count invocations cheaply.
424
- */
425
- /**
426
- * Best-effort load of the distill INPUT asset plus the #608 encoding-time
427
- * salience scoring: read the source, build the once-per-invocation bigram ref
428
- * vocabulary, then score the asset (novelty×0.40 + magnitude×0.35 +
429
- * predictionError×0.25) and mirror the result to both the asset frontmatter and
430
- * `state.db :: asset_salience`. Every write is best-effort. Extracted verbatim
431
- * from `akmDistill`; returns the (possibly salience-stamped) content plus the
432
- * ref vocabulary the caller reuses when scoring the distilled OUTPUT (G4).
433
- */
434
- async function loadAndScoreInputSalience(args) {
435
- const { inputRef, durableInputRef, salienceWriteKey, stash, outcomeWeightEnabled, lookup, ctx } = args;
436
- // Best-effort load: when the asset is not yet indexed we still proceed —
437
- // the LLM is asked to distil from "available signal" (feedback alone).
438
- let assetContent = null;
439
- let assetFilePath = null;
440
- try {
441
- const filePath = await lookup(durableInputRef);
442
- if (filePath && fs.existsSync(filePath)) {
443
- assetFilePath = filePath;
444
- assetContent = ctx.readAsset(filePath);
445
- }
446
- }
447
- catch {
448
- assetContent = null;
449
- }
450
- // ── #608: Encoding-time salience scoring ────────────────────────────────
451
- // Score the source asset with the three-signal model (novelty × 0.40 +
452
- // magnitude × 0.35 + predictionError × 0.25) and persist the result to:
453
- // 1. The asset's frontmatter (human-readable mirror; idempotent delta gate).
454
- // 2. state.db :: asset_salience (canonical; feeds improve's high-salience gate).
455
- // Both writes are best-effort — a DB error never blocks distillation.
456
- //
457
- // The bigram ref vocabulary is built ONCE per invocation — the novelty signal
458
- // reuses it when scoring the distilled OUTPUT at proposal creation (G4).
459
- let existingRefVocabulary = new Set();
460
- try {
461
- const indexDb = openReadonlyExistingDatabase(getDbPath(), { isolatedSnapshot: true });
462
- if (indexDb) {
463
- try {
464
- const allRefs = getAllEntries(indexDb).map((e) => e.itemRef);
465
- existingRefVocabulary = buildRefVocabulary(allRefs);
466
- }
467
- finally {
468
- closeDatabase(indexDb);
469
- }
470
- }
471
- }
472
- catch {
473
- // Index not available — novelty defaults to type-floor.
474
- }
475
- if (args.persistSalience !== false && assetContent && assetFilePath) {
476
- try {
477
- const parsedRef = parseRefInput(inputRef);
478
- // G4: predictionError decays with revision count — the prior hardcoded
479
- // `revisionCount: 0` made it a dead constant 1.0. Use the number of
480
- // proposals ever raised against this ref as the revision proxy.
481
- let revisionCount = 0;
482
- try {
483
- revisionCount = listProposals(stash, { ref: inputRef, includeArchive: true }).length;
484
- }
485
- catch {
486
- // best-effort: unknown history scores as a first encounter
487
- }
488
- const salienceResult = scoreEncodingSalience({
489
- body: assetContent,
490
- type: parsedRef.type,
491
- existingRefVocabulary,
492
- revisionCount,
493
- });
494
- // 1. Write salience to the source asset frontmatter (idempotent).
495
- const updatedContent = writeSalienceToFrontmatter(assetContent, salienceResult.score, salienceResult);
496
- if (updatedContent !== assetContent) {
497
- ctx.writeAsset(assetFilePath, updatedContent);
498
- assetContent = updatedContent;
499
- }
500
- // 2. Persist encoding_salience to state.db.
501
- try {
502
- withStateDb((stateDb) => {
503
- const vector = computeSalience({
504
- ref: inputRef,
505
- type: parsedRef.type,
506
- retrievalFreq: 0,
507
- encodingSalience: salienceResult.score,
508
- outcomeWeightEnabled,
509
- });
510
- upsertAssetSalience(stateDb, salienceWriteKey, vector);
511
- });
512
- }
513
- catch {
514
- // State DB unavailable — frontmatter mirror is the only persistence.
515
- }
516
- }
517
- catch {
518
- // Scoring errors never block distillation.
519
- }
520
- }
521
- return { assetContent, existingRefVocabulary };
522
- }
523
- /**
524
- * Recursive-distillation + secret-material input guard. Distill produces
525
- * *lessons* from non-lesson sources; a lesson input would derive a recursive
526
- * `lessons/lesson-<name>` ref (the 323-archived-proposals defect) and
527
- * env/secret inputs must never be read or sent to the LLM. Emits the
528
- * `distill_invoked(skipped)` event and returns the terminal skipped result,
529
- * or `null` when the input type is allowed. Extracted verbatim from
530
- * `akmDistill` (R25/R31 — the events-ctx threading pushed it over the bar).
531
- */
532
- function refuseDisallowedDistillInput(args) {
533
- const { options, parsedInputRef, inputRef, durableInputRef, eligMeta } = args;
534
- if (!isDistillRefusedInputType(parsedInputRef.type))
535
- return null;
536
- // 08-F2: env/secret are a secret-material refusal (never read the bytes);
537
- // lesson is the recursive-form refusal. Both skip BEFORE any readFileSync.
538
- const isSecretInput = parsedInputRef.type === "env" || parsedInputRef.type === "secret";
539
- const skippedRef = isSecretInput ? inputRef : conceptIdFromTypeName("lesson", parsedInputRef.name);
540
- const message = isSecretInput
541
- ? `Distill refuses ${parsedInputRef.type} inputs — secret material must never be sent to the LLM.`
542
- : "Distill refuses lesson inputs — lessons are the distilled form, not a source.";
543
- appendEvent({
544
- eventType: "distill_invoked",
545
- // Key on item_ref when the planner resolved one, otherwise the conceptId.
546
- ref: options.itemRef ?? durableInputRef,
547
- metadata: {
548
- outcome: "skipped",
549
- proposalRef: skippedRef,
550
- message,
551
- skipReason: isSecretInput ? "refused_secret_input" : "recursive_lesson_input",
552
- ...eligMeta,
553
- },
554
- }, options.eventsCtx);
555
- return {
556
- schemaVersion: 1,
557
- ok: true,
558
- outcome: "skipped",
559
- inputRef,
560
- proposalRef: skippedRef,
561
- skipReason: isSecretInput ? "refused_secret_input" : "recursive_lesson_input",
562
- message,
563
- };
251
+ // ── Invocation ───────────────────────────────────────────────────────────────
252
+ const DISABLED_MESSAGE = "distill is disabled in config; enable processes.distill.enabled to activate.";
253
+ function emitDistill(run, meta) {
254
+ appendEvent({ eventType: "distill_invoked", ref: run.ledgerRef, metadata: { ...meta, ...run.eligMeta } }, run.options.eventsCtx);
564
255
  }
565
- async function prepareDistillExecution(args) {
566
- const { options, config, inputRef, durableInputRef, salienceWriteKey } = args;
567
- const stash = resolveRunStashDir(options.stashDir);
568
- const chat = options.chat;
569
- const executionNotices = new Map();
570
- const collectNotices = (notices) => {
571
- for (const notice of notices)
572
- executionNotices.set(JSON.stringify(notice), notice);
573
- };
574
- const resolvedExecution = !Object.hasOwn(options, "llmRunner")
575
- ? resolveImproveLlmExecution({
576
- config,
577
- profile: options.improveProfile,
578
- process: getImproveProcessConfig("distill", options.improveProfile),
579
- processName: "distill",
580
- })
581
- : null;
582
- if (resolvedExecution)
583
- collectNotices(resolvedExecution.notices);
584
- const distillRunner = Object.hasOwn(options, "llmRunner")
585
- ? (options.llmRunner ?? undefined)
586
- : resolvedExecution?.runner;
587
- const withNotices = (result) => executionNotices.size > 0 ? { ...result, notices: Object.freeze([...executionNotices.values()]) } : result;
588
- const lookup = options.lookupFn ?? ((ref) => defaultLookup(ref, stash));
589
- const readEventsImpl = options.readEventsFn ??
590
- ((readOptions) => readEvents(readOptions, { readOnly: true }));
591
- const outcomeWeightEnabled = config.improve?.salience?.outcomeWeightEnabled !== false;
592
- const fetchSimilarLessonsFn = options.fetchSimilarLessonsFn ?? ((query, n) => fetchTopSimilarLessons(query, n, options.stashDir));
593
- const assetCtx = createRunContext({
594
- stashDir: stash,
595
- config,
596
- eventsCtx: options.eventsCtx ?? {},
597
- proposalsCtx: options.ctx ?? {},
598
- chat,
599
- getLlmRunner: () => distillRunner ?? null,
600
- sourceRun: options.sourceRun ?? `distill-${Date.now()}`,
601
- dryRun: false,
602
- signal: options.signal,
603
- }).withFreshAssetMemo();
604
- const initialSalience = await loadAndScoreInputSalience({
605
- inputRef,
606
- durableInputRef,
607
- salienceWriteKey,
608
- stash,
609
- config,
610
- outcomeWeightEnabled,
611
- lookup,
612
- ctx: assetCtx,
613
- persistSalience: false,
614
- });
615
- const assetState = { ...initialSalience };
616
- const persistInputSalience = async () => {
617
- const scored = await loadAndScoreInputSalience({
618
- inputRef,
619
- durableInputRef,
620
- salienceWriteKey,
621
- stash,
622
- config,
623
- outcomeWeightEnabled,
624
- lookup,
625
- ctx: assetCtx,
626
- });
627
- assetState.assetContent = scored.assetContent ?? assetState.assetContent;
628
- assetState.existingRefVocabulary = scored.existingRefVocabulary;
629
- };
630
- const feedbackState = readDistillFeedback({ readEventsImpl, options, durableInputRef });
631
- const feedback = feedbackState.filteredEvents.slice(-20).map((event) => ({
632
- ts: event.ts,
633
- eventType: event.eventType,
634
- ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
635
- }));
636
- return {
637
- stash,
638
- chat,
639
- collectNotices,
640
- distillRunner,
641
- withNotices,
642
- lookup,
643
- outcomeWeightEnabled,
644
- fetchSimilarLessonsFn,
645
- assetState,
646
- persistInputSalience,
647
- feedback,
648
- ...feedbackState,
649
- };
256
+ /** The exclusion diagnostics for an event (count only) or a result (count + fully-filtered). */
257
+ function exclusionMeta(run, forResult) {
258
+ if (!run.exclusion)
259
+ return {};
260
+ return forResult ? { ...run.exclusion } : { filteredFeedbackCount: run.exclusion.filteredFeedbackCount };
650
261
  }
651
262
  export async function akmDistill(options) {
652
263
  const inputRef = options.ref.trim();
653
264
  if (!inputRef) {
654
265
  throw new UsageError("Asset ref is required. Usage: akm distill <ref>", "MISSING_REQUIRED_ARGUMENT");
655
266
  }
656
- // Validate the ref shape up front so a typo never reaches the LLM.
657
267
  const parsedInputRef = parseRefInput(inputRef);
658
- const durableInputRef = durableImproveRef(inputRef);
659
- // The input asset's durable salience write key is item_ref when resolved,
660
- // otherwise the input conceptId.
661
- const salienceWriteKey = options.itemRef ?? durableInputRef;
662
- const targetKind = options.proposalKind ?? "lesson";
663
268
  const config = options.config ?? loadConfig();
664
- const improveProfile = options.improveProfile ?? resolveImproveStrategy(undefined, config).config;
665
- options = { ...options, improveProfile };
666
- if (!resolveProcessEnabled("distill", improveProfile)) {
667
- const proposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
269
+ const profile = options.improveProfile ?? resolveImproveStrategy(undefined, config).config;
270
+ options = { ...options, improveProfile: profile };
271
+ const targetKind = options.proposalKind ?? "lesson";
272
+ const kind = targetKind === "knowledge" ? "knowledge" : "lesson";
273
+ const outputRef = kind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
274
+ if (!resolveProcessEnabled("distill", profile)) {
668
275
  return {
669
276
  schemaVersion: 1,
670
277
  ok: true,
671
278
  outcome: "config_disabled",
672
279
  inputRef,
673
- proposalRef: proposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef),
674
- proposalKind,
675
- message: "distill is disabled in config; enable processes.distill.enabled to activate.",
280
+ proposalRef: outputRef,
281
+ proposalKind: kind,
282
+ message: DISABLED_MESSAGE,
676
283
  };
677
284
  }
678
- // Attribution tagging: spread into every distill_invoked event's metadata so
679
- // the lane that selected this asset is recorded uniformly across all outcome
680
- // branches. Empty object when no lane was supplied (direct `akm distill`).
681
- const eligMeta = options.eligibilitySource
682
- ? { eligibilitySource: options.eligibilitySource }
683
- : {};
684
- // Recursive-distillation guard (see refuseDisallowedDistillInput). The
685
- // refused-type set is exported as {@link DISTILL_REFUSED_INPUT_TYPES} so
686
- // the improve planner can skip these refs before queuing distill attempts;
687
- // this runtime check stays as a defensive backstop for direct callers.
688
- const refused = refuseDisallowedDistillInput({ options, parsedInputRef, inputRef, durableInputRef, eligMeta });
689
- if (refused)
690
- return refused;
691
- const prepared = await prepareDistillExecution({
285
+ const ledgerRef = options.itemRef ?? inputRef;
286
+ const eligMeta = options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {};
287
+ if (isDistillRefusedInputType(parsedInputRef.type)) {
288
+ const secret = parsedInputRef.type === "env" || parsedInputRef.type === "secret";
289
+ const proposalRef = secret ? inputRef : conceptIdFromTypeName("lesson", parsedInputRef.name);
290
+ const skipReason = secret ? "refused_secret_input" : "recursive_lesson_input";
291
+ const message = secret
292
+ ? `Distill refuses ${parsedInputRef.type} inputs — secret material must never be sent to the LLM.`
293
+ : "Distill refuses lesson inputs — lessons are the distilled form, not a source.";
294
+ emitDistill({ ledgerRef, eligMeta, options }, { outcome: "skipped", proposalRef, message, skipReason });
295
+ return { schemaVersion: 1, ok: true, outcome: "skipped", inputRef, proposalRef, skipReason, message };
296
+ }
297
+ const stash = options.stashDir ?? resolveStashDir();
298
+ const notices = noticeSet();
299
+ const lookup = options.lookupFn ?? ((ref) => defaultLookup(ref, stash));
300
+ const asset = await loadInput(lookup, inputRef);
301
+ const run = {
692
302
  options,
693
- config,
694
- inputRef,
695
- durableInputRef,
696
- salienceWriteKey,
697
- });
698
- const { stash, chat, collectNotices, distillRunner, withNotices, lookup, outcomeWeightEnabled, fetchSimilarLessonsFn, assetState, persistInputSalience, feedback, filteredEvents, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, } = prepared;
699
- const promotionContext = {
700
- targetKind,
701
303
  inputRef,
702
- durableInputRef,
703
- ...(options.itemRef ? { itemRef: options.itemRef } : {}),
704
- assetContent: assetState.assetContent,
705
- filteredEvents,
706
- config,
707
- strategy: options.improveProfile,
708
- llmRunner: distillRunner,
709
- signal: options.signal,
710
- chat,
304
+ ledgerRef,
711
305
  stash,
712
- lookup,
713
- fetchSimilarLessonsFn,
714
- existingRefVocabulary: assetState.existingRefVocabulary,
715
- outcomeWeightEnabled,
306
+ config,
307
+ profile,
308
+ runner: stageRunner(options, config, profile, "distill", notices.add),
309
+ notices,
716
310
  eligMeta,
717
- eligibilitySource: options.eligibilitySource,
718
- sourceRun: options.sourceRun,
719
- proposalsCtx: options.ctx,
720
- eventsCtx: options.eventsCtx,
721
- exclusionSetSize: exclusionSet.size,
722
- filteredFeedbackCount,
723
- feedbackFullyFiltered,
724
- onNotices: collectNotices,
311
+ asset,
312
+ vocabulary: loadRefVocabulary(),
313
+ outcomeWeightEnabled: config.improve?.salience?.outcomeWeightEnabled !== false,
314
+ similar: options.fetchSimilarLessonsFn ?? fetchTopSimilarLessons,
315
+ lookup,
725
316
  };
726
- const promotionPlan = await planMemoryKnowledgePromotion(promotionContext);
727
- let dispatchLease;
728
- try {
729
- // Memory→knowledge promotion branch (D-1/#369). When the target ref is a
730
- // reinforced memory, distill graduates it into a knowledge proposal instead
731
- // of a lesson — the whole branch (LLM contradiction-merge, quality gate,
732
- // proposal creation, event emit) lives in `promoteMemoryToKnowledge` and is
733
- // terminal when it fires. A `null` return means "not a promotion candidate";
734
- // fall through to the ordinary lesson/knowledge distillation path.
735
- if (promotionPlan && distillRunner && memoryKnowledgePromotionRequiresDispatch(promotionContext, promotionPlan)) {
736
- dispatchLease = await preflightStructuredLlmRunner(distillRunner);
737
- }
738
- const promotionResult = await promoteMemoryToKnowledge({ ...promotionContext, ...(dispatchLease ? { lease: dispatchLease } : {}) }, promotionPlan);
739
- if (promotionResult) {
740
- await persistInputSalience();
741
- return withNotices(promotionResult);
742
- }
743
- const effectiveProposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
744
- const effectiveLessonRef = effectiveProposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
745
- const messages = await buildDistillMessages({
746
- options,
747
- stash,
748
- inputRef,
749
- assetContent: assetState.assetContent,
750
- feedback,
751
- effectiveProposalKind,
752
- effectiveLessonRef,
753
- fetchSimilarLessonsFn,
754
- });
755
- if (!dispatchLease && distillRunner)
756
- dispatchLease = await preflightStructuredLlmRunner(distillRunner);
757
- const { raw, fallbackReason } = await runDistillLlmCall({
758
- config,
759
- options,
760
- distillRunner,
761
- lease: dispatchLease,
762
- messages,
763
- effectiveProposalKind,
764
- onNotices: collectNotices,
765
- });
766
- await persistInputSalience();
767
- if (raw === null || raw.trim() === "") {
768
- return withNotices(distillEmptyResponseResult({
769
- fallbackReason,
770
- inputRef,
771
- durableInputRef,
772
- ...(options.itemRef ? { itemRef: options.itemRef } : {}),
773
- effectiveLessonRef,
774
- effectiveProposalKind,
775
- exclusionSet,
776
- filteredFeedbackCount,
777
- feedbackFullyFiltered,
778
- eligMeta,
779
- eventsCtx: options.eventsCtx,
780
- }));
781
- }
782
- const assembled = assembleAndValidateDistillContent({
783
- raw,
784
- effectiveProposalKind,
785
- inputRef,
786
- durableInputRef,
787
- ...(options.itemRef ? { itemRef: options.itemRef } : {}),
788
- effectiveLessonRef,
789
- exclusionSet,
790
- filteredFeedbackCount,
791
- eligMeta,
792
- eventsCtx: options.eventsCtx,
793
- stash,
794
- ...(options.ctx ? { proposalsCtx: options.ctx } : {}),
795
- ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
796
- });
797
- if ("rejection" in assembled)
798
- return withNotices(assembled.rejection);
799
- const { content, descriptionSwapped } = assembled;
800
- const gate = await applyDistillQualityGate({
801
- config,
802
- options,
803
- content,
804
- assetContent: assetState.assetContent,
805
- chat,
806
- distillRunner,
807
- lease: dispatchLease,
808
- fetchSimilarLessonsFn,
809
- stash,
810
- inputRef,
811
- effectiveLessonRef,
812
- exclusionSet,
813
- filteredFeedbackCount,
814
- feedbackFullyFiltered,
815
- onNotices: collectNotices,
816
- });
817
- if ("rejection" in gate)
818
- return withNotices(gate.rejection);
819
- const lessonJudgeConfidence = gate.confidence;
820
- return withNotices(await emitDistillLessonProposal({
821
- content,
822
- options,
823
- distillRunner,
824
- assetContent: assetState.assetContent,
825
- inputRef,
826
- durableInputRef,
827
- effectiveLessonRef,
828
- effectiveProposalKind,
829
- stash,
830
- exclusionSet,
831
- filteredFeedbackCount,
832
- feedbackFullyFiltered,
833
- lessonJudgeConfidence,
834
- existingRefVocabulary: assetState.existingRefVocabulary,
835
- outcomeWeightEnabled,
836
- descriptionSwapped,
837
- eligMeta,
838
- }));
839
- }
840
- finally {
841
- if (dispatchLease)
842
- disposeLoweredExecutionDispatchLease(dispatchLease);
843
- }
317
+ const feedbackEvents = readDistillFeedback(run);
318
+ const result = await distill(run, targetKind, kind, outputRef, feedbackEvents);
319
+ return { ...result, ...notices.fields() };
844
320
  }
845
- // ── Helpers ─────────────────────────────────────────────────────────────────
846
- /**
847
- * The distill-propose pass: the optional WS-3b distill→source fidelity check
848
- * (routes contradictions to human review), provenance xref round-trip, proposal
849
- * creation, and the queued/skipped `distill_invoked` emit + output salience
850
- * scoring (G4). Extracted verbatim from `akmDistill`; every outcome shape and
851
- * event is byte-identical.
852
- */
853
- async function emitDistillLessonProposal(args) {
854
- const { options, distillRunner, assetContent, inputRef, durableInputRef, effectiveLessonRef, effectiveProposalKind, stash, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, lessonJudgeConfidence, existingRefVocabulary, outcomeWeightEnabled, descriptionSwapped, eligMeta, } = args;
855
- let content = args.content;
856
- // WS-3b: Distill→source fidelity check (step 10).
857
- // When fidelityCheck.enabled, check the distill proposal against its cited
858
- // source memories. A contradiction flag routes to human review (not auto-accept).
859
- // DEFAULT OFF. Fail-open: any error is treated as no-contradiction.
860
- const fidelityConfig = getImproveProcessConfig("distill", options.improveProfile)?.fidelityCheck ??
861
- {};
862
- if (fidelityConfig.enabled && assetContent) {
863
- try {
864
- const proposalBody = stripBodyForFidelity(content);
865
- const sourceBodies = [stripBodyForFidelity(assetContent)];
866
- const fidelityResult = checkDistillFidelity(proposalBody, sourceBodies, fidelityConfig);
867
- if (fidelityResult.contradictionDetected) {
868
- // Route to human review by writing a quality rejection with reviewNeeded=true.
869
- return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed
870
- fidelityResult.reason ?? "Proposal may contradict cited source memories.", {
871
- reviewNeeded: true,
872
- fidelityContradiction: true,
873
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
874
- }, options.eligibilitySource, options.eventsCtx, {
875
- ...(options.ctx ? { proposalsCtx: options.ctx } : {}),
876
- ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
877
- ...(distillRunner?.connection.model ? { modelId: distillRunner.connection.model } : {}),
878
- });
879
- }
880
- }
881
- catch {
882
- // Fail open — fidelity check is supplemental.
883
- }
884
- }
885
- // Round-trip the parsed frontmatter so the proposal carries it as a
886
- // structured payload alongside the raw content (matches the shape used by
887
- // other proposal sources).
888
- //
889
- // Serialize canonical provenance into the content that promotion writes.
890
- const parsed = parseFrontmatter(content);
891
- const existingXrefs = Array.isArray(parsed.data.xrefs) ? parsed.data.xrefs.map(String) : [];
892
- const frontmatterWithXrefs = {
893
- ...parsed.data,
894
- xrefs: [...new Set([...existingXrefs, durableInputRef])],
895
- };
896
- delete frontmatterWithXrefs.sources;
897
- content = assembleAsset(frontmatterWithXrefs, parsed.content);
898
- const proposalResult2 = emitProposal({ stashDir: stash, proposalsCtx: options.ctx }, {
899
- ref: effectiveLessonRef,
900
- // §23.6 fingerprint model-id term (WI-6.4). Uses the RESOLVED connection
901
- // (profile/config fallback included), not the raw option — a standalone
902
- // `akm distill` run must fingerprint under the model that actually
903
- // generated the content, matching the promote-memory branch.
904
- ...(distillRunner?.connection.model ? { modelId: distillRunner.connection.model } : {}),
905
- source: "distill",
906
- ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
907
- payload: {
908
- content,
909
- frontmatter: frontmatterWithXrefs,
910
- },
911
- ...(lessonJudgeConfidence !== undefined ? { confidence: lessonJudgeConfidence } : {}),
912
- // Attribution tagging: persist the eligibility lane on the proposal.
913
- ...(options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {}),
914
- });
915
- if (isProposalSkipped(proposalResult2)) {
916
- appendEvent({
917
- eventType: "distill_invoked",
918
- // Use item_ref when resolved, otherwise the input conceptId.
919
- ref: options.itemRef ?? durableInputRef,
920
- metadata: {
921
- outcome: "skipped",
922
- proposalRef: effectiveLessonRef,
923
- message: proposalResult2.message,
924
- skipReason: proposalResult2.reason,
925
- ...eligMeta,
321
+ async function distill(run, targetKind, kind, outputRef, feedbackEvents) {
322
+ // A reinforced memory graduates to knowledge without a generation call.
323
+ const promotion = targetKind === "lesson" ? null : await planPromotion(run, feedbackEvents);
324
+ if (promotion) {
325
+ if (run.runner && (promotion.existing || qualityGateEnabled(run)))
326
+ assertRunnerCredentials(run.runner);
327
+ const promoted = await promoteToKnowledge(run, promotion);
328
+ stampInputSalience(run);
329
+ return promoted;
330
+ }
331
+ const feedback = feedbackEvents.slice(-20).map((event) => ({
332
+ ts: event.ts,
333
+ eventType: event.eventType,
334
+ ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
335
+ }));
336
+ const { system, prompt } = await buildDistillMessages(run, feedback, kind, outputRef);
337
+ const call = run.runner
338
+ ? await callStage({
339
+ feature: "distill",
340
+ runner: run.runner,
341
+ system,
342
+ prompt,
343
+ gate: { config: run.config, enabled: true },
344
+ // The injected test transport never sees the schema.
345
+ request: {
346
+ ...(run.options.chat === undefined
347
+ ? { responseSchema: kind === "knowledge" ? DISTILL_KNOWLEDGE_JSON_SCHEMA : DISTILL_LESSON_JSON_SCHEMA }
348
+ : { chat: run.options.chat }),
349
+ ...(run.options.signal ? { signal: run.options.signal } : {}),
926
350
  },
927
- }, options.eventsCtx);
351
+ onNotices: run.notices.add,
352
+ })
353
+ : { ok: false, reason: "error" };
354
+ // Durable input salience waits until the credential-bearing dispatch returned.
355
+ stampInputSalience(run);
356
+ if (!call.ok || call.raw.trim() === "") {
357
+ if (!call.ok)
358
+ warnVerbose(`[akm] LLM fallback for distill: ${call.reason}`);
359
+ emitDistill(run, {
360
+ outcome: "llm_failed",
361
+ proposalRef: outputRef,
362
+ proposalKind: kind,
363
+ ...exclusionMeta(run, false),
364
+ });
928
365
  return {
929
366
  schemaVersion: 1,
930
367
  ok: true,
931
- outcome: "skipped",
932
- inputRef,
933
- proposalRef: effectiveLessonRef,
934
- skipReason: proposalResult2.reason,
935
- message: proposalResult2.message,
368
+ outcome: "llm_failed",
369
+ inputRef: run.inputRef,
370
+ proposalRef: outputRef,
371
+ proposalKind: kind,
372
+ message: "LLM call returned no usable output (timeout, empty, or error).",
373
+ ...exclusionMeta(run, true),
936
374
  };
937
375
  }
938
- const proposal2 = proposalResult2;
939
- // G4: content-score the distilled OUTPUT so it carries a real encoding
940
- // salience (encoding_source='content') from creation — lessons never get
941
- // another chance (they are refused as distill inputs).
942
- persistOutputEncodingSalience(durableImproveRef(effectiveLessonRef), content, existingRefVocabulary, outcomeWeightEnabled);
943
- appendEvent({
944
- eventType: "distill_invoked",
945
- // Use item_ref when resolved, otherwise the input conceptId.
946
- ref: options.itemRef ?? durableInputRef,
947
- metadata: {
948
- outcome: "queued",
949
- proposalRef: effectiveLessonRef,
950
- proposalKind: effectiveProposalKind,
951
- proposalId: proposal2.id,
952
- // R3: judge verdicts are longitudinally queryable, not just a one-shot
953
- // proposal.confidence write (normalized 1–5 score / 5).
954
- ...(lessonJudgeConfidence !== undefined ? { judgeConfidence: lessonJudgeConfidence } : {}),
955
- ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
956
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
957
- ...(descriptionSwapped > 0 ? { descriptionSwapped } : {}),
958
- ...eligMeta,
959
- },
960
- }, options.eventsCtx);
961
- return {
962
- schemaVersion: 1,
963
- ok: true,
964
- outcome: "queued",
965
- inputRef,
966
- proposalRef: effectiveLessonRef,
967
- proposalKind: effectiveProposalKind,
968
- proposalId: proposal2.id,
969
- proposal: proposal2,
970
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
971
- ...(descriptionSwapped > 0 ? { descriptionSwapped } : {}),
972
- };
376
+ const assembled = assembleDistilledContent(run, call.raw, kind, outputRef);
377
+ if ("rejection" in assembled)
378
+ return assembled.rejection;
379
+ return judgeAndQueue(run, {
380
+ ref: outputRef,
381
+ kind,
382
+ content: assembled.content,
383
+ source: run.asset.content,
384
+ descriptionSwapped: assembled.descriptionSwapped,
385
+ });
973
386
  }
974
- /**
975
- * Turn the raw LLM response into validated proposal content: prefer the
976
- * structured-JSON assembly, else strip markdown fences; run the lesson-only
977
- * auto-repair chain (frontmatter repair, description↔when_to_use swap, truncation
978
- * repair); then lint/validate, emitting `distill_invoked(validation_failed)` and
979
- * throwing a `UsageError` on any finding. Extracted verbatim from `akmDistill`.
980
- */
981
- function assembleAndValidateDistillContent(args) {
982
- const { raw, effectiveProposalKind, inputRef, durableInputRef, itemRef, effectiveLessonRef, exclusionSet, filteredFeedbackCount, eligMeta, eventsCtx, stash, proposalsCtx, sourceRun, } = args;
983
- // Structured-output path: when the provider honoured the JSON schema, `raw`
984
- // is a JSON object string (not a markdown blob). Try to parse it and assemble
985
- // the canonical `---\nfm\n---\n\nbody` form before using the markdown
986
- // response path. Failure here (non-JSON response, missing
987
- // required field, unexpected types) is non-fatal — we drop down to the
988
- // markdown path which has its own auto-repair + lint pass.
989
- let content;
990
- const structuredCandidate = parseEmbeddedJsonResponse(raw);
991
- const structuredAssembled = structuredCandidate && !Array.isArray(structuredCandidate)
992
- ? assembleStructuredDistillMarkdown(structuredCandidate, effectiveProposalKind)
993
- : null;
994
- if (structuredAssembled !== null) {
995
- content = structuredAssembled;
996
- }
997
- else {
998
- // Strip any stray fence the LLM might have added around the markdown.
999
- content = stripMarkdownFences(raw);
1000
- }
1001
- // Lesson-path content normalization (see distill/content-repair): auto-repair
1002
- // missing frontmatter, description↔when_to_use auto-swap, and truncation
1003
- // repair. Knowledge output skips all three (no lesson frontmatter contract).
1004
- if (effectiveProposalKind !== "knowledge") {
1005
- content = autoRepairLessonFrontmatter(content, inputRef);
1006
- }
387
+ /** Turn the response into validated content: structured JSON or markdown, then lesson repairs and lint. */
388
+ function assembleDistilledContent(run, raw, kind, outputRef) {
389
+ const structured = parseEmbeddedJsonResponse(raw);
390
+ let content = (structured && !Array.isArray(structured) ? assembleStructuredDistillMarkdown(structured, kind) : null) ??
391
+ stripMarkdownFences(raw);
1007
392
  let descriptionSwapped = 0;
1008
- if (effectiveProposalKind !== "knowledge") {
1009
- const swapResult = autoSwapDescriptionWhenToUse(content, inputRef);
1010
- content = swapResult.content;
1011
- descriptionSwapped = swapResult.swapped;
1012
- }
1013
- if (effectiveProposalKind !== "knowledge") {
393
+ if (kind === "lesson") {
394
+ content = autoRepairLessonFrontmatter(content, run.inputRef);
395
+ ({ content, swapped: descriptionSwapped } = autoSwapDescriptionWhenToUse(content, run.inputRef));
1014
396
  content = repairLessonDescriptionTruncation(content);
1015
397
  }
1016
- // Parse + lint the lesson before creating the proposal. The lint is the
1017
- // canonical gate for required frontmatter (v1 spec §13): a field that is
1018
- // genuinely missing or empty means there is no valid asset to write, so
1019
- // that failure stays a hard reject — but still emit `distill_invoked` so
1020
- // the failure is observable.
1021
- const structuralFindings = effectiveProposalKind === "knowledge"
1022
- ? validateKnowledgeContent(content, inputRef)
1023
- : lintLessonContent(content, `distill:${inputRef}`).findings;
1024
- // Additional lesson-only quality validators — reject the systematic failure
1025
- // modes seen across 323 archived rejected proposals (see distill/content-repair).
1026
- const qualityFindings = effectiveProposalKind !== "knowledge" && structuralFindings.length === 0
1027
- ? collectLessonQualityFindings(content, inputRef)
1028
- : [];
1029
- if (structuralFindings.length > 0) {
1030
- appendEvent({
1031
- eventType: "distill_invoked",
1032
- // Use item_ref when resolved, otherwise the input conceptId.
1033
- ref: itemRef ?? durableInputRef,
1034
- metadata: {
1035
- outcome: "validation_failed",
1036
- proposalRef: effectiveLessonRef,
1037
- proposalKind: effectiveProposalKind,
1038
- findingKinds: structuralFindings.map((f) => f.kind),
1039
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
1040
- ...eligMeta,
1041
- },
1042
- }, eventsCtx);
1043
- const message = structuralFindings.map((f) => f.message).join("\n");
1044
- throw new UsageError(`Distilled ${effectiveProposalKind} failed validation:\n${message}`, "MISSING_REQUIRED_ARGUMENT", effectiveProposalKind === "knowledge"
398
+ // Required structure missing means there is no asset to write: a hard reject.
399
+ const structural = kind === "knowledge"
400
+ ? validateKnowledgeContent(content, run.inputRef)
401
+ : lintLessonContent(content, `distill:${run.inputRef}`).findings;
402
+ if (structural.length > 0) {
403
+ emitDistill(run, {
404
+ outcome: "validation_failed",
405
+ proposalRef: outputRef,
406
+ proposalKind: kind,
407
+ findingKinds: structural.map((f) => f.kind),
408
+ ...exclusionMeta(run, false),
409
+ });
410
+ throw new UsageError(`Distilled ${kind} failed validation:\n${structural.map((f) => f.message).join("\n")}`, "MISSING_REQUIRED_ARGUMENT", kind === "knowledge"
1045
411
  ? "Knowledge proposals require a non-empty markdown body."
1046
412
  : "Lessons require non-empty `description` and `when_to_use` frontmatter fields. See v1 spec §13.");
1047
413
  }
1048
- if (qualityFindings.length > 0) {
414
+ // Heuristic quality findings go to a human, not the bin.
415
+ const quality = kind === "lesson" ? collectLessonQualityFindings(content, run.inputRef) : [];
416
+ if (quality.length > 0) {
1049
417
  return {
1050
- rejection: writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed — no judge score exists for a structural/heuristic finding
1051
- qualityFindings.map((f) => f.message).join("\n"), {
418
+ rejection: rejectDistilled(run, outputRef, content, 2.0, quality.map((f) => f.message).join("\n"), {
1052
419
  reviewNeeded: true,
1053
- proposalKind: effectiveProposalKind,
1054
- findingKinds: qualityFindings.map((f) => f.kind),
1055
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
1056
- }, eligMeta.eligibilitySource, eventsCtx, {
1057
- ...(proposalsCtx ? { proposalsCtx } : {}),
1058
- ...(sourceRun !== undefined ? { sourceRun } : {}),
420
+ proposalKind: kind,
421
+ findingKinds: quality.map((f) => f.kind),
1059
422
  }),
1060
423
  };
1061
424
  }
1062
425
  return { content, descriptionSwapped };
1063
426
  }
427
+ function qualityGateEnabled(run) {
428
+ return run.profile.processes?.distill?.qualityGate?.enabled ?? true;
429
+ }
1064
430
  /**
1065
- * The single bounded distill LLM call: gate-checked via `callStructured`
1066
- * (R26 migration off the raw `chatCompletion` scaffold), passing the
1067
- * lesson/knowledge JSON schema on the production path and keeping the
1068
- * injected test fake schema-blind. Returns the raw response (or `null`) and
1069
- * the fallback reason.
431
+ * Judge the distilled content, then queue it. A rejected, uncertain or
432
+ * source-contradicting result is recorded instead (see {@link writeQualityRejection}).
1070
433
  */
1071
- async function runDistillLlmCall(args) {
1072
- const { config, options, distillRunner, lease, messages, effectiveProposalKind, onNotices } = args;
1073
- const distillSchema = effectiveProposalKind === "knowledge" ? DISTILL_KNOWLEDGE_JSON_SCHEMA : DISTILL_LESSON_JSON_SCHEMA;
1074
- let fallbackReason;
1075
- const enabled = resolveProcessEnabled("distill", options.improveProfile ?? resolveImproveStrategy(undefined, config).config);
1076
- const recordFallback = (feature, reason) => {
1077
- fallbackReason = reason;
1078
- // Log the fallback reason; the caller (raw === null path) handles
1079
- // emitting the distill_invoked event so we don't double-emit here.
1080
- warnVerbose(`[akm] LLM fallback for ${feature}: ${reason}`);
1081
- };
1082
- if (enabled && !distillRunner) {
1083
- // No LLM connection configured. At HEAD this threw a ConfigError inside
1084
- // the gated fn and tryLlmFeature routed it through the "error" fallback;
1085
- // reproduce that terminal state directly (the gate-disabled case above
1086
- // still dominates: when disabled, callStructured takes the "disabled"
1087
- // fallback before any LLM lookup, exactly as before).
1088
- recordFallback("distill", "error");
1089
- return { raw: null, fallbackReason };
434
+ async function judgeAndQueue(run, out) {
435
+ let content = out.content;
436
+ let confidence;
437
+ if (qualityGateEnabled(run)) {
438
+ const similarLessons = await run.similar(content.slice(0, 500), 3);
439
+ const verdict = await runLessonQualityJudge(run.config, content, out.source ?? "", run.options.chat, {
440
+ ...(similarLessons.length > 0 ? { similarLessons } : {}),
441
+ ...(run.runner ? { llmRunner: run.runner } : {}),
442
+ ...(run.options.signal ? { signal: run.options.signal } : {}),
443
+ onNotices: run.notices.add,
444
+ });
445
+ if (!verdict.pass) {
446
+ return rejectDistilled(run, out.ref, content, verdict.score, verdict.reason, {
447
+ ...(verdict.reviewNeeded ? { reviewNeeded: true } : {}),
448
+ ...(verdict.criteria ? { criteria: verdict.criteria } : {}),
449
+ });
450
+ }
451
+ if (verdict.score > 0)
452
+ confidence = verdict.score / 5;
453
+ }
454
+ let frontmatter;
455
+ if (out.promotion) {
456
+ const data = parseFrontmatter(content).data;
457
+ if (Object.keys(data).length > 0)
458
+ frontmatter = data;
1090
459
  }
1091
- const raw = await callStructured({
1092
- feature: "distill",
1093
- akmConfig: config,
1094
- enabled,
1095
- // Safe: when the gate is open, distillRunner is defined (guard above); when
1096
- // it is closed, the transport never runs and the runner is never read.
1097
- runner: distillRunner,
1098
- ...(lease ? { lease } : {}),
1099
- messages,
1100
- request: options.chat === undefined
1101
- ? // Production path: pass the JSON schema so providers that honour
1102
- // `response_format: json_schema` enforce shape upstream. Providers
1103
- // that ignore the option fall through to the prompt-contract
1104
- // markdown path.
1105
- {
1106
- responseSchema: distillSchema,
1107
- ...(options.signal ? { signal: options.signal } : {}),
460
+ else {
461
+ // Optional check against the cited source; a contradiction goes to a human.
462
+ const fidelity = getImproveProcessConfig("distill", run.profile)?.fidelityCheck ?? {};
463
+ if (fidelity.enabled && out.source) {
464
+ try {
465
+ const verdict = checkDistillFidelity(stripFrontmatterBody(content), [stripFrontmatterBody(out.source)], fidelity);
466
+ if (verdict.contradictionDetected) {
467
+ return rejectDistilled(run, out.ref, content, 2.0, verdict.reason ?? "Proposal may contradict cited source memories.", { reviewNeeded: true, fidelityContradiction: true });
1108
468
  }
1109
- : // Test seam: keep the injected fake as the transport; fakes never
1110
- // see the schema (they return markdown strings).
1111
- { chat: options.chat, ...(options.signal ? { signal: options.signal } : {}) },
1112
- parse: (raw) => raw ?? null,
1113
- onError: (_cls, err) => {
1114
- // At HEAD a transport throw escaped to tryLlmFeature's catch, which
1115
- // fired onFallback("error"); reproduce that observable state.
1116
- void err;
1117
- recordFallback("distill", "error");
1118
- return null;
1119
- },
1120
- fallback: null,
1121
- onFallback: (evt) => recordFallback(evt.feature, evt.reason),
1122
- onNotices,
469
+ }
470
+ catch {
471
+ // The fidelity check is supplemental.
472
+ }
473
+ }
474
+ // Canonical provenance goes into the content promotion writes.
475
+ const parsed = parseFrontmatter(content);
476
+ const xrefs = Array.isArray(parsed.data.xrefs) ? parsed.data.xrefs.map(String) : [];
477
+ frontmatter = { ...parsed.data, xrefs: [...new Set([...xrefs, run.inputRef])] };
478
+ delete frontmatter.sources;
479
+ content = assembleAsset(frontmatter, parsed.content);
480
+ }
481
+ const proposal = mintProposal(run.stash, run.options.ctx, {
482
+ ref: out.ref,
483
+ source: "distill",
484
+ ...(run.options.sourceRun !== undefined ? { sourceRun: run.options.sourceRun } : {}),
485
+ payload: { content, ...(frontmatter ? { frontmatter } : {}) },
486
+ ...(confidence !== undefined ? { confidence } : {}),
487
+ ...(run.options.eligibilitySource ? { eligibilitySource: run.options.eligibilitySource } : {}),
488
+ // The ledger keys the attempt by the input, not the output.
489
+ attemptedRefs: [run.ledgerRef],
490
+ }, { judged: confidence !== undefined });
491
+ persistOutputEncodingSalience(run, out.ref, content);
492
+ const swapped = out.descriptionSwapped ? { descriptionSwapped: out.descriptionSwapped } : {};
493
+ emitDistill(run, {
494
+ outcome: "queued",
495
+ proposalRef: out.ref,
496
+ proposalKind: out.kind,
497
+ proposalId: proposal.id,
498
+ ...(confidence !== undefined ? { judgeConfidence: confidence } : {}),
499
+ ...(run.options.sourceRun !== undefined ? { sourceRun: run.options.sourceRun } : {}),
500
+ ...exclusionMeta(run, false),
501
+ ...swapped,
502
+ });
503
+ return {
504
+ schemaVersion: 1,
505
+ ok: true,
506
+ outcome: "queued",
507
+ inputRef: run.inputRef,
508
+ proposalRef: out.ref,
509
+ proposalKind: out.kind,
510
+ proposalId: proposal.id,
511
+ proposal,
512
+ ...exclusionMeta(run, true),
513
+ ...swapped,
514
+ };
515
+ }
516
+ function rejectDistilled(run, proposalRef, content, score, reason, meta) {
517
+ return writeQualityRejection({
518
+ stash: run.stash,
519
+ inputRef: run.inputRef,
520
+ proposalRef,
521
+ content,
522
+ score,
523
+ reason,
524
+ meta: { ...meta, ...exclusionMeta(run, true) },
525
+ eligibilitySource: run.options.eligibilitySource,
526
+ eventsCtx: run.options.eventsCtx,
527
+ proposalsCtx: run.options.ctx,
528
+ sourceRun: run.options.sourceRun,
529
+ ledgerRef: run.ledgerRef,
1123
530
  });
1124
- return { raw, fallbackReason };
1125
531
  }
1126
532
  /**
1127
- * Build the terminal result for an empty/failed distill LLM response,
1128
- * distinguishing the config-gate-off branch (event suppressed) from a real
1129
- * transport/timeout/empty failure (emits `distill_invoked(llm_failed)`).
1130
- * Extracted verbatim from `akmDistill`.
533
+ * Record a distill quality-gate outcome and return its envelope.
534
+ * `quality_rejected` lands in the improve ledger under the input's key (its
535
+ * rejection window keeps selection from regenerating it); `review_needed`
536
+ * mints a pending proposal for a human, stamped `deferred`/`quality-gate` so
537
+ * the triage drain leaves it alone. Content the mint refuses still records
538
+ * the attempt.
1131
539
  */
1132
- function distillEmptyResponseResult(args) {
1133
- const { fallbackReason, inputRef, durableInputRef, itemRef, effectiveLessonRef, effectiveProposalKind, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, eligMeta, eventsCtx, } = args;
1134
- // Distinguish "config gate disabled" from "LLM call failed". For the
1135
- // config-disabled branch, we ALSO suppress the `distill_invoked` event
1136
- // because no LLM work was actually invoked — emitting the event causes
1137
- // the planner to accumulate phantom invocations that drown out real
1138
- // signal.
1139
- if (fallbackReason === "disabled") {
1140
- return {
1141
- schemaVersion: 1,
1142
- ok: true,
1143
- outcome: "config_disabled",
1144
- inputRef,
1145
- proposalRef: effectiveLessonRef,
1146
- proposalKind: effectiveProposalKind,
1147
- message: "distill is disabled in config; enable processes.distill.enabled to activate.",
1148
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
1149
- };
540
+ export function writeQualityRejection(args) {
541
+ const meta = args.meta ?? {};
542
+ const outcome = meta.reviewNeeded ? "review_needed" : "quality_rejected";
543
+ const ledgerRef = args.ledgerRef ?? args.inputRef;
544
+ const access = { proposalsCtx: args.proposalsCtx, eventsCtx: args.eventsCtx };
545
+ const attempt = { stashDir: args.stash, ref: ledgerRef, source: "distill", detail: args.reason };
546
+ let proposal;
547
+ if (outcome === "quality_rejected") {
548
+ recordLedgerAttempt(access, { ...attempt, outcome: "quality_rejected" });
1150
549
  }
1151
- // LLM was actually invoked but produced nothing usable (transport error,
1152
- // timeout, or empty/whitespace response). Emit the event so the failure
1153
- // is observable.
550
+ else {
551
+ try {
552
+ proposal = mintProposal(args.stash, args.proposalsCtx, {
553
+ ref: args.proposalRef,
554
+ source: "distill",
555
+ ...(args.sourceRun !== undefined ? { sourceRun: args.sourceRun } : {}),
556
+ payload: { content: args.content },
557
+ attemptedRefs: [ledgerRef],
558
+ ...(args.eligibilitySource ? { eligibilitySource: args.eligibilitySource } : {}),
559
+ }, { review: { reason: "quality-review", gate: "quality-gate" } });
560
+ }
561
+ catch (error) {
562
+ warn(`[akm] writeQualityRejection: failed to queue ${args.proposalRef} for review: ${error instanceof Error ? error.message : String(error)}`);
563
+ recordLedgerAttempt(access, { ...attempt, outcome: "review_needed" });
564
+ }
565
+ }
566
+ const eligMeta = args.eligibilitySource ? { eligibilitySource: args.eligibilitySource } : {};
1154
567
  appendEvent({
1155
568
  eventType: "distill_invoked",
1156
- // Use item_ref when resolved, otherwise the input conceptId.
1157
- ref: itemRef ?? durableInputRef,
569
+ ref: ledgerRef,
1158
570
  metadata: {
1159
- outcome: "llm_failed",
1160
- proposalRef: effectiveLessonRef,
1161
- proposalKind: effectiveProposalKind,
1162
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
571
+ outcome,
572
+ proposalRef: args.proposalRef,
573
+ score: args.score,
574
+ reason: args.reason,
575
+ ...meta,
1163
576
  ...eligMeta,
1164
577
  },
1165
- }, eventsCtx);
578
+ }, args.eventsCtx);
1166
579
  return {
1167
580
  schemaVersion: 1,
1168
581
  ok: true,
1169
- outcome: "llm_failed",
1170
- inputRef,
1171
- proposalRef: effectiveLessonRef,
1172
- proposalKind: effectiveProposalKind,
1173
- message: "LLM call returned no usable output (timeout, empty, or error).",
1174
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
582
+ outcome,
583
+ inputRef: args.inputRef,
584
+ proposalRef: args.proposalRef,
585
+ score: args.score,
586
+ reason: args.reason,
587
+ ...(proposal ? { proposalId: proposal.id, proposal } : {}),
588
+ ...meta,
1175
589
  };
1176
590
  }
591
+ async function planPromotion(run, feedbackEvents) {
592
+ const assessment = assessMemoryKnowledgePromotionCandidate({
593
+ inputRef: run.inputRef,
594
+ assetContent: run.asset.content,
595
+ feedbackEvents,
596
+ });
597
+ if (!assessment.promote || !assessment.content)
598
+ return null;
599
+ const existingPath = await run.lookup(assessment.knowledgeRef);
600
+ let existing = null;
601
+ try {
602
+ if (existingPath && fs.existsSync(existingPath))
603
+ existing = fs.readFileSync(existingPath, "utf8");
604
+ }
605
+ catch {
606
+ existing = null;
607
+ }
608
+ return { knowledgeRef: assessment.knowledgeRef, content: assessment.content, existing };
609
+ }
1177
610
  /**
1178
- * The P2-B LLM-as-judge quality gate (fail-CLOSED; D-5/#388 three-band). Returns
1179
- * a terminal rejection result when the judge rejects (or routes to review), or
1180
- * the normalized [0,1] confidence to carry onto the proposal. Extracted verbatim
1181
- * from `akmDistill`.
611
+ * Promote a reinforced memory to knowledge. An existing destination is
612
+ * reconciled by the model (ADD/UPDATE swap content in, NOOP keeps what is
613
+ * there); without a model the existing content is appended for the reviewer.
1182
614
  */
1183
- async function applyDistillQualityGate(args) {
1184
- const { config, options, content, assetContent, chat, distillRunner, lease, fetchSimilarLessonsFn, stash, inputRef, effectiveLessonRef, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, onNotices, } = args;
1185
- if (!(options.improveProfile?.processes?.distill?.qualityGate?.enabled ?? true)) {
1186
- return { confidence: undefined };
1187
- }
1188
- // D-4 / #390: retrieve top-3 similar lessons for dedup check in judge.
1189
- const similarLessons = await fetchSimilarLessonsFn(content.slice(0, 500), 3);
1190
- const judgeResult = await runLessonQualityJudge(config, content, assetContent ?? "", chat, {
1191
- ...(similarLessons.length > 0 ? { similarLessons } : {}),
1192
- ...(distillRunner ? { llmRunner: distillRunner } : {}),
1193
- ...(lease ? { lease } : {}),
1194
- ...(options.signal ? { signal: options.signal } : {}),
1195
- onNotices,
1196
- });
1197
- if (!judgeResult.pass) {
1198
- const proposalOpts = {
1199
- ...(options.ctx ? { proposalsCtx: options.ctx } : {}),
1200
- ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
1201
- ...(distillRunner?.connection.model ? { modelId: distillRunner.connection.model } : {}),
1202
- };
1203
- if (judgeResult.reviewNeeded) {
615
+ async function promoteToKnowledge(run, plan) {
616
+ let content = plan.content;
617
+ if (plan.existing && run.runner) {
618
+ const merged = await callStage({
619
+ feature: "distill",
620
+ runner: run.runner,
621
+ system: "Return only valid JSON. No prose.",
622
+ prompt: [
623
+ "You are merging two versions of a knowledge document.",
624
+ "Existing content is already committed; new content comes from a memory distillation run.",
625
+ "Choose one of: ADD (combine both), UPDATE (replace existing with new), NOOP (keep existing unchanged).",
626
+ 'Return ONLY valid JSON: {"action": "ADD"|"UPDATE"|"NOOP", "content": "<merged markdown if ADD/UPDATE, empty string if NOOP>"}',
627
+ "",
628
+ "## Existing knowledge content",
629
+ "```",
630
+ plan.existing.slice(0, 3000),
631
+ "```",
632
+ "",
633
+ "## New content from distillation",
634
+ "```",
635
+ plan.content.slice(0, 3000),
636
+ "```",
637
+ ].join("\n"),
638
+ request: {
639
+ ...(run.options.signal ? { signal: run.options.signal } : {}),
640
+ ...(run.options.chat ? { chat: run.options.chat } : {}),
641
+ },
642
+ onNotices: run.notices.add,
643
+ });
644
+ const decision = merged.ok
645
+ ? parseEmbeddedJsonResponse(merged.raw)
646
+ : undefined;
647
+ if (decision?.action === "NOOP") {
648
+ emitDistill(run, {
649
+ outcome: "skipped",
650
+ proposalRef: plan.knowledgeRef,
651
+ message: "D-1: LLM resolved destination conflict as NOOP — existing content kept",
652
+ });
1204
653
  return {
1205
- rejection: writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, {
1206
- reviewNeeded: true,
1207
- ...(judgeResult.criteria ? { criteria: judgeResult.criteria } : {}),
1208
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
1209
- }, options.eligibilitySource, options.eventsCtx, proposalOpts),
654
+ schemaVersion: 1,
655
+ ok: true,
656
+ outcome: "skipped",
657
+ inputRef: run.inputRef,
658
+ proposalRef: plan.knowledgeRef,
659
+ skipReason: "conflict_noop",
660
+ message: "Existing knowledge content unchanged (contradiction resolution: NOOP)",
1210
661
  };
1211
662
  }
1212
- return {
1213
- rejection: writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, {
1214
- ...(judgeResult.criteria ? { criteria: judgeResult.criteria } : {}),
1215
- ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
1216
- }, options.eligibilitySource, options.eventsCtx, proposalOpts),
1217
- };
663
+ if ((decision?.action === "ADD" || decision?.action === "UPDATE") && decision.content?.trim()) {
664
+ content = decision.content;
665
+ }
666
+ }
667
+ else if (plan.existing) {
668
+ content = [
669
+ plan.content,
670
+ "",
671
+ "---",
672
+ "<!-- D-1 / #369: Existing knowledge content is shown below for reviewer reference. -->",
673
+ "<!-- Review: decide whether to ADD (merge), UPDATE (replace), or NOOP (keep existing). -->",
674
+ "",
675
+ "## Existing content (for reviewer reference)",
676
+ "",
677
+ plan.existing,
678
+ ].join("\n");
679
+ }
680
+ return judgeAndQueue(run, {
681
+ ref: plan.knowledgeRef,
682
+ kind: "knowledge",
683
+ content,
684
+ source: run.asset.content,
685
+ promotion: true,
686
+ });
687
+ }
688
+ // ── Inputs ───────────────────────────────────────────────────────────────────
689
+ /** Read the input asset (best-effort: an unindexed asset distils from feedback alone). */
690
+ async function loadInput(lookup, inputRef) {
691
+ try {
692
+ const filePath = await lookup(inputRef);
693
+ if (filePath && fs.existsSync(filePath))
694
+ return { path: filePath, content: fs.readFileSync(filePath, "utf8") };
695
+ }
696
+ catch {
697
+ // An index miss is not fatal.
698
+ }
699
+ return { path: null, content: null };
700
+ }
701
+ /** The index's ref bigram vocabulary, for the novelty term of encoding salience. */
702
+ function loadRefVocabulary() {
703
+ try {
704
+ const db = openReadonlyExistingDatabase(getDbPath(), { isolatedSnapshot: true });
705
+ if (!db)
706
+ return new Set();
707
+ try {
708
+ return buildRefVocabulary(getAllEntries(db).map((e) => e.itemRef));
709
+ }
710
+ finally {
711
+ closeDatabase(db);
712
+ }
713
+ }
714
+ catch {
715
+ return new Set();
1218
716
  }
1219
- // Normalize 1-5 judge score to [0, 1]. Only a real passing verdict
1220
- // reaches here (07 P0-2: the judge now fails CLOSED on no-LLM / timeout /
1221
- // parse failure, so those return pass:false and never fall through to
1222
- // this line). A defensive score>0 guard keeps confidence undefined for any
1223
- // non-positive score the auto-accept gate should treat as unscored.
1224
- return { confidence: judgeResult.score > 0 ? judgeResult.score / 5 : undefined };
1225
717
  }
1226
718
  /**
1227
- * Read the target ref's `feedback` events and apply the #267 exclusion filter.
1228
- * Returns the filtered events plus the exclusion tallies the outcome branches
1229
- * carry. Extracted verbatim from `akmDistill`.
719
+ * Score the input's encoding salience and mirror it to the asset frontmatter
720
+ * and `asset_salience` (keyed by the ledger ref). Best-effort throughout.
1230
721
  */
1231
- function readDistillFeedback(args) {
1232
- const { readEventsImpl, options, durableInputRef } = args;
1233
- const { events: unfilteredEvents } = readEventsImpl({
1234
- ref: options.itemRef ?? durableInputRef,
1235
- type: "feedback",
1236
- excludeTags: options.excludeTags,
1237
- includeTags: options.includeTags,
1238
- });
1239
- const events = unfilteredEvents;
1240
- // #267 — feedback exclusion. Filter events whose `ref` matches the
1241
- // exclusion list BEFORE the prompt is built. The original event stream
1242
- // is never mutated; only the `feedback` slice that reaches the LLM is
1243
- // affected. Exclusion refs are compared with event refs exactly.
1244
- const exclusionList = options.excludeFeedbackFromRefs ?? [];
1245
- const exclusionSet = new Set(exclusionList.map((ref) => ref.trim()).filter((ref) => ref.length > 0));
1246
- const originalEventCount = events.length;
1247
- const filteredEvents = exclusionSet.size > 0 ? events.filter((e) => !(e.ref !== undefined && exclusionSet.has(e.ref))) : events;
1248
- const filteredFeedbackCount = originalEventCount - filteredEvents.length;
1249
- const feedbackFullyFiltered = exclusionSet.size > 0 && originalEventCount > 0 && filteredEvents.length === 0;
1250
- return { filteredEvents, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered };
722
+ function stampInputSalience(run) {
723
+ const { content, path: filePath } = run.asset;
724
+ if (!content || !filePath)
725
+ return;
726
+ try {
727
+ const type = parseRefInput(run.inputRef).type;
728
+ let revisionCount = 0;
729
+ try {
730
+ // Revisions so far: every proposal raised against this ref.
731
+ revisionCount = listProposals(run.stash, { ref: run.inputRef, includeArchive: true }).length;
732
+ }
733
+ catch {
734
+ // Unknown history scores as a first encounter.
735
+ }
736
+ const scored = scoreEncodingSalience({ body: content, type, existingRefVocabulary: run.vocabulary, revisionCount });
737
+ const updated = writeSalienceToFrontmatter(content, scored.score, scored);
738
+ if (updated !== content) {
739
+ fs.writeFileSync(filePath, updated, "utf8");
740
+ recordWrittenPath(filePath);
741
+ run.asset.content = updated;
742
+ }
743
+ try {
744
+ withStateDb((stateDb) => upsertAssetSalience(stateDb, run.ledgerRef, computeSalience({
745
+ ref: run.inputRef,
746
+ type,
747
+ retrievalFreq: 0,
748
+ encodingSalience: scored.score,
749
+ outcomeWeightEnabled: run.outcomeWeightEnabled,
750
+ })));
751
+ }
752
+ catch {
753
+ // The frontmatter mirror is the only persistence then.
754
+ }
755
+ }
756
+ catch {
757
+ // Scoring never blocks distillation.
758
+ }
1251
759
  }
1252
760
  /**
1253
- * Build the distill chat messages: inject the last 1–3 rejected proposals
1254
- * (Reflexion verbal-RL), the optional WS-3b CLS adjacent-context, and the stash
1255
- * authoring standards, then assemble the system+user prompt. Extracted verbatim
1256
- * from `akmDistill`.
761
+ * Content-score a distilled output so it carries a real encoding salience from
762
+ * creation — lessons are refused as inputs, so this is their only chance.
1257
763
  */
1258
- async function buildDistillMessages(args) {
1259
- const { options, stash, inputRef, assetContent, feedback, effectiveProposalKind, effectiveLessonRef, fetchSimilarLessonsFn, } = args;
1260
- // Inject last 1–3 rejected proposals for this ref as Reflexion-style
1261
- // verbal-RL context so the LLM avoids regenerating refused proposals.
1262
- // Exclude the drain's stale-target auto-rejects (STALE, R20): those are a
1263
- // procedural refusal (the target changed after mint), not a judgement on
1264
- // the content, and would mislead this Reflexion-style "don't repeat this"
1265
- // context.
1266
- const rejectedForRef = listProposalsReadOnly(stash, { ref: inputRef, status: "rejected", includeArchive: true }, options.ctx)
1267
- .filter((p) => !isStaleTargetRejection(p))
1268
- .sort((a, b) => new Date(b.updatedAt ?? 0).getTime() - new Date(a.updatedAt ?? 0).getTime())
1269
- .slice(0, MAX_REJECTED_PROPOSALS)
1270
- .map((p) => ({
1271
- reason: p.review?.reason ?? "no reason given",
1272
- // #legacy: `changes` is empty for pre-existing rows (storedToChanges),
1273
- // which makes `proposalContent` throw before reflect dispatch even
1274
- // runs. `payload.content` is populated for every row regardless, so
1275
- // read the preview from there instead.
1276
- contentPreview: p.payload.content.slice(0, 500),
1277
- }));
1278
- // WS-3b CLS interleaving (step 9).
1279
- // When cls.enabled, inject embedding-retrieved adjacent lessons/knowledge
1280
- // into the distill prompt so the LLM avoids overwriting prior generalizations
1281
- // (catastrophic interference). DEFAULT OFF.
1282
- const clsConfig = getImproveProcessConfig("distill", options.improveProfile)?.cls ?? {};
764
+ function persistOutputEncodingSalience(run, ref, body) {
765
+ try {
766
+ const type = parseRefInput(ref).type;
767
+ const scored = scoreEncodingSalience({ body, type, existingRefVocabulary: run.vocabulary, revisionCount: 0 });
768
+ withStateDb((stateDb) => upsertAssetSalience(stateDb, ref, computeSalience({
769
+ ref,
770
+ type,
771
+ retrievalFreq: 0,
772
+ encodingSalience: scored.score,
773
+ outcomeWeightEnabled: run.outcomeWeightEnabled,
774
+ })));
775
+ }
776
+ catch {
777
+ // Scoring never blocks proposal creation.
778
+ }
779
+ }
780
+ /** The ref's feedback events, minus any `excludeFeedbackFromRefs` matches. */
781
+ function readDistillFeedback(run) {
782
+ const read = run.options.readEventsFn ??
783
+ ((readOptions) => readEvents(readOptions, { readOnly: true }));
784
+ const { events } = read({
785
+ ref: run.ledgerRef,
786
+ type: "feedback",
787
+ excludeTags: run.options.excludeTags,
788
+ includeTags: run.options.includeTags,
789
+ });
790
+ const excluded = new Set((run.options.excludeFeedbackFromRefs ?? []).map((ref) => ref.trim()).filter((ref) => ref.length > 0));
791
+ if (excluded.size === 0)
792
+ return events;
793
+ const kept = events.filter((e) => !(e.ref !== undefined && excluded.has(e.ref)));
794
+ run.exclusion = {
795
+ filteredFeedbackCount: events.length - kept.length,
796
+ feedbackFullyFiltered: events.length > 0 && kept.length === 0,
797
+ };
798
+ return kept;
799
+ }
800
+ /** System + user prompt: rejected-proposal context, optional CLS neighbours, stash standards. */
801
+ async function buildDistillMessages(run, feedback, kind, outputRef) {
802
+ const rejectedProposals = rejectedProposalContext(run.stash, run.inputRef, run.options.ctx);
803
+ // CLS interleaving (default off): show related lessons so the model does not overwrite them.
804
+ const cls = getImproveProcessConfig("distill", run.profile)?.cls ?? {};
1283
805
  let clsContext = "";
1284
- if (clsConfig.enabled) {
806
+ if (cls.enabled) {
1285
807
  try {
1286
- const adjacentCount = clsConfig.adjacentCount ?? DEFAULT_CLS_ADJACENT_COUNT;
1287
- // Use the asset content or input ref as the query for adjacent retrieval.
1288
- const clsQuery = assetContent ? assetContent.slice(0, 500) : inputRef;
1289
- const adjacentItems = await fetchSimilarLessonsFn(clsQuery, adjacentCount);
1290
- clsContext = buildClsContext(adjacentItems, clsConfig);
808
+ const query = run.asset.content ? run.asset.content.slice(0, 500) : run.inputRef;
809
+ clsContext = buildClsContext(await run.similar(query, cls.adjacentCount ?? DEFAULT_CLS_ADJACENT_COUNT), cls);
1291
810
  }
1292
811
  catch {
1293
- // Fail open — CLS is supplemental, never required.
812
+ // CLS context is supplemental.
1294
813
  }
1295
814
  }
1296
- // Distill output is a lesson/knowledge (non-wiki) → stash authoring
1297
- // standards. Resolved once for this single call.
1298
- const standardsContext = resolveStandardsContext(effectiveLessonRef, stash);
1299
- const baseUserPrompt = buildDistillPrompt({
1300
- inputRef,
1301
- assetContent,
815
+ const standardsContext = resolveStandardsContext(outputRef, run.stash);
816
+ const prompt = buildDistillPrompt({
817
+ inputRef: run.inputRef,
818
+ assetContent: run.asset.content,
1302
819
  feedback,
1303
- proposalKind: effectiveProposalKind,
1304
- ...(rejectedForRef.length > 0 ? { rejectedProposals: rejectedForRef } : {}),
820
+ proposalKind: kind,
821
+ ...(rejectedProposals.length > 0 ? { rejectedProposals } : {}),
1305
822
  ...(standardsContext.trim() ? { standardsContext } : {}),
1306
823
  });
1307
- const userPrompt = clsContext ? `${baseUserPrompt}${clsContext}` : baseUserPrompt;
1308
- return [
1309
- { role: "system", content: effectiveProposalKind === "knowledge" ? KNOWLEDGE_SYSTEM_PROMPT : LESSON_SYSTEM_PROMPT },
1310
- { role: "user", content: userPrompt },
1311
- ];
824
+ return {
825
+ system: kind === "knowledge" ? distillKnowledgeSystemPrompt : distillLessonSystemPrompt,
826
+ prompt: `${prompt}${clsContext}`,
827
+ };
1312
828
  }
1313
- /**
1314
- * Exported (PRECHECK, tier3-0917) so the improve loop's distill
1315
- * pre-generation guard (`loop-stages.ts`) can resolve the same asset path
1316
- * `akmDistill` would when checking whether a memory promotes to knowledge —
1317
- * without duplicating the resolution logic.
1318
- */
1319
- export async function defaultLookup(ref, stashDir) {
829
+ async function defaultLookup(ref, stashDir) {
1320
830
  return resolveAssetPath(ref, {
1321
831
  stashDir,
1322
832
  mode: "disk-only",
@@ -1325,3 +835,26 @@ export async function defaultLookup(ref, stashDir) {
1325
835
  honorOrigin: false,
1326
836
  });
1327
837
  }
838
+ /** Top-N existing lessons similar to `query` (empty when search is unavailable). */
839
+ async function fetchTopSimilarLessons(query, n) {
840
+ try {
841
+ const result = await akmSearch({ query, type: "lesson", limit: n, skipLogging: true, eventSource: "improve" });
842
+ return (result?.hits ?? [])
843
+ .filter((h) => "path" in h && typeof h.path === "string")
844
+ .slice(0, n)
845
+ .map((h) => {
846
+ let content = "";
847
+ try {
848
+ if (h.path && fs.existsSync(h.path))
849
+ content = fs.readFileSync(h.path, "utf8");
850
+ }
851
+ catch {
852
+ // best-effort
853
+ }
854
+ return { ref: h.ref, content };
855
+ });
856
+ }
857
+ catch {
858
+ return [];
859
+ }
860
+ }