akm-cli 0.9.0-rc.0 → 0.9.0-rc.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (598) hide show
  1. package/CHANGELOG.md +1283 -22
  2. package/README.md +62 -37
  3. package/SECURITY.md +46 -31
  4. package/dist/akm +162 -38
  5. package/dist/akm-migrate +44 -0
  6. package/dist/assets/backends/schtasks-template.xml +2 -1
  7. package/dist/assets/hints/cli-hints-full.md +268 -118
  8. package/dist/assets/hints/cli-hints-short.md +87 -24
  9. package/dist/assets/{profiles → improve-strategies}/catchup.json +3 -1
  10. package/dist/assets/{profiles → improve-strategies}/consolidate.json +3 -1
  11. package/dist/assets/{profiles → improve-strategies}/default.json +6 -7
  12. package/dist/assets/improve-strategies/frequent.json +15 -0
  13. package/dist/assets/{profiles → improve-strategies}/graph-refresh.json +4 -2
  14. package/dist/assets/{profiles → improve-strategies}/memory-focus.json +4 -1
  15. package/dist/assets/{profiles → improve-strategies}/proactive-maintenance.json +5 -5
  16. package/dist/assets/{profiles → improve-strategies}/quick.json +4 -2
  17. package/dist/assets/improve-strategies/reflect-distill.json +30 -0
  18. package/dist/assets/{profiles → improve-strategies}/thorough.json +1 -1
  19. package/dist/assets/prompts/consolidate-system.md +5 -5
  20. package/dist/assets/prompts/extract-session.md +2 -6
  21. package/dist/assets/prompts/memory-infer-user.md +2 -3
  22. package/dist/assets/prompts/reflect-llm-framed-contract.md +11 -0
  23. package/dist/assets/prompts/reflect-llm-schema-contract.md +3 -0
  24. package/dist/assets/prompts/reflect-output-repair.md +3 -0
  25. package/dist/assets/prompts/workflow-unit-preamble.md +26 -0
  26. package/dist/assets/stash-skeleton/README.md +38 -10
  27. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +8 -0
  28. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +8 -0
  29. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +14 -1
  30. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +13 -1
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +9 -1
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +11 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +9 -0
  34. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +9 -0
  35. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +8 -0
  36. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +100 -0
  37. package/dist/assets/stash-skeleton/facts/conventions/domains.md +64 -0
  38. package/dist/assets/stash-skeleton/facts/conventions/organization.md +136 -0
  39. package/dist/assets/tasks/core/extract.yml +3 -2
  40. package/dist/assets/tasks/core/improve.yml +2 -1
  41. package/dist/assets/tasks/core/index-refresh.yml +1 -0
  42. package/dist/assets/tasks/core/sync.yml +1 -0
  43. package/dist/assets/tasks/core/version-check.yml +2 -1
  44. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +5 -0
  45. package/dist/assets/tasks/improve/akm-improve-catchup.yml +8 -0
  46. package/dist/assets/tasks/improve/akm-improve-consolidate.yml +5 -0
  47. package/dist/assets/tasks/improve/akm-improve-frequent.yml +5 -0
  48. package/dist/assets/tasks/improve/akm-improve-nightly.yml +5 -0
  49. package/dist/assets/templates/html/health.html +5 -4
  50. package/dist/assets/workflows/workflow-template.md +31 -15
  51. package/dist/cli/invocation.js +279 -0
  52. package/dist/cli/parse-args.js +5 -90
  53. package/dist/cli/retired-commands.js +78 -0
  54. package/dist/cli/shared.js +158 -48
  55. package/dist/cli-node.mjs +2 -1
  56. package/dist/cli.js +747 -293
  57. package/dist/commands/agent/agent-dispatch.js +19 -18
  58. package/dist/commands/agent/agent-support.js +0 -24
  59. package/dist/commands/agent/contribute-cli.js +43 -97
  60. package/dist/commands/completions.js +80 -23
  61. package/dist/commands/config-cli.js +44 -281
  62. package/dist/commands/env/env-binding.js +99 -0
  63. package/dist/commands/env/env-cli.js +84 -224
  64. package/dist/commands/env/env.js +12 -163
  65. package/dist/commands/env/marker-path.js +6 -0
  66. package/dist/commands/env/secret-cli.js +45 -61
  67. package/dist/commands/env/secret.js +32 -62
  68. package/dist/commands/feedback-cli.js +179 -85
  69. package/dist/commands/health/accept-rate.js +58 -0
  70. package/dist/commands/health/advisories.js +7 -8
  71. package/dist/commands/health/checks.js +279 -94
  72. package/dist/commands/health/html-report.js +197 -578
  73. package/dist/commands/health/improve-metrics.js +277 -246
  74. package/dist/commands/health/llm-usage.js +19 -19
  75. package/dist/commands/health/md-report.js +16 -7
  76. package/dist/commands/health/metrics.js +67 -32
  77. package/dist/commands/health/renderers.js +47 -0
  78. package/dist/commands/health/report-view-model.js +508 -0
  79. package/dist/commands/health/stash-exposure.js +1 -1
  80. package/dist/commands/health/surfaces.js +16 -56
  81. package/dist/commands/health/task-runs.js +3 -67
  82. package/dist/{migrate-storage-node.mjs → commands/health/types-checks.js} +1 -5
  83. package/dist/commands/health/types-improve.js +29 -0
  84. package/dist/{output/text/save.js → commands/health/types-metrics.js} +1 -2
  85. package/dist/commands/health/types-result.js +7 -0
  86. package/dist/commands/health/types-runs.js +4 -0
  87. package/dist/commands/health/types-session-log.js +4 -0
  88. package/dist/commands/health/types-windows.js +4 -0
  89. package/dist/commands/health/types.js +26 -21
  90. package/dist/commands/health/windows.js +2 -3
  91. package/dist/commands/health.js +296 -167
  92. package/dist/commands/improve/anti-collapse.js +5 -5
  93. package/dist/commands/improve/autonomy-gate.js +68 -0
  94. package/dist/commands/improve/collapse-detector.js +65 -52
  95. package/dist/commands/improve/consolidate/chunking.js +9 -7
  96. package/dist/commands/improve/consolidate/eligibility.js +1 -23
  97. package/dist/commands/improve/consolidate/merge.js +4 -0
  98. package/dist/commands/improve/consolidate.js +454 -1354
  99. package/dist/commands/improve/content-hash.js +39 -0
  100. package/dist/commands/improve/distill/content-repair.js +4 -10
  101. package/dist/commands/improve/distill/promote-memory.js +89 -64
  102. package/dist/commands/improve/distill/quality-gate.js +118 -42
  103. package/dist/commands/improve/distill-guards.js +1 -1
  104. package/dist/commands/improve/distill-promotion-policy.js +33 -888
  105. package/dist/commands/improve/distill.js +607 -363
  106. package/dist/commands/improve/eligibility.js +165 -79
  107. package/dist/commands/improve/extract-cli.js +35 -126
  108. package/dist/commands/improve/extract-prompt.js +6 -35
  109. package/dist/commands/improve/extract.js +640 -391
  110. package/dist/commands/improve/feedback-valence.js +2 -12
  111. package/dist/commands/improve/improve-cli.js +134 -135
  112. package/dist/commands/improve/improve-result-file.js +30 -50
  113. package/dist/commands/improve/improve-run-types.js +4 -0
  114. package/dist/commands/improve/improve-strategies.js +135 -0
  115. package/dist/commands/improve/improve.js +904 -701
  116. package/dist/commands/improve/locks.js +64 -111
  117. package/dist/commands/improve/loop-stages.js +1110 -923
  118. package/dist/commands/improve/memory/derived-ref.js +124 -0
  119. package/dist/commands/improve/memory/memory-belief.js +79 -7
  120. package/dist/commands/improve/memory/memory-contradiction-detect.js +49 -52
  121. package/dist/commands/improve/memory/memory-improve.js +25 -37
  122. package/dist/commands/improve/outcome-loop.js +25 -88
  123. package/dist/commands/improve/preparation.js +1034 -813
  124. package/dist/commands/improve/proactive-maintenance.js +34 -9
  125. package/dist/commands/improve/proposal-envelope.js +31 -0
  126. package/dist/commands/improve/reflect.js +983 -794
  127. package/dist/commands/improve/run-context.js +119 -0
  128. package/dist/commands/improve/salience.js +24 -127
  129. package/dist/commands/improve/session-asset.js +7 -3
  130. package/dist/commands/improve/shared.js +14 -34
  131. package/dist/commands/improve/source-identity.js +28 -0
  132. package/dist/commands/improve/triage.js +20 -17
  133. package/dist/commands/lint/base-linter.js +340 -313
  134. package/dist/commands/lint/env-key-rules.js +31 -47
  135. package/dist/commands/lint/index.js +185 -30
  136. package/dist/commands/{events.js → log.js} +28 -38
  137. package/dist/commands/migrate-cli.js +54 -0
  138. package/dist/commands/migration-tool.js +55 -0
  139. package/dist/commands/observability-cli.js +70 -208
  140. package/dist/commands/proposal/diff-format.js +50 -0
  141. package/dist/commands/proposal/drain-policies.js +0 -6
  142. package/dist/commands/proposal/drain.js +91 -40
  143. package/dist/commands/proposal/proposal-cli.js +134 -132
  144. package/dist/commands/proposal/proposal-types.js +56 -0
  145. package/dist/commands/proposal/proposal.js +83 -65
  146. package/dist/commands/proposal/propose-cli.js +88 -0
  147. package/dist/commands/proposal/propose.js +105 -88
  148. package/dist/commands/proposal/repository.js +1303 -278
  149. package/dist/commands/proposal/validators/proposal-quality-validators.js +16 -6
  150. package/dist/commands/proposal/validators/proposal-validators.js +61 -12
  151. package/dist/commands/proposal/validators/proposals.js +6 -8
  152. package/dist/commands/read/curate.js +78 -73
  153. package/dist/commands/read/knowledge.js +510 -13
  154. package/dist/commands/read/registry-search.js +2 -2
  155. package/dist/commands/read/remember-cli.js +84 -15
  156. package/dist/commands/read/search-cli.js +203 -96
  157. package/dist/commands/read/search.js +126 -94
  158. package/dist/commands/read/show.js +226 -250
  159. package/dist/commands/registry-cli.js +34 -60
  160. package/dist/commands/remember.js +18 -57
  161. package/dist/commands/sources/add-cli.js +104 -49
  162. package/dist/commands/sources/bundle-cli.js +166 -0
  163. package/dist/commands/sources/bundle-config-ops.js +63 -0
  164. package/dist/commands/sources/info.js +27 -15
  165. package/dist/commands/sources/init.js +30 -40
  166. package/dist/commands/sources/installed-stashes.js +469 -172
  167. package/dist/commands/sources/migration-help.js +7 -4
  168. package/dist/commands/sources/schema-repair.js +10 -9
  169. package/dist/commands/sources/self-update.js +182 -121
  170. package/dist/commands/sources/source-add.js +169 -178
  171. package/dist/commands/sources/source-clone.js +144 -41
  172. package/dist/commands/sources/source-manage.js +94 -59
  173. package/dist/commands/sources/sources-cli.js +64 -205
  174. package/dist/commands/sources/stash-cli.js +91 -54
  175. package/dist/commands/sources/stash-skeleton.js +1 -1
  176. package/dist/commands/tasks/tasks-cli.js +106 -104
  177. package/dist/commands/tasks/tasks.js +445 -262
  178. package/dist/commands/workflow-cli.js +232 -121
  179. package/dist/core/action-contributors.js +1 -1
  180. package/dist/core/activation-policy.js +49 -0
  181. package/dist/core/adapter/adapters/agent-skills-adapter.js +181 -0
  182. package/dist/core/adapter/adapters/akm-adapter.js +528 -0
  183. package/dist/core/adapter/adapters/akm-lint.js +392 -0
  184. package/dist/core/adapter/adapters/akm-metadata.js +387 -0
  185. package/dist/core/adapter/adapters/akm-task-adapter.js +149 -0
  186. package/dist/core/adapter/adapters/akm-workflow-adapter.js +180 -0
  187. package/dist/core/adapter/adapters/claude-adapter.js +61 -0
  188. package/dist/core/adapter/adapters/dotenv-adapter.js +187 -0
  189. package/dist/core/adapter/adapters/generic-files-adapter.js +119 -0
  190. package/dist/core/adapter/adapters/index.js +80 -0
  191. package/dist/core/adapter/adapters/llm-wiki-adapter.js +419 -0
  192. package/dist/core/adapter/adapters/okf-adapter.js +391 -0
  193. package/dist/core/adapter/adapters/opencode-adapter.js +68 -0
  194. package/dist/core/adapter/adapters/shared.js +286 -0
  195. package/dist/core/adapter/adapters/tool-dir-shared.js +217 -0
  196. package/dist/core/adapter/adapters/website-snapshot-adapter.js +155 -0
  197. package/dist/core/adapter/bundle-adapter.js +4 -0
  198. package/dist/core/adapter/detect-adapter.js +17 -0
  199. package/dist/core/adapter/recognize-match.js +44 -0
  200. package/dist/core/adapter/registry.js +56 -0
  201. package/dist/core/adapter/types.js +4 -0
  202. package/dist/core/asset/akm-markdown.js +30 -0
  203. package/dist/core/asset/asset-placement.js +243 -0
  204. package/dist/core/asset/asset-ref.js +110 -79
  205. package/dist/core/asset/asset-serialize.js +20 -0
  206. package/dist/core/asset/frontmatter.js +28 -12
  207. package/dist/core/asset/markdown.js +40 -51
  208. package/dist/core/asset/resolve-ref.js +274 -0
  209. package/dist/core/asset/stash-meta.js +2 -2
  210. package/dist/core/bundle-id.js +51 -0
  211. package/dist/core/common.js +281 -86
  212. package/dist/core/config/config-io.js +42 -128
  213. package/dist/core/config/config-schema.js +233 -834
  214. package/dist/core/config/config-sources.js +162 -39
  215. package/dist/core/config/config-types.js +16 -11
  216. package/dist/core/config/config-version.js +29 -0
  217. package/dist/core/config/config-walker.js +126 -37
  218. package/dist/core/config/config.js +154 -331
  219. package/dist/core/config/deep-merge.js +41 -0
  220. package/dist/core/config/engine-semantics.js +28 -0
  221. package/dist/core/config/experimental.js +21 -0
  222. package/dist/core/config/schema/embedding.js +38 -0
  223. package/dist/core/config/schema/engines.js +116 -0
  224. package/dist/core/config/schema/experimental.js +47 -0
  225. package/dist/core/config/schema/feedback.js +31 -0
  226. package/dist/core/config/schema/improve-processes.js +389 -0
  227. package/dist/core/config/schema/improve.js +94 -0
  228. package/dist/core/config/schema/index-config.js +176 -0
  229. package/dist/core/config/schema/output.js +18 -0
  230. package/dist/core/config/schema/primitives.js +94 -0
  231. package/dist/core/config/schema/search.js +30 -0
  232. package/dist/core/config/schema/setup.js +18 -0
  233. package/dist/core/config/schema/sources-bundles.js +169 -0
  234. package/dist/core/config/schema/workflow.js +29 -0
  235. package/dist/core/env-secret-ref.js +155 -20
  236. package/dist/core/errors.js +17 -15
  237. package/dist/core/events-types.js +4 -0
  238. package/dist/core/events.js +46 -128
  239. package/dist/core/extra-params.js +62 -0
  240. package/dist/core/file-change.js +17 -0
  241. package/dist/core/file-lock.js +202 -57
  242. package/dist/core/fs-txn.js +392 -0
  243. package/dist/core/git-message.js +59 -0
  244. package/dist/core/improve-result.js +167 -0
  245. package/dist/core/json-schema.js +142 -0
  246. package/dist/core/lesson-lint.js +1 -17
  247. package/dist/core/logs-db.js +1 -1
  248. package/dist/core/maintenance-barrier.js +135 -0
  249. package/dist/core/migration-operation.js +44 -0
  250. package/dist/core/mutation-target.js +78 -0
  251. package/dist/core/paths.js +22 -25
  252. package/dist/core/platform.js +10 -0
  253. package/dist/core/recognition-util.js +128 -0
  254. package/dist/core/redaction.js +392 -0
  255. package/dist/core/standards/resolve-standards-context.js +36 -65
  256. package/dist/core/standards/resolve-stash-standards.js +2 -2
  257. package/dist/core/standards/resolve-type-conventions.js +5 -5
  258. package/dist/core/state/migrations.js +242 -11
  259. package/dist/core/state-db.js +98 -10
  260. package/dist/core/structured.js +1 -1
  261. package/dist/core/subprocess.js +303 -0
  262. package/dist/core/text-truncation.js +9 -5
  263. package/dist/core/time.js +20 -0
  264. package/dist/core/type-presentation.js +130 -0
  265. package/dist/core/warn.js +0 -3
  266. package/dist/core/write-source.js +834 -118
  267. package/dist/indexer/bundle-identity-guard.js +92 -0
  268. package/dist/indexer/db/graph-db.js +1 -25
  269. package/dist/indexer/db/llm-cache.js +1 -1
  270. package/dist/indexer/ensure-index.js +30 -9
  271. package/dist/indexer/graph/graph-boost.js +9 -30
  272. package/dist/indexer/graph/graph-extraction.js +41 -27
  273. package/dist/indexer/graph/graph-types.js +4 -0
  274. package/dist/indexer/index-writer-lock.js +93 -49
  275. package/dist/indexer/index-written-assets.js +100 -53
  276. package/dist/indexer/indexer.js +746 -329
  277. package/dist/indexer/init.js +18 -25
  278. package/dist/indexer/installations.js +142 -0
  279. package/dist/indexer/passes/dir-staleness.js +18 -10
  280. package/dist/indexer/passes/memory-inference.js +25 -15
  281. package/dist/indexer/passes/metadata.js +412 -243
  282. package/dist/indexer/scan/doc-to-entry.js +160 -0
  283. package/dist/indexer/scan/drain-dir.js +134 -0
  284. package/dist/indexer/search/db-search.js +292 -108
  285. package/dist/indexer/search/fts-query.js +64 -0
  286. package/dist/indexer/search/ranking-contributors.js +145 -25
  287. package/dist/indexer/search/ranking-types.js +4 -0
  288. package/dist/indexer/search/ranking.js +28 -71
  289. package/dist/indexer/search/search-attribution.js +67 -0
  290. package/dist/indexer/search/search-fields.js +18 -3
  291. package/dist/indexer/search/search-hit-enrichers.js +30 -40
  292. package/dist/indexer/search/search-source.js +157 -111
  293. package/dist/indexer/search/semantic-status.js +4 -1
  294. package/dist/indexer/usage/usage-events.js +10 -30
  295. package/dist/indexer/walk/file-context.js +3 -45
  296. package/dist/indexer/walk/matchers.js +42 -34
  297. package/dist/indexer/walk/path-resolver.js +11 -5
  298. package/dist/indexer/walk/walker.js +42 -14
  299. package/dist/integrations/agent/builder-shared.js +7 -0
  300. package/dist/integrations/agent/builders.js +5 -56
  301. package/dist/integrations/agent/config.js +3 -143
  302. package/dist/integrations/agent/detect.js +17 -2
  303. package/dist/integrations/agent/engine-resolution.js +231 -0
  304. package/dist/integrations/agent/index.js +1 -2
  305. package/dist/integrations/agent/model-aliases.js +16 -2
  306. package/dist/integrations/agent/profiles.js +36 -62
  307. package/dist/integrations/agent/prompts.js +46 -18
  308. package/dist/integrations/agent/runner-dispatch.js +93 -4
  309. package/dist/integrations/agent/runner.js +76 -208
  310. package/dist/integrations/agent/spawn.js +88 -196
  311. package/dist/integrations/harnesses/aider/agent-builder.js +114 -0
  312. package/dist/integrations/harnesses/aider/index.js +48 -0
  313. package/dist/integrations/harnesses/aider/result-extractor.js +53 -0
  314. package/dist/integrations/harnesses/amazonq/agent-builder.js +147 -0
  315. package/dist/integrations/harnesses/amazonq/index.js +45 -0
  316. package/dist/integrations/harnesses/amazonq/result-extractor.js +48 -0
  317. package/dist/integrations/harnesses/claude/agent-builder.js +46 -8
  318. package/dist/integrations/harnesses/claude/config-import.js +1 -3
  319. package/dist/integrations/harnesses/claude/index.js +24 -35
  320. package/dist/integrations/harnesses/claude/result-extractor.js +52 -0
  321. package/dist/integrations/harnesses/claude/session-log.js +27 -75
  322. package/dist/integrations/harnesses/codex/agent-builder.js +138 -0
  323. package/dist/integrations/harnesses/codex/index.js +52 -0
  324. package/dist/integrations/harnesses/codex/result-extractor.js +73 -0
  325. package/dist/integrations/harnesses/copilot/agent-builder.js +122 -0
  326. package/dist/integrations/harnesses/copilot/index.js +48 -0
  327. package/dist/integrations/harnesses/copilot/result-extractor.js +151 -0
  328. package/dist/integrations/harnesses/gemini/agent-builder.js +120 -0
  329. package/dist/integrations/harnesses/gemini/index.js +48 -0
  330. package/dist/integrations/harnesses/gemini/result-extractor.js +121 -0
  331. package/dist/integrations/harnesses/ids.js +24 -0
  332. package/dist/integrations/harnesses/index.js +54 -34
  333. package/dist/integrations/harnesses/opencode/agent-builder.js +23 -5
  334. package/dist/integrations/harnesses/opencode/config-import.js +1 -3
  335. package/dist/integrations/harnesses/opencode/index.js +14 -32
  336. package/dist/integrations/harnesses/opencode/session-log.js +67 -125
  337. package/dist/integrations/harnesses/opencode-sdk/harness.js +51 -0
  338. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +681 -108
  339. package/dist/integrations/harnesses/openhands/agent-builder.js +128 -0
  340. package/dist/integrations/harnesses/openhands/index.js +48 -0
  341. package/dist/integrations/harnesses/openhands/result-extractor.js +103 -0
  342. package/dist/integrations/harnesses/pi/agent-builder.js +97 -0
  343. package/dist/integrations/harnesses/pi/index.js +45 -0
  344. package/dist/integrations/harnesses/pi/result-extractor.js +135 -0
  345. package/dist/integrations/harnesses/shared.js +17 -0
  346. package/dist/integrations/harnesses/types.js +43 -32
  347. package/dist/integrations/lockfile.js +211 -24
  348. package/dist/integrations/session-logs/index.js +36 -39
  349. package/dist/integrations/session-logs/provider-base.js +113 -0
  350. package/dist/llm/client.js +182 -110
  351. package/dist/llm/embedders/deterministic.js +2 -2
  352. package/dist/llm/embedders/remote.js +21 -9
  353. package/dist/llm/feature-gate.js +17 -57
  354. package/dist/llm/graph-extract.js +12 -13
  355. package/dist/llm/index-passes.js +8 -42
  356. package/dist/llm/memory-infer.js +144 -1
  357. package/dist/llm/metadata-enhance.js +45 -30
  358. package/dist/llm/structured-call.js +16 -8
  359. package/dist/llm/usage-persist.js +30 -5
  360. package/dist/llm/usage-telemetry.js +59 -6
  361. package/dist/output/cli-hints.js +1 -2
  362. package/dist/output/command-registry.js +27 -0
  363. package/dist/output/context.js +22 -7
  364. package/dist/output/format-exempt.js +80 -0
  365. package/dist/output/generic-render.js +251 -0
  366. package/dist/output/html-render.js +11 -16
  367. package/dist/output/render-registry.js +57 -0
  368. package/dist/output/renderers.js +14 -279
  369. package/dist/output/shapes/curate.js +10 -1
  370. package/dist/output/shapes/events.js +12 -7
  371. package/dist/output/shapes/helpers.js +58 -84
  372. package/dist/output/shapes/passthrough.js +11 -39
  373. package/dist/output/shapes/proposal/producer.js +15 -7
  374. package/dist/output/shapes/registry.js +12 -6
  375. package/dist/output/shapes.js +0 -9
  376. package/dist/output/text/{init.js → bundle-create.js} +3 -1
  377. package/dist/output/text/bundle-show.js +7 -0
  378. package/dist/output/text/command-format.js +562 -0
  379. package/dist/output/text/env.js +1 -3
  380. package/dist/output/text/events.js +8 -7
  381. package/dist/output/text/helpers.js +15 -1164
  382. package/dist/output/text/proposal/producer.js +4 -2
  383. package/dist/output/text/proposal-format.js +202 -0
  384. package/dist/output/text/registry-commands.js +1 -2
  385. package/dist/output/text/registry.js +12 -6
  386. package/dist/output/text/show-directives.js +117 -0
  387. package/dist/output/text/show-format.js +103 -0
  388. package/dist/output/text/sync.js +5 -0
  389. package/dist/output/text/workflow-format.js +332 -0
  390. package/dist/output/text/workflow.js +3 -2
  391. package/dist/output/text.js +10 -19
  392. package/dist/registry/factory.js +4 -6
  393. package/dist/registry/origin-resolve.js +16 -27
  394. package/dist/registry/providers/skills-sh.js +3 -3
  395. package/dist/registry/providers/static-index.js +15 -25
  396. package/dist/registry/resolve.js +43 -94
  397. package/dist/registry/semver.js +43 -0
  398. package/dist/runtime.js +81 -12
  399. package/dist/scripts/akm-migrate.js +35529 -0
  400. package/dist/setup/detect.js +5 -7
  401. package/dist/setup/detected-engines.js +136 -0
  402. package/dist/setup/engine-config.js +100 -0
  403. package/dist/setup/registry-stash-loader.js +3 -3
  404. package/dist/setup/semantic-assets.js +12 -9
  405. package/dist/setup/setup.js +444 -208
  406. package/dist/setup/steps/connection-shared.js +120 -0
  407. package/dist/setup/steps/connection.js +108 -305
  408. package/dist/setup/steps/platforms.js +13 -12
  409. package/dist/setup/steps/semantic.js +15 -3
  410. package/dist/setup/steps/sources.js +21 -15
  411. package/dist/setup/steps/stashdir.js +6 -4
  412. package/dist/setup/steps/tasks.js +236 -119
  413. package/dist/setup/steps.js +3 -2
  414. package/dist/sources/freshness.js +39 -0
  415. package/dist/sources/provider-factory.js +11 -17
  416. package/dist/sources/providers/filesystem.js +2 -3
  417. package/dist/sources/providers/git-install.js +278 -34
  418. package/dist/sources/providers/git-provider.js +54 -56
  419. package/dist/sources/providers/git-stash.js +420 -91
  420. package/dist/sources/providers/git.js +2 -2
  421. package/dist/sources/providers/npm.js +16 -19
  422. package/dist/sources/providers/provider-utils.js +47 -22
  423. package/dist/sources/providers/sync-from-ref.js +3 -9
  424. package/dist/sources/providers/website.js +2 -2
  425. package/dist/sources/resolve.js +11 -10
  426. package/dist/sources/snapshot-fetchers/types.js +4 -0
  427. package/dist/sources/{website-ingest.js → snapshot-fetchers/website-ingest.js} +110 -41
  428. package/dist/storage/database.js +60 -4
  429. package/dist/storage/engines/sqlite-migrations.js +156 -5
  430. package/dist/storage/locations.js +1 -2
  431. package/dist/storage/repositories/canaries-repository.js +1 -1
  432. package/dist/storage/repositories/events-repository.js +51 -11
  433. package/dist/storage/repositories/improve-runs-repository.js +6 -32
  434. package/dist/storage/repositories/index-connection.js +79 -0
  435. package/dist/storage/repositories/index-db.js +4 -3
  436. package/dist/storage/repositories/index-entries-repository.js +863 -0
  437. package/dist/{indexer/db/entry-mapper.js → storage/repositories/index-entry-mapper.js} +19 -2
  438. package/dist/storage/repositories/index-entry-types.js +4 -0
  439. package/dist/storage/repositories/index-fts-repository.js +167 -0
  440. package/dist/storage/repositories/index-llm-cache-repository.js +108 -0
  441. package/dist/storage/repositories/index-meta-repository.js +49 -0
  442. package/dist/{indexer/db/schema.js → storage/repositories/index-schema.js} +226 -100
  443. package/dist/storage/repositories/index-sql.js +12 -0
  444. package/dist/storage/repositories/index-utility-repository.js +356 -0
  445. package/dist/storage/repositories/index-vec-repository.js +250 -0
  446. package/dist/storage/repositories/outcome-repository.js +119 -0
  447. package/dist/storage/repositories/proposals-repository.js +317 -75
  448. package/dist/storage/repositories/registry-cache.js +1 -1
  449. package/dist/storage/repositories/salience-repository.js +172 -0
  450. package/dist/storage/repositories/task-history-repository.js +110 -3
  451. package/dist/storage/repositories/workflow-runs-repository.js +240 -19
  452. package/dist/tasks/backends/cron.js +169 -46
  453. package/dist/tasks/backends/exec-utils.js +76 -3
  454. package/dist/tasks/backends/index.js +6 -9
  455. package/dist/tasks/backends/launchd.js +292 -55
  456. package/dist/tasks/backends/schtasks.js +557 -70
  457. package/dist/tasks/backends/types.js +4 -0
  458. package/dist/tasks/command-executable.js +93 -0
  459. package/dist/tasks/embedded.js +56 -38
  460. package/dist/tasks/parser.js +156 -64
  461. package/dist/tasks/resolve-akm-bin.js +144 -51
  462. package/dist/tasks/runner.js +377 -209
  463. package/dist/tasks/schedule.js +108 -19
  464. package/dist/tasks/scheduler-invocation.js +296 -0
  465. package/dist/tasks/schema.js +1 -1
  466. package/dist/tasks/task-id.js +35 -0
  467. package/dist/tasks/validator.js +30 -16
  468. package/dist/text-import-hook.mjs +1 -1
  469. package/dist/workflows/authoring/authoring.js +104 -43
  470. package/dist/workflows/authoring/scope-key.js +1 -1
  471. package/dist/workflows/cli.js +0 -16
  472. package/dist/workflows/concurrency-policy.js +15 -0
  473. package/dist/workflows/exec/brief.js +450 -0
  474. package/dist/workflows/exec/frozen-judge.js +47 -0
  475. package/dist/workflows/exec/native-executor.js +1038 -0
  476. package/dist/workflows/exec/param-secrets.js +115 -0
  477. package/dist/workflows/exec/report.js +1460 -0
  478. package/dist/workflows/exec/run-workflow.js +602 -0
  479. package/dist/workflows/exec/scheduler.js +71 -0
  480. package/dist/workflows/exec/step-work.js +1190 -0
  481. package/dist/workflows/exec/unit-writer.js +23 -0
  482. package/dist/workflows/exec/workflow-engine-gate.js +67 -0
  483. package/dist/workflows/exec/worktree.js +171 -0
  484. package/dist/workflows/ir/compile.js +246 -0
  485. package/dist/workflows/ir/freeze.js +233 -0
  486. package/dist/workflows/ir/params.js +54 -0
  487. package/dist/workflows/ir/plan-hash.js +68 -0
  488. package/dist/workflows/ir/schema.js +540 -0
  489. package/dist/workflows/parser.js +878 -304
  490. package/dist/workflows/program/expressions.js +181 -0
  491. package/dist/workflows/program/schema.js +51 -0
  492. package/dist/workflows/renderer.js +100 -45
  493. package/dist/workflows/resource-limits.js +22 -0
  494. package/dist/workflows/runtime/agent-identity.js +59 -14
  495. package/dist/workflows/runtime/checkin.js +1 -1
  496. package/dist/workflows/runtime/plan-classifier.js +131 -0
  497. package/dist/workflows/runtime/runs.js +376 -119
  498. package/dist/workflows/runtime/unit-checkin.js +45 -0
  499. package/dist/workflows/runtime/unit-phases.js +20 -0
  500. package/dist/workflows/runtime/workflow-asset-loader.js +241 -40
  501. package/dist/workflows/schema.js +1 -11
  502. package/dist/workflows/validate-summary.js +2 -3
  503. package/dist/workflows/validator.js +52 -30
  504. package/docs/README.md +42 -78
  505. package/docs/migration/README.md +8 -0
  506. package/docs/migration/release-notes/0.6.0.md +1 -1
  507. package/docs/migration/release-notes/0.7.0.md +9 -8
  508. package/docs/migration/release-notes/0.9.0.md +158 -14
  509. package/docs/migration/v0.7-to-v0.8.md +46 -47
  510. package/docs/migration/v0.8-to-v0.9.md +844 -0
  511. package/docs/reference/README.md +12 -0
  512. package/docs/reference/data-and-telemetry.md +333 -0
  513. package/package.json +21 -17
  514. package/schemas/akm-asset-envelope.json +93 -0
  515. package/schemas/akm-config.json +4636 -0
  516. package/schemas/akm-task.json +87 -0
  517. package/schemas/akm-workflow.json +373 -0
  518. package/dist/akm-migrate-storage +0 -38
  519. package/dist/assets/help/help-accept.md +0 -12
  520. package/dist/assets/help/help-improve.md +0 -84
  521. package/dist/assets/help/help-proposals.md +0 -17
  522. package/dist/assets/help/help-propose.md +0 -17
  523. package/dist/assets/help/help-reject.md +0 -11
  524. package/dist/assets/profiles/frequent.json +0 -13
  525. package/dist/assets/profiles/recombine-only.json +0 -21
  526. package/dist/assets/profiles/reflect-distill.json +0 -30
  527. package/dist/assets/profiles/synthesize.json +0 -15
  528. package/dist/assets/prompts/procedural-system.md +0 -44
  529. package/dist/assets/prompts/recombine-system.md +0 -40
  530. package/dist/assets/prompts/staleness-detect-system.md +0 -6
  531. package/dist/assets/tasks/core/backup.yml +0 -4
  532. package/dist/assets/tasks/graph-refresh-weekly.yml +0 -10
  533. package/dist/assets/templates/html/default.html +0 -78
  534. package/dist/assets/templates/html/vendor/echarts.min.js +0 -45
  535. package/dist/assets/wiki/index-template.md +0 -12
  536. package/dist/assets/wiki/ingest-workflow-template.md +0 -83
  537. package/dist/assets/wiki/log-template.md +0 -8
  538. package/dist/assets/wiki/schema-template.md +0 -61
  539. package/dist/cli/config-migrate.js +0 -150
  540. package/dist/cli/config-validate.js +0 -39
  541. package/dist/commands/graph/graph-cli.js +0 -124
  542. package/dist/commands/graph/graph.js +0 -487
  543. package/dist/commands/improve/calibration.js +0 -161
  544. package/dist/commands/improve/dedup.js +0 -482
  545. package/dist/commands/improve/extract-watch.js +0 -140
  546. package/dist/commands/improve/hot-probation.js +0 -45
  547. package/dist/commands/improve/improve-auto-accept.js +0 -276
  548. package/dist/commands/improve/improve-profiles.js +0 -168
  549. package/dist/commands/improve/procedural.js +0 -398
  550. package/dist/commands/improve/recombine.js +0 -818
  551. package/dist/commands/improve/schema-similarity-gate.js +0 -168
  552. package/dist/commands/lint/agent-linter.js +0 -44
  553. package/dist/commands/lint/command-linter.js +0 -44
  554. package/dist/commands/lint/default-linter.js +0 -16
  555. package/dist/commands/lint/fact-linter.js +0 -39
  556. package/dist/commands/lint/knowledge-linter.js +0 -16
  557. package/dist/commands/lint/memory-linter.js +0 -61
  558. package/dist/commands/lint/registry.js +0 -41
  559. package/dist/commands/lint/skill-linter.js +0 -45
  560. package/dist/commands/lint/task-linter.js +0 -50
  561. package/dist/commands/lint/workflow-linter.js +0 -81
  562. package/dist/commands/proposal/legacy-import.js +0 -115
  563. package/dist/commands/sources/history.js +0 -196
  564. package/dist/commands/tasks/default-tasks.js +0 -186
  565. package/dist/commands/wiki-cli.js +0 -292
  566. package/dist/core/asset/asset-registry.js +0 -76
  567. package/dist/core/asset/asset-spec.js +0 -259
  568. package/dist/core/config/config-migration.js +0 -602
  569. package/dist/core/deep-merge.js +0 -38
  570. package/dist/core/eval/rank-metrics.js +0 -113
  571. package/dist/core/ripgrep/install.js +0 -163
  572. package/dist/core/ripgrep/resolve.js +0 -81
  573. package/dist/indexer/db/db.js +0 -1413
  574. package/dist/indexer/manifest.js +0 -170
  575. package/dist/indexer/passes/metadata-contributors.js +0 -31
  576. package/dist/indexer/usage/unmigrated-vaults-guard.js +0 -94
  577. package/dist/integrations/harnesses/opencode-sdk/index.js +0 -49
  578. package/dist/llm/call-ai.js +0 -62
  579. package/dist/llm/memory-infer-impl.js +0 -138
  580. package/dist/output/shapes/distill.js +0 -14
  581. package/dist/output/shapes/history.js +0 -11
  582. package/dist/output/text/distill.js +0 -6
  583. package/dist/output/text/enable-disable.js +0 -8
  584. package/dist/output/text/history.js +0 -6
  585. package/dist/output/text/wiki.js +0 -16
  586. package/dist/registry/build-index.js +0 -386
  587. package/dist/scripts/migrate-storage.js +0 -19108
  588. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +0 -9411
  589. package/dist/scripts/migrations/v16-to-v17.js +0 -141
  590. package/dist/setup/legacy-config.js +0 -106
  591. package/dist/storage/repositories/consolidation-repository.js +0 -38
  592. package/dist/storage/repositories/recombine-repository.js +0 -213
  593. package/dist/wiki/wiki-templates.js +0 -15
  594. package/dist/wiki/wiki.js +0 -1012
  595. package/dist/workflows/db.js +0 -215
  596. package/docs/data-and-telemetry.md +0 -226
  597. /package/dist/sources/{wiki-fetchers → snapshot-fetchers}/registry.js +0 -0
  598. /package/dist/sources/{wiki-fetchers → snapshot-fetchers}/youtube.js +0 -0
@@ -1,28 +1,32 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
4
5
  import path from "node:path";
5
- import { parseAssetRef } from "../../core/asset/asset-ref.js";
6
+ import { parseRefInput } from "../../core/asset/resolve-ref.js";
6
7
  import { daysToMs } from "../../core/common.js";
7
- import { loadConfig } from "../../core/config/config.js";
8
+ import { DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE, loadConfig } from "../../core/config/config.js";
8
9
  import { UsageError } from "../../core/errors.js";
9
10
  import { appendEvent } from "../../core/events.js";
10
11
  import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
11
12
  import { getDbPath } from "../../core/paths.js";
12
13
  import { withStateDb } from "../../core/state-db.js";
13
- import { info, warn } from "../../core/warn.js";
14
- import { closeDatabase, openIndexDatabase } from "../../indexer/db/db.js";
15
- import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
14
+ import { info } from "../../core/warn.js";
15
+ import { DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES, runGraphExtractionPass, } from "../../indexer/graph/graph-extraction.js";
16
16
  import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
17
17
  import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
18
18
  import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
19
- import { resolveImproveProcessRunnerFromProfile } from "../../integrations/agent/runner.js";
19
+ import { materializeLlmRunnerConnection } from "../../integrations/agent/runner.js";
20
20
  import { isProcessEnabled } from "../../llm/feature-gate.js";
21
21
  import { withLlmStage } from "../../llm/usage-telemetry.js";
22
22
  import { purgeOldCycleMetrics } from "../../storage/repositories/canaries-repository.js";
23
23
  import { purgeOldEvents } from "../../storage/repositories/events-repository.js";
24
24
  import { purgeOldImproveRuns } from "../../storage/repositories/improve-runs-repository.js";
25
- import { createProposal, expireStaleProposals, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/repository.js";
25
+ import { closeDatabase, openIndexDatabase } from "../../storage/repositories/index-connection.js";
26
+ import { getEntryByRef } from "../../storage/repositories/index-entries-repository.js";
27
+ import { clearAssetOutcomeMissing, countAssetOutcomeMissing, deleteAssetOutcomeMissingBefore, listAssetOutcomeMissingState, stampAssetOutcomeMissing, } from "../../storage/repositories/outcome-repository.js";
28
+ import { clearAssetSalienceMissing, countAssetSalienceMissing, deleteAssetSalienceMissingBefore, listAssetSalienceMissingState, stampAssetSalienceMissing, } from "../../storage/repositories/salience-repository.js";
29
+ import { expireStaleProposals, listProposals, purgeOrphanProposals } from "../proposal/repository.js";
26
30
  import { checkDeadUrls } from "../url-checker.js";
27
31
  import { DEFAULT_RETENTION_DAYS as CYCLE_METRICS_RETENTION_DAYS, runCollapseDetector } from "./collapse-detector.js";
28
32
  import { deriveLessonRef } from "./distill.js";
@@ -30,570 +34,527 @@ import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
30
34
  // Eligibility / candidate-selection predicates live in ./eligibility.
31
35
  import { findAssetFilePath, isDistillCandidateRef } from "./eligibility.js";
32
36
  import { writeEvalCase } from "./eval-cases.js";
33
- import { makeGateConfig, runAutoAcceptGate } from "./improve-auto-accept.js";
34
- import { resolveProcessEnabled, shouldSkipRef } from "./improve-profiles.js";
35
- // The pre-loop preparation pipeline lives in ./preparation.
36
- import { maybeAutoTuneThreshold } from "./preparation.js";
37
- import { akmProcedural } from "./procedural.js";
38
- import { akmRecombine } from "./recombine.js";
37
+ import { shouldSkipRef } from "./improve-strategies.js";
39
38
  import { recordNoOp, resetConsecutiveNoOps } from "./salience.js";
40
39
  import { errMessage, refSlug } from "./shared.js";
40
+ import { bareImproveRef, durableImproveRef } from "./source-identity.js";
41
41
  // ── improve loop / post-loop / maintenance stages ───────────────────
42
42
  // The cycle stages run by akmImprove, extracted from improve.ts.
43
- export async function runImproveLoopStage(args) {
44
- const { scope, options, primaryStashDir, reflectFn, distillFn, loopRefs, actions, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, utilityMap, startMs, budgetMs, eventsCtx, improveProfile, } = args;
43
+ /** O-5 / #378: rolling per-originator error-window cap. */
44
+ const RECENT_ERRORS_CAP = 3;
45
+ /** O-5 / #378: push a per-originator error into the rolling window. */
46
+ function pushRecentError(recentErrors, originator, msg) {
47
+ if (!recentErrors[originator])
48
+ recentErrors[originator] = [];
49
+ recentErrors[originator].push(msg);
50
+ if (recentErrors[originator].length > RECENT_ERRORS_CAP)
51
+ recentErrors[originator].shift();
52
+ }
53
+ /**
54
+ * Build the per-run loop environment from the run context: the derived guards
55
+ * and the pending-proposal preload.
56
+ */
57
+ export function prepareImproveLoopEnv(args) {
58
+ const { ctx, scope, options, reflectFn, distillFn, loopRefs, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, startMs, budgetMs, improveProfile, resolvedPlan, } = args;
59
+ // WI-9.10: the legacy dual context's optional eventsCtx/budgetSignal are
60
+ // now RunContext's required eventsCtx / optional signal — renamed local
61
+ // aliases so the rest of this function (and the ImproveLoopEnv object
62
+ // literal below) is unchanged. `primaryStashDir` deliberately comes from
63
+ // the state's honest optional field, NOT `ctx.stashDir`: the rare
64
+ // unresolvable-primary path must keep skipping the `if (primaryStashDir)`
65
+ // guards below (see the field's doc in ./improve-run-types).
66
+ const primaryStashDir = args.primaryStashDir;
67
+ const eventsCtx = ctx.eventsCtx;
68
+ const budgetSignal = ctx.signal;
45
69
  // O-1 (#364): compute remaining budget at call time so each sub-call
46
70
  // receives only its fair share of the wall-clock budget.
47
71
  const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
48
- const RECENT_ERRORS_CAP = 3;
72
+ // Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
73
+ const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
49
74
  // requirePlannedRefs guard: when the distill profile sets this flag, skip
50
75
  // distill for distill-only refs if the reflect phase produced no planned refs.
51
76
  // Prevents the distill loop from generating hundreds of distill-skipped events
52
77
  // on quiet passes (all refs on reflect cooldown, no new signal to distill).
53
78
  const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
54
- const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
55
- const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
79
+ const hasReflectEligibleRefs = loopRefs.some((r) => !distillOnlyRefSet.has(r.ref));
56
80
  const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
57
- // R-2 / #389: Self-Consistency multi-sample voting helpers.
58
- // Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
59
- const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
60
- const SC_N = Math.min(Math.max(2, options.selfConsistencyN ?? 3), 5);
61
- /**
62
- * Compute Jaccard token overlap between two strings.
63
- * Tokenizes by whitespace; returns 0 when both are empty.
64
- */
65
- function jaccardSimilarity(a, b) {
66
- const tokensA = new Set(a.split(/\s+/).filter(Boolean));
67
- const tokensB = new Set(b.split(/\s+/).filter(Boolean));
68
- if (tokensA.size === 0 && tokensB.size === 0)
69
- return 1;
70
- let intersection = 0;
71
- for (const t of tokensA) {
72
- if (tokensB.has(t))
73
- intersection++;
74
- }
75
- const union = tokensA.size + tokensB.size - intersection;
76
- return union > 0 ? intersection / union : 0;
77
- }
78
- /**
79
- * Given N reflect results, return the one with the highest average Jaccard
80
- * similarity to all other successful results (majority-vote winner).
81
- * Falls back to the first successful result when N < 2.
82
- */
83
- function pickMajorityVote(results) {
84
- const successful = results.filter((r) => r.ok);
85
- if (successful.length === 0)
86
- return (results[0] ?? {
87
- schemaVersion: 1,
88
- ok: false,
89
- reason: "non_zero_exit",
90
- error: "all samples failed",
91
- exitCode: null,
92
- });
93
- if (successful.length === 1)
94
- return successful[0];
95
- let bestIdx = 0;
96
- let bestScore = -1;
97
- for (let i = 0; i < successful.length; i++) {
98
- let totalSim = 0;
99
- for (let j = 0; j < successful.length; j++) {
100
- if (i === j)
101
- continue;
102
- totalSim += jaccardSimilarity(successful[i].proposal.payload.content ?? "", successful[j].proposal.payload.content ?? "");
103
- }
104
- const avgSim = totalSim / (successful.length - 1);
105
- if (avgSim > bestScore) {
106
- bestScore = avgSim;
107
- bestIdx = i;
108
- }
109
- }
110
- return successful[bestIdx] ?? successful[0];
111
- }
112
- // O-5 / #378: helper to push per-originator errors into the rolling window.
113
- function pushRecentError(originator, msg) {
114
- if (!recentErrors[originator])
115
- recentErrors[originator] = [];
116
- recentErrors[originator].push(msg);
117
- if (recentErrors[originator].length > RECENT_ERRORS_CAP)
118
- recentErrors[originator].shift();
119
- }
120
- // Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
121
- const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
122
- let completedCount = 0;
123
- let reflectsWithErrorContext = 0;
124
- const memoryRefsForInference = new Set();
125
81
  // Pre-load all pending proposals once instead of querying per asset in the loop.
126
82
  const dedupeStashDirForProposals = primaryStashDir ?? options.stashDir;
127
83
  const pendingProposalRefSet = new Set(dedupeStashDirForProposals
128
84
  ? listProposals(dedupeStashDirForProposals, { status: "pending" }).map((p) => p.ref)
129
85
  : []);
130
- let gateAutoAcceptedCount = 0;
131
- let gateAutoAcceptFailedCount = 0;
132
- const reflectGateCfg = makeGateConfig("reflect", {
133
- globalThreshold: options.autoAccept,
134
- dryRun: options.dryRun ?? false,
135
- stashDir: primaryStashDir,
136
- config: options.config ?? loadConfig(),
137
- eventsCtx,
138
- stateDbPath: eventsCtx?.dbPath,
139
- // candidateCount drives the exploration budget. loopRefs is the per-phase
140
- // set for reflect/distill; pass it so exploration budget is proportional.
141
- candidateCount: loopRefs.length,
142
- });
143
- const distillGateCfg = makeGateConfig("distill", {
144
- globalThreshold: options.autoAccept,
145
- dryRun: options.dryRun ?? false,
146
- stashDir: primaryStashDir,
147
- config: options.config ?? loadConfig(),
86
+ return {
87
+ scope,
88
+ options,
89
+ primaryStashDir,
90
+ reflectFn,
91
+ distillFn,
92
+ signalBearingSet,
93
+ distillCooledRefs,
94
+ distillOnlyRefSet,
95
+ recentErrors,
96
+ rejectedProposalsByRef,
148
97
  eventsCtx,
149
- stateDbPath: eventsCtx?.dbPath,
150
- candidateCount: loopRefs.length,
151
- });
152
- for (const planned of loopRefs) {
153
- if (Date.now() - startMs >= budgetMs) {
154
- const remaining = loopRefs.length - completedCount;
155
- info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
98
+ improveProfile,
99
+ resolvedPlan,
100
+ budgetSignal,
101
+ skipDistillDueToRequirePlannedRefs,
102
+ pendingProposalRefSet,
103
+ remainingBudgetMs,
104
+ };
105
+ }
106
+ /**
107
+ * One improve-loop iteration for a single planned ref: the reflect pass, then
108
+ * the distill pass, with the per-ref error classification (B7) around both.
109
+ * `continue` in the old inline loop body is an early `return` inside the
110
+ * passes; the orchestrator folds the returned tally and owns the run counters.
111
+ */
112
+ export async function processImproveLoopRef(planned, env) {
113
+ const tally = {
114
+ actions: [],
115
+ reflectsWithErrorContext: 0,
116
+ recentErrorPushes: [],
117
+ memoryRefsForInference: [],
118
+ };
119
+ try {
120
+ // Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
121
+ // Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
122
+ // have their synthetic actions emitted in runImprovePreparationStage.
123
+ const isDistillOnly = env.distillOnlyRefSet.has(planned.ref);
124
+ const parsedPlannedRef = parseRefInput(planned.ref);
125
+ await runLoopReflectPass(planned, isDistillOnly, env, tally);
126
+ // isDistillOnly refs: no reflect action emitted — proceed directly to the distill pass.
127
+ await runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally);
128
+ }
129
+ catch (err) {
130
+ // B7: UsageError thrown by akmDistill on validation_failed should be recorded
131
+ // as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
132
+ // The distill_invoked event was already emitted inside akmDistill before the throw.
133
+ if (err instanceof UsageError) {
134
+ tally.actions.push({
135
+ ref: planned.ref,
136
+ mode: "distill",
137
+ result: { ok: false, outcome: "validation_failed", error: err.message },
138
+ });
139
+ }
140
+ else {
141
+ tally.actions.push({
142
+ ref: planned.ref,
143
+ mode: "error",
144
+ result: { ok: false, error: errMessage(err) },
145
+ });
146
+ }
147
+ }
148
+ return tally;
149
+ }
150
+ /**
151
+ * Reflect half of one loop iteration: type/profile gates, the reflect call with
152
+ * recent-error avoidPatterns, outcome classification (cooldown / guard-reject /
153
+ * type-refused / noise-gate), and plasticity counters. Records onto the per-ref
154
+ * tally only.
155
+ */
156
+ async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
157
+ const { options, primaryStashDir, reflectFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
158
+ // B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
159
+ // shouldDistillMemoryRef already returns false for .derived refs, so the distill
160
+ // path is also a no-op for them — we just avoid unnecessary agent spawns.
161
+ // D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
162
+ if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
163
+ // Type guard: skip reflect for unsupported types (script, env, task, etc.)
164
+ // and raw wiki directories, driven by the active improve profile.
165
+ const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
166
+ if (reflectSkip.skip) {
167
+ tally.actions.push({
168
+ ref: planned.ref,
169
+ mode: "reflect-skipped",
170
+ result: { ok: true, reason: reflectSkip.reason },
171
+ });
172
+ }
173
+ else {
174
+ // O-5 / #378: only inject reflect-originator errors into the reflect call.
175
+ // Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
176
+ const reflectErrors = env.recentErrors.reflect ?? [];
177
+ if (reflectErrors.length > 0)
178
+ tally.reflectsWithErrorContext++;
179
+ // O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
180
+ // bounded by the wall-clock deadline rather than the default per-profile timeout.
181
+ const reflectBudgetMs = env.remainingBudgetMs();
182
+ // Use the runner frozen in the invocation plan; no leaf re-resolution.
183
+ const reflectProfileRunner = resolvedPlan.processes.reflect.runner;
184
+ const reflectCallArgs = {
185
+ ref: planned.ref,
186
+ // Carry the resolved item_ref so reflect uses the same durable key for
187
+ // events and state reads.
188
+ ...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
189
+ task: options.task,
190
+ // Active strategy supplies non-engine process tuning.
191
+ ...(improveProfile ? { improveProfile } : {}),
192
+ config: options.config,
193
+ ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
194
+ ...(options.sourceName && primaryStashDir
195
+ ? { target: { source: options.sourceName, root: primaryStashDir } }
196
+ : {}),
197
+ ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
198
+ eventSource: "improve",
199
+ // #639 — resolve the low-value filter from the ACTIVE improve profile
200
+ // (default off when unset), so the running strategy decides.
201
+ lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
202
+ ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
203
+ signal: budgetSignal,
204
+ runner: reflectProfileRunner ?? null,
205
+ // R25: reflect's event emits reuse the run's long-lived state.db handle.
206
+ eventsCtx: env.eventsCtx,
207
+ // Attribution: carry the eligibility lane so reflect stamps it on
208
+ // the reflect_invoked event and the persisted proposal.
209
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
210
+ };
211
+ const reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs), {
212
+ engine: resolvedPlan.processes.reflect.runner?.engine,
213
+ process: "reflect",
214
+ });
215
+ const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
216
+ // Content-policy guard hits (reflect size-rail rejections) are NOT
217
+ // LLM faults — the agent responded fine, the downstream guard
218
+ // blocked the output. Route them to a distinct `reflect-guard-rejected`
219
+ // mode so health metrics can split deterministic guard hits out of
220
+ // true LLM failures. See
221
+ // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
222
+ const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
223
+ // Type-guard rejection (reflect refused a script/env/task ref) is
224
+ // also NOT an LLM failure — the LLM is never invoked. Route to the
225
+ // existing `reflect-skipped` bucket so it does not inflate the
226
+ // failure-rate numerator. ~9% of `reflect-failed` events in the
227
+ // user's stack were this case; see review §1a row "Reflect refused
228
+ // asset type".
229
+ const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
230
+ // Noise-gate suppression (#580): the candidate edit was an empty
231
+ // diff or a cosmetic-only reformat of the current asset. Like
232
+ // `unsupported_type`, this is a deterministic skip — not an LLM
233
+ // fault — so it routes to the `reflect-skipped` bucket and stays
234
+ // out of recentErrors/avoidPatterns.
235
+ const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
236
+ tally.actions.push({
237
+ ref: planned.ref,
238
+ mode: reflectResult.ok
239
+ ? "reflect"
240
+ : isCooldown
241
+ ? "reflect-cooldown"
242
+ : isGuardReject
243
+ ? "reflect-guard-rejected"
244
+ : isTypeRefused || isNoChange
245
+ ? "reflect-skipped"
246
+ : "reflect-failed",
247
+ result: reflectResult,
248
+ });
249
+ // Cooldown skips, guard rejects, type-refused skips, and noise-gate
250
+ // skips are not failures — do not pollute recentErrors with them
251
+ // (those get injected as `avoidPatterns` into the next reflect
252
+ // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
253
+ // so the next iteration sees "your last expansion was too large";
254
+ // type-refused and no-change are deterministic and add no learning
255
+ // signal.
256
+ if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
257
+ const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
258
+ tally.recentErrorPushes.push({ originator: "reflect", message: errMsg });
259
+ }
260
+ // improve_reflect_outcome — per-asset metric for tuning the reflect path.
156
261
  appendEvent({
157
- eventType: "improve_skipped",
262
+ eventType: "improve_reflect_outcome",
158
263
  ref: planned.ref,
159
264
  metadata: {
160
- reason: "budget_exhausted",
161
- remaining,
265
+ ok: reflectResult.ok,
266
+ durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
267
+ engine: reflectResult.engine,
268
+ reason: reflectResult.ok ? undefined : reflectResult.reason,
162
269
  },
163
270
  }, eventsCtx);
164
- // B11: Emit improve_skipped for all remaining assets that will not be processed.
165
- for (const remainingRef of loopRefs.slice(completedCount + 1)) {
166
- appendEvent({
167
- eventType: "improve_skipped",
168
- ref: remainingRef.ref,
169
- metadata: { reason: "budget_exhausted_batch", remaining: loopRefs.length - completedCount - 1 },
170
- }, eventsCtx);
271
+ // Plasticity counter (plan §WS-1 step 8): record no-ops so the
272
+ // WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
273
+ // repeatedly-silent assets during consolidation-selection.
274
+ // A no_change reflect means the LLM was invoked but found nothing to
275
+ // improve — the asset is stable. Track it. A successful reflect means
276
+ // the asset changed; reset the counter so the dampener lifts.
277
+ // Use the same item_ref-or-conceptId salience key as preparation/distill.
278
+ const plasticityKey = planned.itemRef ?? durableImproveRef(planned.ref);
279
+ if (isNoChange && eventsCtx?.db) {
280
+ try {
281
+ recordNoOp(eventsCtx.db, plasticityKey);
282
+ }
283
+ catch {
284
+ // best-effort: plasticity counter failure never blocks the run
285
+ }
171
286
  }
172
- actions.push({
173
- ref: planned.ref,
174
- mode: "error",
175
- result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
176
- });
177
- break;
178
- }
179
- try {
180
- // Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
181
- // Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
182
- // have their synthetic actions emitted in runImprovePreparationStage.
183
- const isDistillOnly = distillOnlyRefSet.has(planned.ref);
184
- const parsedPlannedRef = parseAssetRef(planned.ref);
185
- // B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
186
- // shouldDistillMemoryRef already returns false for .derived refs, so the distill
187
- // path is also a no-op for them — we just avoid unnecessary agent spawns.
188
- // D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
189
- if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
190
- // Type guard: skip reflect for unsupported types (script, env, task, etc.)
191
- // and raw wiki directories, driven by the active improve profile.
192
- const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
193
- if (reflectSkip.skip) {
194
- actions.push({
195
- ref: planned.ref,
196
- mode: "reflect-skipped",
197
- result: { ok: true, reason: reflectSkip.reason },
198
- });
287
+ else if (reflectResult.ok && eventsCtx?.db) {
288
+ try {
289
+ resetConsecutiveNoOps(eventsCtx.db, plasticityKey);
290
+ }
291
+ catch {
292
+ // best-effort
199
293
  }
200
- else {
201
- // O-5 / #378: only inject reflect-originator errors into the reflect call.
202
- // Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
203
- const reflectErrors = recentErrors.reflect ?? [];
204
- if (reflectErrors.length > 0)
205
- reflectsWithErrorContext++;
206
- // O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
207
- // bounded by the wall-clock deadline rather than the default per-profile timeout.
208
- const reflectBudgetMs = remainingBudgetMs();
209
- // Wire profile.processes.reflect.{mode, profile, timeoutMs} into the reflect
210
- // dispatch when present. Falls back to akmReflect's own config-based resolution
211
- // (profiles.improve.<name>.processes.reflect → defaults.llm) when the profile
212
- // does not specify.
213
- const reflectProfileRunner = resolveImproveProcessRunnerFromProfile(improveProfile.processes?.reflect, options.config ?? loadConfig());
214
- const reflectCallArgs = {
215
- ref: planned.ref,
216
- task: options.task,
217
- // Active profile so reflect's per-process reads honor `--profile`.
218
- ...(improveProfile ? { improveProfile } : {}),
219
- ...(options.stashDir ? { stashDir: options.stashDir } : {}),
220
- ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
221
- agentProcess: options.agentProcess ?? "reflect",
222
- eventSource: "improve",
223
- // #639 — resolve the low-value filter from the ACTIVE improve profile
224
- // (default off when unset), so the running profile decides instead of
225
- // a hardcoded profiles.improve.default path.
226
- lowValueFilter: improveProfile.processes?.reflect?.lowValueFilter?.enabled === true,
227
- ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
228
- ...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
229
- // Attribution: carry the eligibility lane so reflect stamps it on
230
- // the reflect_invoked event and the persisted proposal.
231
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
232
- };
233
- // R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
234
- // Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
235
- const refUtility = utilityMap.get(planned.ref) ?? 0;
236
- const useConsistency = refUtility >= SC_THRESHOLD && SC_N >= 2;
237
- let reflectResult;
238
- if (useConsistency) {
239
- const samples = [];
240
- for (let s = 0; s < SC_N; s++) {
241
- if (remainingBudgetMs() <= 0)
242
- break;
243
- // draftMode: skip DB write so each sample doesn't create a proposal.
244
- samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
245
- }
246
- const winner = pickMajorityVote(samples.length > 0
247
- ? samples
248
- : [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
249
- // Persist only the majority-vote winner as a single real proposal.
250
- if (winner.ok && primaryStashDir) {
251
- const persistResult = createProposal(primaryStashDir, {
252
- ref: winner.proposal.ref,
253
- source: "reflect",
254
- sourceRun: `reflect-sc-${Date.now()}`,
255
- payload: winner.proposal.payload,
256
- // Attribution: the self-consistency path persists the winner here
257
- // (draftMode skips reflect's own createProposal), so stamp the lane.
258
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
259
- });
260
- reflectResult = isProposalSkipped(persistResult)
261
- ? {
262
- schemaVersion: 1,
263
- ok: false,
264
- reason: "cooldown",
265
- error: `SC proposal skipped: ${persistResult.message}`,
266
- ref: winner.ref,
267
- exitCode: null,
268
- }
269
- : { ...winner, proposal: persistResult };
270
- }
271
- else {
272
- reflectResult = winner;
273
- }
274
- }
275
- else {
276
- reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
277
- }
278
- const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
279
- // Content-policy guard hits (reflect size-rail rejections) are NOT
280
- // LLM faults — the agent responded fine, the downstream guard
281
- // blocked the output. Route them to a distinct `reflect-guard-rejected`
282
- // mode so health metrics can split deterministic guard hits out of
283
- // true LLM failures. See
284
- // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
285
- const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
286
- // Type-guard rejection (reflect refused a script/env/task ref) is
287
- // also NOT an LLM failure — the LLM is never invoked. Route to the
288
- // existing `reflect-skipped` bucket so it does not inflate the
289
- // failure-rate numerator. ~9% of `reflect-failed` events in the
290
- // user's stack were this case; see review §1a row "Reflect refused
291
- // asset type".
292
- const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
293
- // Noise-gate suppression (#580): the candidate edit was an empty
294
- // diff or a cosmetic-only reformat of the current asset. Like
295
- // `unsupported_type`, this is a deterministic skip — not an LLM
296
- // fault — so it routes to the `reflect-skipped` bucket and stays
297
- // out of recentErrors/avoidPatterns.
298
- const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
299
- actions.push({
300
- ref: planned.ref,
301
- mode: reflectResult.ok
302
- ? "reflect"
303
- : isCooldown
304
- ? "reflect-cooldown"
305
- : isGuardReject
306
- ? "reflect-guard-rejected"
307
- : isTypeRefused || isNoChange
308
- ? "reflect-skipped"
309
- : "reflect-failed",
310
- result: reflectResult,
311
- });
312
- // Cooldown skips, guard rejects, type-refused skips, and noise-gate
313
- // skips are not failures — do not pollute recentErrors with them
314
- // (those get injected as `avoidPatterns` into the next reflect
315
- // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
316
- // so the next iteration sees "your last expansion was too large";
317
- // type-refused and no-change are deterministic and add no learning
318
- // signal.
319
- if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
320
- const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
321
- pushRecentError("reflect", errMsg);
322
- }
323
- // improve_reflect_outcome — per-asset metric for tuning the reflect path.
324
- appendEvent({
325
- eventType: "improve_reflect_outcome",
326
- ref: planned.ref,
327
- metadata: {
328
- ok: reflectResult.ok,
329
- durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
330
- agentProfile: reflectResult.ok ? reflectResult.agentProfile : undefined,
331
- reason: reflectResult.ok ? undefined : reflectResult.reason,
332
- },
333
- }, eventsCtx);
334
- // Plasticity counter (plan §WS-1 step 8): record no-ops so the
335
- // WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
336
- // repeatedly-silent assets during consolidation-selection.
337
- // A no_change reflect means the LLM was invoked but found nothing to
338
- // improve — the asset is stable. Track it. A successful reflect means
339
- // the asset changed; reset the counter so the dampener lifts.
340
- if (isNoChange && eventsCtx?.db) {
341
- try {
342
- recordNoOp(eventsCtx.db, planned.ref);
343
- }
344
- catch {
345
- // best-effort: plasticity counter failure never blocks the run
346
- }
347
- }
348
- else if (reflectResult.ok && eventsCtx?.db) {
349
- try {
350
- resetConsecutiveNoOps(eventsCtx.db, planned.ref);
351
- }
352
- catch {
353
- // best-effort
354
- }
355
- }
356
- if (reflectResult.ok) {
357
- const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
358
- gateAutoAcceptedCount += reflectGr.promoted.length;
359
- gateAutoAcceptFailedCount += reflectGr.failed.length;
360
- }
361
- } // end else (reflect type/profile check)
362
294
  }
363
- else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
364
- // B6: .derived refs skip reflect; record synthetic skip action.
365
- actions.push({
295
+ } // end else (reflect type/profile check)
296
+ }
297
+ else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
298
+ // B6: .derived refs skip reflect; record synthetic skip action.
299
+ tally.actions.push({
300
+ ref: planned.ref,
301
+ mode: "distill-skipped",
302
+ result: { ok: true, reason: "derived-memory-reflect-skipped" },
303
+ });
304
+ appendEvent({
305
+ eventType: "improve_skipped",
306
+ ref: planned.ref,
307
+ metadata: { reason: "derived_memory_reflect_skipped" },
308
+ }, eventsCtx);
309
+ }
310
+ }
311
+ /**
312
+ * Distill half of one loop iteration: the profile / requirePlannedRefs /
313
+ * candidate-type / weak-signal / cooldown gates, then the pending-proposal and
314
+ * reject-grace dedup checks, then {@link invokeDistillAndRecord}. Each gate
315
+ * that was a `continue` in the old inline loop body is an early `return` here.
316
+ */
317
+ async function runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally) {
318
+ const { options, primaryStashDir, eventsCtx, improveProfile } = env;
319
+ const hasRecentFeedbackSignal = env.signalBearingSet.has(planned.ref);
320
+ const explicitRefScope = env.scope.mode === "ref";
321
+ // Profile gate: apply the full type-filter / raw-wiki / disabled rules to
322
+ // distill so callers who configure `profile.processes.distill.allowedTypes`
323
+ // or land on raw-wiki refs get a recorded skip action instead of silently
324
+ // proceeding.
325
+ const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
326
+ if (distillSkip.skip) {
327
+ tally.actions.push({
328
+ ref: planned.ref,
329
+ mode: "distill-skipped",
330
+ result: { ok: true, reason: distillSkip.reason },
331
+ });
332
+ return;
333
+ }
334
+ // requirePlannedRefs guard: skip distill for distill-only refs when no
335
+ // reflect-eligible refs were planned this run, preventing mass skip events.
336
+ if (env.skipDistillDueToRequirePlannedRefs && isDistillOnly) {
337
+ tally.actions.push({
338
+ ref: planned.ref,
339
+ mode: "distill-skipped",
340
+ result: { ok: true, reason: "require_planned_refs" },
341
+ });
342
+ return;
343
+ }
344
+ // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
345
+ // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
346
+ // it will refuse.
347
+ const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
348
+ const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
349
+ // distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
350
+ // refs; non-candidate refs in the set are blocked here.
351
+ // O-2 (#365): bypass the distill cooldown when the user explicitly targeted
352
+ // this ref via --scope — their intent overrides unattended-run policies.
353
+ if (shouldAttemptDistill &&
354
+ !skipMemoryDistillForWeakSignal &&
355
+ (!env.distillCooledRefs.has(planned.ref) || explicitRefScope)) {
356
+ // TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
357
+ const lessonRef = deriveLessonRef(planned.ref);
358
+ const knowledgeRef = deriveKnowledgeRef(planned.ref);
359
+ const dedupeStashDir = primaryStashDir ?? options.stashDir;
360
+ if (dedupeStashDir) {
361
+ // B2: check both lesson ref and knowledge ref since auto-promoted memories
362
+ // create knowledge: proposals, not lesson: proposals.
363
+ const hasExistingPending = env.pendingProposalRefSet.has(lessonRef) || env.pendingProposalRefSet.has(knowledgeRef);
364
+ if (hasExistingPending) {
365
+ tally.actions.push({
366
366
  ref: planned.ref,
367
367
  mode: "distill-skipped",
368
- result: { ok: true, reason: "derived-memory-reflect-skipped" },
368
+ result: { ok: true, reason: "pending proposal exists" },
369
369
  });
370
370
  appendEvent({
371
371
  eventType: "improve_skipped",
372
372
  ref: planned.ref,
373
- metadata: { reason: "derived_memory_reflect_skipped" },
373
+ metadata: { reason: "pending_proposal_exists" },
374
374
  }, eventsCtx);
375
+ return;
375
376
  }
376
- // isDistillOnly refs: no reflect action emitted — proceed directly to distill path below.
377
- const hasRecentFeedbackSignal = signalBearingSet.has(planned.ref);
378
- const explicitRefScope = scope.mode === "ref";
379
- // Profile gate: apply the full type-filter / raw-wiki / disabled rules to
380
- // distill so callers who configure `profile.processes.distill.allowedTypes`
381
- // or land on raw-wiki refs get a recorded skip action instead of silently
382
- // proceeding.
383
- const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
384
- if (distillSkip.skip) {
385
- actions.push({
386
- ref: planned.ref,
387
- mode: "distill-skipped",
388
- result: { ok: true, reason: distillSkip.reason },
389
- });
390
- completedCount++;
391
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
392
- continue;
393
- }
394
- // requirePlannedRefs guard: skip distill for distill-only refs when no
395
- // reflect-eligible refs were planned this run, preventing mass skip events.
396
- if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
397
- actions.push({
398
- ref: planned.ref,
399
- mode: "distill-skipped",
400
- result: { ok: true, reason: "require_planned_refs" },
401
- });
402
- completedCount++;
403
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
404
- continue;
405
- }
406
- // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
407
- // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
408
- // it will refuse.
409
- const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
410
- const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
411
- // distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
412
- // refs; non-candidate refs in the set are blocked here.
413
- // O-2 (#365): bypass the distill cooldown when the user explicitly targeted
414
- // this ref via --scope — their intent overrides unattended-run policies.
415
- if (shouldAttemptDistill &&
416
- !skipMemoryDistillForWeakSignal &&
417
- (!distillCooledRefs.has(planned.ref) || explicitRefScope)) {
418
- // TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
419
- const lessonRef = deriveLessonRef(planned.ref);
420
- const knowledgeRef = deriveKnowledgeRef(planned.ref);
421
- const dedupeStashDir = primaryStashDir ?? options.stashDir;
422
- if (dedupeStashDir) {
423
- // B2: check both lesson ref and knowledge ref since auto-promoted memories
424
- // create knowledge: proposals, not lesson: proposals.
425
- const hasExistingPending = pendingProposalRefSet.has(lessonRef) || pendingProposalRefSet.has(knowledgeRef);
426
- if (hasExistingPending) {
427
- actions.push({
428
- ref: planned.ref,
429
- mode: "distill-skipped",
430
- result: { ok: true, reason: "pending proposal exists" },
431
- });
432
- appendEvent({
433
- eventType: "improve_skipped",
434
- ref: planned.ref,
435
- metadata: { reason: "pending_proposal_exists" },
436
- }, eventsCtx);
437
- completedCount++;
438
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
439
- continue;
440
- }
441
- // D-2 (#370): reject-aware cooldown for distill. When the reviewer
442
- // recently rejected a distilled lesson or knowledge proposal for this
443
- // asset, skip re-distillation for a 1-day grace window. Prevents the
444
- // same rejected proposal from being regenerated immediately. The
445
- // window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
446
- // signal-delta gates and dropped --distill-cooldown-days; a short
447
- // reject grace is preserved here so a fresh rejection isn't
448
- // overridden by the same run).
449
- // References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
450
- const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
451
- const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
452
- (rejectedProposalsByRef.has(lessonRef) || rejectedProposalsByRef.has(knowledgeRef));
453
- if (recentlyRejectedLesson) {
454
- const rejectedEntry = rejectedProposalsByRef.get(lessonRef) ?? rejectedProposalsByRef.get(knowledgeRef);
455
- const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
456
- if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
457
- actions.push({
458
- ref: planned.ref,
459
- mode: "distill-skipped",
460
- result: { ok: true, reason: "distill reject grace window" },
461
- });
462
- appendEvent({
463
- eventType: "improve_skipped",
464
- ref: planned.ref,
465
- metadata: {
466
- reason: "distill_reject_grace_window",
467
- },
468
- }, eventsCtx);
469
- completedCount++;
470
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
471
- continue;
472
- }
473
- }
474
- }
475
- const distillResult = await withLlmStage("distill", () => distillFn({
476
- ref: planned.ref,
477
- ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
478
- ...(options.stashDir ? { stashDir: options.stashDir } : {}),
479
- // Active profile so distill's per-process reads honor `--profile`.
480
- ...(improveProfile ? { improveProfile } : {}),
481
- // Attribution: carry the eligibility lane so distill stamps it on the
482
- // distill_invoked event and the persisted proposal.
483
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
484
- }));
485
- actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
486
- if (distillResult.outcome === "queued" && distillResult.proposal) {
487
- const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
488
- gateAutoAcceptedCount += distillGr.promoted.length;
489
- gateAutoAcceptFailedCount += distillGr.failed.length;
490
- }
491
- if (parsedPlannedRef.type === "memory") {
492
- const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
493
- if (!promotedToKnowledge)
494
- memoryRefsForInference.add(planned.ref);
495
- }
496
- // Plasticity counter (plan §WS-1 step 8) for the distill path.
497
- // quality_rejected: the LLM ran but produced output that didn't pass the
498
- // quality gate — the asset is not yielding useful distill output.
499
- // queued: a proposal was produced; reset the no-op counter.
500
- if (eventsCtx?.db) {
501
- try {
502
- if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
503
- recordNoOp(eventsCtx.db, planned.ref);
504
- }
505
- else if (distillResult.outcome === "queued") {
506
- resetConsecutiveNoOps(eventsCtx.db, planned.ref);
507
- }
508
- }
509
- catch {
510
- // best-effort: plasticity counter failure never blocks the run
511
- }
512
- }
513
- if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
514
- const slug = refSlug(planned.ref);
515
- writeEvalCase(primaryStashDir, {
377
+ // D-2 (#370): reject-aware cooldown for distill. When the reviewer
378
+ // recently rejected a distilled lesson or knowledge proposal for this
379
+ // asset, skip re-distillation for a 1-day grace window. Prevents the
380
+ // same rejected proposal from being regenerated immediately. The
381
+ // window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
382
+ // signal-delta gates and dropped --distill-cooldown-days; a short
383
+ // reject grace is preserved here so a fresh rejection isn't
384
+ // overridden by the same run).
385
+ // References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
386
+ const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
387
+ const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
388
+ (env.rejectedProposalsByRef.has(lessonRef) || env.rejectedProposalsByRef.has(knowledgeRef));
389
+ if (recentlyRejectedLesson) {
390
+ const rejectedEntry = env.rejectedProposalsByRef.get(lessonRef) ?? env.rejectedProposalsByRef.get(knowledgeRef);
391
+ const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
392
+ if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
393
+ tally.actions.push({
516
394
  ref: planned.ref,
517
- failureReason: distillResult.reason ?? "quality gate rejected",
518
- assetType: parseAssetRef(planned.ref).type ?? "unknown",
519
- rejectedAt: Date.now(),
520
- source: "distill_quality_rejected",
521
- slug: `${slug}-${Date.now()}`,
395
+ mode: "distill-skipped",
396
+ result: { ok: true, reason: "distill reject grace window" },
522
397
  });
523
- }
524
- // D6: use pre-loaded map instead of per-iteration DB query
525
- const rejectedProposalEvent = rejectedProposalsByRef.get(planned.ref);
526
- if (rejectedProposalEvent && primaryStashDir) {
527
- const slug = refSlug(planned.ref);
528
- writeEvalCase(primaryStashDir, {
398
+ appendEvent({
399
+ eventType: "improve_skipped",
529
400
  ref: planned.ref,
530
- failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
531
- assetType: parseAssetRef(planned.ref).type ?? "unknown",
532
- rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
533
- source: "proposal_rejected",
534
- slug: `${slug}-rejected`,
535
- });
401
+ metadata: {
402
+ reason: "distill_reject_grace_window",
403
+ },
404
+ }, eventsCtx);
405
+ return;
536
406
  }
537
407
  }
538
- else if (skipMemoryDistillForWeakSignal) {
539
- actions.push({
540
- ref: planned.ref,
541
- mode: "distill-skipped",
542
- result: { ok: true, reason: "memory requires recent feedback signal" },
543
- });
544
- appendEvent({
545
- eventType: "improve_skipped",
546
- ref: planned.ref,
547
- metadata: { reason: "memory_distill_requires_feedback" },
548
- }, eventsCtx);
549
- }
550
408
  }
551
- catch (err) {
552
- // B7: UsageError thrown by akmDistill on validation_failed should be recorded
553
- // as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
554
- // The distill_invoked event was already emitted inside akmDistill before the throw.
555
- if (err instanceof UsageError) {
556
- actions.push({
557
- ref: planned.ref,
558
- mode: "distill",
559
- result: { ok: false, outcome: "validation_failed", error: err.message },
560
- });
409
+ await invokeDistillAndRecord(planned, parsedPlannedRef, env, tally);
410
+ }
411
+ else if (skipMemoryDistillForWeakSignal) {
412
+ tally.actions.push({
413
+ ref: planned.ref,
414
+ mode: "distill-skipped",
415
+ result: { ok: true, reason: "memory requires recent feedback signal" },
416
+ });
417
+ appendEvent({
418
+ eventType: "improve_skipped",
419
+ ref: planned.ref,
420
+ metadata: { reason: "memory_distill_requires_feedback" },
421
+ }, eventsCtx);
422
+ }
423
+ }
424
+ /**
425
+ * The distill invocation for one ref that passed every gate: the `distillFn`
426
+ * call, memory-inference queueing, plasticity counters, and the
427
+ * quality-rejected / proposal-rejected eval-case writes.
428
+ */
429
+ async function invokeDistillAndRecord(planned, parsedPlannedRef, env, tally) {
430
+ const { options, primaryStashDir, distillFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
431
+ const distillResult = await withLlmStage("distill", () => distillFn({
432
+ ref: planned.ref,
433
+ // Carry the resolved item_ref so distill matches preparation's state key.
434
+ ...(planned.itemRef ? { itemRef: planned.itemRef } : {}),
435
+ ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
436
+ ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
437
+ // Active profile so distill's per-process reads honor `--profile`.
438
+ ...(improveProfile ? { improveProfile } : {}),
439
+ config: options.config,
440
+ llmConfig: resolvedPlan.processes.distill.runner
441
+ ? materializeLlmRunnerConnection(resolvedPlan.processes.distill.runner)
442
+ : null,
443
+ signal: budgetSignal,
444
+ // R25: distill's event emits reuse the run's long-lived state.db handle.
445
+ eventsCtx,
446
+ // Attribution: carry the eligibility lane so distill stamps it on the
447
+ // distill_invoked event and the persisted proposal.
448
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
449
+ }), { engine: resolvedPlan.processes.distill.runner?.engine, process: "distill" });
450
+ tally.actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
451
+ if (parsedPlannedRef.type === "memory") {
452
+ const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
453
+ if (!promotedToKnowledge)
454
+ tally.memoryRefsForInference.push(planned.ref);
455
+ }
456
+ // Plasticity counter (plan §WS-1 step 8) for the distill path.
457
+ // quality_rejected: the LLM ran but produced output that didn't pass the
458
+ // quality gate — the asset is not yielding useful distill output.
459
+ // queued: a proposal was produced; reset the no-op counter.
460
+ if (eventsCtx?.db) {
461
+ // Use the same item_ref-or-conceptId key as the distill/preparation writers.
462
+ const plasticityKey = planned.itemRef ?? durableImproveRef(planned.ref);
463
+ try {
464
+ if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
465
+ recordNoOp(eventsCtx.db, plasticityKey);
561
466
  }
562
- else {
563
- actions.push({
564
- ref: planned.ref,
565
- mode: "error",
566
- result: { ok: false, error: errMessage(err) },
567
- });
467
+ else if (distillResult.outcome === "queued") {
468
+ resetConsecutiveNoOps(eventsCtx.db, plasticityKey);
568
469
  }
569
470
  }
570
- completedCount++;
571
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
471
+ catch {
472
+ // best-effort: plasticity counter failure never blocks the run
473
+ }
572
474
  }
573
- // WS-4: Per-phase threshold auto-tune — runs AFTER the loop so the gate
574
- // has processed all candidates for this run. Persists each phase's tuned
575
- // threshold to state.db for the NEXT run's makeGateConfig to read.
576
- // Best-effort: a tune failure must never fail the improve run.
577
- const stateDbPathForTune = eventsCtx?.dbPath;
578
- if (options.autoAccept !== undefined && stateDbPathForTune) {
579
- const phaseGateCfgMap = {
580
- reflect: reflectGateCfg,
581
- distill: distillGateCfg,
582
- };
583
- for (const phase of ["reflect", "distill"]) {
584
- const phaseCfg = phaseGateCfgMap[phase];
585
- try {
586
- maybeAutoTuneThreshold(phaseCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), stateDbPathForTune, undefined, phase);
587
- }
588
- catch (err) {
589
- warn(`[improve] calibration auto-tune (${phase}) skipped: ${errMessage(err)}`);
590
- }
475
+ if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
476
+ const slug = refSlug(planned.ref);
477
+ writeEvalCase(primaryStashDir, {
478
+ ref: planned.ref,
479
+ failureReason: distillResult.reason ?? "quality gate rejected",
480
+ assetType: parseRefInput(planned.ref).type ?? "unknown",
481
+ rejectedAt: Date.now(),
482
+ source: "distill_quality_rejected",
483
+ slug: `${slug}-${Date.now()}`,
484
+ });
485
+ }
486
+ // D6: use pre-loaded map instead of per-iteration DB query
487
+ const rejectedProposalEvent = env.rejectedProposalsByRef.get(planned.ref);
488
+ if (rejectedProposalEvent && primaryStashDir) {
489
+ const slug = refSlug(planned.ref);
490
+ writeEvalCase(primaryStashDir, {
491
+ ref: planned.ref,
492
+ failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
493
+ assetType: parseRefInput(planned.ref).type ?? "unknown",
494
+ rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
495
+ source: "proposal_rejected",
496
+ slug: `${slug}-rejected`,
497
+ });
498
+ }
499
+ }
500
+ /**
501
+ * Wall-clock budget exhausted mid-loop (O-1 / #364): emit the improve_skipped
502
+ * events for the current and remaining refs (B11) and return the terminal
503
+ * error action for the orchestrator to record before breaking out of the loop.
504
+ */
505
+ function recordBudgetExhausted(args) {
506
+ const { planned, loopRefs, completedCount, startMs, eventsCtx } = args;
507
+ const remaining = loopRefs.length - completedCount;
508
+ info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
509
+ appendEvent({
510
+ eventType: "improve_skipped",
511
+ ref: planned.ref,
512
+ metadata: {
513
+ reason: "budget_exhausted",
514
+ remaining,
515
+ },
516
+ }, eventsCtx);
517
+ // B11: Emit improve_skipped for all remaining assets that will not be processed.
518
+ for (const remainingRef of loopRefs.slice(completedCount + 1)) {
519
+ appendEvent({
520
+ eventType: "improve_skipped",
521
+ ref: remainingRef.ref,
522
+ metadata: { reason: "budget_exhausted_batch", remaining: loopRefs.length - completedCount - 1 },
523
+ }, eventsCtx);
524
+ }
525
+ return {
526
+ ref: planned.ref,
527
+ mode: "error",
528
+ result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
529
+ };
530
+ }
531
+ export async function runImproveLoopStage(args) {
532
+ const { ctx, loopRefs, actions, recentErrors, startMs, budgetMs } = args;
533
+ const eventsCtx = ctx.eventsCtx;
534
+ const env = prepareImproveLoopEnv(args);
535
+ let completedCount = 0;
536
+ let reflectsWithErrorContext = 0;
537
+ const memoryRefsForInference = new Set();
538
+ for (const planned of loopRefs) {
539
+ if (Date.now() - startMs >= budgetMs) {
540
+ actions.push(recordBudgetExhausted({ planned, loopRefs, completedCount, startMs, eventsCtx }));
541
+ break;
591
542
  }
543
+ const tally = await processImproveLoopRef(planned, env);
544
+ // Fold the per-ref tally into run-level state — the passes never touch it.
545
+ actions.push(...tally.actions);
546
+ for (const push of tally.recentErrorPushes)
547
+ pushRecentError(recentErrors, push.originator, push.message);
548
+ reflectsWithErrorContext += tally.reflectsWithErrorContext;
549
+ for (const ref of tally.memoryRefsForInference)
550
+ memoryRefsForInference.add(ref);
551
+ completedCount++;
552
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
592
553
  }
593
- return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
554
+ return { reflectsWithErrorContext, memoryRefsForInference };
594
555
  }
595
556
  export async function runImprovePostLoopStage(args) {
596
- const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, consolidationRan, } = args;
557
+ const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, resolvedPlan, consolidationRan, } = args;
597
558
  const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
598
559
  info("[improve] post-loop maintenance starting");
599
560
  const maintenanceResult = await runImproveMaintenancePasses({
@@ -608,6 +569,7 @@ export async function runImprovePostLoopStage(args) {
608
569
  budgetSignal,
609
570
  eventsCtx,
610
571
  improveProfile,
572
+ resolvedPlan,
611
573
  });
612
574
  let deadUrls;
613
575
  if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
@@ -615,14 +577,28 @@ export async function runImprovePostLoopStage(args) {
615
577
  const knowledgeEntries = actionableRefs
616
578
  .filter((r) => {
617
579
  try {
618
- return parseAssetRef(r.ref).type === "knowledge";
580
+ return parseRefInput(r.ref).type === "knowledge";
619
581
  }
620
582
  catch {
621
583
  return false;
622
584
  }
623
585
  })
624
586
  .slice(0, 10)
625
- .map((r) => ({ ref: r.ref, body: "" }));
587
+ .map((r) => {
588
+ // The URL scan needs the document body; filePath is pre-resolved on
589
+ // eligible refs at planning time (#591). Best-effort — an unreadable
590
+ // or unresolved file contributes no URLs, same as before.
591
+ let body = "";
592
+ if (r.filePath) {
593
+ try {
594
+ body = fs.readFileSync(r.filePath, "utf8");
595
+ }
596
+ catch {
597
+ // best-effort
598
+ }
599
+ }
600
+ return { ref: r.ref, body };
601
+ });
626
602
  if (knowledgeEntries.length > 0) {
627
603
  info(`[improve] checking URLs in ${knowledgeEntries.length} knowledge refs`);
628
604
  deadUrls = await checkDeadUrls(primaryStashDir, knowledgeEntries);
@@ -633,86 +609,17 @@ export async function runImprovePostLoopStage(args) {
633
609
  // best-effort
634
610
  }
635
611
  }
636
- // #609 — recombine / synthesize pass. Whole-corpus cross-episodic
637
- // generalization. Runs in the post-loop stage under consolidate.lock (it
638
- // reads the consolidated corpus and writes proposals). Opt-in: gated on the
639
- // `recombine` process being enabled, whole-stash / type scope (never `ref`),
640
- // and not a dry run. Mirrors the proactiveMaintenance opt-in wiring.
641
- let recombination;
642
- if (primaryStashDir &&
643
- improveProfile &&
644
- resolveProcessEnabled("recombine", improveProfile) &&
645
- scope.mode !== "ref" &&
646
- !options.dryRun) {
647
- const recombineFn = options.recombineFn ?? akmRecombine;
648
- try {
649
- recombination = await recombineFn({
650
- stashDir: primaryStashDir,
651
- config: options.config ?? loadConfig(),
652
- improveProfile,
653
- ...(options.runId ? { sourceRun: options.runId } : {}),
654
- ...(budgetSignal ? { signal: budgetSignal } : {}),
655
- eligibilitySource: "recombine",
656
- ...(eventsCtx ? { ctx: eventsCtx } : {}),
657
- minClusterSize: improveProfile.processes?.recombine?.minClusterSize,
658
- maxClustersPerRun: improveProfile.processes?.recombine?.maxClustersPerRun,
659
- relatednessSource: improveProfile.processes?.recombine?.relatednessSource,
660
- confirmThreshold: improveProfile.processes?.recombine?.confirmThreshold,
661
- // #632 — clustering-tuning knobs. UNSET = pre-#632 behaviour.
662
- maxClusterSize: improveProfile.processes?.recombine?.maxClusterSize,
663
- excludeTags: improveProfile.processes?.recombine?.excludeTags,
664
- excludeEntities: improveProfile.processes?.recombine?.excludeEntities,
665
- });
666
- }
667
- catch (e) {
668
- allWarnings.push(`recombine: ${String(e)}`);
669
- }
670
- }
671
- // #615 — procedural-compilation pass. Detects recurring successful ordered
672
- // action sequences and compiles them into workflow proposals. Opt-in: gated
673
- // on the `procedural` process being enabled, whole-stash / type scope (never
674
- // `ref`), and not a dry run. Mirrors the recombine opt-in wiring.
675
- let proceduralCompilation;
676
- if (primaryStashDir &&
677
- improveProfile &&
678
- resolveProcessEnabled("procedural", improveProfile) &&
679
- scope.mode !== "ref" &&
680
- !options.dryRun) {
681
- const proceduralFn = options.proceduralFn ?? akmProcedural;
682
- try {
683
- proceduralCompilation = await proceduralFn({
684
- stashDir: primaryStashDir,
685
- config: options.config ?? loadConfig(),
686
- ...(improveProfile ? { improveProfile } : {}),
687
- ...(options.runId ? { sourceRun: options.runId } : {}),
688
- ...(budgetSignal ? { signal: budgetSignal } : {}),
689
- eligibilitySource: "procedural",
690
- ...(eventsCtx ? { ctx: eventsCtx } : {}),
691
- minRecurrence: improveProfile.processes?.procedural?.minRecurrence,
692
- maxProposalsPerRun: improveProfile.processes?.procedural?.maxProposalsPerRun,
693
- });
694
- }
695
- catch (e) {
696
- allWarnings.push(`procedural: ${String(e)}`);
697
- }
698
- }
699
612
  // ── R5: collapse/churn detector ────────────────────────────────────────────
700
- // One snapshot per QUALIFYING cycle: consolidate processed work and/or
701
- // recombine formed clusters. Runs AFTER the maintenance reindex so FTS sees
702
- // the post-merge index; one call site covers both passes. Deterministic,
613
+ // One snapshot per QUALIFYING cycle: consolidate processed work. Runs AFTER
614
+ // the maintenance reindex so FTS sees the post-merge index. Deterministic,
703
615
  // observe-only, fail-open (the orchestrator catches everything) — and inert
704
616
  // on the ~9-in-10 default-profile runs that touch no merges.
705
617
  let cycleMetrics;
706
- const recombineWorked = (recombination?.clustersFormed ?? 0) > 0;
707
- if (!options.dryRun && (consolidationRan || recombineWorked)) {
618
+ if (!options.dryRun && consolidationRan) {
708
619
  cycleMetrics = runCollapseDetector({
709
620
  runId: options.runId ?? "improve-adhoc",
710
621
  ...(improveProfile ? { improveProfile } : {}),
711
- pass: consolidationRan && recombineWorked ? "both" : consolidationRan ? "consolidate" : "recombine",
712
- // prep+loop gate accepts, PLUS recombine's confirmed-lesson promotions —
713
- // recombine churn is the historically observed failure mode and its
714
- // promotions never flow through the prep/loop gates.
715
- acceptedActions: (args.acceptedActions ?? 0) + (recombination?.lessonsPromoted ?? 0),
622
+ pass: "consolidate",
716
623
  mergeFloorViolations: args.consolidationMergeFloorViolations ?? 0,
717
624
  config: options.config ?? loadConfig(),
718
625
  ...(eventsCtx ? { eventsCtx } : {}),
@@ -722,8 +629,6 @@ export async function runImprovePostLoopStage(args) {
722
629
  allWarnings,
723
630
  deadUrls,
724
631
  ...(cycleMetrics ? { cycleMetrics } : {}),
725
- ...(recombination ? { recombination } : {}),
726
- ...(proceduralCompilation ? { proceduralCompilation } : {}),
727
632
  ...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
728
633
  ...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
729
634
  ...(maintenanceResult.actions && maintenanceResult.actions.length > 0
@@ -733,368 +638,650 @@ export async function runImprovePostLoopStage(args) {
733
638
  graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
734
639
  orphansPurged: maintenanceResult.orphansPurged,
735
640
  proposalsExpired: maintenanceResult.proposalsExpired,
736
- // Consolidation's auto-accept gate counts now accrue in the preparation
737
- // stage (#551); post-loop no longer runs an auto-accept gate of its own.
738
- gateAutoAcceptedCount: 0,
739
- gateAutoAcceptFailedCount: 0,
740
641
  };
741
642
  }
742
- // TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
743
- // Exported for tests (#584/#585 DB-locking regression coverage); production
744
- // callers reach it only through akmImprove → runImprovePostLoopStage.
745
- // TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
746
643
  // Exported for tests (#584/#585 DB-locking regression coverage); production
747
644
  // callers reach it only through akmImprove → runImprovePostLoopStage.
748
645
  export async function runImproveMaintenancePasses(args) {
749
- const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
646
+ const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, budgetSignal, eventsCtx } = args;
750
647
  if (!primaryStashDir)
751
648
  return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
649
+ if (budgetSignal?.aborted)
650
+ return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
752
651
  const config = options.config ?? loadConfig();
753
652
  const sources = resolveSourceEntries(options.stashDir, config);
754
653
  const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
755
654
  const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
756
- let db;
757
- let memoryInference;
758
- let graphExtraction;
759
- let reindexedAfterInference = false;
760
- const actions = [];
761
- let memoryInferenceDurationMs = 0;
762
- let graphExtractionDurationMs = 0;
763
- let orphansPurged = 0;
764
- let proposalsExpired = 0;
765
655
  const openIndexDb = () => openIndexDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
766
- // #584: reindexFn opens its own write handle on the same index.db WAL file.
767
- // Holding our handle across that call produced SQLITE_BUSY / "database is
768
- // locked" failures in production, so the handle is closed BEFORE every
769
- // reindex and reopened after — the fresh handle also sees the post-reindex
770
- // state that graph extraction below relies on. The
771
- // reopen runs in `finally` so a failed reindex still leaves a usable handle.
656
+ const dbCell = {};
657
+ // #584: see the MaintenanceCtx.reindexWithIndexDbReleased doc — close before
658
+ // every reindex, reopen in `finally` so a failed reindex still leaves a
659
+ // usable handle in the cell.
772
660
  const reindexWithIndexDbReleased = async (stashDir) => {
773
- if (db) {
774
- closeDatabase(db);
775
- db = undefined;
661
+ if (dbCell.current) {
662
+ closeDatabase(dbCell.current);
663
+ dbCell.current = undefined;
776
664
  }
777
665
  try {
778
- await reindexFn({ stashDir });
666
+ await reindexFn({ stashDir, signal: budgetSignal });
779
667
  }
780
668
  finally {
781
- db = openIndexDb();
669
+ dbCell.current = openIndexDb();
782
670
  }
783
671
  };
784
- await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
785
- try {
786
- db = openIndexDb();
787
- // Memory inference candidate-discovery (post-Item 9 fix from
788
- // memory:akm-improve-critical-review-2026-05-20). Previously this pass
789
- // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
790
- // candidateRefs filter. But memoryRefsForInference is populated from refs
791
- // distilled THIS RUN — by the time that happens, those parents are
792
- // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
793
- // them. The genuinely-pending parents in the stash never entered the
794
- // filter. Result: 0/0/0 for 25 consecutive runs.
795
- //
796
- // Fix: always run the pass when the feature is enabled; let the pass's
797
- // own `collectPendingMemories` + `isPendingMemory` predicate find
798
- // candidates from the filesystem-of-truth. The this-run set is still
799
- // logged as a hint but no longer used as a filter.
800
- const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
801
- const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
802
- const pendingBelowMinCount = (() => {
803
- if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
804
- return false;
805
- const pending = collectPendingMemories(primaryStashDir).length;
806
- if (pending < minPendingCount) {
807
- info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
808
- return true;
809
- }
810
- return false;
811
- })();
812
- if (memoryInferenceDisabledByProfile) {
813
- info("[improve] memory inference skipped (disabled by improve profile)");
814
- }
815
- else if (pendingBelowMinCount) {
816
- // skipped — message already emitted above
672
+ const ctx = {
673
+ config,
674
+ sources,
675
+ primaryStashDir,
676
+ eventsCtx,
677
+ budgetSignal,
678
+ improveProfile: args.improveProfile,
679
+ resolvedPlan: args.resolvedPlan,
680
+ memoryInferenceFn,
681
+ graphExtractionFn,
682
+ reindexWithIndexDbReleased,
683
+ };
684
+ const collected = await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, () => runMaintenancePassesUnderLease(ctx, dbCell, {
685
+ actionableRefs: args.actionableRefs,
686
+ memoryRefsForInference,
687
+ consolidationRan: args.consolidationRan,
688
+ allWarnings,
689
+ openIndexDb,
690
+ }));
691
+ return {
692
+ ...(collected.memoryInference ? { memoryInference: collected.memoryInference } : {}),
693
+ ...(collected.graphExtraction ? { graphExtraction: collected.graphExtraction } : {}),
694
+ ...(collected.actions.length > 0 ? { actions: collected.actions } : {}),
695
+ memoryInferenceDurationMs: collected.memoryInferenceDurationMs,
696
+ graphExtractionDurationMs: collected.graphExtractionDurationMs,
697
+ orphansPurged: collected.orphansPurged,
698
+ proposalsExpired: collected.proposalsExpired,
699
+ };
700
+ }
701
+ /**
702
+ * The maintenance sequence run under the index-writer lease (formerly the
703
+ * ~389-line anonymous `withIndexWriterLease` callback): memory inference →
704
+ * reindex-after-inference → graph extraction → proposal hygiene (orphan purge,
705
+ * expiration) → retention purges. Each pass returns its results and warnings;
706
+ * this orchestrator folds warnings into the caller's `allWarnings` sink at the
707
+ * same points the inline code pushed them.
708
+ */
709
+ async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
710
+ const { allWarnings } = args;
711
+ const actions = [];
712
+ let reindexedAfterInference = false;
713
+ try {
714
+ dbCell.current = args.openIndexDb();
715
+ const inference = await runMemoryInferenceMaintenancePass(ctx, dbCell, args.memoryRefsForInference);
716
+ if (inference.action)
717
+ actions.push(inference.action);
718
+ allWarnings.push(...inference.warnings);
719
+ const memoryInference = inference.memoryInference;
720
+ if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
721
+ info("[improve] reindexing after memory inference writes");
722
+ try {
723
+ await ctx.reindexWithIndexDbReleased(ctx.primaryStashDir);
724
+ reindexedAfterInference = true;
725
+ info("[improve] reindex after memory inference complete");
817
726
  }
818
- else {
819
- const hintRefs = memoryRefsForInference.size;
820
- info(hintRefs > 0
821
- ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
822
- : "[improve] memory inference starting (discovering pending parents)");
823
- const inferenceStart = Date.now();
824
- try {
825
- // O-1 (#364): pass budget signal so a hung inference call is cancelled.
826
- memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
827
- config,
828
- sources,
829
- signal: budgetSignal,
830
- db,
831
- reEnrich: false,
832
- onProgress: (event) => {
833
- const current = event.currentRef ? ` ${event.currentRef}` : "";
834
- info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
835
- },
836
- }));
837
- memoryInferenceDurationMs = Date.now() - inferenceStart;
838
- actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
839
- info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
840
- }
841
- catch (err) {
842
- memoryInferenceDurationMs = Date.now() - inferenceStart;
843
- allWarnings.push(`memory inference failed: ${errMessage(err)}`);
844
- }
727
+ catch (err) {
728
+ allWarnings.push(`reindex after memory inference failed: ${errMessage(err)}`);
845
729
  }
846
- if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
847
- info("[improve] reindexing after memory inference writes");
730
+ }
731
+ const graph = await runGraphExtractionMaintenancePass(ctx, dbCell, {
732
+ actionableRefs: args.actionableRefs,
733
+ memoryRefsForInference: args.memoryRefsForInference,
734
+ consolidationRan: args.consolidationRan,
735
+ reindexedAfterInference,
736
+ });
737
+ if (graph.action)
738
+ actions.push(graph.action);
739
+ allWarnings.push(...graph.warnings);
740
+ const orphan = runOrphanProposalPurgePass(ctx);
741
+ allWarnings.push(...orphan.warnings);
742
+ // #733: orphan-state GC — stamps/clears/(optionally) collects
743
+ // asset_salience/asset_outcome rows whose ref no longer resolves in
744
+ // index.db. Needs the SAME already-open index.db handle (dbCell.current)
745
+ // the passes above share.
746
+ const stateGc = runOrphanStateGcPass(ctx, dbCell);
747
+ allWarnings.push(...stateGc.warnings);
748
+ const expiration = runProposalExpirationPass(ctx);
749
+ allWarnings.push(...expiration.warnings);
750
+ allWarnings.push(...runRetentionPurgePass(ctx).warnings);
751
+ return {
752
+ memoryInference,
753
+ graphExtraction: graph.graphExtraction,
754
+ actions,
755
+ memoryInferenceDurationMs: inference.durationMs,
756
+ graphExtractionDurationMs: graph.durationMs,
757
+ orphansPurged: orphan.orphansPurged,
758
+ proposalsExpired: expiration.proposalsExpired,
759
+ };
760
+ }
761
+ finally {
762
+ if (dbCell.current)
763
+ closeDatabase(dbCell.current);
764
+ }
765
+ }
766
+ /**
767
+ * Memory inference candidate-discovery (post-Item 9 fix from
768
+ * memories/akm-improve-critical-review-2026-05-20). Previously this pass
769
+ * was gated on memoryRefsForInference.size > 0 AND passed those refs as a
770
+ * candidateRefs filter. But memoryRefsForInference is populated from refs
771
+ * distilled THIS RUN — by the time that happens, those parents are
772
+ * already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
773
+ * them. The genuinely-pending parents in the stash never entered the
774
+ * filter. Result: 0/0/0 for 25 consecutive runs.
775
+ *
776
+ * Fix: always run the pass when the feature is enabled; let the pass's
777
+ * own `collectPendingMemories` + `isPendingMemory` predicate find
778
+ * candidates from the filesystem-of-truth. The this-run set is still
779
+ * logged as a hint but no longer used as a filter.
780
+ */
781
+ export async function runMemoryInferenceMaintenancePass(ctx, dbCell, memoryRefsForInference) {
782
+ const { config, sources, primaryStashDir, budgetSignal, improveProfile, resolvedPlan, memoryInferenceFn } = ctx;
783
+ const warnings = [];
784
+ let memoryInference;
785
+ let durationMs = 0;
786
+ let action;
787
+ const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
788
+ const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
789
+ const pendingBelowMinCount = (() => {
790
+ if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
791
+ return false;
792
+ const pending = collectPendingMemories(primaryStashDir).length;
793
+ if (pending < minPendingCount) {
794
+ info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
795
+ return true;
796
+ }
797
+ return false;
798
+ })();
799
+ if (memoryInferenceDisabledByProfile) {
800
+ info("[improve] memory inference skipped (disabled by improve profile)");
801
+ }
802
+ else if (pendingBelowMinCount) {
803
+ // skipped — message already emitted above
804
+ }
805
+ else {
806
+ const hintRefs = memoryRefsForInference.size;
807
+ info(hintRefs > 0
808
+ ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
809
+ : "[improve] memory inference starting (discovering pending parents)");
810
+ const inferenceStart = Date.now();
811
+ try {
812
+ // O-1 (#364): pass budget signal so a hung inference call is cancelled.
813
+ memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
814
+ config,
815
+ ...(resolvedPlan
816
+ ? {
817
+ llmConfig: resolvedPlan.processes.memoryInference.runner
818
+ ? materializeLlmRunnerConnection(resolvedPlan.processes.memoryInference.runner)
819
+ : null,
820
+ }
821
+ : {}),
822
+ sources,
823
+ signal: budgetSignal,
824
+ db: dbCell.current,
825
+ reEnrich: false,
826
+ onProgress: (event) => {
827
+ const current = event.currentRef ? ` ${event.currentRef}` : "";
828
+ info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
829
+ },
830
+ }), { engine: resolvedPlan?.processes.memoryInference.runner?.engine, process: "memoryInference" });
831
+ durationMs = Date.now() - inferenceStart;
832
+ // Synthetic sentinel ref (ref-grammar decision D-R3): a colon-free
833
+ // `<domain>/_<marker>` label on the event row, never parsed as an asset
834
+ // ref. The domain is the asset stash-subdir for asset-scoped sentinels
835
+ // (`memories/…`) and the subsystem name for maintenance/artifact sentinels
836
+ // (`graph/…`, `events/…`, `proposals/…`, `health/…`, …). Readers match the
837
+ // event by `eventType`, never by this string.
838
+ action = { ref: "memories/_inference", mode: "memory-inference", result: memoryInference };
839
+ info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
840
+ }
841
+ catch (err) {
842
+ durationMs = Date.now() - inferenceStart;
843
+ warnings.push(`memory inference failed: ${errMessage(err)}`);
844
+ }
845
+ }
846
+ return { memoryInference, durationMs, action, warnings };
847
+ }
848
+ /**
849
+ * Graph-extraction maintenance pass.
850
+ *
851
+ * INVARIANT: graph extraction normally runs only on files touched by
852
+ * actionable refs (candidatePaths). Full-corpus scans are opt-in via
853
+ * profile.processes.graphExtraction.fullScan = true (used by the
854
+ * `graph-refresh` built-in profile and its weekly scheduled task).
855
+ * The empty-Set fallback is intentional when no refs were touched —
856
+ * the extractor's filter rejects every file and returns empty, keeping
857
+ * the pass invoked so the action is recorded and tests stay exercised.
858
+ */
859
+ export async function runGraphExtractionMaintenancePass(ctx, dbCell, args) {
860
+ const { config, sources, primaryStashDir, budgetSignal, improveProfile, resolvedPlan, graphExtractionFn } = ctx;
861
+ const warnings = [];
862
+ let graphExtraction;
863
+ let durationMs = 0;
864
+ let action;
865
+ let reindexedAfterInference = args.reindexedAfterInference;
866
+ const graphEnabled = resolvedPlan ? true : isProcessEnabled("index", "graph_extraction", config);
867
+ const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
868
+ const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
869
+ // #624 P2: optional incremental high-signal-first cap. Unset = process all
870
+ // eligible (byte-identical to today; no ranking/slice).
871
+ const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
872
+ const graphExtractionIncludeTypes = improveProfile?.processes?.graphExtraction?.includeTypes ?? [
873
+ ...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES,
874
+ ];
875
+ const graphExtractionBatchSize = improveProfile?.processes?.graphExtraction?.batchSize ?? DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE;
876
+ // Build the set of refs actually touched this run.
877
+ const touchedRefs = new Set();
878
+ for (const r of args.actionableRefs)
879
+ touchedRefs.add(r.ref);
880
+ for (const r of args.memoryRefsForInference)
881
+ touchedRefs.add(r);
882
+ if (graphExtractionDisabledByProfile) {
883
+ info("[improve] graph extraction skipped (disabled by improve profile)");
884
+ }
885
+ else if (sources.length > 0 && graphEnabled) {
886
+ info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
887
+ const extractionStart = Date.now();
888
+ try {
889
+ // D9: if consolidation ran but memory inference did not reindex, force a reindex
890
+ // so graph extraction sees current DB state after consolidation writes.
891
+ if (args.consolidationRan && !reindexedAfterInference) {
892
+ info("[improve] reindexing after consolidation (graph extraction needs current state)");
848
893
  try {
849
- await reindexWithIndexDbReleased(primaryStashDir);
894
+ await ctx.reindexWithIndexDbReleased(primaryStashDir);
850
895
  reindexedAfterInference = true;
851
- info("[improve] reindex after memory inference complete");
896
+ info("[improve] reindex after consolidation complete");
852
897
  }
853
898
  catch (err) {
854
- allWarnings.push(`reindex after memory inference failed: ${errMessage(err)}`);
899
+ warnings.push(`reindex after consolidation failed: ${errMessage(err)}`);
855
900
  }
856
901
  }
857
- const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
858
- const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
859
- const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
860
- // #624 P2: optional incremental high-signal-first cap. Unset = process all
861
- // eligible (byte-identical to today; no ranking/slice).
862
- const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
863
- // Build the set of refs actually touched this run.
864
- const touchedRefs = new Set();
865
- for (const r of args.actionableRefs)
866
- touchedRefs.add(r.ref);
867
- for (const r of memoryRefsForInference)
868
- touchedRefs.add(r);
869
- // INVARIANT: graph extraction normally runs only on files touched by
870
- // actionable refs (candidatePaths). Full-corpus scans are opt-in via
871
- // profile.processes.graphExtraction.fullScan = true (used by the
872
- // `graph-refresh` built-in profile and its weekly scheduled task).
873
- // The empty-Set fallback is intentional when no refs were touched —
874
- // the extractor's filter rejects every file and returns empty, keeping
875
- // the pass invoked so the action is recorded and tests stay exercised.
876
- if (graphExtractionDisabledByProfile) {
877
- info("[improve] graph extraction skipped (disabled by improve profile)");
878
- }
879
- else if (sources.length > 0 && graphEnabled) {
880
- info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
881
- const extractionStart = Date.now();
882
- try {
883
- // D9: if consolidation ran but memory inference did not reindex, force a reindex
884
- // so graph extraction sees current DB state after consolidation writes.
885
- if (consolidationRan && !reindexedAfterInference) {
886
- info("[improve] reindexing after consolidation (graph extraction needs current state)");
887
- try {
888
- await reindexWithIndexDbReleased(primaryStashDir);
889
- reindexedAfterInference = true;
890
- info("[improve] reindex after consolidation complete");
891
- }
892
- catch (err) {
893
- allWarnings.push(`reindex after consolidation failed: ${errMessage(err)}`);
894
- }
895
- }
896
- // #584: no close/reopen needed here — reindexWithIndexDbReleased
897
- // already swapped in a fresh post-reindex handle.
898
- // Resolve touched refs to absolute file paths. Skipped for fullScan
899
- // (candidatePaths stays undefined → extractor processes all files).
900
- let candidatePaths;
901
- if (!graphExtractionFullScan) {
902
- candidatePaths = new Set();
903
- if (primaryStashDir && touchedRefs.size > 0) {
904
- const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
905
- const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
906
- for (const p of resolved) {
907
- if (typeof p === "string" && p.length > 0)
908
- candidatePaths.add(p);
909
- }
910
- }
902
+ // #584: no close/reopen needed here — reindexWithIndexDbReleased
903
+ // already swapped in a fresh post-reindex handle.
904
+ // Resolve touched refs to absolute file paths. Skipped for fullScan
905
+ // (candidatePaths stays undefined → extractor processes all files).
906
+ let candidatePaths;
907
+ if (!graphExtractionFullScan) {
908
+ candidatePaths = new Set();
909
+ if (primaryStashDir && touchedRefs.size > 0) {
910
+ const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
911
+ const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
912
+ for (const p of resolved) {
913
+ if (typeof p === "string" && p.length > 0)
914
+ candidatePaths.add(p);
911
915
  }
912
- const progressHandler = (event) => {
913
- const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
914
- info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
915
- };
916
- // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
917
- graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
918
- config,
919
- sources,
920
- signal: budgetSignal,
921
- db,
922
- reEnrich: false,
923
- onProgress: progressHandler,
924
- options: { candidatePaths, ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}) },
925
- }));
926
- graphExtractionDurationMs = Date.now() - extractionStart;
927
- actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
928
- info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
929
916
  }
930
- catch (err) {
931
- graphExtractionDurationMs = Date.now() - extractionStart;
932
- allWarnings.push(`graph extraction failed: ${errMessage(err)}`);
933
- }
934
- }
935
- else if (sources.length > 0 && !graphEnabled) {
936
- info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
937
917
  }
938
- // Orphan proposal purge — reject pending reflect proposals whose target
939
- // asset no longer exists on disk. Runs after graph extraction so newly
940
- // promoted assets from accept flows during this run are already present.
941
- if (primaryStashDir) {
942
- try {
943
- const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
944
- orphansPurged = purgeResult.rejected;
945
- if (purgeResult.rejected > 0) {
946
- info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
918
+ const progressHandler = (event) => {
919
+ const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
920
+ info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
921
+ };
922
+ // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
923
+ graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
924
+ config,
925
+ ...(resolvedPlan
926
+ ? {
927
+ llmConfig: resolvedPlan.processes.graphExtraction.runner
928
+ ? materializeLlmRunnerConnection(resolvedPlan.processes.graphExtraction.runner)
929
+ : null,
947
930
  }
948
- appendEvent({
949
- eventType: "proposal_orphan_purge",
950
- ref: "proposals:_orphan-purge",
951
- metadata: {
952
- checked: purgeResult.checked,
953
- rejected: purgeResult.rejected,
954
- durationMs: purgeResult.durationMs,
955
- byType: purgeResult.byType,
956
- orphans: purgeResult.orphans.map((o) => o.ref),
957
- },
958
- }, eventsCtx);
931
+ : {}),
932
+ sources,
933
+ signal: budgetSignal,
934
+ db: dbCell.current,
935
+ reEnrich: false,
936
+ onProgress: progressHandler,
937
+ options: {
938
+ candidatePaths,
939
+ includeTypes: graphExtractionIncludeTypes,
940
+ batchSize: graphExtractionBatchSize,
941
+ ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}),
942
+ },
943
+ }), { engine: resolvedPlan?.processes.graphExtraction.runner?.engine, process: "graphExtraction" });
944
+ durationMs = Date.now() - extractionStart;
945
+ // Synthetic sentinel ref (D-R3): `graph` has no asset stash-subdir, so the
946
+ // colon-free `graph/_artifact` names the subsystem, per the sentinel
947
+ // convention documented at the memory-inference writer above.
948
+ action = { ref: "graph/_artifact", mode: "graph-extraction", result: graphExtraction };
949
+ info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
950
+ }
951
+ catch (err) {
952
+ durationMs = Date.now() - extractionStart;
953
+ warnings.push(`graph extraction failed: ${errMessage(err)}`);
954
+ }
955
+ }
956
+ else if (sources.length > 0 && !graphEnabled) {
957
+ info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
958
+ }
959
+ return { graphExtraction, durationMs, action, warnings };
960
+ }
961
+ /**
962
+ * Orphan proposal purge — reject pending reflect proposals whose target
963
+ * asset no longer exists on disk. Runs after graph extraction so newly
964
+ * promoted assets from accept flows during this run are already present.
965
+ */
966
+ function runOrphanProposalPurgePass(ctx) {
967
+ const { primaryStashDir, sources, eventsCtx } = ctx;
968
+ const warnings = [];
969
+ let orphansPurged = 0;
970
+ try {
971
+ const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
972
+ orphansPurged = purgeResult.rejected;
973
+ if (purgeResult.rejected > 0) {
974
+ info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
975
+ }
976
+ appendEvent({
977
+ eventType: "proposal_orphan_purge",
978
+ ref: "proposals/_orphan-purge",
979
+ metadata: {
980
+ checked: purgeResult.checked,
981
+ rejected: purgeResult.rejected,
982
+ durationMs: purgeResult.durationMs,
983
+ byType: purgeResult.byType,
984
+ orphans: purgeResult.orphans.map((o) => o.ref),
985
+ },
986
+ }, eventsCtx);
987
+ }
988
+ catch (err) {
989
+ warnings.push(`orphan purge failed: ${errMessage(err)}`);
990
+ }
991
+ return { orphansPurged, warnings };
992
+ }
993
+ /**
994
+ * Phase 6B (Advantage D6b): expire pending proposals that have aged past
995
+ * the retention window. Runs AFTER orphan purge so we never double-archive
996
+ * a proposal that orphan-purge already moved. `expireStaleProposals` emits
997
+ * its own per-proposal `proposal_expired` events; we additionally emit a
998
+ * single roll-up event here for parity with the orphan-purge surface.
999
+ */
1000
+ function runProposalExpirationPass(ctx) {
1001
+ const { primaryStashDir, config, eventsCtx } = ctx;
1002
+ const warnings = [];
1003
+ let proposalsExpired = 0;
1004
+ try {
1005
+ const expireResult = expireStaleProposals(primaryStashDir, config);
1006
+ proposalsExpired = expireResult.expired;
1007
+ if (expireResult.expired > 0) {
1008
+ info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
1009
+ `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
1010
+ }
1011
+ appendEvent({
1012
+ eventType: "proposal_expiration_pass",
1013
+ ref: "proposals/_expiration",
1014
+ metadata: {
1015
+ checked: expireResult.checked,
1016
+ expired: expireResult.expired,
1017
+ durationMs: expireResult.durationMs,
1018
+ retentionDays: expireResult.retentionDays,
1019
+ expiredProposals: expireResult.expiredProposals,
1020
+ },
1021
+ }, eventsCtx);
1022
+ }
1023
+ catch (err) {
1024
+ warnings.push(`proposal expiration failed: ${errMessage(err)}`);
1025
+ }
1026
+ return { proposalsExpired, warnings };
1027
+ }
1028
+ /**
1029
+ * Fix #2 (observability 0.8.0): trim the events table in state.db so it
1030
+ * doesn't grow unbounded. `akm health` writes a `health_probe` row on every
1031
+ * invocation, and every command surface emits at least one event besides —
1032
+ * without this trim, state.db is a permanent append-only log. Config key
1033
+ * `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
1034
+ * window. The purge runs against state.db (a different SQLite file from
1035
+ * the index handle the other passes use).
1036
+ */
1037
+ export function runRetentionPurgePass(ctx) {
1038
+ const { config, eventsCtx } = ctx;
1039
+ const warnings = [];
1040
+ const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
1041
+ if (retentionDays > 0) {
1042
+ // #585: reuse the long-lived eventsCtx.db connection when akmImprove
1043
+ // opened one — opening a second state.db write connection while
1044
+ // eventsDb is still live made two simultaneous writers contend on the
1045
+ // same WAL file ("database is locked"). Only the eventsCtx.dbPath
1046
+ // fallback path (state.db failed to open up-front) opens — and then
1047
+ // owns and closes — its own handle. C2 still holds: the fallback uses
1048
+ // the boundary-pinned path, never a live `process.env` re-read.
1049
+ try {
1050
+ withStateDb((stateDb) => {
1051
+ const purgedCount = purgeOldEvents(stateDb, retentionDays);
1052
+ if (purgedCount > 0) {
1053
+ info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
959
1054
  }
960
- catch (err) {
961
- allWarnings.push(`orphan purge failed: ${errMessage(err)}`);
1055
+ appendEvent({
1056
+ eventType: "events_purged",
1057
+ ref: "events/_purge",
1058
+ metadata: { purgedCount, retentionDays },
1059
+ }, eventsCtx);
1060
+ // improve_runs uses the same retention window as events — both are
1061
+ // observability/audit data, both grow append-only, both have a
1062
+ // dedicated purge helper. Mirroring the events purge here means a
1063
+ // single retention knob (improve.eventRetentionDays) governs both.
1064
+ const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
1065
+ if (improveRunsPurged > 0) {
1066
+ info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
962
1067
  }
963
- // Phase 6B (Advantage D6b): expire pending proposals that have aged past
964
- // the retention window. Runs AFTER orphan purge so we never double-archive
965
- // a proposal that orphan-purge already moved. `expireStaleProposals` emits
966
- // its own per-proposal `proposal_expired` events; we additionally emit a
967
- // single roll-up event here for parity with the orphan-purge surface.
968
- try {
969
- const expireResult = expireStaleProposals(primaryStashDir, config);
970
- proposalsExpired = expireResult.expired;
971
- if (expireResult.expired > 0) {
972
- info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
973
- `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
974
- }
1068
+ appendEvent({
1069
+ eventType: "improve_runs_purged",
1070
+ ref: "improve_runs/_purge",
1071
+ metadata: { purgedCount: improveRunsPurged, retentionDays },
1072
+ }, eventsCtx);
1073
+ // R5: improve_cycle_metrics has its OWN retention window
1074
+ // (default 365d — a slow collapse needs a longer trend than
1075
+ // the 90d events window). canary_queries rows are never purged.
1076
+ const cycleRetention = config.improve?.collapseDetector?.retentionDays ?? CYCLE_METRICS_RETENTION_DAYS;
1077
+ const cycleMetricsPurged = purgeOldCycleMetrics(stateDb, cycleRetention);
1078
+ if (cycleMetricsPurged > 0) {
1079
+ info(`[improve] cycle-metrics purge: ${cycleMetricsPurged} row(s) older than ${cycleRetention}d removed from state.db`);
975
1080
  appendEvent({
976
- eventType: "proposal_expiration_pass",
977
- ref: "proposals:_expiration",
978
- metadata: {
979
- checked: expireResult.checked,
980
- expired: expireResult.expired,
981
- durationMs: expireResult.durationMs,
982
- retentionDays: expireResult.retentionDays,
983
- expiredProposals: expireResult.expiredProposals,
984
- },
1081
+ // Dedicated type (mirrors improve_runs_purged) so consumers
1082
+ // never have to disambiguate purge targets via the ref string.
1083
+ eventType: "improve_cycle_metrics_purged",
1084
+ ref: "improve_cycle_metrics/_purge",
1085
+ metadata: { purgedCount: cycleMetricsPurged, retentionDays: cycleRetention },
985
1086
  }, eventsCtx);
986
1087
  }
987
- catch (err) {
988
- allWarnings.push(`proposal expiration failed: ${errMessage(err)}`);
989
- }
990
- }
991
- // Fix #2 (observability 0.8.0): trim the events table in state.db so it
992
- // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
993
- // invocation, and every command surface emits at least one event besides —
994
- // without this trim, state.db is a permanent append-only log. Config key
995
- // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
996
- // window. The purge runs against state.db (a different SQLite file from
997
- // the index `db` above).
998
- {
999
- const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
1000
- if (retentionDays > 0) {
1001
- // #585: reuse the long-lived eventsCtx.db connection when akmImprove
1002
- // opened one — opening a second state.db write connection while
1003
- // eventsDb is still live made two simultaneous writers contend on the
1004
- // same WAL file ("database is locked"). Only the eventsCtx.dbPath
1005
- // fallback path (state.db failed to open up-front) opens — and then
1006
- // owns and closes — its own handle. C2 still holds: the fallback uses
1007
- // the boundary-pinned path, never a live `process.env` re-read.
1008
- try {
1009
- withStateDb((stateDb) => {
1010
- const purgedCount = purgeOldEvents(stateDb, retentionDays);
1011
- if (purgedCount > 0) {
1012
- info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
1013
- }
1014
- appendEvent({
1015
- eventType: "events_purged",
1016
- ref: "events:_purge",
1017
- metadata: { purgedCount, retentionDays },
1018
- }, eventsCtx);
1019
- // improve_runs uses the same retention window as events — both are
1020
- // observability/audit data, both grow append-only, both have a
1021
- // dedicated purge helper. Mirroring the events purge here means a
1022
- // single retention knob (improve.eventRetentionDays) governs both.
1023
- const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
1024
- if (improveRunsPurged > 0) {
1025
- info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
1026
- }
1027
- appendEvent({
1028
- eventType: "improve_runs_purged",
1029
- ref: "improve_runs:_purge",
1030
- metadata: { purgedCount: improveRunsPurged, retentionDays },
1031
- }, eventsCtx);
1032
- // R5: improve_cycle_metrics has its OWN retention window
1033
- // (default 365d — a slow collapse needs a longer trend than
1034
- // the 90d events window). canary_queries rows are never purged.
1035
- const cycleRetention = config.improve?.collapseDetector?.retentionDays ?? CYCLE_METRICS_RETENTION_DAYS;
1036
- const cycleMetricsPurged = purgeOldCycleMetrics(stateDb, cycleRetention);
1037
- if (cycleMetricsPurged > 0) {
1038
- info(`[improve] cycle-metrics purge: ${cycleMetricsPurged} row(s) older than ${cycleRetention}d removed from state.db`);
1039
- appendEvent({
1040
- // Dedicated type (mirrors improve_runs_purged) so consumers
1041
- // never have to disambiguate purge targets via the ref string.
1042
- eventType: "improve_cycle_metrics_purged",
1043
- ref: "improve_cycle_metrics:_purge",
1044
- metadata: { purgedCount: cycleMetricsPurged, retentionDays: cycleRetention },
1045
- }, eventsCtx);
1046
- }
1047
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1048
- }
1049
- catch (err) {
1050
- allWarnings.push(`events purge failed: ${errMessage(err)}`);
1051
- }
1052
- // task_logs in logs.db (#579) shares the same retention window as
1053
- // events/improve_runs — all three are observability data governed by
1054
- // the single improve.eventRetentionDays knob. Separate try/finally
1055
- // because logs.db is a different file: a locked/missing logs.db must
1056
- // not block the state.db purges above.
1057
- let logsDb;
1058
- try {
1059
- logsDb = openLogsDatabase();
1060
- const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
1061
- if (taskLogsPurged > 0) {
1062
- info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
1063
- }
1064
- appendEvent({
1065
- eventType: "task_logs_purged",
1066
- ref: "task_logs:_purge",
1067
- metadata: { purgedCount: taskLogsPurged, retentionDays },
1068
- }, eventsCtx);
1069
- }
1070
- catch (err) {
1071
- allWarnings.push(`task_logs purge failed: ${errMessage(err)}`);
1072
- }
1073
- finally {
1074
- if (logsDb) {
1075
- try {
1076
- logsDb.close();
1077
- }
1078
- catch {
1079
- // best-effort
1080
- }
1081
- }
1082
- }
1083
- }
1088
+ }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1089
+ }
1090
+ catch (err) {
1091
+ warnings.push(`events purge failed: ${errMessage(err)}`);
1092
+ }
1093
+ // task_logs in logs.db (#579) shares the same retention window as
1094
+ // events/improve_runs — all three are observability data governed by
1095
+ // the single improve.eventRetentionDays knob. Separate try/finally
1096
+ // because logs.db is a different file: a locked/missing logs.db must
1097
+ // not block the state.db purges above.
1098
+ let logsDb;
1099
+ try {
1100
+ logsDb = openLogsDatabase();
1101
+ const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
1102
+ if (taskLogsPurged > 0) {
1103
+ info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
1084
1104
  }
1105
+ appendEvent({
1106
+ eventType: "task_logs_purged",
1107
+ ref: "task_logs/_purge",
1108
+ metadata: { purgedCount: taskLogsPurged, retentionDays },
1109
+ }, eventsCtx);
1110
+ }
1111
+ catch (err) {
1112
+ warnings.push(`task_logs purge failed: ${errMessage(err)}`);
1085
1113
  }
1086
1114
  finally {
1087
- if (db)
1088
- closeDatabase(db);
1115
+ if (logsDb) {
1116
+ try {
1117
+ logsDb.close();
1118
+ }
1119
+ catch {
1120
+ // best-effort
1121
+ }
1122
+ }
1089
1123
  }
1090
- });
1091
- return {
1092
- ...(memoryInference ? { memoryInference } : {}),
1093
- ...(graphExtraction ? { graphExtraction } : {}),
1094
- ...(actions.length > 0 ? { actions } : {}),
1095
- memoryInferenceDurationMs,
1096
- graphExtractionDurationMs,
1097
- orphansPurged,
1098
- proposalsExpired,
1099
- };
1124
+ }
1125
+ return { warnings };
1126
+ }
1127
+ // ── #733 — orphan-state GC pass (Workstream C) ──────────────────────────────
1128
+ //
1129
+ // Deliberately lean: one maintenance pass, one additive migration (021), one
1130
+ // event type (asset_state_gc), one config gate (improve.stateGc.collect,
1131
+ // default false). See docs/architecture/specs/0.9.0-close-out-plan.md
1132
+ // Workstream C for the full design rationale. No quarantine archive, no
1133
+ // circuit breaker, no health-advisory plumbing, no new tables.
1134
+ /**
1135
+ * Grace window (ms) before an unresolved `asset_salience` / `asset_outcome`
1136
+ * row becomes delete-eligible — only when `improve.stateGc.collect` is true.
1137
+ * A named constant, not a config knob (owner ruling — see the close-out
1138
+ * plan's Workstream C). Mirrors `TXN_SWEEP_GRACE_MS` (src/core/fs-txn.ts:298).
1139
+ */
1140
+ export const STATE_GC_GRACE_MS = daysToMs(7);
1141
+ /**
1142
+ * Resolve one state-table's stored `asset_ref` against the live index.
1143
+ *
1144
+ * "ref not present in entries.item_ref" is the authoritative-deletion
1145
+ * predicate (see the pass doc comment below), so this is a thin wrapper
1146
+ * around the same single-ref probe the rest of improve uses
1147
+ * (`getEntryByRef`, index-entries-repository.ts) — which already resolves
1148
+ * both storage spellings a write can produce (`salienceWriteKey`/
1149
+ * `outcomeWriteKey` = `itemRef ?? ref`): an exact bundle-qualified item_ref,
1150
+ * or a bare conceptId matched by suffix across all bundles.
1151
+ *
1152
+ * On top of that, falls back to the BARE conceptId form (`bareImproveRef` —
1153
+ * the same primitive `preparation.ts`'s `normalizeStoredKey` map is built
1154
+ * from via `improveStateReadRefs`) when the stored ref carries a bundle
1155
+ * prefix that no longer matches exactly. This is the legacy-spelling
1156
+ * normalization trap: a naive `asset_ref NOT IN (SELECT item_ref FROM
1157
+ * entries)` would treat a live asset whose row predates bundle-qualification
1158
+ * (or whose bundle prefix is stale) as an orphan and delete it. Preferring
1159
+ * "never delete a live row" over "never miss a genuinely dead one" mirrors
1160
+ * `getEntryByRef`'s own bare-conceptId suffix-match trade-off.
1161
+ */
1162
+ function isStateRefLive(indexDb, storedRef) {
1163
+ if (getEntryByRef(indexDb, storedRef) !== null)
1164
+ return true;
1165
+ const bare = bareImproveRef(storedRef);
1166
+ return bare !== storedRef && getEntryByRef(indexDb, bare) !== null;
1167
+ }
1168
+ /**
1169
+ * Sweep ONE state table: stamp refs that just went unresolved, clear refs
1170
+ * that resolved again, and — only when `collect` is true — delete rows whose
1171
+ * `missing_since` is older than {@link STATE_GC_GRACE_MS}. `pending` is a
1172
+ * point-in-time snapshot taken AFTER stamp/clear/delete (re-queried, not
1173
+ * accumulated), so it reflects the current backlog rather than this run's
1174
+ * delta — "every run emits the counts either way, so live data accumulates
1175
+ * proof" (close-out plan, Workstream C).
1176
+ */
1177
+ function gcOneStateTable(args) {
1178
+ const { refRows, indexDb, now, collect, stamp, clear, deleteOlderThan, countPending } = args;
1179
+ const toStamp = [];
1180
+ const toClear = [];
1181
+ for (const row of refRows) {
1182
+ const live = isStateRefLive(indexDb, row.asset_ref);
1183
+ if (!live && row.missing_since == null)
1184
+ toStamp.push(row.asset_ref);
1185
+ else if (live && row.missing_since != null)
1186
+ toClear.push(row.asset_ref);
1187
+ }
1188
+ if (toStamp.length > 0)
1189
+ stamp(toStamp, now);
1190
+ if (toClear.length > 0)
1191
+ clear(toClear);
1192
+ const collected = collect ? deleteOlderThan(now - STATE_GC_GRACE_MS) : 0;
1193
+ const pending = countPending();
1194
+ return { pending, collected };
1195
+ }
1196
+ /**
1197
+ * Orphan-state GC — #733 (Workstream C). For each of the two per-asset state
1198
+ * tables (`asset_salience`, `asset_outcome`), stamps `missing_since` on refs
1199
+ * that no longer resolve against `entries.item_ref` in index.db, clears the
1200
+ * stamp on refs that resolve again, and — only when `improve.stateGc.collect`
1201
+ * is true — deletes rows whose stamp is older than {@link STATE_GC_GRACE_MS}.
1202
+ *
1203
+ * "ref not present in entries.item_ref" IS the authoritative-deletion
1204
+ * predicate: "absent ≠ deleted" is inherited from the indexer, not
1205
+ * re-implemented here — an incomplete or failed source scan preserves that
1206
+ * source's last-known-good `entries` rows (indexer.ts ~1195-1199), and
1207
+ * mass-wipe is already gated upstream (`preserveExistingIndex` +
1208
+ * `fullDelete && scanComplete`). A temporarily unreachable source therefore
1209
+ * never surfaces candidates; no separate scan-status tracking is needed.
1210
+ *
1211
+ * Runs under the SAME index-writer lease / borrowed-state.db-connection
1212
+ * discipline as the neighboring maintenance passes: `dbCell.current` supplies
1213
+ * the already-open index.db handle (#584), and state.db access goes through
1214
+ * `withStateDb(..., { borrowed: eventsCtx?.db })` so this never opens a
1215
+ * second live writer alongside a long-lived `eventsCtx.db` connection — see
1216
+ * the #585 comment on {@link runRetentionPurgePass}'s events-purge call for
1217
+ * why that matters ("database is locked").
1218
+ *
1219
+ * Exported for direct test coverage (tests/integration/commands/improve/
1220
+ * state-gc.test.ts), mirroring the `runMemoryInferenceMaintenancePass` /
1221
+ * `runGraphExtractionMaintenancePass` / `runRetentionPurgePass` precedent;
1222
+ * production callers reach it only through `runMaintenancePassesUnderLease`.
1223
+ */
1224
+ export function runOrphanStateGcPass(ctx, dbCell) {
1225
+ const { eventsCtx, config } = ctx;
1226
+ const warnings = [];
1227
+ const indexDb = dbCell.current;
1228
+ if (!indexDb) {
1229
+ warnings.push("orphan state GC skipped: no index.db handle available");
1230
+ return { pending: 0, collected: 0, warnings };
1231
+ }
1232
+ const collect = config.improve?.stateGc?.collect === true;
1233
+ const now = Date.now();
1234
+ let pending = 0;
1235
+ let collected = 0;
1236
+ try {
1237
+ withStateDb((stateDb) => {
1238
+ const salienceResult = gcOneStateTable({
1239
+ refRows: listAssetSalienceMissingState(stateDb),
1240
+ indexDb,
1241
+ now,
1242
+ collect,
1243
+ stamp: (refs, ts) => stampAssetSalienceMissing(stateDb, refs, ts),
1244
+ clear: (refs) => clearAssetSalienceMissing(stateDb, refs),
1245
+ deleteOlderThan: (cutoffMs) => deleteAssetSalienceMissingBefore(stateDb, cutoffMs),
1246
+ countPending: () => countAssetSalienceMissing(stateDb),
1247
+ });
1248
+ const outcomeResult = gcOneStateTable({
1249
+ refRows: listAssetOutcomeMissingState(stateDb),
1250
+ indexDb,
1251
+ now,
1252
+ collect,
1253
+ stamp: (refs, ts) => stampAssetOutcomeMissing(stateDb, refs, ts),
1254
+ clear: (refs) => clearAssetOutcomeMissing(stateDb, refs),
1255
+ deleteOlderThan: (cutoffMs) => deleteAssetOutcomeMissingBefore(stateDb, cutoffMs),
1256
+ countPending: () => countAssetOutcomeMissing(stateDb),
1257
+ });
1258
+ // Table-name-shaped keys ("salience"/"outcome", not the guarded
1259
+ // "asset_salience"/"asset_outcome" table names) — this file sits
1260
+ // outside src/storage/repositories/**, where the state-table-sql
1261
+ // lint rule (#672) forbids naming those tables even in a log string
1262
+ // or an object key, not just in raw SQL.
1263
+ const byTable = { salience: salienceResult, outcome: outcomeResult };
1264
+ pending = salienceResult.pending + outcomeResult.pending;
1265
+ collected = salienceResult.collected + outcomeResult.collected;
1266
+ if (pending > 0 || collected > 0) {
1267
+ info(`[improve] orphan state GC: ${pending} pending, ${collected} collected ` +
1268
+ `(salience ${salienceResult.pending}/${salienceResult.collected}, ` +
1269
+ `outcome ${outcomeResult.pending}/${outcomeResult.collected})`);
1270
+ // #733 — asset_state_gc reports the current per-table backlog
1271
+ // snapshot (`pending`) plus this run's deletions (`collected`);
1272
+ // emitted only when there is something to report (mirrors the
1273
+ // rekey script's no-op-stays-silent precedent) so a perpetually
1274
+ // clean stash never accumulates events.
1275
+ appendEvent({
1276
+ eventType: "asset_state_gc",
1277
+ ref: "asset_state/_gc",
1278
+ metadata: { pending, collected, byTable },
1279
+ }, eventsCtx);
1280
+ }
1281
+ }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1282
+ }
1283
+ catch (err) {
1284
+ warnings.push(`orphan state GC failed: ${errMessage(err)}`);
1285
+ }
1286
+ return { pending, collected, warnings };
1100
1287
  }