akm-cli 0.9.16 → 0.9.17-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (403) hide show
  1. package/CHANGELOG.md +2101 -0
  2. package/STABILITY.md +11 -10
  3. package/dist/akm +124 -193
  4. package/dist/akm-migrate +38 -19
  5. package/dist/assets/hints/cli-hints-full.md +6 -7
  6. package/dist/assets/improve-strategies/catchup.json +0 -3
  7. package/dist/assets/improve-strategies/consolidate.json +0 -1
  8. package/dist/assets/improve-strategies/default.json +1 -2
  9. package/dist/assets/improve-strategies/proactive-maintenance.json +1 -2
  10. package/dist/assets/improve-strategies/quick.json +1 -2
  11. package/dist/assets/improve-strategies/reflect-distill.json +1 -2
  12. package/dist/assets/improve-strategies/thorough.json +0 -3
  13. package/dist/assets/prompts/consolidate-pair.md +20 -0
  14. package/dist/assets/prompts/consolidate-system.md +4 -11
  15. package/dist/assets/prompts/retrieval-relevance-judge.md +6 -0
  16. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +20 -20
  17. package/dist/assets/stash-skeleton/facts/conventions/domains.md +2 -2
  18. package/dist/assets/templates/html/health.html +3 -5
  19. package/dist/cli/retired-commands.js +1 -1
  20. package/dist/cli/shared.js +6 -2
  21. package/dist/cli/unknown-flags.js +24 -1
  22. package/dist/cli.js +68 -10
  23. package/dist/commands/agent/agent-dispatch.js +1 -1
  24. package/dist/commands/command/command-execution.js +24 -62
  25. package/dist/commands/feedback-cli.js +0 -1
  26. package/dist/commands/health/accept-rate.js +6 -0
  27. package/dist/commands/health/archive-usage.js +92 -0
  28. package/dist/commands/health/checks.js +83 -74
  29. package/dist/commands/health/config-skew.js +38 -0
  30. package/dist/commands/health/data-dir-usage.js +25 -13
  31. package/dist/commands/health/egress.js +54 -0
  32. package/dist/commands/health/html-report.js +1 -42
  33. package/dist/commands/health/improve-metrics.js +136 -591
  34. package/dist/commands/health/md-report.js +1 -6
  35. package/dist/commands/health/plugin-staleness.js +53 -3
  36. package/dist/commands/health/renderers.js +12 -4
  37. package/dist/commands/health/report-view-model.js +14 -120
  38. package/dist/commands/health/types-improve.js +4 -19
  39. package/dist/commands/health/windows.js +64 -74
  40. package/dist/commands/health.js +145 -143
  41. package/dist/commands/improve/consolidate/chunking.js +26 -117
  42. package/dist/commands/improve/consolidate/continuity-check.js +137 -0
  43. package/dist/commands/improve/consolidate/pair-pass.js +791 -0
  44. package/dist/commands/improve/consolidate/sanitize.js +54 -149
  45. package/dist/commands/improve/consolidate.js +589 -1127
  46. package/dist/commands/improve/content-hash.js +16 -24
  47. package/dist/commands/improve/distill/content-repair.js +18 -100
  48. package/dist/commands/improve/distill-guards.js +20 -81
  49. package/dist/commands/improve/distill-promotion-policy.js +23 -243
  50. package/dist/commands/improve/distill.js +608 -1041
  51. package/dist/commands/improve/eligibility.js +126 -390
  52. package/dist/commands/improve/execution.js +8 -10
  53. package/dist/commands/improve/extract-prompt.js +1 -2
  54. package/dist/commands/improve/extract.js +487 -1046
  55. package/dist/commands/improve/feedback-valence.js +0 -25
  56. package/dist/commands/improve/improve-cli.js +75 -169
  57. package/dist/commands/improve/improve-result-file.js +10 -66
  58. package/dist/commands/improve/improve-strategies.js +52 -4
  59. package/dist/commands/improve/improve-usage-report.js +18 -64
  60. package/dist/commands/improve/improve.js +480 -1074
  61. package/dist/commands/improve/ledger.js +119 -0
  62. package/dist/commands/improve/locks.js +2 -8
  63. package/dist/commands/improve/loop-stages.js +415 -1073
  64. package/dist/commands/improve/memory/derived-ref.js +12 -77
  65. package/dist/commands/improve/memory/memory-belief.js +16 -118
  66. package/dist/commands/improve/memory/memory-improve.js +266 -14
  67. package/dist/commands/improve/outcome-loop.js +28 -156
  68. package/dist/commands/improve/planner.js +5 -15
  69. package/dist/commands/improve/preparation.js +779 -2319
  70. package/dist/commands/improve/proactive-maintenance.js +34 -101
  71. package/dist/commands/improve/reflect-noise.js +104 -280
  72. package/dist/commands/improve/reflect.js +642 -1353
  73. package/dist/commands/improve/retrieval-gate.js +127 -0
  74. package/dist/commands/improve/retrieval-scope.js +92 -0
  75. package/dist/commands/improve/salience.js +41 -240
  76. package/dist/commands/improve/session-asset.js +19 -100
  77. package/dist/commands/improve/stage.js +322 -0
  78. package/dist/commands/lint/base-linter.js +37 -15
  79. package/dist/commands/proposal/drain.js +261 -578
  80. package/dist/commands/proposal/proposal-cli.js +19 -20
  81. package/dist/commands/proposal/proposal-types.js +31 -24
  82. package/dist/commands/proposal/proposal.js +38 -8
  83. package/dist/commands/proposal/propose.js +134 -160
  84. package/dist/commands/proposal/repository.js +1097 -1394
  85. package/dist/commands/proposal/validators/proposal-quality-validators.js +71 -174
  86. package/dist/commands/proposal/validators/proposal-validators.js +1 -1
  87. package/dist/commands/proposal/validators/proposals.js +22 -89
  88. package/dist/commands/read/curate.js +105 -462
  89. package/dist/commands/read/knowledge.js +3 -2
  90. package/dist/commands/read/search-cli.js +16 -33
  91. package/dist/commands/read/search.js +17 -23
  92. package/dist/commands/read/show.js +57 -108
  93. package/dist/commands/sources/bundle-cli.js +25 -2
  94. package/dist/commands/sources/bundle-config-ops.js +4 -0
  95. package/dist/commands/sources/dangerous-env-audit.js +1 -2
  96. package/dist/commands/sources/info.js +127 -29
  97. package/dist/commands/sources/installed-stashes.js +197 -746
  98. package/dist/commands/sources/schema-repair.js +98 -129
  99. package/dist/commands/sources/source-add.js +62 -12
  100. package/dist/commands/sources/source-manage.js +9 -2
  101. package/dist/commands/sources/stash-cli.js +24 -4
  102. package/dist/commands/tasks/explain.js +10 -13
  103. package/dist/commands/tasks/tasks-cli.js +12 -13
  104. package/dist/commands/tasks/tasks.js +350 -936
  105. package/dist/commands/tasks/validate.js +26 -24
  106. package/dist/commands/workflow/plan.js +22 -29
  107. package/dist/commands/workflow-cli.js +4 -4
  108. package/dist/core/adapter/adapters/akm-adapter.js +2 -1
  109. package/dist/core/adapter/adapters/akm-lint.js +2 -3
  110. package/dist/core/adapter/adapters/akm-metadata.js +42 -12
  111. package/dist/core/adapter/adapters/akm-task-adapter.js +29 -8
  112. package/dist/core/adapter/adapters/akm-workflow-adapter.js +1 -1
  113. package/dist/core/adapter/execution-source.js +17 -29
  114. package/dist/core/asset/asset-placement.js +4 -13
  115. package/dist/core/asset/frontmatter.js +106 -1
  116. package/dist/core/asset/resolve-ref.js +1 -1
  117. package/dist/core/bundle-id.js +42 -5
  118. package/dist/core/bundle-rename.js +285 -0
  119. package/dist/core/config/config-io.js +1 -2
  120. package/dist/core/config/config-schema.js +9 -34
  121. package/dist/core/config/config-walker.js +1 -1
  122. package/dist/core/config/config.js +184 -111
  123. package/dist/core/config/engine-semantics.js +0 -2
  124. package/dist/core/config/legacy-source-shape-shim.js +38 -9
  125. package/dist/core/config/schema/embedding.js +20 -5
  126. package/dist/core/config/schema/engines.js +5 -0
  127. package/dist/core/config/schema/execution.js +1 -1
  128. package/dist/core/config/schema/experimental.js +1 -1
  129. package/dist/core/config/schema/improve-processes.js +54 -125
  130. package/dist/core/config/schema/improve.js +4 -42
  131. package/dist/core/config/schema/index-config.js +9 -48
  132. package/dist/core/config/schema/scheduler.js +12 -12
  133. package/dist/core/config/schema/search.js +6 -22
  134. package/dist/core/env-secret-ref.js +0 -1
  135. package/dist/core/errors.js +8 -9
  136. package/dist/core/file-change.js +13 -5
  137. package/dist/core/file-lock.js +76 -173
  138. package/dist/core/improve-result.js +35 -7
  139. package/dist/core/improve-types.js +0 -1
  140. package/dist/core/logs-db.js +2 -2
  141. package/dist/core/loopback.js +7 -12
  142. package/dist/core/non-task-input.js +20 -0
  143. package/dist/core/parse.js +13 -16
  144. package/dist/core/paths.js +0 -24
  145. package/dist/core/redaction.js +109 -2
  146. package/dist/core/run-lock.js +2 -5
  147. package/dist/core/spawn-env.js +1 -1
  148. package/dist/core/state/migrations.js +123 -61
  149. package/dist/core/state-db-scope.js +2 -4
  150. package/dist/core/state-db.js +126 -692
  151. package/dist/core/time.js +0 -20
  152. package/dist/core/type-presentation.js +1 -9
  153. package/dist/core/write-source.js +294 -1005
  154. package/dist/execution/input-contract.js +1 -1
  155. package/dist/execution/resolved-request.js +135 -689
  156. package/dist/execution/source.js +63 -257
  157. package/dist/execution/target-ref.js +1 -1
  158. package/dist/indexer/bundle-identity-guard.js +2 -2
  159. package/dist/indexer/db/llm-cache.js +2 -2
  160. package/dist/indexer/ensure-index.js +77 -73
  161. package/dist/indexer/index-rebuild-lock.js +3 -11
  162. package/dist/indexer/index-writer-lock.js +8 -17
  163. package/dist/indexer/index-written-assets.js +141 -154
  164. package/dist/indexer/indexer.js +400 -1124
  165. package/dist/indexer/links/declared-links.js +90 -0
  166. package/dist/indexer/materialize-embeddings.js +60 -397
  167. package/dist/indexer/passes/memory-inference.js +96 -90
  168. package/dist/indexer/passes/metadata.js +132 -219
  169. package/dist/indexer/read-preflight.js +0 -7
  170. package/dist/indexer/scan/doc-to-entry.js +2 -3
  171. package/dist/indexer/scan/drain-dir.js +1 -1
  172. package/dist/indexer/search/db-search.js +190 -590
  173. package/dist/indexer/search/fts-query.js +30 -41
  174. package/dist/indexer/search/ranking.js +28 -154
  175. package/dist/indexer/search/search-attribution.js +12 -32
  176. package/dist/indexer/search/search-fields.js +11 -15
  177. package/dist/indexer/search/search-hit-enrichers.js +54 -85
  178. package/dist/indexer/search/search-source.js +1 -4
  179. package/dist/indexer/usage/usage-events.js +36 -7
  180. package/dist/indexer/walk/walker.js +3 -4
  181. package/dist/integrations/agent/engine-fallback.js +23 -40
  182. package/dist/integrations/agent/engine-resolution.js +93 -183
  183. package/dist/integrations/agent/execution.js +507 -0
  184. package/dist/integrations/agent/model-map.js +28 -156
  185. package/dist/integrations/agent/request-lowering.js +66 -141
  186. package/dist/integrations/agent/runner-dispatch.js +143 -321
  187. package/dist/integrations/agent/runner.js +54 -14
  188. package/dist/integrations/lockfile.js +53 -101
  189. package/dist/llm/client.js +18 -6
  190. package/dist/llm/embedders/deterministic.js +2 -3
  191. package/dist/llm/embedders/profile.js +71 -0
  192. package/dist/llm/embedders/remote.js +11 -17
  193. package/dist/llm/feature-gate.js +0 -8
  194. package/dist/llm/index-passes.js +3 -5
  195. package/dist/llm/memory-infer.js +1 -2
  196. package/dist/llm/structured-call.js +5 -24
  197. package/dist/output/generic-render.js +23 -11
  198. package/dist/output/html-render.js +13 -10
  199. package/dist/output/render-registry.js +3 -32
  200. package/dist/output/shapes/helpers.js +25 -38
  201. package/dist/output/shapes/passthrough.js +1 -9
  202. package/dist/{indexer/graph/graph-types.js → output/text/bundle-rename.js} +4 -1
  203. package/dist/output/text/command-format.js +69 -31
  204. package/dist/output/text/helpers.js +1 -1
  205. package/dist/output/text/migrate.js +5 -14
  206. package/dist/output/text/proposal-format.js +48 -3
  207. package/dist/output/text/show-format.js +13 -17
  208. package/dist/output/text/workflow-format.js +0 -32
  209. package/dist/output/text.js +2 -0
  210. package/dist/registry/factory.js +4 -19
  211. package/dist/registry/network.js +66 -220
  212. package/dist/registry/providers/index.js +0 -2
  213. package/dist/registry/providers/skills-sh.js +3 -14
  214. package/dist/registry/providers/static-index.js +24 -26
  215. package/dist/registry/resolve.js +55 -131
  216. package/dist/scripts/akm-migrate-node.js +42948 -92369
  217. package/dist/scripts/akm-migrate.js +42935 -92354
  218. package/dist/setup/registry-stash-loader.js +4 -13
  219. package/dist/setup/semantic-assets.js +3 -44
  220. package/dist/setup/setup.js +1 -1
  221. package/dist/setup/steps/connection.js +5 -6
  222. package/dist/setup/steps/platforms.js +2 -2
  223. package/dist/setup/steps/tasks.js +25 -15
  224. package/dist/sources/provider-factory.js +17 -18
  225. package/dist/sources/providers/filesystem.js +2 -3
  226. package/dist/sources/providers/git-install.js +7 -1
  227. package/dist/sources/providers/git-provider.js +0 -3
  228. package/dist/sources/providers/git-stash.js +83 -21
  229. package/dist/sources/providers/npm.js +2 -4
  230. package/dist/sources/providers/provider-utils.js +5 -10
  231. package/dist/sources/providers/website.js +0 -2
  232. package/dist/sources/snapshot-fetchers/website-ingest.js +1 -1
  233. package/dist/sources/website-url.js +2 -2
  234. package/dist/storage/database.js +9 -35
  235. package/dist/storage/repositories/improve-ledger-repository.js +209 -0
  236. package/dist/storage/repositories/index-connection.js +39 -72
  237. package/dist/storage/repositories/index-entries-repository.js +131 -129
  238. package/dist/storage/repositories/index-entry-mapper.js +1 -2
  239. package/dist/storage/repositories/index-entry-schema.js +101 -268
  240. package/dist/storage/repositories/index-fts-repository.js +86 -256
  241. package/dist/storage/repositories/index-links-repository.js +143 -0
  242. package/dist/storage/repositories/index-llm-cache-repository.js +7 -9
  243. package/dist/storage/repositories/index-meta-repository.js +6 -4
  244. package/dist/storage/repositories/index-schema.js +257 -325
  245. package/dist/storage/repositories/index-utility-repository.js +8 -29
  246. package/dist/storage/repositories/index-vec-repository.js +133 -414
  247. package/dist/storage/repositories/outcome-repository.js +2 -1
  248. package/dist/storage/repositories/proposals-repository.js +104 -1
  249. package/dist/storage/repositories/registry-index-cache-repository.js +100 -0
  250. package/dist/storage/repositories/salience-repository.js +1 -19
  251. package/dist/storage/repositories/task-history-repository.js +26 -4
  252. package/dist/storage/repositories/workflow-runs-repository.js +53 -244
  253. package/dist/storage/sqlite-migrations.js +136 -0
  254. package/dist/storage/sqlite-pragmas.js +11 -9
  255. package/dist/storage/sqlite-transaction.js +170 -0
  256. package/dist/storage/state-db-integrity.js +130 -0
  257. package/dist/tasks/activation-config.js +134 -62
  258. package/dist/tasks/backends/cron.js +191 -302
  259. package/dist/tasks/backends/exec-utils.js +2 -5
  260. package/dist/tasks/backends/launchd.js +141 -748
  261. package/dist/tasks/backends/schtasks.js +119 -623
  262. package/dist/tasks/prepare/prepare-support.js +5 -15
  263. package/dist/tasks/prepare/prepare.js +0 -2
  264. package/dist/tasks/resolve-akm-bin.js +20 -79
  265. package/dist/tasks/run/attempt-lifecycle.js +0 -1
  266. package/dist/tasks/run/load-task.js +1 -1
  267. package/dist/tasks/scheduler-binding.js +20 -238
  268. package/dist/tasks/scheduler-invocation.js +136 -244
  269. package/dist/tasks/scheduler-lock.js +53 -0
  270. package/dist/tasks/scheduler-sync.js +368 -679
  271. package/dist/tasks/source/parse-task-source.js +55 -9
  272. package/dist/tasks/source/task-source-v3-frozen.js +3 -4
  273. package/dist/tasks/source/task-to-v4.js +464 -88
  274. package/dist/workflows/authoring/authoring.js +3 -12
  275. package/dist/workflows/compile.js +211 -0
  276. package/dist/workflows/concurrency-policy.js +13 -74
  277. package/dist/workflows/exec/child-invocation.js +3 -17
  278. package/dist/workflows/exec/child-workflow.js +32 -141
  279. package/dist/workflows/exec/dispatch-redaction.js +13 -53
  280. package/dist/workflows/exec/environment.js +98 -0
  281. package/dist/workflows/exec/exec-unit.js +33 -140
  282. package/dist/workflows/exec/frozen-judge.js +7 -59
  283. package/dist/workflows/exec/native-executor.js +82 -341
  284. package/dist/workflows/exec/param-secrets.js +29 -47
  285. package/dist/workflows/exec/run-workflow.js +154 -387
  286. package/dist/workflows/exec/scheduler.js +9 -36
  287. package/dist/workflows/exec/step-work.js +127 -430
  288. package/dist/workflows/exec/unit-dispatch.js +11 -63
  289. package/dist/workflows/exec/unit-writer.js +8 -52
  290. package/dist/workflows/exec/worktree.js +39 -273
  291. package/dist/workflows/freeze/child-output-references.js +4 -15
  292. package/dist/workflows/freeze/environment.js +99 -92
  293. package/dist/workflows/freeze/freeze.js +172 -0
  294. package/dist/workflows/freeze/step-values.js +19 -21
  295. package/dist/workflows/freeze/targets/child-workflow.js +23 -92
  296. package/dist/workflows/freeze/targets/command.js +10 -33
  297. package/dist/workflows/freeze/targets/script.js +5 -12
  298. package/dist/workflows/freeze/targets/shell.js +3 -6
  299. package/dist/workflows/freeze/targets/task.js +25 -80
  300. package/dist/workflows/freeze/task-bindings.js +20 -67
  301. package/dist/workflows/{source-ir/github-yaml.js → github-yaml.js} +88 -206
  302. package/dist/workflows/ir/params.js +6 -51
  303. package/dist/workflows/ir/plan-hash.js +2 -34
  304. package/dist/workflows/parser.js +140 -43
  305. package/dist/{commands/improve/consolidate/types.js → workflows/plan.js} +2 -1
  306. package/dist/workflows/renderer.js +36 -69
  307. package/dist/workflows/resource-limits.js +12 -120
  308. package/dist/workflows/runtime/agent-identity.js +8 -40
  309. package/dist/workflows/runtime/run-outputs.js +3 -6
  310. package/dist/workflows/runtime/run-plan.js +316 -0
  311. package/dist/workflows/runtime/runs.js +48 -200
  312. package/dist/workflows/runtime/workflow-asset-loader.js +24 -57
  313. package/dist/workflows/{source-ir/semantics.js → source-semantics.js} +16 -20
  314. package/dist/workflows/validate-summary.js +2 -7
  315. package/docs/integration/bundling-akm.md +49 -42
  316. package/docs/migration/README.md +1 -0
  317. package/docs/migration/release-notes/0.9.17.md +43 -0
  318. package/docs/migration/v0.9.1-to-v0.9.2.md +23 -7
  319. package/docs/reference/cli.md +232 -135
  320. package/docs/reference/configuration.md +71 -57
  321. package/docs/reference/data-and-telemetry.md +20 -21
  322. package/docs/reference/tasks.md +105 -39
  323. package/docs/reference/workflow-schema.md +14 -18
  324. package/docs/reference/workflows.md +6 -9
  325. package/package.json +1 -1
  326. package/schemas/akm-config.json +115 -738
  327. package/schemas/akm-workflow.json +1 -0
  328. package/dist/assets/improve-strategies/graph-refresh.json +0 -15
  329. package/dist/assets/prompts/contradiction-judge.md +0 -33
  330. package/dist/assets/prompts/graph-extract-system.md +0 -1
  331. package/dist/assets/prompts/graph-extract-user-prompt.md +0 -35
  332. package/dist/assets/prompts/metadata-enhance-system.md +0 -1
  333. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +0 -4
  334. package/dist/commands/health/advisories.js +0 -150
  335. package/dist/commands/health/metrics.js +0 -329
  336. package/dist/commands/health/surfaces.js +0 -102
  337. package/dist/commands/improve/anti-collapse.js +0 -83
  338. package/dist/commands/improve/collapse-detector.js +0 -432
  339. package/dist/commands/improve/consolidate/eligibility.js +0 -48
  340. package/dist/commands/improve/consolidate/merge.js +0 -149
  341. package/dist/commands/improve/distill/promote-memory.js +0 -291
  342. package/dist/commands/improve/distill/quality-gate.js +0 -337
  343. package/dist/commands/improve/eval-cases.js +0 -52
  344. package/dist/commands/improve/memory/memory-contradiction-detect.js +0 -291
  345. package/dist/commands/improve/proposal-envelope.js +0 -31
  346. package/dist/commands/improve/run-context.js +0 -123
  347. package/dist/commands/improve/shared.js +0 -31
  348. package/dist/commands/improve/source-identity.js +0 -28
  349. package/dist/commands/improve/triage.js +0 -96
  350. package/dist/commands/proposal/drain-policies.js +0 -151
  351. package/dist/commands/sources/update-transaction.js +0 -220
  352. package/dist/core/action-contributors.js +0 -28
  353. package/dist/core/config/config-version-shim.js +0 -101
  354. package/dist/core/fs-txn.js +0 -405
  355. package/dist/core/lexical-score.js +0 -25
  356. package/dist/core/maintenance-barrier.js +0 -167
  357. package/dist/execution/executable-identity.js +0 -105
  358. package/dist/execution/guarded-source.js +0 -427
  359. package/dist/indexer/db/graph-db.js +0 -444
  360. package/dist/indexer/graph/graph-boost.js +0 -427
  361. package/dist/indexer/graph/graph-dedup.js +0 -95
  362. package/dist/indexer/graph/graph-extraction.js +0 -1108
  363. package/dist/indexer/search/name-match.js +0 -35
  364. package/dist/indexer/search/ranking-contributors.js +0 -515
  365. package/dist/indexer/search/ranking-types.js +0 -4
  366. package/dist/indexer/walk/project-context.js +0 -192
  367. package/dist/integrations/agent/execution-cascade.js +0 -566
  368. package/dist/integrations/agent/execution-definitions.js +0 -202
  369. package/dist/integrations/agent/execution-lowering.js +0 -841
  370. package/dist/integrations/agent/execution-preparation.js +0 -98
  371. package/dist/integrations/agent/inline-execution.js +0 -74
  372. package/dist/llm/graph-extract.js +0 -728
  373. package/dist/llm/metadata-enhance.js +0 -96
  374. package/dist/registry/create-provider-registry.js +0 -29
  375. package/dist/registry/pinned-request-helper.js +0 -247
  376. package/dist/registry/pinned-transport.js +0 -717
  377. package/dist/sources/providers/index.js +0 -14
  378. package/dist/storage/engines/sqlite-migrations.js +0 -271
  379. package/dist/storage/repositories/canaries-repository.js +0 -107
  380. package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
  381. package/dist/storage/repositories/registry-cache.js +0 -113
  382. package/dist/tasks/scheduler-sync-preview.js +0 -52
  383. package/dist/tasks/source/task-to-v3.js +0 -507
  384. package/dist/workflows/freeze/resolve-steps.js +0 -86
  385. package/dist/workflows/freeze/source-freeze.js +0 -64
  386. package/dist/workflows/ir/compile.js +0 -321
  387. package/dist/workflows/ir/environment-v4.js +0 -330
  388. package/dist/workflows/ir/freeze-v4.js +0 -153
  389. package/dist/workflows/ir/schema-v4.js +0 -745
  390. package/dist/workflows/ir/schema.js +0 -354
  391. package/dist/workflows/program/schema.js +0 -77
  392. package/dist/workflows/runtime/checkin.js +0 -57
  393. package/dist/workflows/runtime/plan-classifier.js +0 -196
  394. package/dist/workflows/runtime/unit-checkin.js +0 -45
  395. package/dist/workflows/runtime/unit-phases.js +0 -20
  396. package/dist/workflows/schema.js +0 -4
  397. package/dist/workflows/source-ir/compile.js +0 -200
  398. package/dist/workflows/source-ir/program.js +0 -50
  399. package/dist/workflows/source-ir/result.js +0 -26
  400. package/dist/workflows/source-ir/schema.js +0 -786
  401. package/dist/workflows/source-ir/triggers.js +0 -79
  402. package/dist/workflows/source-ir/uses.js +0 -40
  403. package/dist/workflows/validator.js +0 -60
@@ -1,271 +1,140 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * The improve preparation stage: consolidation and session extraction (which
6
+ * run before the loop), memory cleanup, structural validation, and candidate
7
+ * selection for the reflect/distill loop.
8
+ *
9
+ * Candidate selection reads the improve ledger plus one set of signals: a ref
10
+ * is eligible for a source when feedback newer than its last attempt landed and
11
+ * no ledger window holds it. Refs without recent feedback can still be picked
12
+ * by the fallback lanes (proactive maintenance, high salience); the survivors
13
+ * are ranked by salience, checked on disk and capped. A plan-only run
14
+ * evaluates the same selectors against read snapshots and writes nothing.
15
+ */
4
16
  import fs from "node:fs";
5
17
  import path from "node:path";
6
- import { makeBundleRef } from "../../core/asset/asset-ref.js";
7
18
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
8
- import { typeNameFromConceptId } from "../../core/asset/resolve-ref.js";
9
19
  import { daysToMs } from "../../core/common.js";
10
20
  import { loadConfig } from "../../core/config/config.js";
11
21
  import { ConfigError, rethrowIfTestIsolationError } from "../../core/errors.js";
12
22
  import { appendEvent, readEvents } from "../../core/events.js";
13
- import { openStateDatabase, withStateDb } from "../../core/state-db.js";
23
+ import { withStateDb } from "../../core/state-db.js";
14
24
  import { info, warn } from "../../core/warn.js";
15
- import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
25
+ import { countUsageEventsByType, USAGE_EVENT_RETENTION_DAYS } from "../../indexer/usage/usage-events.js";
16
26
  import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
17
- import { withLlmStage } from "../../llm/usage-telemetry.js";
18
- import { closeDatabase, openExistingDatabase, openReadonlyExistingDatabase, } from "../../storage/repositories/index-connection.js";
19
27
  import { getZeroResultSearches } from "../../storage/repositories/index-entries-repository.js";
20
28
  import { getRetrievalCounts } from "../../storage/repositories/index-utility-repository.js";
21
29
  import { listStateProposals } from "../../storage/repositories/proposals-repository.js";
22
30
  import { akmLint } from "../lint/index.js";
23
31
  import { runSchemaRepairPass } from "../sources/schema-repair.js";
24
32
  import { isAutonomyLaneAllowed } from "./autonomy-gate.js";
25
- import { akmConsolidate, inspectConsolidationPool } from "./consolidate.js";
33
+ import { akmConsolidate, inspectConsolidationPool, loadExistingKnowledgeBodyHashes, makeConsolidateResult, } from "./consolidate.js";
26
34
  import { computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
27
- // Eligibility / candidate-selection predicates live in ./eligibility.
28
- import { buildLatestFeedbackTsMap, buildLatestProposalTsMap, buildUtilityMap, dedupeRefs, findAssetFilePath, isDistillCandidateRef, isLessonCandidate, isSignalDeltaEligible, resolveImproveScope, } from "./eligibility.js";
35
+ import { assetTypeOf, buildUtilityMap, dedupeRefs, findAssetFilePath, isDistillCandidateRef, isLessonCandidate, resolveImproveScope, withIndexDb, } from "./eligibility.js";
29
36
  import { akmExtract, countNewExtractCandidates } from "./extract.js";
30
- import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
37
+ import { computeValenceScore } from "./feedback-valence.js";
38
+ import { isLedgerBlocked, lastAttemptByRef, ledgerRowFor, loadLedgerSnapshot, stateKey, } from "./ledger.js";
31
39
  import { applyMemoryCleanup } from "./memory/memory-improve.js";
32
- import { computeProxyAdequacy, getAllAssetOutcomes, getAssetOutcome, getOutcomeScoresByRef, OUTCOME_SCORE_MAX, outcomeScoreToSalience, projectAssetOutcome, updateAssetOutcome, } from "./outcome-loop.js";
40
+ import { getAllAssetOutcomes, getAssetOutcome, getOutcomeScoresByRef, OUTCOME_SCORE_MAX, outcomeScoreToSalience, projectAssetOutcome, updateAssetOutcome, } from "./outcome-loop.js";
33
41
  import { projectMemoryCleanup, selectEffectiveImproveRefs } from "./planner.js";
34
42
  import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
35
- import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getLastUseMsByRef, isContentEncodingRow, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
36
- import { bareImproveRef, improveStateReadRefs } from "./source-identity.js";
37
- function readAssetSalienceForImproveRef(db, ref, itemRef) {
38
- for (const key of improveStateReadRefs(ref, itemRef)) {
39
- const row = getAssetSalience(db, key);
40
- if (row)
41
- return row;
43
+ import { isInRetrievalScope, loadRetrievalScope } from "./retrieval-scope.js";
44
+ import { computeSalience, getAssetSalience, getLastUseMsByRef, isContentEncodingRow, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
45
+ import { attributeStage, errMessage } from "./stage.js";
46
+ /** The candidate's durable state key (salience, outcome, ledger). */
47
+ const keyOf = (r) => stateKey(r.ref, r.itemRef);
48
+ /**
49
+ * Run `fn` against the run's state.db (its long-lived handle when there is
50
+ * one). A plan-only run without a handle reads nothing. Best-effort.
51
+ */
52
+ function withRunState(eventsCtx, persist, fn) {
53
+ if (!persist && !eventsCtx?.db)
54
+ return undefined;
55
+ try {
56
+ return withStateDb(fn, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
57
+ }
58
+ catch (err) {
59
+ rethrowIfTestIsolationError(err);
60
+ return undefined;
42
61
  }
43
- return undefined;
44
- }
45
- function readConsecutiveNoOpsForImproveRef(db, ref, itemRef) {
46
- return readAssetSalienceForImproveRef(db, ref, itemRef)?.consecutive_no_ops ?? 0;
47
62
  }
48
- // ── Durable-state write keys ──────────────────────────────────────────────────
49
- //
50
- // The durable improve-state writers (salience + outcome) key by the resolved
51
- // index entry's `item_ref` (`<bundle>//<conceptId>`) when the planner resolved
52
- // one (`ImproveEligibleRef.itemRef`). Both writers share one key expression,
53
- // `itemRefByRef.get(ref) ?? ref`.
54
- //
55
- // A direct `--scope <ref>` candidate does not flow through
56
- // `collectEligibleRefsFromIndex`, so its conceptId `ref` is the write key.
57
- //
58
- // itemRefByRef is `ref → item_ref | undefined`, built once per pass from the
59
- // candidate set.
60
- /** `ref → item_ref | undefined` for a run's candidate set. */
61
- function buildItemRefByRef(refs) {
62
- const m = new Map();
63
- for (const r of refs)
64
- m.set(r.ref, r.itemRef);
65
- return m;
63
+ function fileSize(filePath) {
64
+ if (!filePath)
65
+ return undefined;
66
+ try {
67
+ return fs.statSync(filePath).size;
68
+ }
69
+ catch {
70
+ return undefined;
71
+ }
66
72
  }
67
- /** Durable `asset_salience` write key: the entry's item_ref, else its conceptId `ref` (scope-ref fallback). */
68
- function salienceWriteKey(ref, itemRefByRef) {
69
- return itemRefByRef.get(ref) ?? ref;
73
+ /** `{ key: value }` for each key `source` defines. */
74
+ export function pickDefined(source, keys) {
75
+ const out = {};
76
+ for (const key of keys)
77
+ if (source?.[key] !== undefined)
78
+ out[key] = source[key];
79
+ return out;
70
80
  }
71
- /** Durable `asset_outcome` write key: the entry's item_ref, else its conceptId `ref` (scope-ref fallback). */
72
- function outcomeWriteKey(ref, itemRefByRef) {
73
- return itemRefByRef.get(ref) ?? ref;
81
+ export const CONSOLIDATION_CONFIG_KEYS = ["enabled", "minPoolSize", "limit", "maxChunkSize"];
82
+ /** Emit an aggregate `improve_skipped` row (never one per ref). */
83
+ export function recordImproveSkip(eventsCtx, ref, metadata) {
84
+ appendEvent({ eventType: "improve_skipped", ref, metadata }, eventsCtx);
74
85
  }
75
- /** Resolve an AKM asset type from a short or bundle-qualified conceptId. */
76
- function assetTypeOf(ref) {
77
- const tail = ref.includes("//") ? ref.slice(ref.indexOf("//") + 2) : ref;
78
- return typeNameFromConceptId(tail)?.type ?? "";
86
+ /** Per-originator rolling error windows (3 each) shown to later prompts as patterns to avoid. */
87
+ export function pushRecentError(recentErrors, originator, msg) {
88
+ const window = recentErrors[originator] ?? [];
89
+ window.push(msg);
90
+ if (window.length > 3)
91
+ window.shift();
92
+ recentErrors[originator] = window;
79
93
  }
94
+ // ── Consolidation ────────────────────────────────────────────────────────────
80
95
  /**
81
- * Evaluate the consolidation gate flags (volume trigger, #551 pool-delta
82
- * cooldown, profile disable, #553 min-pool-size) up front, before any LLM call.
83
- * Extracted verbatim from `runConsolidationPass` — logic is byte-identical.
96
+ * The consolidation gates and pool, with no model call: the profile toggle,
97
+ * `minPoolSize` (not for a named strategy or ref scope, nor once the pool is
98
+ * over the 100-memory volume trigger), and the ledger delta (every memory
99
+ * judged recently and unchanged since means nothing to do).
84
100
  */
85
- function evaluateConsolidationEligibility(args) {
86
- const { options, primaryStashDir, memorySummary, improveProfile, resolvedPlan, eventsCtx } = args;
87
- const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
88
- const hasLlm = resolvedPlan.processes.consolidate.runner !== null;
89
- const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
90
- // 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
91
- // memory file has been updated since the most recent successful
92
- // consolidate_completed event. Time-based cooldowns produced the same
93
- // synchronised-wave failure mode the reflect/distill cooldowns did; the
94
- // pool-delta gate ties consolidation to actual work-to-do.
95
- const sourceName = options.sourceName ?? options.writeTarget?.source.name ?? options.config?.defaultBundle ?? "stash";
96
- const recentConsolidations = readEvents({ type: "consolidate_completed" }, eventsCtx);
97
- const lastConsolidation = recentConsolidations.events
98
- .filter((e) => e.metadata?.source === sourceName && Number(e.metadata?.processed) > 0)
99
- .sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
100
- const lastConsolidateTs = typeof lastConsolidation?.metadata?.completedThrough === "string"
101
- ? lastConsolidation.metadata.completedThrough
102
- : lastConsolidation?.ts;
103
- // #551 smarter gate: build the set of memory asset paths whose only delta
104
- // since the last consolidate is their OWN promotion. Those files
105
- // have not had a full improve cycle to settle, so they offer no merge /
106
- // contradiction candidates yet — excluding them stops the gate firing on
107
- // freshly-promoted single-source memories. We read `promoted` events emitted
108
- // after the last consolidate; each carries the written `assetPath`.
109
- const promotedSinceConsolidate = (() => {
110
- const paths = new Set();
111
- try {
112
- const promoted = readEvents({
113
- type: "promoted",
114
- ...(lastConsolidateTs ? { since: lastConsolidateTs } : {}),
115
- }, eventsCtx).events;
116
- for (const e of promoted) {
117
- const ap = e.metadata?.assetPath;
118
- if (typeof ap === "string" && ap.length > 0)
119
- paths.add(path.resolve(ap));
120
- }
121
- }
122
- catch {
123
- // best-effort: if the events query fails, fall back to no exclusions
124
- // (preserves pre-#551 behaviour rather than over-skipping).
125
- }
126
- return paths;
127
- })();
128
- // Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do,
129
- // EXCEPT files whose only post-consolidate change was their own promotion.
130
- // Using file mtime keeps this query DB-free and matches what the indexer
131
- // already uses as the canonical `memory.updated_at` proxy.
132
- //
133
- // Bootstrap: when no successful consolidate_completed event has ever been
134
- // recorded, we cannot evaluate the pool-delta — treat as eligible so a
135
- // fresh stash runs consolidate once before the steady-state gate kicks in.
136
- const memoryUpdatedAfterLastConsolidate = (() => {
137
- if (volumeTriggered)
138
- return true; // volume override forces the run regardless.
139
- if (!lastConsolidateTs)
140
- return true; // bootstrap path: never consolidated.
141
- if (!primaryStashDir)
142
- return false;
143
- const memoriesDir = path.join(primaryStashDir, "memories");
144
- if (!fs.existsSync(memoriesDir))
145
- return false;
146
- try {
147
- const pending = [memoriesDir];
148
- while (pending.length > 0) {
149
- const current = pending.pop();
150
- for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
151
- const filePath = path.join(current, entry.name);
152
- if (entry.isDirectory()) {
153
- pending.push(filePath);
154
- continue;
155
- }
156
- if (!entry.isFile() || !entry.name.endsWith(".md"))
157
- continue;
158
- if (promotedSinceConsolidate.has(path.resolve(filePath)))
159
- continue;
160
- try {
161
- if (fs.statSync(filePath).mtime.toISOString() > lastConsolidateTs)
162
- return true;
163
- }
164
- catch {
165
- // Ignore files that disappear during the scan.
166
- }
167
- }
168
- }
169
- return false;
170
- }
171
- catch {
172
- return false;
173
- }
174
- })();
175
- const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
176
- // Profile gate: if profile explicitly disables consolidate, skip the entire pass.
177
- const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
178
- // #553 minPoolSize guard: skip consolidation when the eligible memory pool is
179
- // below a minimum size, rather than spending an LLM pass on a handful of
180
- // memories. This is an INDEPENDENT skip condition from #551's mtime pool-delta
181
- // gate — either can skip. Default 0 (disabled) — every built-in strategy
182
- // used to ship 500, which meant `akm improve --strategy consolidate`, typed
183
- // by a human, silently did nothing on almost every real install. Evaluated
184
- // against the eligible-pool count BEFORE entering the LLM loop so a skip
185
- // costs ZERO LLM calls when an operator opts back into a floor.
186
- const CONSOLIDATE_DEFAULT_MIN_POOL_SIZE = 0;
187
- const configuredMinPoolSize = improveProfile?.processes?.consolidate?.minPoolSize;
188
- const minPoolSize = typeof configuredMinPoolSize === "number" ? configuredMinPoolSize : CONSOLIDATE_DEFAULT_MIN_POOL_SIZE;
189
- const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
190
- const userNamedStrategyOrScope = options.strategy !== undefined || resolveImproveScope(options.scope).mode === "ref";
191
- // volumeTriggered means the pool already exceeds the volume threshold (100),
192
- // so a force-triggered run never trips the pool-size guard. The guard only
193
- // engages when minPoolSize > 0 and the eligible pool is strictly below it.
194
- const poolBelowMinSize = !volumeTriggered && !userNamedStrategyOrScope && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
195
- return {
196
- volumeTriggered,
197
- consolidationOnCooldown,
198
- consolidateDisabledByProfile,
199
- poolBelowMinSize,
200
- eligiblePoolSize,
201
- minPoolSize,
202
- ...(lastConsolidateTs ? { lastConsolidationTs: lastConsolidateTs } : {}),
203
- };
204
- }
205
- /** Build the no-dispatch consolidation projection consumed by dry and live. */
206
101
  function planConsolidationPass(args) {
207
- const { options, primaryStashDir, memorySummary, improveProfile, resolvedPlan, eventsCtx } = args;
208
- const processConfig = improveProfile?.processes?.consolidate;
209
- const eligibility = evaluateConsolidationEligibility({
210
- options,
211
- primaryStashDir,
212
- memorySummary,
213
- improveProfile,
214
- resolvedPlan,
215
- eventsCtx,
216
- });
217
- const effectiveOptions = {
218
- ...options.consolidateOptions,
219
- config: options.config,
220
- stashDir: options.stashDir,
221
- writeTarget: options.writeTarget,
222
- limit: processConfig?.limit,
223
- incrementalSince: processConfig?.incrementalSince,
224
- neighborsPerChanged: processConfig?.neighborsPerChanged,
225
- maxChunkSize: processConfig?.maxChunkSize,
226
- };
227
- const poolWarnings = [];
102
+ const { options, primaryStashDir, memorySummary, resolvedPlan } = args;
103
+ const processConfig = args.improveProfile?.processes?.consolidate;
104
+ const volumeTriggered = memorySummary.eligible > 100 && resolvedPlan.processes.consolidate.runner !== null;
105
+ const minPoolSize = typeof processConfig?.minPoolSize === "number" ? processConfig.minPoolSize : 0;
106
+ const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
107
+ const userNamed = options.strategy !== undefined || resolveImproveScope(options.scope).mode === "ref";
108
+ const poolBelowMinSize = !volumeTriggered && !userNamed && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
228
109
  const pool = primaryStashDir
229
- ? inspectConsolidationPool(effectiveOptions, primaryStashDir, poolWarnings, {
230
- readOnly: eventsCtx?.readOnly === true,
231
- })
232
- : { poolSize: 0, candidatePoolSize: 0, dedupPoolSize: 0, memories: [] };
233
- // #800/#957 round 3 — a credential-unavailable consolidate engine still
234
- // resolved a context length structurally; read it off the `engineUnavailable`
235
- // entry instead of falling back to the generic default, so a dry-run
236
- // preview reflects the real engine even when its credential isn't
237
- // materialized here.
238
- const consolidateUnavailable = resolvedPlan.engineUnavailable.find((item) => item.process === "consolidate");
110
+ ? inspectConsolidationPool({
111
+ config: options.config,
112
+ stashDir: options.stashDir,
113
+ writeTarget: options.writeTarget,
114
+ target: options.target,
115
+ limit: processConfig?.limit,
116
+ maxChunkSize: processConfig?.maxChunkSize,
117
+ }, primaryStashDir, [], args.existingKnowledgeBodyHashes ?? loadExistingKnowledgeBodyHashes(primaryStashDir), { readOnly: args.eventsCtx?.readOnly === true })
118
+ : { poolSize: 0, candidatePoolSize: 0, judgedUnchanged: 0 };
119
+ // A credential-unavailable engine still resolved its context length.
120
+ const unavailable = resolvedPlan.engineUnavailable.find((item) => item.process === "consolidate");
239
121
  const chunkSize = computeSafeChunkSize(resolvedPlan.processes.consolidate.runner?.connection.contextLength ??
240
- consolidateUnavailable?.contextLength ??
122
+ unavailable?.contextLength ??
241
123
  DEFAULT_CONTEXT_LENGTH_TOKENS, 500, processConfig?.maxChunkSize);
242
- const profilePassed = !eligibility.consolidateDisabledByProfile;
243
- const minimumPoolPassed = !eligibility.poolBelowMinSize;
244
- const deltaPassed = !eligibility.consolidationOnCooldown;
245
- const nonEmptyPool = pool.candidatePoolSize > 0;
246
- const wouldRun = profilePassed && minimumPoolPassed && deltaPassed && nonEmptyPool;
247
- const reason = !profilePassed
248
- ? "disabled by improve profile"
249
- : !minimumPoolPassed
250
- ? `pool ${eligibility.eligiblePoolSize} is below minPoolSize ${eligibility.minPoolSize}`
251
- : !deltaPassed
252
- ? "no memory updates since the last completed consolidation"
253
- : !nonEmptyPool
254
- ? "candidate pool is empty after narrowing"
255
- : "all consolidation gates pass";
124
+ const profilePassed = processConfig?.enabled !== false;
125
+ const deltaPassed = pool.candidatePoolSize > 0 || pool.judgedUnchanged === 0;
126
+ const wouldRun = profilePassed && !poolBelowMinSize && deltaPassed && pool.candidatePoolSize > 0;
127
+ const belowMin = `pool ${eligiblePoolSize} is below minPoolSize ${minPoolSize}`;
128
+ const unchanged = "every memory was judged recently and is unchanged since";
256
129
  return {
257
- eligibility,
130
+ poolBelowMinSize,
131
+ eligiblePoolSize,
132
+ minPoolSize,
258
133
  plan: {
259
- configured: {
260
- ...(processConfig?.enabled !== undefined ? { enabled: processConfig.enabled } : {}),
261
- ...(processConfig?.minPoolSize !== undefined ? { minPoolSize: processConfig.minPoolSize } : {}),
262
- ...(processConfig?.limit !== undefined ? { limit: processConfig.limit } : {}),
263
- ...(processConfig?.maxChunkSize !== undefined ? { maxChunkSize: processConfig.maxChunkSize } : {}),
264
- ...(processConfig?.incrementalSince !== undefined ? { incrementalSince: processConfig.incrementalSince } : {}),
265
- },
134
+ configured: pickDefined(processConfig, CONSOLIDATION_CONFIG_KEYS),
266
135
  effective: {
267
136
  enabled: profilePassed,
268
- minPoolSize: eligibility.minPoolSize,
137
+ minPoolSize,
269
138
  ...(processConfig?.limit !== undefined ? { limit: processConfig.limit } : {}),
270
139
  chunkSize,
271
140
  },
@@ -277,314 +146,180 @@ function planConsolidationPass(args) {
277
146
  reason: profilePassed ? "consolidation enabled" : "disabled by improve profile",
278
147
  },
279
148
  minimumPool: {
280
- passed: minimumPoolPassed,
281
- reason: minimumPoolPassed
282
- ? `pool satisfies minPoolSize ${eligibility.minPoolSize}`
283
- : `pool ${eligibility.eligiblePoolSize} is below minPoolSize ${eligibility.minPoolSize}`,
149
+ passed: !poolBelowMinSize,
150
+ reason: poolBelowMinSize ? belowMin : `pool satisfies minPoolSize ${minPoolSize}`,
284
151
  },
285
152
  delta: {
286
153
  passed: deltaPassed,
287
- reason: deltaPassed ? "memory pool has work" : "no updates since the last completed consolidation",
154
+ reason: !deltaPassed
155
+ ? unchanged
156
+ : pool.judgedUnchanged > 0
157
+ ? `${pool.judgedUnchanged} recently judged, unchanged memories skipped`
158
+ : "no memory was judged recently",
288
159
  },
289
160
  },
290
161
  wouldRun,
291
- reason,
162
+ reason: !profilePassed
163
+ ? "disabled by improve profile"
164
+ : poolBelowMinSize
165
+ ? belowMin
166
+ : !deltaPassed
167
+ ? unchanged
168
+ : pool.candidatePoolSize === 0
169
+ ? "candidate pool is empty after narrowing"
170
+ : "all consolidation gates pass",
292
171
  estimatedChunks: wouldRun ? Math.ceil(pool.candidatePoolSize / chunkSize) : 0,
293
172
  },
294
173
  };
295
174
  }
296
- export async function runConsolidationPass(args) {
297
- const { options, primaryStashDir, memorySummary, improveProfile, resolvedPlan, eventsCtx, budgetSignal, runBudgetMs, } = args;
298
- const baseConfig = options.config ?? loadConfig();
299
- const consolidationConfig = baseConfig;
300
- const planned = planConsolidationPass({
301
- options,
302
- primaryStashDir,
303
- memorySummary,
304
- improveProfile,
305
- resolvedPlan,
306
- eventsCtx,
307
- });
308
- const { volumeTriggered, consolidationOnCooldown, consolidateDisabledByProfile, poolBelowMinSize, eligiblePoolSize, minPoolSize, lastConsolidationTs, } = planned.eligibility;
309
- let consolidation = {
310
- schemaVersion: 1,
311
- ok: true,
312
- shape: "consolidate-result",
313
- dryRun: false,
314
- previewOnly: false,
315
- target: "",
316
- processed: 0,
317
- merged: 0,
318
- deleted: 0,
319
- promoted: [],
320
- contradicted: 0,
321
- warnings: [],
322
- durationMs: 0,
323
- };
324
- if (consolidateDisabledByProfile) {
175
+ async function runConsolidationPass(args) {
176
+ const { options, primaryStashDir, improveProfile, resolvedPlan, eventsCtx } = args;
177
+ // Walked once and shared with the live pass.
178
+ const existingKnowledgeBodyHashes = primaryStashDir ? loadExistingKnowledgeBodyHashes(primaryStashDir) : undefined;
179
+ const planned = planConsolidationPass({ ...args, existingKnowledgeBodyHashes });
180
+ const processConfig = improveProfile?.processes?.consolidate;
181
+ let consolidation = makeConsolidateResult({ target: "", durationMs: 0 });
182
+ if (!planned.plan.gates.profile.passed) {
325
183
  info("[improve] consolidation skipped (disabled by improve profile)");
326
184
  }
327
- else if (poolBelowMinSize) {
328
- // #553: eligible pool below the configured minimum — skip with zero LLM
329
- // calls. Reuse the #551 `improve_skipped` emission path so health surfaces
330
- // it via the dynamic skipReasons aggregation under `pool_below_min_size`.
331
- appendEvent({
332
- eventType: "improve_skipped",
333
- ref: "memories/_consolidation",
334
- metadata: {
335
- reason: "pool_below_min_size",
336
- poolSize: eligiblePoolSize,
337
- minPoolSize,
338
- },
339
- }, eventsCtx);
340
- info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
185
+ else if (planned.poolBelowMinSize) {
186
+ recordImproveSkip(eventsCtx, "memories/_consolidation", {
187
+ reason: "pool_below_min_size",
188
+ poolSize: planned.eligiblePoolSize,
189
+ minPoolSize: planned.minPoolSize,
190
+ });
191
+ info(`[improve] consolidation skipped (pool ${planned.eligiblePoolSize} < minPoolSize ${planned.minPoolSize})`);
341
192
  }
342
- else if (!consolidationOnCooldown) {
343
- const consolidationStartedAt = new Date().toISOString();
344
- consolidation = await withLlmStage("consolidate", () => akmConsolidate({
345
- ...options.consolidateOptions,
346
- config: consolidationConfig,
193
+ else if (!planned.plan.gates.delta.passed) {
194
+ recordImproveSkip(eventsCtx, "memories/_consolidation", { reason: "consolidation_no_memory_updates" });
195
+ info("[improve] consolidation skipped (every memory was judged recently and is unchanged)");
196
+ }
197
+ else {
198
+ consolidation = await attributeStage(resolvedPlan, "consolidate", () => akmConsolidate({
199
+ target: options.target,
200
+ ...(options.writeTarget ? { writeTarget: options.writeTarget } : {}),
201
+ config: options.config ?? loadConfig(),
347
202
  dryRun: options.dryRun ?? false,
348
203
  stashDir: options.stashDir,
349
- // Active profile for this improve run — lets consolidate's secondary
350
- // process-config reads honor `--profile <name>` instead of `default`.
351
204
  improveProfile,
352
205
  llmRunner: resolvedPlan.processes.consolidate.runner,
353
- autoTriggered: volumeTriggered,
354
- // Tie consolidate proposals back to this improve invocation so
355
- // accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
206
+ existingKnowledgeBodyHashes,
356
207
  sourceRun: `consolidate-${Date.now()}`,
357
- // Pass profile-configured options. incrementalSince narrows the pool to
358
- // recently-changed memories + graph neighbours — use this for frequent
359
- // passes (quick-shredder). Leave absent in the nightly default profile for
360
- // a full-pool sweep that catches stale-but-unmerged duplicates.
361
- limit: improveProfile?.processes?.consolidate?.limit,
362
- incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
363
- neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
364
- maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
365
- // WS-3a: forward budget signal for graceful abort on timeout, and pass
366
- // the profile's p90 estimate for cold-start budget reduction.
367
- signal: budgetSignal,
368
- p90ChunkSecondsDefault: improveProfile?.processes?.consolidate?.p90ChunkSecondsDefault,
369
- // WS-5: pass total run budget so perfTelemetry.estimatedBudgetFractionUsed
370
- // can flag when consolidation alone exceeded the budget.
371
- runBudgetMs,
372
- }), { engine: resolvedPlan.processes.consolidate.runner?.engine, process: "consolidate" });
373
- const sourceName = options.sourceName ?? options.writeTarget?.source.name ?? baseConfig.defaultBundle ?? "stash";
374
- const complete = (consolidation.failedChunks ?? 0) === 0 &&
375
- (consolidation.failedChunkMemories ?? 0) === 0 &&
376
- (consolidation.failedPromotions ?? 0) === 0 &&
377
- (consolidation.deferredMemories ?? 0) === 0;
378
- const hasUnappliedAdvisoryOperations = consolidation.planned?.some((op) => op.op !== "promote") ?? false;
379
- if (consolidation.ok &&
380
- !consolidation.dryRun &&
381
- complete &&
382
- !hasUnappliedAdvisoryOperations &&
383
- consolidation.processed > 0) {
384
- appendEvent({
385
- eventType: "consolidate_completed",
386
- ref: makeBundleRef(sourceName, "memories/_consolidation"),
387
- metadata: {
388
- processed: consolidation.processed,
389
- source: sourceName,
390
- completedThrough: consolidationStartedAt,
391
- merged: consolidation.merged,
392
- deleted: consolidation.deleted,
393
- contradicted: consolidation.contradicted,
394
- failedChunks: consolidation.failedChunks ?? 0,
395
- durationMs: consolidation.durationMs,
396
- },
397
- }, eventsCtx);
398
- }
208
+ limit: processConfig?.limit,
209
+ maxChunkSize: processConfig?.maxChunkSize,
210
+ signal: args.budgetSignal,
211
+ p90ChunkSecondsDefault: processConfig?.p90ChunkSecondsDefault,
212
+ }));
399
213
  }
400
- else {
401
- appendEvent({
402
- eventType: "improve_skipped",
403
- ref: "memories/_consolidation",
404
- metadata: {
405
- reason: "consolidation_no_memory_updates",
406
- lastEventTs: lastConsolidationTs ?? null,
407
- },
408
- }, eventsCtx);
409
- info("[improve] consolidation skipped (no memory updates since last run)");
410
- }
411
- // D9: track whether consolidation wrote any data so graph extraction can reindex if needed
412
- const consolidationRan = !consolidateDisabledByProfile &&
413
- !poolBelowMinSize &&
414
- !consolidationOnCooldown &&
415
- !consolidation.previewOnly &&
416
- consolidation.processed > 0;
417
- return { consolidation, consolidationRan, plan: planned.plan };
214
+ return { consolidation, plan: planned.plan };
418
215
  }
419
- /**
420
- * Evaluate the exact pre-dispatch extract gates. Live execution consumes this
421
- * snapshot and dry-run only reports it, so neither path reconstructs the
422
- * selector independently.
423
- */
424
- function inspectExtractPass(args) {
425
- const { options, improveProfile, resolvedPlan, eventsCtx, readOnly } = args;
216
+ /** The extract gates, evaluated once for both the live pass and the dry-run report. */
217
+ function inspectExtractPass(args, readOnly) {
218
+ const { options, improveProfile, resolvedPlan, eventsCtx } = args;
426
219
  const enabled = resolvedPlan.processes.extract.enabled;
427
220
  const hasRunner = resolvedPlan.processes.extract.runner?.engine !== undefined;
428
- const availableHarnesses = (options.extractHarnesses ?? getAvailableHarnesses()).filter((harness) => harness.isAvailable());
429
- const configuredMinNewSessions = improveProfile.processes?.extract?.minNewSessions;
430
- const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : 0;
221
+ const availableHarnesses = (options.extractHarnesses ?? getAvailableHarnesses()).filter((h) => h.isAvailable());
222
+ const configured = improveProfile.processes?.extract?.minNewSessions;
223
+ const minNewSessions = typeof configured === "number" ? configured : 0;
431
224
  let newCandidateCount;
432
225
  if (enabled && hasRunner && availableHarnesses.length > 0 && minNewSessions > 0) {
433
- const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
434
- newCandidateCount = countFn(options.config ?? loadConfig(), {
226
+ const defaultSince = improveProfile.processes?.extract?.defaultSince;
227
+ newCandidateCount = (options.extractCandidateCountFn ?? countNewExtractCandidates)(options.config ?? loadConfig(), {
435
228
  harnesses: availableHarnesses,
436
229
  improveProfile,
437
- ...(improveProfile.processes?.extract?.defaultSince
438
- ? { since: improveProfile.processes.extract.defaultSince }
439
- : {}),
230
+ ...(defaultSince ? { since: defaultSince } : {}),
440
231
  ...(eventsCtx?.db ? { stateDb: eventsCtx.db } : {}),
441
232
  ...(!readOnly && eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
442
233
  ...(readOnly ? { readOnly: true } : {}),
443
234
  });
444
235
  }
445
236
  const belowMinNewSessions = minNewSessions > 0 && newCandidateCount !== undefined && newCandidateCount < minNewSessions;
446
- const wouldRun = enabled && hasRunner && availableHarnesses.length > 0 && !belowMinNewSessions;
447
- const reason = !enabled
448
- ? "disabled"
449
- : !hasRunner
450
- ? "enabled but no runner is resolved"
451
- : availableHarnesses.length === 0
452
- ? "enabled but no session-log harness is available"
453
- : belowMinNewSessions
454
- ? `${newCandidateCount ?? 0} new sessions is below minNewSessions ${minNewSessions}`
455
- : minNewSessions > 0
456
- ? `${newCandidateCount ?? 0} new sessions satisfies minNewSessions ${minNewSessions}`
457
- : `enabled with ${availableHarnesses.length} available session-log harness(es); minNewSessions is disabled`;
237
+ const count = `${newCandidateCount ?? 0} new sessions`;
458
238
  return {
459
239
  availableHarnesses,
460
240
  minNewSessions,
461
241
  ...(newCandidateCount !== undefined ? { newCandidateCount } : {}),
462
242
  belowMinNewSessions,
463
- wouldRun,
464
- reason,
243
+ wouldRun: enabled && hasRunner && availableHarnesses.length > 0 && !belowMinNewSessions,
244
+ reason: !enabled
245
+ ? "disabled"
246
+ : !hasRunner
247
+ ? "enabled but no runner is resolved"
248
+ : availableHarnesses.length === 0
249
+ ? "enabled but no session-log harness is available"
250
+ : belowMinNewSessions
251
+ ? `${count} is below minNewSessions ${minNewSessions}`
252
+ : minNewSessions > 0
253
+ ? `${count} satisfies minNewSessions ${minNewSessions}`
254
+ : `enabled with ${availableHarnesses.length} available session-log harness(es); minNewSessions is disabled`,
465
255
  };
466
256
  }
467
257
  /**
468
- * Phase 0.4 — session-extract pass. Reads native session files through the
469
- * SessionLogHarness registry, and asks a bounded LLM for candidate proposals.
470
- * Failures are non-fatal (collected into `warnings`). Returns the extract
471
- * results + any warnings collected along the way.
258
+ * One `akmExtract` per available harness under the strategy's frozen plan. A
259
+ * harness that throws is a warning; the `minNewSessions` gate skips the whole
260
+ * pass with no model call.
472
261
  */
473
- async function runSessionExtractPass(args) {
474
- const { options, primaryStashDir, improveProfile, resolvedPlan, eventsCtx, budgetSignal } = args;
262
+ async function runSessionExtractPass(args, plan) {
263
+ const { options, primaryStashDir, resolvedPlan, eventsCtx } = args;
475
264
  const warnings = [];
476
- // Phase 0.4 — session-extract pass.
477
- //
478
- // Reads native session files (claude JSONL, opencode storage tree)
479
- // through the SessionLogHarness registry, pre-filters noise, and asks a
480
- // bounded in-tree LLM to produce candidate memory/lesson/knowledge
481
- // proposals for content the agent did NOT preserve via inline `akm remember`
482
- // / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
483
- // hook with an on-demand pull pipeline.
484
- //
485
- // Runs only when the ACTIVE strategy resolves
486
- // `processes.extract.enabled: true` (#593: the gate respects the resolved
487
- // improve strategy, not just the hardcoded `default` path the legacy feature
488
- // flag read). Shipped `default` and `frequent` strategies leave this off.
489
- // Each available harness gets one call with the default --since window;
490
- // already-seen sessions (tracked in state.db.extract_sessions_seen) are
491
- // skipped automatically so re-runs don't burn LLM calls on unchanged data.
492
- //
493
- // Failures are non-fatal — one harness throwing doesn't abort improve.
494
- // The extract envelope's own `warnings` field surfaces what went wrong.
495
- let extractResults;
496
- const extractConfig = options.config ?? loadConfig();
497
- // #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
498
- // already done upstream; here we elide every akmExtract/processSession call)
499
- // when the NEW (unseen, in-window) candidate-session pool is below a minimum.
500
- // 22% of improve runs produce zero memory-inference writes because extract
501
- // finds no new sessions, yet still burns the full extract pipeline. Default 0
502
- // (disabled) preserves existing always-run behaviour; only opted-in profiles
503
- // (e.g. a user-enabled `frequent` strategy) set it. Evaluated BEFORE any LLM
504
- // call so a skip costs zero LLM work AND writes nothing. A skipped extract
505
- // never flags work for the NEXT run's consolidation mtime-gate (the
506
- // downstream trigger #554 asks us to suppress).
507
- const plan = args.plan ?? inspectExtractPass({ options, improveProfile, resolvedPlan, eventsCtx, readOnly: false });
508
- // #593/#594: the ACTIVE resolved improve profile is the single source of
509
- // truth for whether extract runs. (Previously this also ANDed in the legacy
510
- // `session_extraction` feature flag, which only reads
511
- // a retired global feature path; the selected strategy is authoritative.)
512
- // `akmExtract` re-checks the same active profile internally via `improveProfile`.
513
- if (resolvedPlan.processes.extract.enabled) {
514
- const extractRunner = resolvedPlan.processes.extract.runner;
515
- if (!extractRunner?.engine) {
516
- throw new ConfigError("Resolved improve plan has no runner for enabled extract process.", "LLM_NOT_CONFIGURED");
517
- }
518
- const extractPlan = Object.freeze({
519
- strategy: resolvedPlan.strategy.name,
520
- engine: extractRunner.engine,
521
- enabled: true,
522
- process: resolvedPlan.processes.extract.config,
523
- runner: extractRunner,
524
- timeoutMs: extractRunner.timeoutMs === undefined ? 600_000 : extractRunner.timeoutMs,
525
- embeddingConfig: Object.freeze(structuredClone(extractConfig.embedding)),
526
- ...(resolvedPlan.processes.extract.notices?.length ? { notices: resolvedPlan.processes.extract.notices } : {}),
265
+ if (!resolvedPlan.processes.extract.enabled)
266
+ return { warnings };
267
+ const runner = resolvedPlan.processes.extract.runner;
268
+ if (!runner?.engine) {
269
+ throw new ConfigError("Resolved improve plan has no runner for enabled extract process.", "LLM_NOT_CONFIGURED");
270
+ }
271
+ const config = options.config ?? loadConfig();
272
+ const extractPlan = Object.freeze({
273
+ strategy: resolvedPlan.strategy.name,
274
+ engine: runner.engine,
275
+ enabled: true,
276
+ process: resolvedPlan.processes.extract.config,
277
+ runner,
278
+ timeoutMs: runner.timeoutMs === undefined ? 600_000 : runner.timeoutMs,
279
+ embeddingConfig: Object.freeze(structuredClone(config.embedding)),
280
+ ...(resolvedPlan.processes.extract.notices?.length ? { notices: resolvedPlan.processes.extract.notices } : {}),
281
+ });
282
+ if (plan.belowMinNewSessions) {
283
+ recordImproveSkip(eventsCtx, "memories/_extract", {
284
+ reason: "below_min_new_sessions",
285
+ newSessions: plan.newCandidateCount ?? 0,
286
+ minNewSessions: plan.minNewSessions,
527
287
  });
528
- const availableHarnesses = plan.availableHarnesses;
529
- if (plan.belowMinNewSessions) {
530
- // Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
531
- // skipReasons aggregation surfaces this under `below_min_new_sessions`.
532
- appendEvent({
533
- eventType: "improve_skipped",
534
- ref: "memories/_extract",
535
- metadata: {
536
- reason: "below_min_new_sessions",
537
- newSessions: plan.newCandidateCount ?? 0,
538
- minNewSessions: plan.minNewSessions,
539
- },
540
- }, eventsCtx);
541
- info(`[improve] extract skipped (new sessions ${plan.newCandidateCount ?? 0} < minNewSessions ${plan.minNewSessions})`);
288
+ info(`[improve] extract skipped (new sessions ${plan.newCandidateCount ?? 0} < minNewSessions ${plan.minNewSessions})`);
289
+ }
290
+ if (!plan.wouldRun)
291
+ return { warnings };
292
+ const extractResults = [];
293
+ for (const harness of plan.availableHarnesses) {
294
+ try {
295
+ extractResults.push(await attributeStage(resolvedPlan, "extract", () => akmExtract({
296
+ type: harness.name,
297
+ ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
298
+ config,
299
+ resolvedPlan: extractPlan,
300
+ dryRun: options.dryRun ?? false,
301
+ signal: args.budgetSignal,
302
+ ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
303
+ ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
304
+ eventsCtx,
305
+ })));
542
306
  }
543
- if (plan.wouldRun) {
544
- extractResults = [];
545
- for (const h of availableHarnesses) {
546
- try {
547
- const result = await withLlmStage("session-extraction", () => akmExtract({
548
- type: h.name,
549
- ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
550
- config: extractConfig,
551
- resolvedPlan: extractPlan,
552
- dryRun: options.dryRun ?? false,
553
- signal: budgetSignal,
554
- ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
555
- // C2: pin extract's skip-tracking state.db open to the boundary path.
556
- ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
557
- // R25: extract's event emits reuse the run's events context
558
- // (fast path when it carries the long-lived handle).
559
- eventsCtx,
560
- }), { engine: resolvedPlan.processes.extract.runner?.engine, process: "extract" });
561
- extractResults.push(result);
562
- }
563
- catch (err) {
564
- const msg = err instanceof Error ? err.message : String(err);
565
- warnings.push(`extract(${h.name}) failed: ${msg}`);
566
- }
567
- }
568
- if (extractResults.length === 0) {
569
- // All harnesses threw — clear so the envelope's `extract` field is
570
- // absent rather than misleadingly empty.
571
- extractResults = undefined;
572
- }
307
+ catch (err) {
308
+ warnings.push(`extract(${harness.name}) failed: ${errMessage(err)}`);
573
309
  }
574
310
  }
575
- return {
576
- extractResults,
577
- warnings,
578
- };
311
+ // Every harness threw: no `extract` field rather than a misleadingly empty one.
312
+ return { ...(extractResults.length > 0 ? { extractResults } : {}), warnings };
579
313
  }
314
+ // ── Validation ───────────────────────────────────────────────────────────────
580
315
  /**
581
- * Phase 1 — validation + schema-repair pass. Scans postCleanupRefs for assets
582
- * with structural problems (missing file, missing lesson description), attempts
583
- * LLM schema repair, and returns the still-failing ref set + the repair records.
316
+ * Structural validation (file on disk, lesson description) with optional LLM
317
+ * schema repair. A repair is advisory: a ref leaves the failure set only when a
318
+ * fresh read of the live asset passes.
584
319
  */
585
320
  export async function runValidationAndRepairPass(args) {
586
- const { postCleanupRefs, options, startMs, budgetMs, primaryStashDir, resolvedPlan, repairValidationFailures, schemaRepairFn = runSchemaRepairPass, } = args;
587
- const validateCandidate = async (candidate) => {
321
+ const { postCleanupRefs, options, resolvedPlan, repairValidationFailures } = args;
322
+ const validate = async (candidate) => {
588
323
  try {
589
324
  const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
590
325
  ? candidate.filePath
@@ -593,10 +328,8 @@ export async function runValidationAndRepairPass(args) {
593
328
  return "file not found on disk";
594
329
  if (path.extname(filePath).toLowerCase() !== ".md")
595
330
  return undefined;
596
- if (isLessonCandidate(candidate.ref)) {
597
- const fm = parseFrontmatter(fs.readFileSync(filePath, "utf8")).data;
598
- if (!fm.description)
599
- return "missing description";
331
+ if (isLessonCandidate(candidate.ref) && !parseFrontmatter(fs.readFileSync(filePath, "utf8")).data.description) {
332
+ return "missing description";
600
333
  }
601
334
  return undefined;
602
335
  }
@@ -606,7 +339,7 @@ export async function runValidationAndRepairPass(args) {
606
339
  };
607
340
  const validationFailures = [];
608
341
  for (const candidate of postCleanupRefs) {
609
- const reason = await validateCandidate(candidate);
342
+ const reason = await validate(candidate);
610
343
  if (reason)
611
344
  validationFailures.push({ ref: candidate.ref, reason });
612
345
  }
@@ -616,166 +349,122 @@ export async function runValidationAndRepairPass(args) {
616
349
  info(` ${f.ref}: ${f.reason}`);
617
350
  }
618
351
  let schemaRepairs = [];
619
- const repairedRefs = new Set();
620
- // Schema repair pass: attempt to fix validation failures via LLM before skipping.
621
- if (repairValidationFailures && validationFailures.length > 0) {
622
- const validationRunner = resolvedPlan.processes.validation.runner;
623
- if (validationRunner) {
624
- const result = await withLlmStage("validation", () => schemaRepairFn(validationFailures, {
625
- startMs,
626
- budgetMs,
627
- llmRunner: validationRunner,
628
- // #591/#379 regression: options.stashDir is the raw, unresolved CLI
629
- // flag (only set when --stash-dir is passed explicitly — never true
630
- // for the scheduled tasks). primaryStashDir is the already-resolved
631
- // source path and is what runSchemaRepairPass's `stashDir` param
632
- // documents itself as needing ("proposal-queue writes"). Passing
633
- // options.stashDir here made every schema-repair attempt throw
634
- // `runSchemaRepairPass requires stashDir` on every cron invocation.
635
- stashDir: primaryStashDir,
636
- findFilePath: findAssetFilePath,
637
- isLessonCandidateFn: isLessonCandidate,
638
- }), { engine: resolvedPlan.processes.validation.runner?.engine, process: "validation" });
639
- schemaRepairs = result.repairs;
640
- // A repair result is advisory. Only a fresh structural read of the live
641
- // asset can remove it from the failure set; queued content is not live.
642
- const failedRefs = new Set(validationFailures.map((failure) => failure.ref));
643
- const candidatesByRef = new Map(postCleanupRefs.map((candidate) => [candidate.ref, candidate]));
644
- for (const ref of failedRefs) {
645
- const candidate = candidatesByRef.get(ref);
646
- if (candidate && !(await validateCandidate(candidate)))
647
- repairedRefs.add(ref);
648
- }
352
+ const repaired = new Set();
353
+ const runner = resolvedPlan.processes.validation.runner;
354
+ if (repairValidationFailures && validationFailures.length > 0 && runner) {
355
+ const result = await attributeStage(resolvedPlan, "validation", () => (args.schemaRepairFn ?? runSchemaRepairPass)(validationFailures, {
356
+ startMs: args.startMs,
357
+ budgetMs: args.budgetMs,
358
+ llmRunner: runner,
359
+ // The resolved source path, not the raw `--stash-dir` flag.
360
+ stashDir: args.primaryStashDir,
361
+ findFilePath: findAssetFilePath,
362
+ isLessonCandidateFn: isLessonCandidate,
363
+ }));
364
+ schemaRepairs = result.repairs;
365
+ const byRef = new Map(postCleanupRefs.map((candidate) => [candidate.ref, candidate]));
366
+ for (const { ref } of validationFailures) {
367
+ const candidate = byRef.get(ref);
368
+ if (candidate && !(await validate(candidate)))
369
+ repaired.add(ref);
649
370
  }
650
371
  }
651
- const validationFailureRefs = new Set(validationFailures.filter((f) => !repairedRefs.has(f.ref)).map((f) => f.ref));
652
- if (repairedRefs.size > 0) {
653
- info(`[improve] schema repair fixed ${repairedRefs.size}/${validationFailures.length} validation failures; ${validationFailureRefs.size} remain`);
372
+ const validationFailureRefs = new Set(validationFailures.filter((f) => !repaired.has(f.ref)).map((f) => f.ref));
373
+ if (repaired.size > 0) {
374
+ info(`[improve] schema repair fixed ${repaired.size}/${validationFailures.length} validation failures; ${validationFailureRefs.size} remain`);
654
375
  }
655
376
  return { validationFailures, validationFailureRefs, schemaRepairs };
656
377
  }
657
- /**
658
- * Resolve the preparation stages that precede candidate ranking. Keeping these
659
- * lifecycle decisions in one named pass preserves the 220-line orchestrator
660
- * ratchet while giving dry and live execution one implementation.
661
- */
662
- async function runPreparationPrelude(args) {
663
- const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, improveProfile, resolvedPlan, strategyName, budgetSignal, planOnly, actions, cleanupWarnings, } = args;
664
- const memoryBudget = assessMemoryIndexBudget(primaryStashDir);
665
- if (memoryBudget.warning)
666
- cleanupWarnings.push(memoryBudget.warning);
667
- // Consolidation intentionally precedes extract so current-run promotions
668
- // cannot force the pool-delta gate open (#551).
378
+ export async function runImprovePreparationStage(args) {
379
+ const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, eventsCtx, resolvedPlan } = args;
380
+ const planOnly = args.planOnly ?? options.dryRun === true;
381
+ const persist = !planOnly;
382
+ const actions = [];
383
+ const cleanupWarnings = [...(args.initialCleanupWarnings ?? [])];
384
+ const memoryIndexHealth = assessMemoryIndex(primaryStashDir, cleanupWarnings);
385
+ // Consolidation precedes extract, so it only judges memories from earlier runs.
669
386
  const consolidationPass = planOnly
670
- ? (() => {
671
- const planned = planConsolidationPass({
672
- options,
673
- primaryStashDir,
674
- memorySummary,
675
- improveProfile,
676
- resolvedPlan,
677
- eventsCtx,
678
- });
679
- return {
680
- consolidation: {
681
- schemaVersion: 1,
682
- ok: true,
683
- shape: "consolidate-result",
684
- dryRun: true,
685
- previewOnly: true,
686
- target: options.target ?? options.stashDir ?? "",
687
- processed: 0,
688
- merged: 0,
689
- deleted: 0,
690
- promoted: [],
691
- contradicted: 0,
692
- warnings: [],
693
- durationMs: 0,
694
- },
695
- consolidationRan: false,
696
- plan: planned.plan,
697
- };
698
- })()
699
- : await runConsolidationPass({
700
- options,
701
- primaryStashDir,
702
- memorySummary,
703
- improveProfile,
704
- resolvedPlan,
705
- eventsCtx,
706
- budgetSignal,
707
- runBudgetMs: budgetMs,
708
- });
709
- const extractPlan = inspectExtractPass({ options, improveProfile, resolvedPlan, eventsCtx, readOnly: planOnly });
710
- const extractPass = planOnly
711
- ? { extractResults: undefined, warnings: [] }
712
- : await runSessionExtractPass({
713
- options,
714
- primaryStashDir,
715
- improveProfile,
716
- resolvedPlan,
717
- eventsCtx,
718
- budgetSignal,
719
- plan: extractPlan,
720
- });
721
- if (extractPass.warnings.length > 0)
722
- cleanupWarnings.push(...extractPass.warnings);
723
- if (!planOnly) {
387
+ ? {
388
+ consolidation: makeConsolidateResult({
389
+ dryRun: true,
390
+ previewOnly: true,
391
+ target: options.target ?? options.stashDir ?? "",
392
+ durationMs: 0,
393
+ }),
394
+ plan: planConsolidationPass(args).plan,
395
+ }
396
+ : await runConsolidationPass(args);
397
+ const extractPlan = inspectExtractPass(args, planOnly);
398
+ const extractPass = planOnly ? { warnings: [] } : await runSessionExtractPass(args, extractPlan);
399
+ cleanupWarnings.push(...extractPass.warnings);
400
+ if (persist) {
724
401
  appendEvent({
725
402
  eventType: "improve_invoked",
726
403
  ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
727
- metadata: { strategy: strategyName, scope, dryRun: options.dryRun ?? false, eligibleCount: plannedRefs.length },
404
+ metadata: {
405
+ strategy: args.strategyName,
406
+ scope,
407
+ dryRun: options.dryRun ?? false,
408
+ eligibleCount: plannedRefs.length,
409
+ },
728
410
  }, eventsCtx);
729
411
  }
730
- const allowCleanupApply = isAutonomyLaneAllowed("memoryCleanup", options.config ?? loadConfig());
731
- const cleanup = planOnly
732
- ? {
733
- ...projectMemoryCleanup({
734
- mode: "estimate",
735
- plannedRefs,
736
- candidateRefs: memoryCleanupPlan?.pruneCandidates.map((candidate) => candidate.ref) ?? [],
737
- allowApply: allowCleanupApply,
738
- }),
739
- pruneActions: [],
740
- warnings: [],
741
- appliedCleanup: undefined,
412
+ // Memory cleanup: archive redundant derived memories (autonomy-gated).
413
+ const allowCleanup = isAutonomyLaneAllowed("memoryCleanup", options.config ?? loadConfig());
414
+ let appliedCleanup;
415
+ if (persist) {
416
+ try {
417
+ appliedCleanup =
418
+ primaryStashDir && memoryCleanupPlan && allowCleanup
419
+ ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan)
420
+ : undefined;
421
+ }
422
+ catch (err) {
423
+ cleanupWarnings.push(`applyMemoryCleanup failed: ${errMessage(err)}`);
742
424
  }
743
- : await applyCleanupPass({
744
- primaryStashDir,
745
- memoryCleanupPlan,
425
+ }
426
+ const cleanup = planOnly
427
+ ? projectMemoryCleanup({
428
+ mode: "estimate",
429
+ plannedRefs,
430
+ candidateRefs: memoryCleanupPlan?.pruneCandidates.map((candidate) => candidate.ref) ?? [],
431
+ allowApply: allowCleanup,
432
+ })
433
+ : projectMemoryCleanup({
434
+ mode: "execution",
746
435
  plannedRefs,
747
- reindexFn,
748
- budgetSignal,
749
- allowApply: allowCleanupApply,
436
+ archivedRefs: appliedCleanup?.archived.map((record) => record.ref) ?? [],
437
+ allowApply: allowCleanup,
750
438
  });
751
- actions.push(...cleanup.pruneActions);
752
- cleanupWarnings.push(...cleanup.warnings);
753
- const validation = await runValidationAndRepairPass({
754
- postCleanupRefs: cleanup.postCleanupRefs,
439
+ if (appliedCleanup) {
440
+ for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
441
+ if (!appliedCleanup.archived.some((record) => record.ref === candidate.ref))
442
+ continue;
443
+ actions.push({
444
+ ref: candidate.ref,
445
+ mode: "memory-prune",
446
+ result: { ok: true, pruned: true, reason: candidate.reason },
447
+ });
448
+ }
449
+ if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
450
+ try {
451
+ await args.reindexFn({ stashDir: primaryStashDir, signal: args.budgetSignal });
452
+ }
453
+ catch (err) {
454
+ cleanupWarnings.push(`reindex after cleanup failed: ${errMessage(err)}`);
455
+ }
456
+ }
457
+ }
458
+ const { postCleanupRefs } = cleanup;
459
+ const { validationFailures, validationFailureRefs, schemaRepairs } = await runValidationAndRepairPass({
460
+ postCleanupRefs,
755
461
  options,
756
- startMs,
757
- budgetMs,
462
+ startMs: args.startMs,
463
+ budgetMs: args.budgetMs,
758
464
  primaryStashDir,
759
465
  resolvedPlan,
760
- repairValidationFailures: !planOnly && resolvedPlan.processes.validation.enabled && options.repairValidationFailures !== false,
466
+ repairValidationFailures: persist && resolvedPlan.processes.validation.enabled && options.repairValidationFailures !== false,
761
467
  });
762
- return {
763
- memoryIndexHealth: memoryBudget.memoryIndexHealth,
764
- consolidationPass,
765
- extractPlan,
766
- extractResults: extractPass.extractResults,
767
- appliedCleanup: cleanup.appliedCleanup,
768
- postCleanupRefs: cleanup.postCleanupRefs,
769
- cleanupGate: cleanup.gate,
770
- ...validation,
771
- };
772
- }
773
- export async function runImprovePreparationStage(args) {
774
- const { scope, options, primaryStashDir, eventsCtx, initialCleanupWarnings, improveProfile, resolvedPlan, planOnly = options.dryRun === true, } = args;
775
- const actions = [];
776
- const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
777
- const { memoryIndexHealth, consolidationPass, extractPlan, extractResults, appliedCleanup, postCleanupRefs, cleanupGate, validationFailures, validationFailureRefs, schemaRepairs, } = await runPreparationPrelude({ ...args, planOnly, actions, cleanupWarnings });
778
- // Phase 0.5 — structural hygiene pass
779
468
  let lintSummary;
780
469
  if (primaryStashDir) {
781
470
  try {
@@ -783,1796 +472,567 @@ export async function runImprovePreparationStage(args) {
783
472
  lintSummary = { fixed: lintResult.summary.fixed, flagged: lintResult.summary.flagged };
784
473
  }
785
474
  catch {
786
- // lint is best-effort; never block improve
475
+ // lint never blocks improve
787
476
  }
788
477
  }
789
- const recentErrors = seedRecentErrorWindows(schemaRepairs);
790
- const snapshot = buildSnapshotManifest({ postCleanupRefs, validationFailureRefs, eventsCtx });
791
- const gathered = gatherCandidates({
792
- scope,
793
- options,
794
- primaryStashDir,
795
- eventsCtx,
796
- improveProfile,
797
- resolvedPlan,
798
- postCleanupRefs,
799
- validationFailureRefs,
800
- snapshot,
801
- persist: !planOnly,
802
- });
803
- const eligibilitySourceByRef = stampEligibilitySource({
804
- scope,
805
- processableRefs: gathered.processableRefs,
806
- mergedRefs: gathered.mergedRefs,
807
- signalFiltered: gathered.signalFiltered,
808
- proactiveRefs: gathered.proactiveRefs,
809
- highSalienceRefs: gathered.highSalienceRefs,
810
- });
811
- // Shared admission boundary for every synthetic fallback lane. Cleanup and
812
- // structural validation are exclusive selectors: no later rank/replay state
813
- // may re-create a candidate they removed. Keep the exact surviving objects
814
- // so any admitted fallback preserves its index-resolved file/item provenance.
815
- const fallbackEligibleRefs = postCleanupRefs.filter((candidate) => !validationFailureRefs.has(candidate.ref));
816
- const scored = scoreSalience({
817
- scope,
818
- options,
819
- primaryStashDir,
820
- eventsCtx,
821
- mergedRefs: gathered.mergedRefs,
822
- eligibilitySourceByRef,
823
- feedbackSummary: gathered.feedbackSummary,
824
- retrievalCounts: gathered.retrievalCounts,
825
- signalFiltered: gathered.signalFiltered,
826
- proactiveRefs: gathered.proactiveRefs,
827
- highSalienceRefs: gathered.highSalienceRefs,
828
- forgettingEligibleRefs: fallbackEligibleRefs,
829
- persist: !planOnly,
830
- });
831
- // Replay is additive to the signal lanes, but it must not bypass selectors
832
- // that have already removed a ref. Use the exact surviving objects so a
833
- // replay admission preserves the index-resolved file/item provenance while
834
- // excluding cleanup-pruned and structurally-invalid candidates.
835
- const filtered = await filterEligibility({
836
- scope,
837
- options,
838
- replayEligibleRefs: fallbackEligibleRefs,
839
- eventsCtx,
840
- mergedRefs: scored.mergedRefs,
841
- salienceMap: scored.salienceMap,
842
- eligibilitySourceByRef,
843
- distillOnlyRefs: gathered.distillOnlyRefs,
844
- validationFailureRefs,
845
- summary: {
846
- signalAndRetrievalRefs: gathered.signalAndRetrievalRefs,
847
- signalFiltered: gathered.signalFiltered,
848
- },
849
- persist: !planOnly,
850
- });
851
- const preDiskRefSet = new Set(filtered.preDiskRefs.map((candidate) => candidate.ref));
852
- const terminalSignalSkippedRefs = fallbackEligibleRefs.filter((candidate) => !preDiskRefSet.has(candidate.ref));
853
- recordSignalSkipObservability({
854
- actions,
855
- terminalSignalSkippedRefs,
856
- distillCooledRefs: gathered.distillCooledRefs,
857
- eventsCtx,
858
- persist: !planOnly,
859
- });
860
- // Gate counts are an exclusive, sequential accounting of the raw pool.
861
- // Replay, proactive maintenance, high-salience, and forgetting-safety are
862
- // legitimate signal-gate fallback lanes, so a ref admitted by any of them
863
- // was not removed by the signal gate. Derive this count from the actual
864
- // pre-disk survivor set instead of the earlier lane-rescue snapshot; the
865
- // latter is intentionally assembled before replay and is also broader than
866
- // the effective pool when --require-feedback-signal suppresses fallbacks.
867
- const signalRemoved = terminalSignalSkippedRefs.length;
868
- const totalReflectBlocked = terminalSignalSkippedRefs.length + gathered.distillOnlyRefs.length;
869
- if (totalReflectBlocked > 0) {
870
- info(`[improve] ${totalReflectBlocked} of ${gathered.preCooldownCount} indexed refs blocked by reflect signal-delta ` +
871
- `(${terminalSignalSkippedRefs.length} fully skipped, ${gathered.distillOnlyRefs.length} routed to distill-only)`);
478
+ // Schema-repair errors get their own window; they are never shown to reflect.
479
+ const recentErrors = {};
480
+ for (const repair of schemaRepairs) {
481
+ if (repair.outcome !== "error")
482
+ continue;
483
+ pushRecentError(recentErrors, "schema-repair", repair.error ?? `schema repair error: ${repair.reason}`);
872
484
  }
873
- const planningGates = [
874
- cleanupGate,
875
- {
876
- name: "validation",
877
- removed: validationFailureRefs.size,
878
- reason: "structural validation failures",
879
- },
880
- {
881
- name: "signal",
882
- removed: signalRemoved,
883
- reason: "no fresh signal and no fallback lane selected the ref",
884
- },
885
- {
886
- name: "disk",
887
- removed: filtered.missingDiskCount,
888
- reason: "backing asset is absent on disk",
889
- },
890
- {
891
- name: "limit",
892
- removed: filtered.limitRemoved,
893
- reason: "deferred by the effective run limit",
894
- },
895
- ];
485
+ const selection = await selectLoopCandidates(args, postCleanupRefs, validationFailureRefs, actions, persist);
896
486
  return {
897
487
  actions,
898
488
  cleanupWarnings,
899
489
  appliedCleanup,
900
490
  memoryIndexHealth,
901
- extract: extractResults,
902
- actionableRefs: filtered.actionableRefs,
903
- signalBearingSet: gathered.signalBearingSet,
491
+ extract: extractPass.extractResults,
492
+ actionableRefs: selection.actionableRefs,
493
+ signalBearingSet: selection.signalBearingSet,
904
494
  validationFailures,
905
495
  schemaRepairs,
906
496
  lintSummary,
907
- loopRefs: filtered.loopRefs,
908
- distillCooledRefs: gathered.distillCooledRefs,
909
- distillOnlyRefs: filtered.distillOnlyRefs,
910
- coverageGaps: filtered.coverageGaps,
497
+ loopRefs: selection.loopRefs,
498
+ distillCooledRefs: selection.distillCooledRefs,
499
+ distillOnlyRefs: selection.distillOnlyRefs,
500
+ coverageGaps: selection.coverageGaps,
911
501
  recentErrors,
912
- utilityMap: scored.utilityMap,
913
502
  consolidation: consolidationPass.consolidation,
914
- consolidationRan: consolidationPass.consolidationRan,
915
- ...(gathered.proactiveMaintenanceSummary ? { proactiveMaintenance: gathered.proactiveMaintenanceSummary } : {}),
503
+ ...(selection.proactive.proactiveMaintenanceSummary
504
+ ? { proactiveMaintenance: selection.proactive.proactiveMaintenanceSummary }
505
+ : {}),
916
506
  planning: {
917
- gates: planningGates,
918
- replayBudget: filtered.replayBudget,
919
- ...(gathered.proactivePlan ? { proactive: gathered.proactivePlan } : {}),
507
+ gates: [
508
+ cleanup.gate,
509
+ { name: "validation", removed: validationFailureRefs.size, reason: "structural validation failures" },
510
+ ...selection.gates,
511
+ ],
512
+ ...(selection.proactive.proactivePlan ? { proactive: selection.proactive.proactivePlan } : {}),
920
513
  consolidation: consolidationPass.plan,
921
514
  extract: { wouldRun: extractPlan.wouldRun, reason: extractPlan.reason },
922
515
  },
923
516
  };
924
517
  }
925
- // ── preparation-stage passes (WI-7.6 decomposition, R31) ────────────────────
926
- // The six-pass split prescribed by the chunk-7 brief §WI-7.6, adapted to the
927
- // code as it exists at HEAD (anchors re-measured; see the chunk-7 ledger):
928
- // snapshot-manifest → buildSnapshotManifest
929
- // candidate-gather → gatherCandidates (+ its five lane/sub-passes)
930
- // salience-score → scoreSalience (+ outcome/vector/persist sub-passes)
931
- // valence-score → the two computeValenceScore call sites move VERBATIM
932
- // inside the salience passes (pure fn; no separate pass)
933
- // standards-context → does not exist in preparation.ts (assembly lives in
934
- // extract.ts — recorded in the ledger, no empty pass)
935
- // eligibility-filter → filterEligibility (+ replay/disk-check sub-passes)
936
- // Every pass takes an args object and returns its results; the orchestrator
937
- // folds them. Shared-by-reference structures (the ImproveEligibleRef objects,
938
- // eligibilitySourceByRef, salienceMap, actions, recentErrors) keep their
939
- // identity — attribution stamps must travel with the ref objects into the
940
- // loop stage exactly as before.
941
- /** Phase 0 — MEMORY.md budget check (200-line cap; warn at 180). */
942
- function assessMemoryIndexBudget(primaryStashDir) {
943
- let warning;
944
- // Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
945
- let memoryIndexHealth;
946
- if (primaryStashDir) {
947
- const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
948
- if (fs.existsSync(memoryMdPath)) {
949
- try {
950
- const lines = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
951
- const overBudget = lines >= 180;
952
- memoryIndexHealth = { lineCount: lines, overBudget };
953
- if (overBudget) {
954
- warning = `MEMORY.md has ${lines} lines (budget: 200). Consolidation strongly recommended.`;
955
- }
956
- }
957
- catch {
958
- // best-effort
959
- }
960
- }
961
- }
962
- return { memoryIndexHealth, warning };
963
- }
964
- /**
965
- * Memory-cleanup apply + prune-action recording + the post-cleanup reindex.
966
- * Returns the surviving ref set and the prune actions/warnings for the
967
- * orchestrator to fold (same order as the old inline pushes).
968
- */
969
- async function applyCleanupPass(args) {
970
- const { primaryStashDir, memoryCleanupPlan, plannedRefs, reindexFn, budgetSignal, allowApply } = args;
971
- const pruneActions = [];
972
- const warnings = [];
973
- let appliedCleanup;
518
+ /** MEMORY.md line budget: warn at 180 of 200 lines. */
519
+ function assessMemoryIndex(primaryStashDir, warnings) {
520
+ if (!primaryStashDir)
521
+ return undefined;
522
+ const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
523
+ if (!fs.existsSync(memoryMdPath))
524
+ return undefined;
974
525
  try {
975
- appliedCleanup =
976
- primaryStashDir && memoryCleanupPlan && allowApply
977
- ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan)
978
- : undefined;
979
- }
980
- catch (err) {
981
- warnings.push(`applyMemoryCleanup failed: ${err instanceof Error ? err.message : String(err)}`);
982
- }
983
- const projection = projectMemoryCleanup({
984
- mode: "execution",
985
- plannedRefs,
986
- archivedRefs: appliedCleanup?.archived.map((record) => record.ref) ?? [],
987
- allowApply,
988
- });
989
- // ── Phase 1: validation pass + schema repair (run on full postCleanupRefs) ──
990
- // Identifies refs whose on-disk asset has structural problems. Validation
991
- // failures are excluded from every downstream bucket. Run early so the
992
- // cooldown partition operates on a clean set.
993
- if (appliedCleanup) {
994
- for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
995
- const archived = appliedCleanup.archived.find((record) => record.ref === candidate.ref);
996
- if (!archived)
997
- continue;
998
- pruneActions.push({
999
- ref: candidate.ref,
1000
- mode: "memory-prune",
1001
- result: { ok: true, pruned: true, reason: candidate.reason },
1002
- });
1003
- }
1004
- if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
1005
- try {
1006
- await reindexFn({ stashDir: primaryStashDir, signal: budgetSignal });
1007
- }
1008
- catch (err) {
1009
- warnings.push(`reindex after cleanup failed: ${err instanceof Error ? err.message : String(err)}`);
1010
- }
526
+ const lineCount = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
527
+ if (lineCount >= 180) {
528
+ warnings.push(`MEMORY.md has ${lineCount} lines (budget: 200). Consolidation strongly recommended.`);
1011
529
  }
530
+ return { lineCount, overBudget: lineCount >= 180 };
1012
531
  }
1013
- return { appliedCleanup, ...projection, pruneActions, warnings };
1014
- }
1015
- /** Seed the per-originator rolling error windows from schema-repair errors. */
1016
- function seedRecentErrorWindows(schemaRepairs) {
1017
- // O-5 / #378: Per-originator rolling error windows.
1018
- // Reflexion (arXiv:2303.11366) warns that cross-task verbal critique
1019
- // contamination degrades below single-shot baseline. Each originator key
1020
- // ("schema-repair", "reflect") maintains its own rolling window so that
1021
- // schema-repair failures are not injected as avoidPatterns into reflect calls.
1022
- const recentErrors = {};
1023
- const RECENT_ERRORS_CAP = 3;
1024
- // Helper: push an error onto an originator's rolling window.
1025
- function pushRecentError(originator, msg) {
1026
- if (!recentErrors[originator])
1027
- recentErrors[originator] = [];
1028
- recentErrors[originator].push(msg);
1029
- if (recentErrors[originator].length > RECENT_ERRORS_CAP)
1030
- recentErrors[originator].shift();
1031
- }
1032
- // Seed schema-repair originator window from any schema-repair errors.
1033
- for (const repair of schemaRepairs) {
1034
- if (repair.outcome === "error") {
1035
- const errMsg = repair.error ?? `schema repair error: ${repair.reason}`;
1036
- pushRecentError("schema-repair", errMsg);
1037
- }
532
+ catch {
533
+ return undefined;
1038
534
  }
1039
- return recentErrors;
1040
535
  }
1041
- /** Pass: snapshot-manifest — the three timestamp maps + the 30-day signal window. */
536
+ // ── Candidate selection ──────────────────────────────────────────────────────
537
+ const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
538
+ /** Feedback that counts as a signal carries a signal or a note (a bare `akm feedback` does not). */
539
+ function isSignalEvent(metadata) {
540
+ const meta = metadata;
541
+ return meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
542
+ }
543
+ /** One read of the feedback events and the ledger's reflect/distill rows. */
1042
544
  export function buildSnapshotManifest(args) {
1043
- const { postCleanupRefs, validationFailureRefs, eventsCtx } = args;
1044
- // ── Phase 2: signal-delta eligibility sets built EARLY ────────────────────
1045
- // 0.8.0 replaces the flat time-based cooldowns (which produced synchronised
1046
- // waves whenever many refs cooled at the same instant — see the 2026-05-26
1047
- // 54-ref simultaneous-reflect incident) with a *signal-delta* gate:
1048
- //
1049
- // reflectEligible(ref) ≡ latestFeedbackTs(ref) > lastReflectProposalTs(ref)
1050
- // distillEligible(ref) ≡ latestFeedbackTs(ref) > lastDistillProposalTs(ref)
1051
- //
1052
- // i.e. a ref is re-eligible iff new feedback has landed since the last
1053
- // proposal was generated for it. Stable content with no new signal stays
1054
- // out of the queue regardless of clock time; a sudden burst of feedback
1055
- // surfaces only the refs that the burst actually touches.
1056
- //
1057
- // The 30-day FEEDBACK_SIGNAL_WINDOW_DAYS bound still applies — only feedback
1058
- // events newer than that count as "current signal". Ancient one-off
1059
- // negatives don't permanently lock a ref into every run.
1060
- const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
545
+ const { eventsCtx, stashDir } = args;
1061
546
  const feedbackSinceCutoff = new Date(Date.now() - daysToMs(FEEDBACK_SIGNAL_WINDOW_DAYS)).toISOString();
1062
- // Build the three timestamp maps once across the entire postCleanupRefs set.
1063
- // Per-ref queries would be N+1 and the planner is already the hottest path
1064
- // in `akm improve`.
1065
- const candidateRefs = postCleanupRefs.filter((r) => !validationFailureRefs.has(r.ref)).map((r) => r.ref);
1066
- // Carry each candidate's item_ref into the feedback/proposal timestamp reads.
1067
- const itemRefByRef = buildItemRefByRef(postCleanupRefs);
1068
- const latestFeedbackTs = buildLatestFeedbackTsMap(candidateRefs, feedbackSinceCutoff, itemRefByRef, eventsCtx);
1069
- const lastReflectProposalTs = buildLatestProposalTsMap(candidateRefs, "reflect", itemRefByRef, eventsCtx);
1070
- const lastDistillProposalTs = buildLatestProposalTsMap(candidateRefs, "distill", itemRefByRef, eventsCtx);
1071
- return { feedbackSinceCutoff, latestFeedbackTs, lastReflectProposalTs, lastDistillProposalTs };
1072
- }
1073
- /**
1074
- * Pass: candidate-gather — the signal-delta partition, the bulk feedback
1075
- * summary, retrieval signals, the Layer-2 proactive and Layer-3 high-salience
1076
- * rescue lanes, and the merged candidate set. Lane attribution stamping is a
1077
- * separate pass — see `stampEligibilitySource` — run by the caller once
1078
- * `mergedRefs` is known.
1079
- */
1080
- function gatherCandidates(args) {
1081
- const { scope, options, primaryStashDir, eventsCtx, improveProfile, resolvedPlan, postCleanupRefs, persist } = args;
1082
- const { feedbackSinceCutoff, lastReflectProposalTs, lastDistillProposalTs } = args.snapshot;
1083
- const partition = partitionBySignalDelta({
1084
- scope,
1085
- options,
1086
- postCleanupRefs,
1087
- validationFailureRefs: args.validationFailureRefs,
1088
- snapshot: args.snapshot,
1089
- });
1090
- const { distillCooledRefs, preCooldownCount, eligibleRefs, distillOnlyRefs, noFeedbackPool } = partition;
1091
- // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
1092
- // Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
1093
- // deferred noFeedbackPool that may be rescued by the proactive-maintenance
1094
- // (Layer 2) or high-salience (Layer 3) fallbacks below. The fully-skipped
1095
- // bucket is retained as partition metadata only; terminal skip observability
1096
- // is delayed until every fallback lane has finalized. We deliberately avoid
1097
- // spending DB/CPU on refs that the signal-delta gate rejected with feedback
1098
- // already on record.
1099
- const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
1100
- const feedbackSummary = buildFeedbackSummaryMap({
1101
- processableRefs,
1102
- noFeedbackPool,
1103
- eventsCtx,
1104
- feedbackSinceCutoff,
1105
- });
1106
- const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
1107
- const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
1108
- // Zero-feedback candidates for the proactive/high-salience fallbacks:
1109
- // processableRefs without a recent signal, plus the deferred noFeedbackPool.
1110
- // Dedupe by ref (the two sources are disjoint by construction, but guard
1111
- // against overlap defensively).
1112
- const noFeedbackSeen = new Set();
1113
- const noFeedbackCandidates = [];
1114
- for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
1115
- if (noFeedbackSeen.has(r.ref))
1116
- continue;
1117
- noFeedbackSeen.add(r.ref);
1118
- noFeedbackCandidates.push(r);
547
+ const candidates = args.postCleanupRefs.filter((r) => !args.validationFailureRefs.has(r.ref));
548
+ const refByKey = new Map(candidates.map((r) => [keyOf(r), r.ref]));
549
+ const latestFeedbackTs = new Map();
550
+ const feedback = new Map(candidates.map((r) => [r.ref, { hasSignal: false, positive: 0, negative: 0 }]));
551
+ if (candidates.length > 0) {
552
+ for (const e of readEvents({ type: "feedback" }, eventsCtx).events) {
553
+ const ref = e.ref ? refByKey.get(e.ref) : undefined;
554
+ const entry = ref ? feedback.get(ref) : undefined;
555
+ if (!ref || !entry)
556
+ continue;
557
+ const ts = e.ts ?? "";
558
+ if (ts >= feedbackSinceCutoff && isSignalEvent(e.metadata)) {
559
+ entry.hasSignal = true;
560
+ if (ts > (latestFeedbackTs.get(ref) ?? ""))
561
+ latestFeedbackTs.set(ref, ts);
562
+ }
563
+ const signal = e.metadata?.signal;
564
+ if (signal === "positive")
565
+ entry.positive++;
566
+ else if (signal === "negative")
567
+ entry.negative++;
568
+ }
1119
569
  }
1120
- const { retrievalCounts, lastUseMsForProactive } = fetchRetrievalSignals({
1121
- options,
1122
- primaryStashDir,
1123
- signalFiltered,
1124
- noFeedbackCandidates,
1125
- eventsCtx,
1126
- persist,
1127
- });
1128
- // `--require-feedback-signal` is a hard policy boundary, not merely a final
1129
- // list filter. Do not run or report fallback selectors that the invocation
1130
- // explicitly disabled (and do not emit their live selection events).
1131
- const allowFallbacks = options.requireFeedbackSignal !== true;
1132
- const proactive = allowFallbacks
1133
- ? selectProactiveMaintenanceLane({
1134
- scope,
1135
- improveProfile,
1136
- resolvedPlan,
1137
- eventsCtx,
1138
- noFeedbackCandidates,
1139
- lastReflectProposalTs,
1140
- lastDistillProposalTs,
1141
- retrievalCounts,
1142
- lastUseMsForProactive,
1143
- persist,
1144
- })
1145
- : { proactiveRefs: [] };
1146
- const proactiveRefs = proactive.proactiveRefs;
1147
- const proactiveMaintenanceSummary = proactive.proactiveMaintenanceSummary;
1148
- const highSalienceRefs = allowFallbacks
1149
- ? selectHighSalienceLane({
1150
- options,
1151
- improveProfile,
1152
- eventsCtx,
1153
- noFeedbackCandidates,
1154
- proactiveRefs,
1155
- lastReflectProposalTs,
1156
- persist,
1157
- })
1158
- : [];
1159
- // If the user explicitly scoped to a single ref, always act on it —
1160
- // skip the signal/retrieval filter entirely. The filter exists to avoid
1161
- // noisy "improve everything" runs; it should not gate an intentional
1162
- // per-ref invocation where the user's explicit choice is the signal.
1163
- //
1164
- // For type/all scope: only process refs with usage signals (recent feedback
1165
- // or a proactive/high-salience rescue). A stash with no signals has 0
1166
- // eligible refs — usage is the gate. Run `akm feedback <ref> --positive` or
1167
- // retrieve assets to bring them into the eligible pool.
1168
- // Layer-2 proactive refs join the eligible set alongside feedback-signal
1169
- // refs. The three sources are disjoint by construction (proactive draws from
1170
- // noFeedbackCandidates, and high-salience draws from the remainder), but
1171
- // dedupe defensively so a ref can never enter the loop twice.
1172
- // `requireFeedbackSignal` still suppresses all fallback sources for callers
1173
- // that want feedback-only runs.
1174
- const signalAndRetrievalRefs = dedupeRefs([...signalFiltered, ...proactiveRefs, ...highSalienceRefs]);
1175
- const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
570
+ const ledger = stashDir
571
+ ? loadLedgerSnapshot({ eventsCtx, ...(args.readOnly ? { readOnly: true } : {}) }, stashDir, ["reflect", "distill"])
572
+ : new Map();
1176
573
  return {
1177
- distillCooledRefs,
1178
- preCooldownCount,
1179
- distillOnlyRefs,
1180
- feedbackSummary,
1181
- signalFiltered,
1182
- signalBearingSet,
1183
- retrievalCounts,
1184
- proactiveRefs,
1185
- proactiveMaintenanceSummary,
1186
- proactivePlan: proactive.proactivePlan,
1187
- highSalienceRefs,
1188
- signalAndRetrievalRefs,
1189
- mergedRefs,
1190
- processableRefs,
574
+ feedbackSinceCutoff,
575
+ nowIso: new Date().toISOString(),
576
+ latestFeedbackTs,
577
+ ledger,
578
+ lastReflectAttemptAt: lastAttemptByRef(ledger, "reflect", candidates),
579
+ lastDistillAttemptAt: lastAttemptByRef(ledger, "distill", candidates),
580
+ feedback,
1191
581
  };
1192
582
  }
1193
583
  /**
1194
- * Attribution tagging: stamp each ref with the eligibility lane that selected
1195
- * it. Every reflect/distill proposal must record WHICH lane chose its source
1196
- * asset so downstream accept/reject/revert/retrieval outcomes can be sliced by
1197
- * lane (does the PROACTIVE lane produce value vs the reactive lanes?). We
1198
- * build the lane map here — the one place all three lanes are known — and
1199
- * stamp it onto each ImproveEligibleRef object. Because the ref objects are
1200
- * shared by reference across buckets, the stamp travels with the ref through
1201
- * the sort, disk-check, and loop stages down to the reflect/distill event
1202
- * emit sites and createProposal calls. See EligibilitySource for the lane
1203
- * vocabulary.
1204
- *
1205
- * Precedence (prefer the most specific reactive signal):
1206
- * scope > signal-delta > proactive > high-salience
1207
- * A ref with real feedback is attributed to feedback even if it was also due
1208
- * for proactive maintenance or had high encoding salience. We apply lanes
1209
- * weakest-first so the strongest overwrites; the explicit --scope <ref> bypass
1210
- * wins outright (user intent).
1211
- */
1212
- function stampEligibilitySource(args) {
1213
- const { scope, processableRefs, mergedRefs, signalFiltered, proactiveRefs, highSalienceRefs } = args;
1214
- const eligibilitySourceByRef = new Map();
1215
- for (const r of highSalienceRefs)
1216
- eligibilitySourceByRef.set(r.ref, "high-salience");
1217
- for (const r of proactiveRefs)
1218
- eligibilitySourceByRef.set(r.ref, "proactive");
1219
- for (const r of signalFiltered)
1220
- eligibilitySourceByRef.set(r.ref, "signal-delta");
1221
- if (scope.mode === "ref") {
1222
- // O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
1223
- // arrived via the scopeRefBypass branch, so attribute the whole set to scope.
1224
- for (const r of processableRefs)
1225
- eligibilitySourceByRef.set(r.ref, "scope");
1226
- }
1227
- for (const r of mergedRefs) {
1228
- // "unknown" is a genuine fallback, never a silent alias for signal-delta:
1229
- // only refs we truly cannot attribute land here (none in practice, since
1230
- // mergedRefs is always a subset of the four lanes above).
1231
- r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
1232
- }
1233
- return eligibilitySourceByRef;
1234
- }
1235
- /**
1236
- * The signal-delta partition of postCleanupRefs into the four buckets (pass:
1237
- * candidate-gather, phase 3). The 2026-05-26 54-ref incident semantics move
1238
- * VERBATIM — see the phase-2/3 comments inside.
584
+ * Partition the post-cleanup refs against the ledger:
585
+ * - eligibleRefs: reflect's signal delta passes (distill may still be cooled);
586
+ * - distillOnlyRefs: only distill's passes, on a distill candidate;
587
+ * - noFeedbackPool: no recent feedback and no reflect window, left to the
588
+ * fallback lanes;
589
+ * - fullySkippedCount: feedback on record but nothing new, or a live window.
590
+ * An explicit `--scope <ref>` bypasses every gate.
1239
591
  */
1240
592
  export function partitionBySignalDelta(args) {
1241
- const { scope, options, postCleanupRefs, validationFailureRefs } = args;
1242
- const { latestFeedbackTs, lastReflectProposalTs, lastDistillProposalTs } = args.snapshot;
1243
- // Refs the distill signal-delta gate rejected at planning time. The main
1244
- // loop reads this to skip distill for these refs without re-checking
1245
- // eligibility per iteration.
1246
- const distillCooledRefs = new Set();
1247
- const preCooldownCount = postCleanupRefs.length;
1248
- // ── Phase 3: partition postCleanupRefs by signal-delta eligibility ────────
1249
- // Three buckets (validation failures are excluded entirely):
1250
- // eligibleRefs — reflect signal-delta passes (full reflect+distill
1251
- // loop path; distill guard remains in the loop for
1252
- // refs that fail the distill signal-delta gate).
1253
- // distillOnlyRefs — reflect blocked but distill signal-delta passes
1254
- // AND ref is a distill candidate.
1255
- // noFeedbackPool — neither signal-delta gate passes *and* the ref has
1256
- // no recent feedback signal at all. These are NOT
1257
- // skipped here: they are handed to the proactive
1258
- // (Layer 2) and high-salience (Layer 3) fallbacks
1259
- // below so never-rated assets can still be improved.
1260
- // Only refs those lanes decline are fully skipped.
1261
- // fullySkippedCount — has stale feedback but no signal delta → genuine
1262
- // skip candidate, excluded from sort. Final skip
1263
- // observability is emitted only after fallbacks.
1264
- const eligibleRefs = [];
1265
- const distillOnlyRefs = [];
1266
- // Zero-(recent-)feedback refs deferred to the proactive/high-salience fallbacks.
1267
- const noFeedbackPool = [];
1268
- let fullySkippedCount = 0;
1269
- // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
1270
- const scopeRefBypass = scope.mode === "ref";
593
+ const { postCleanupRefs, validationFailureRefs } = args;
594
+ const { latestFeedbackTs, ledger, nowIso } = args.snapshot;
595
+ // Newer feedback lifts a revisit window, never a rejection.
596
+ const deltaPasses = (candidate, source) => {
597
+ const feedbackAt = latestFeedbackTs.get(candidate.ref);
598
+ if (!feedbackAt)
599
+ return false;
600
+ const row = ledgerRowFor(ledger, source, candidate.ref, candidate.itemRef);
601
+ return feedbackAt > (row?.lastAttemptAt ?? "") && !isLedgerBlocked(row, nowIso, feedbackAt);
602
+ };
603
+ const out = {
604
+ distillCooledRefs: new Set(),
605
+ preCooldownCount: postCleanupRefs.length,
606
+ eligibleRefs: [],
607
+ distillOnlyRefs: [],
608
+ noFeedbackPool: [],
609
+ fullySkippedCount: 0,
610
+ };
1271
611
  for (const r of postCleanupRefs) {
1272
612
  if (validationFailureRefs.has(r.ref))
1273
613
  continue;
1274
- if (scopeRefBypass) {
1275
- eligibleRefs.push(r);
614
+ if (args.scope.mode === "ref") {
615
+ out.eligibleRefs.push(r);
1276
616
  continue;
1277
617
  }
1278
- const reflectOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastReflectProposalTs);
1279
- const distillOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastDistillProposalTs);
1280
- const isDistillCandidate = isDistillCandidateRef(r.ref, options.stashDir);
618
+ const reflectOk = deltaPasses(r, "reflect");
619
+ const distillOk = deltaPasses(r, "distill");
1281
620
  if (reflectOk) {
1282
- if (!distillOk && isDistillCandidate) {
1283
- // Reflect passes the gate, distill does not. Record only partition
1284
- // metadata here; observability is emitted after every fallback selector
1285
- // has finalized the invocation's terminal skipped set.
1286
- distillCooledRefs.add(r.ref);
1287
- }
1288
- else if (!distillOk) {
1289
- // Not a distill candidate AND distill gate doesn't pass — just mark
1290
- // distillCooled so the loop's distill section is a no-op.
1291
- distillCooledRefs.add(r.ref);
1292
- }
1293
- eligibleRefs.push(r);
621
+ if (!distillOk)
622
+ out.distillCooledRefs.add(r.ref);
623
+ out.eligibleRefs.push(r);
1294
624
  }
1295
- else if (distillOk && isDistillCandidate) {
1296
- // Reflect blocked but distill passes → distill-only bucket.
1297
- distillOnlyRefs.push(r);
625
+ else if (distillOk && isDistillCandidateRef(r.ref, args.options.stashDir)) {
626
+ out.distillOnlyRefs.push(r);
1298
627
  }
1299
- else if (!latestFeedbackTs.has(r.ref)) {
1300
- // Neither signal-delta gate passes AND there is no recent feedback signal
1301
- // at all. Rather than skip outright, defer to the proactive-maintenance
1302
- // and high-salience fallbacks below: a never-rated asset is exactly what
1303
- // those lanes are meant to rescue. Refs those lanes decline are skipped there.
1304
- noFeedbackPool.push(r);
628
+ else if (!latestFeedbackTs.has(r.ref) &&
629
+ !isLedgerBlocked(ledgerRowFor(ledger, "reflect", r.ref, r.itemRef), nowIso)) {
630
+ out.noFeedbackPool.push(r);
1305
631
  }
1306
632
  else {
1307
- // Has feedback on record but no signal delta since the last proposal —
1308
- // genuinely a fully-skipped candidate. Count it as partition metadata;
1309
- // final observability waits until replay and every other fallback lane
1310
- // has had a chance to rescue it.
1311
- fullySkippedCount++;
633
+ out.fullySkippedCount++;
1312
634
  }
1313
635
  }
1314
- return {
1315
- distillCooledRefs,
1316
- preCooldownCount,
1317
- eligibleRefs,
1318
- distillOnlyRefs,
1319
- noFeedbackPool,
1320
- fullySkippedCount,
1321
- };
636
+ return out;
1322
637
  }
1323
638
  /**
1324
- * Emit signal-delta skip observability only after every fallback lane has
1325
- * finalized the pre-disk survivor set. This prevents replay, proactive,
1326
- * high-salience, or forgetting-safety winners from also being recorded as
1327
- * terminally skipped work.
639
+ * Pick the loop's refs: signal delta, the fallback lanes (unless
640
+ * `--require-feedback-signal`), lane attribution, salience, the
641
+ * no-op-dampened ranking, the disk check and the limit.
1328
642
  */
1329
- function recordSignalSkipObservability(args) {
1330
- const { actions, terminalSignalSkippedRefs, distillCooledRefs, eventsCtx, persist } = args;
1331
- for (const ref of distillCooledRefs) {
643
+ async function selectLoopCandidates(args, postCleanupRefs, validationFailureRefs, actions, persist) {
644
+ const { scope, options, primaryStashDir, eventsCtx, improveProfile } = args;
645
+ const snapshot = buildSnapshotManifest({
646
+ postCleanupRefs,
647
+ validationFailureRefs,
648
+ eventsCtx,
649
+ stashDir: primaryStashDir ?? options.stashDir,
650
+ readOnly: !persist,
651
+ });
652
+ const partition = partitionBySignalDelta({ scope, options, postCleanupRefs, validationFailureRefs, snapshot });
653
+ const processableRefs = [...partition.eligibleRefs, ...partition.distillOnlyRefs];
654
+ const signalFiltered = processableRefs.filter((c) => snapshot.feedback.get(c.ref)?.hasSignal === true);
655
+ const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
656
+ // The fallback lanes (proactive, high salience) have no usage evidence of
657
+ // their own: they pick only what retrieval returned or new material improve
658
+ // never processed (#986). Evaluated once per candidate.
659
+ const fallbackEligible = postCleanupRefs.filter((c) => !validationFailureRefs.has(c.ref));
660
+ const allowFallbacks = options.requireFeedbackSignal !== true;
661
+ const retrievalScope = scope.mode === "ref" || !allowFallbacks
662
+ ? undefined
663
+ : loadRetrievalScope({ eventsCtx, ...(persist ? {} : { readOnly: true }) }, primaryStashDir ?? options.stashDir);
664
+ const unscoped = new Set(fallbackEligible.filter((c) => !isInRetrievalScope(retrievalScope, c.ref, c.filePath)).map((c) => c.ref));
665
+ const noFeedbackPool = dedupeRefs([
666
+ ...processableRefs.filter((r) => !signalBearingSet.has(r.ref)),
667
+ ...partition.noFeedbackPool,
668
+ ]);
669
+ const noFeedbackCandidates = noFeedbackPool.filter((r) => !unscoped.has(r.ref));
670
+ // Only a ref no fallback lane may pick anymore is charged to the retrieval gate.
671
+ const outOfScope = new Set(noFeedbackPool.filter((r) => unscoped.has(r.ref)).map((r) => r.ref));
672
+ const retrieval = fetchRetrievalSignals(options, signalFiltered, noFeedbackCandidates, eventsCtx, persist);
673
+ const proactive = allowFallbacks
674
+ ? selectProactiveMaintenanceLane(args, noFeedbackCandidates, snapshot, retrieval, persist)
675
+ : { proactiveRefs: [] };
676
+ const highSalienceRefs = allowFallbacks
677
+ ? selectHighSalienceLane(options, improveProfile, eventsCtx, noFeedbackCandidates.filter((r) => !proactive.proactiveRefs.some((p) => p.ref === r.ref)), snapshot.lastReflectAttemptAt, persist)
678
+ : [];
679
+ // An explicit ref scope always acts on its ref; otherwise usage signals gate the pool.
680
+ const signalAndRetrievalRefs = dedupeRefs([...signalFiltered, ...proactive.proactiveRefs, ...highSalienceRefs]);
681
+ const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
682
+ // Lane attribution, weakest first so the strongest wins: high-salience <
683
+ // proactive < signal-delta, and an explicit ref scope over everything.
684
+ const sourceByRef = new Map();
685
+ for (const r of highSalienceRefs)
686
+ sourceByRef.set(r.ref, "high-salience");
687
+ for (const r of proactive.proactiveRefs)
688
+ sourceByRef.set(r.ref, "proactive");
689
+ for (const r of signalFiltered)
690
+ sourceByRef.set(r.ref, "signal-delta");
691
+ if (scope.mode === "ref")
692
+ for (const r of processableRefs)
693
+ sourceByRef.set(r.ref, "scope");
694
+ for (const r of mergedRefs)
695
+ r.eligibilitySource = sourceByRef.get(r.ref) ?? "unknown";
696
+ const salienceMap = scoreSalience(args, mergedRefs, snapshot.feedback, retrieval.retrievalCounts, persist);
697
+ // Rank by salience; a ref skipped as a no-op repeatedly sorts lower (its stored rank is untouched).
698
+ const noOps = new Map();
699
+ withRunState(eventsCtx, persist, (db) => {
700
+ for (const r of mergedRefs)
701
+ noOps.set(r.ref, getAssetSalience(db, keyOf(r))?.consecutive_no_ops ?? 0);
702
+ });
703
+ const effectiveScore = (ref) => {
704
+ const rank = salienceMap.get(ref)?.rankScore ?? 0;
705
+ return (noOps.get(ref) ?? 0) >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rank * SALIENCE_NO_OP_DAMPEN_FACTOR : rank;
706
+ };
707
+ const sorted = [...mergedRefs].sort((a, b) => effectiveScore(b.ref) - effectiveScore(a.ref) || (a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0));
708
+ const coverageGaps = withIndexDb(!persist, getZeroResultSearches) ?? [];
709
+ const { actionableRefs, missing } = await dropRefsMissingOnDisk(sorted, options, eventsCtx, persist);
710
+ const selection = selectEffectiveImproveRefs({
711
+ rankedRefs: actionableRefs,
712
+ distillOnlyRefs: partition.distillOnlyRefs,
713
+ limit: options.limit,
714
+ });
715
+ if (signalAndRetrievalRefs.length > 0) {
716
+ info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback)`);
717
+ }
718
+ if (validationFailureRefs.size > 0)
719
+ info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
720
+ if (persist && missing.length > 0)
721
+ info(`[improve] ${missing.length} candidates dropped — file not on disk`);
722
+ const deferred = actionableRefs.length - selection.loopRefs.length;
723
+ info(`[improve] ${actionableRefs.length} actionable; ${selection.loopRefs.length} will be processed` +
724
+ (options.limit && deferred > 0 ? ` (--limit ${options.limit} applied; ${deferred} deferred)` : ""));
725
+ // Skip observability waits until every fallback lane has finalized the
726
+ // survivors, so a rescued ref is never also reported skipped.
727
+ const survivors = new Set(sorted.map((c) => c.ref));
728
+ const skipped = fallbackEligible.filter((c) => !survivors.has(c.ref));
729
+ const retrievalSkipped = skipped.filter((c) => outOfScope.has(c.ref));
730
+ const signalSkipped = skipped.filter((c) => !outOfScope.has(c.ref));
731
+ for (const ref of partition.distillCooledRefs) {
1332
732
  actions.push({ ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
1333
- if (persist) {
1334
- appendEvent({
1335
- eventType: "improve_skipped",
1336
- ref,
1337
- metadata: { reason: "distill_no_new_signal" },
1338
- }, eventsCtx);
1339
- }
733
+ if (persist)
734
+ recordImproveSkip(eventsCtx, ref, { reason: "distill_no_new_signal" });
1340
735
  }
1341
- for (const candidate of terminalSignalSkippedRefs) {
736
+ for (const candidate of skipped) {
1342
737
  actions.push({
1343
738
  ref: candidate.ref,
1344
739
  mode: "distill-skipped",
1345
- result: { ok: true, reason: "no new signal since last proposal" },
1346
- });
1347
- }
1348
- // One aggregate row preserves health accounting without restoring the old
1349
- // O(n) event-write path. The count now exactly matches the signal gate.
1350
- if (persist && terminalSignalSkippedRefs.length > 0) {
1351
- appendEvent({
1352
- eventType: "improve_skipped",
1353
- ref: undefined,
1354
- metadata: {
1355
- reason: "no_new_signal",
1356
- count: terminalSignalSkippedRefs.length,
740
+ result: {
741
+ ok: true,
742
+ reason: outOfScope.has(candidate.ref)
743
+ ? "not retrieved inside the usage window"
744
+ : "no new signal since last proposal",
1357
745
  },
1358
- }, eventsCtx);
746
+ });
1359
747
  }
1360
- }
1361
- /** Bulk per-ref feedback summary in a SINGLE readEvents pass (candidate-gather). */
1362
- function buildFeedbackSummaryMap(args) {
1363
- const { processableRefs, noFeedbackPool, eventsCtx, feedbackSinceCutoff } = args;
1364
- // Gap 6: only surface feedback signals from the last 30 days so that
1365
- // ancient one-off feedback events don't permanently lock an asset into
1366
- // every improve run. Assets with only stale signals fall through to the
1367
- // proactive/high-salience fallbacks or are skipped until new signals arrive.
1368
- // (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
1369
- // Phase 2 above for the signal-delta gate; we reuse them here.)
1370
- // Pre-compute feedback summary per ref in a SINGLE bulk read so we don't
1371
- // open state.db once per asset (which caused 5000+ accumulated FDs and a
1372
- // 2-hour runaway on a 13K-asset stash). Pattern mirrors buildLatestFeedbackTsMap
1373
- // above: one readEvents() call fetches ALL feedback events, then we aggregate
1374
- // in-memory by ref — O(1) DB opens regardless of candidate set size.
1375
- // Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
1376
- // ratios are available for any noFeedbackPool ref the fallback lanes rescue below.
1377
- //
1378
- // Behavioral note: positive/negative COUNTS are all-time (same as the old
1379
- // per-ref readEvents call which had no `since` filter); hasSignal is bounded
1380
- // to feedbackSinceCutoff (same as the old inline `(e.ts ?? "") >= cutoff` guard).
1381
- const feedbackSummary = new Map();
1382
- {
1383
- const feedbackCandidates = [...processableRefs, ...noFeedbackPool];
1384
- const feedbackCandidateSet = new Set(feedbackCandidates.map((r) => r.ref));
1385
- // Map each candidate's single durable event key back to its display ref.
1386
- const feedbackRefByDurableKey = new Map(feedbackCandidates.flatMap((r) => improveStateReadRefs(r.ref, r.itemRef).map((key) => [key, r.ref])));
1387
- if (feedbackCandidateSet.size > 0) {
1388
- // Fetch ALL feedback events in one query (no ref filter, no since filter =
1389
- // single full table scan). Filtering per-ref in memory avoids N sequential
1390
- // state.db opens — the dominant FD-leak path on large stashes.
1391
- const { events: allFeedbackEvents } = readEvents({ type: "feedback" }, eventsCtx);
1392
- for (const e of allFeedbackEvents) {
1393
- const ref = e.ref ? feedbackRefByDurableKey.get(e.ref) : undefined;
1394
- if (!ref)
1395
- continue;
1396
- const entry = feedbackSummary.get(ref) ?? { hasSignal: false, positive: 0, negative: 0 };
1397
- const meta = e.metadata;
1398
- // hasSignal: only count feedback events within the 30-day window.
1399
- if (!entry.hasSignal &&
1400
- (e.ts ?? "") >= feedbackSinceCutoff &&
1401
- meta !== undefined &&
1402
- (typeof meta.signal === "string" || typeof meta.note === "string")) {
1403
- entry.hasSignal = true;
1404
- }
1405
- // positive/negative: all-time counts (no since filter, matching prior behaviour).
1406
- if (meta?.signal === "positive")
1407
- entry.positive++;
1408
- else if (meta?.signal === "negative")
1409
- entry.negative++;
1410
- feedbackSummary.set(ref, entry);
1411
- }
1412
- // Ensure every candidate has an entry (even refs with zero feedback events).
1413
- for (const ref of feedbackCandidateSet) {
1414
- if (!feedbackSummary.has(ref)) {
1415
- feedbackSummary.set(ref, { hasSignal: false, positive: 0, negative: 0 });
1416
- }
1417
- }
1418
- }
748
+ if (persist && signalSkipped.length > 0) {
749
+ recordImproveSkip(eventsCtx, undefined, { reason: "no_new_signal", count: signalSkipped.length });
1419
750
  }
1420
- return feedbackSummary;
1421
- }
1422
- /** Retrieval counts + last-use timestamps for the candidate pools (candidate-gather). */
1423
- function fetchRetrievalSignals(args) {
1424
- const { options, signalFiltered, noFeedbackCandidates, eventsCtx, persist } = args;
1425
- // Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
1426
- // proactive-maintenance selector below can reuse them without a second DB pass.
1427
- // Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
1428
- // step 2: recency is MANDATORY — never pinned to floor).
1429
- let retrievalCounts = new Map();
1430
- let lastUseMsForProactive = new Map();
1431
- let dbForRetrieval;
1432
- try {
1433
- dbForRetrieval = persist
1434
- ? openExistingDatabase()
1435
- : openReadonlyExistingDatabase(undefined, { isolatedSnapshot: true });
1436
- if (!dbForRetrieval)
1437
- return { retrievalCounts, lastUseMsForProactive };
1438
- // usage_events lives in state.db (Chunk-8 WI-8.3); entries stay in index.db,
1439
- // so the retrieval-count reads take both handles.
1440
- const dbForRetrievalIndex = dbForRetrieval;
1441
- if (persist || eventsCtx?.db) {
1442
- withStateDb((stateDb) => {
1443
- const showEventCount = countUsageEventsByType(stateDb, "show");
1444
- if (showEventCount === 0) {
1445
- warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
1446
- }
1447
- // Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
1448
- // Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
1449
- // had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
1450
- // to 0 regardless of actual use. Two assets of the same type — one
1451
- // heavily-retrieved, one never-touched — would receive identical rankScores.
1452
- // Fix (WS-1 blocker 3): union the feedback pool into the lookup.
1453
- const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
1454
- retrievalCounts = getRetrievalCounts(dbForRetrievalIndex, stateDb, allCandidateRefs, {
1455
- sourceName: options.sourceName,
1456
- });
1457
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1458
- }
1459
- lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates);
751
+ if (persist && retrievalSkipped.length > 0) {
752
+ recordImproveSkip(eventsCtx, undefined, { reason: "not_retrieved", count: retrievalSkipped.length });
1460
753
  }
1461
- catch (err) {
1462
- rethrowIfTestIsolationError(err);
1463
- // best-effort: if DB unavailable, retrievalCounts/lastUseMsForProactive stay empty
754
+ const blocked = signalSkipped.length + partition.distillOnlyRefs.length;
755
+ if (blocked > 0) {
756
+ info(`[improve] ${blocked} of ${partition.preCooldownCount} indexed refs blocked by reflect signal-delta ` +
757
+ `(${signalSkipped.length} fully skipped, ${partition.distillOnlyRefs.length} routed to distill-only)`);
1464
758
  }
1465
- finally {
1466
- if (dbForRetrieval)
1467
- closeDatabase(dbForRetrieval);
759
+ if (retrievalSkipped.length > 0) {
760
+ info(`[improve] ${retrievalSkipped.length} refs left out: not retrieved in the last ${USAGE_EVENT_RETENTION_DAYS} days, and not new material`);
1468
761
  }
1469
- return { retrievalCounts, lastUseMsForProactive };
762
+ const gates = [
763
+ {
764
+ name: "retrieval",
765
+ removed: retrievalSkipped.length,
766
+ reason: `no feedback, and neither returned by search, curate or show in the last ${USAGE_EVENT_RETENTION_DAYS} days nor new material improve never processed`,
767
+ },
768
+ {
769
+ name: "signal",
770
+ removed: signalSkipped.length,
771
+ reason: "no fresh signal since the last attempt (or an improve-ledger window) and no fallback lane selected the ref",
772
+ },
773
+ { name: "disk", removed: missing.length, reason: "backing asset is absent on disk" },
774
+ { name: "limit", removed: selection.limitRemoved, reason: "deferred by the effective run limit" },
775
+ ];
776
+ return {
777
+ actionableRefs,
778
+ loopRefs: selection.loopRefs,
779
+ distillOnlyRefs: selection.distillOnlyRefs,
780
+ distillCooledRefs: partition.distillCooledRefs,
781
+ signalBearingSet,
782
+ coverageGaps,
783
+ gates,
784
+ proactive,
785
+ };
1470
786
  }
1471
- /** Layer 2 — the proactive-maintenance selector lane (candidate-gather). */
1472
- function selectProactiveMaintenanceLane(args) {
1473
- const { scope, improveProfile, resolvedPlan, eventsCtx, noFeedbackCandidates, lastReflectProposalTs, lastDistillProposalTs, retrievalCounts, lastUseMsForProactive, persist, } = args;
1474
- // ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (second eligibility source) ────
1475
- // The signal-delta gate only surfaces assets with fresh feedback. It never
1476
- // revisits a stable, high-value asset on a schedule, so on a quiet stash
1477
- // useful assets drift stale and are never refreshed. When the
1478
- // `proactiveMaintenance` process is enabled (DEFAULT OFF)
1479
- // and the run is whole-stash / type scope, this selector ranks the eligible
1480
- // population by a composite maintenance priority, gates on staleness ("due"),
1481
- // bounds to top-N, and folds the winners into the SAME candidate set the other
1482
- // sources feed — so they flow through the existing #580 empty-diff /
1483
- // cosmetic suppression and additive-distill gates. It adds no new mutation
1484
- // logic of its own. The due gate doubles as the rotation cooldown: a freshly
1485
- // reflected asset is excluded until it ages back past `dueDays`, so successive
1486
- // runs rotate through the due pool rather than re-selecting the same heads.
1487
- let proactiveRefs = [];
1488
- let proactiveMaintenanceSummary;
1489
- let proactivePlan;
1490
- const proactiveEnabled = scope.mode !== "ref" && resolvedPlan.processes.proactiveMaintenance.enabled;
1491
- if (proactiveEnabled) {
1492
- const pmCfg = improveProfile.processes?.proactiveMaintenance;
1493
- const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
1494
- const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
1495
- // Candidate population: the zero-feedback / non-signal pool — exactly the
1496
- // assets the signal-delta gate would NOT pick this run.
1497
- const pmCandidates = noFeedbackCandidates;
1498
- const selection = selectProactiveMaintenanceRefs({
1499
- candidates: pmCandidates,
1500
- lastReflectTs: lastReflectProposalTs,
1501
- lastDistillTs: lastDistillProposalTs,
1502
- retrievalCounts,
1503
- // WS-1: wire lastUseMs so the recency decay term is genuine (plan §step 2).
1504
- lastUseMs: lastUseMsForProactive,
1505
- sizeBytesOf: (r) => {
1506
- const fp = r.filePath;
1507
- if (!fp)
1508
- return undefined;
1509
- try {
1510
- return fs.statSync(fp).size;
1511
- }
1512
- catch {
1513
- return undefined;
1514
- }
1515
- },
1516
- dueDays,
1517
- maxPerRun,
787
+ /** Retrieval counts for every candidate, and last-use times for the zero-feedback pool. */
788
+ function fetchRetrievalSignals(options, signalFiltered, noFeedbackCandidates, eventsCtx, persist) {
789
+ const out = { retrievalCounts: new Map(), lastUseMs: new Map() };
790
+ withIndexDb(!persist, (indexDb) => {
791
+ // usage_events live in state.db, entries in index.db.
792
+ withRunState(eventsCtx, persist, (stateDb) => {
793
+ if (countUsageEventsByType(stateDb, "show") === 0) {
794
+ warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
795
+ }
796
+ const refs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
797
+ out.retrievalCounts = getRetrievalCounts(indexDb, stateDb, refs, { sourceName: options.sourceName });
1518
798
  });
1519
- proactiveRefs = selection.selected;
1520
- proactiveMaintenanceSummary = {
1521
- selected: selection.selected.length,
1522
- dueTotal: selection.dueTotal,
1523
- neverReflected: selection.neverReflected,
1524
- selectedRefs: selection.selected.map((entry) => entry.ref),
1525
- };
1526
- proactivePlan = {
1527
- configured: {
1528
- ...(pmCfg?.dueDays !== undefined ? { dueDays: pmCfg.dueDays } : {}),
1529
- ...(pmCfg?.maxPerRun !== undefined ? { maxPerRun: pmCfg.maxPerRun } : {}),
1530
- ...(pmCfg?.limit !== undefined ? { limit: pmCfg.limit } : {}),
1531
- },
1532
- effective: { dueDays, maxPerRun },
1533
- candidatePool: pmCandidates.length,
1534
- dueTotal: selection.dueTotal,
1535
- neverReflected: selection.neverReflected,
1536
- selected: selection.selected.length,
1537
- selectedRefs: selection.selected.map((entry) => entry.ref),
1538
- };
1539
- // Aggregated observability event (never per-ref — avoids the event flood the
1540
- // Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
1541
- if (persist) {
1542
- appendEvent({
1543
- eventType: "proactive_selected",
1544
- ref: undefined,
1545
- metadata: {
1546
- count: selection.selected.length,
1547
- dueTotal: selection.dueTotal,
1548
- neverReflected: selection.neverReflected,
1549
- },
1550
- }, eventsCtx);
1551
- }
1552
- if (selection.selected.length > 0) {
1553
- info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
1554
- `(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
1555
- }
1556
- }
1557
- return { proactiveRefs, proactiveMaintenanceSummary, proactivePlan };
799
+ out.lastUseMs = getLastUseMsByRef(indexDb, noFeedbackCandidates);
800
+ });
801
+ return out;
1558
802
  }
1559
- /** Layer 3 — the high-salience admission gate (#608/#644; candidate-gather). */
1560
- function selectHighSalienceLane(args) {
1561
- const { options, improveProfile, eventsCtx, noFeedbackCandidates, proactiveRefs, lastReflectProposalTs, persist } = args;
1562
- // ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
1563
- // Zero-feedback refs whose encoding_salience (set at distill time by
1564
- // scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
1565
- // into the improve run even without retrieval or feedback signal. This rescues
1566
- // newly distilled assets that the stash has not yet surfaced to users.
1567
- //
1568
- // Cap: at most 10% of the effective run limit so the lane cannot crowd out
1569
- // reactive feedback. Requires state.db to have an asset_salience row — refs
1570
- // without a row (pre-#608 assets still on the type-weight stub) are skipped.
1571
- //
1572
- // Cooldown: a ref qualifies at most once — when no prior reflect proposal
1573
- // exists for it (`!lastReflectProposalTs.has`). Without this guard the lane
1574
- // re-selects the same high-salience refs on EVERY run (promotion emits a
1575
- // `promoted` event, not `feedback`, so the ref never leaves
1576
- // noFeedbackCandidates), burning LLM calls and churning the asset. This
1577
- // mirrors the same `!lastReflectProposalTs.has(r.ref)` once-per-asset
1578
- // semantics the other "rescue" lanes share.
1579
- //
1580
- // Content-provenance gate (#644 follow-up): the row must ALSO carry a genuine
1581
- // content-derived encoding score (`isContentEncodingRow`). Otherwise the lane
1582
- // admits the per-type WEIGHT STUB (skill/agent 0.9, command/workflow 0.8,
1583
- // lesson 0.75 from DEFAULT_TYPE_ENCODING_WEIGHTS) for every distill-unscored
1584
- // asset — i.e. "high-salience" degenerates into "is a skill/agent/command/
1585
- // lesson", which selected the lore-writer type-stub agent on every run. Only
1586
- // content-scored assets earn the high-salience rescue; type-stub rows must
1587
- // earn retrieval/feedback signal via the other lanes. This PRESERVES #608's
1588
- // intent — distilled assets (the lane's real targets) keep their real content
1589
- // score and still qualify — while cutting the type-stub waste. See §5 F1 of
1590
- // #608/#644.
1591
- const highSalienceRefs = [];
1592
- const salienceCfg = (options.config ?? loadConfig()).improve?.salience;
1593
- const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
1594
- const proactiveSelectedSet = new Set(proactiveRefs.map((r) => r.ref));
1595
- try {
1596
- if (!persist && !eventsCtx?.db)
1597
- return highSalienceRefs;
1598
- withStateDb((dbForHighSalience) => {
1599
- // Derive the cap from the resolved reflect limit (mirrors improve.ts's
1600
- // options.limit resolution) so an unbounded whole-stash run does not
1601
- // collapse the lane to exactly 1 ref via the bare `?? 10` fallback.
1602
- const effectiveLimit = options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit ?? 10;
1603
- const highSalienceCap = Math.max(1, Math.floor(effectiveLimit * 0.1));
1604
- const candidates = noFeedbackCandidates.filter((r) => !proactiveSelectedSet.has(r.ref));
1605
- // Collect ALL qualifying candidates, then take the top-N BY SCORE — the
1606
- // previous first-N-in-scan-order break meant a higher-salience candidate
1607
- // found later in the scan lost its slot to an earlier lower-scoring one.
1608
- const qualifying = [];
1609
- for (const r of candidates) {
1610
- const row = readAssetSalienceForImproveRef(dbForHighSalience, r.ref, r.itemRef);
1611
- if (row &&
1612
- isContentEncodingRow(row) &&
1613
- row.encoding_salience >= salienceThreshold &&
1614
- !lastReflectProposalTs.has(r.ref)) {
1615
- qualifying.push({ ref: r, score: row.encoding_salience });
1616
- }
1617
- }
1618
- qualifying.sort((a, b) => b.score - a.score);
1619
- for (const q of qualifying.slice(0, highSalienceCap)) {
1620
- highSalienceRefs.push(q.ref);
1621
- }
1622
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
803
+ /**
804
+ * Proactive maintenance (default off, whole-stash/type runs): revisit stable
805
+ * assets on a schedule. The due gate doubles as the rotation cooldown: a
806
+ * freshly reflected asset waits `dueDays` before it is picked again.
807
+ */
808
+ function selectProactiveMaintenanceLane(args, candidates, snapshot, retrieval, persist) {
809
+ if (args.scope.mode === "ref" || !args.resolvedPlan.processes.proactiveMaintenance.enabled) {
810
+ return { proactiveRefs: [] };
1623
811
  }
1624
- catch (err) {
1625
- rethrowIfTestIsolationError(err);
1626
- // best-effort: if DB unavailable, highSalienceRefs stays empty
812
+ const pmCfg = args.improveProfile.processes?.proactiveMaintenance;
813
+ const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
814
+ const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
815
+ const selection = selectProactiveMaintenanceRefs({
816
+ candidates,
817
+ lastReflectTs: snapshot.lastReflectAttemptAt,
818
+ lastDistillTs: snapshot.lastDistillAttemptAt,
819
+ retrievalCounts: retrieval.retrievalCounts,
820
+ lastUseMs: retrieval.lastUseMs,
821
+ sizeBytesOf: (r) => fileSize(r.filePath),
822
+ dueDays,
823
+ maxPerRun,
824
+ });
825
+ const summary = {
826
+ selected: selection.selected.length,
827
+ dueTotal: selection.dueTotal,
828
+ neverReflected: selection.neverReflected,
829
+ };
830
+ if (persist) {
831
+ appendEvent({
832
+ eventType: "proactive_selected",
833
+ ref: undefined,
834
+ metadata: { count: summary.selected, dueTotal: summary.dueTotal, neverReflected: summary.neverReflected },
835
+ }, args.eventsCtx);
1627
836
  }
1628
- if (highSalienceRefs.length > 0) {
1629
- info(`[improve] high-salience lane admitted ${highSalienceRefs.length} content-scored ref(s) ` +
1630
- `(threshold=${salienceThreshold}, requires content-derived encoding_source)`);
837
+ if (summary.selected > 0) {
838
+ info(`[improve] proactive maintenance selected ${summary.selected}/${summary.dueTotal} due refs ` +
839
+ `(${summary.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
1631
840
  }
1632
- return highSalienceRefs;
841
+ const selectedRefs = selection.selected.map((entry) => entry.ref);
842
+ return {
843
+ proactiveRefs: selection.selected,
844
+ proactiveMaintenanceSummary: { ...summary, selectedRefs },
845
+ proactivePlan: {
846
+ configured: pickDefined(pmCfg, ["dueDays", "maxPerRun", "limit"]),
847
+ effective: { dueDays, maxPerRun },
848
+ candidatePool: candidates.length,
849
+ ...summary,
850
+ selectedRefs,
851
+ },
852
+ };
1633
853
  }
1634
854
  /**
1635
- * Pass: salience-score — the WS-2 outcome loop, the WS-1 salience vector
1636
- * computation (#644 provenance preserved), persistence + rank-change report,
1637
- * and the forgetting-safety injection. The valence-score call sites
1638
- * (computeValenceScore) live VERBATIM inside the outcome/persist sub-passes.
1639
- * Mutates the shared eligibilitySourceByRef map and ref objects in place —
1640
- * attribution identity is load-bearing (see the candidate-gather comments).
855
+ * High salience: zero-feedback refs whose content-derived encoding score (not
856
+ * a per-type stub) reaches `salienceThreshold` and that were never reflected,
857
+ * top-N by score, capped at 10% of the effective limit.
1641
858
  */
1642
- function scoreSalience(args) {
1643
- const { scope, options, primaryStashDir, eventsCtx, eligibilitySourceByRef, feedbackSummary, retrievalCounts, signalFiltered, proactiveRefs, highSalienceRefs, forgettingEligibleRefs, persist, } = args;
1644
- const mergedRefs = args.mergedRefs;
1645
- // Chunk-5 flip F5e — resolve each candidate's durable item_ref ONCE for this
1646
- // pass (the write/read key source for the outcome + salience state writers).
1647
- const itemRefByRef = buildItemRefByRef(mergedRefs);
1648
- // WS-1 — Unified salience vector (S1 seam).
1649
- //
1650
- // WS-1 converges utility, valence, and proactive-maintenance signals into one
1651
- // `computeSalience()` call per ref, with
1652
- // three independently-stored sub-scores and one documented rankScore projection.
1653
- //
1654
- // Fetch last-use timestamps from the index DB for the full merged set so the
1655
- // recency term in retrievalSalience is genuinely decayable (plan §WS-1 step 2).
1656
- // This reuses the index DB opened earlier for retrieval counts; a separate
1657
- // lightweight open is used here to avoid holding the connection longer than needed.
1658
- let lastUseMsByRef = new Map();
1659
- // Health and outcome reporting consume the utility projection.
1660
- const utilityMap = buildUtilityMap(mergedRefs, !persist);
1661
- let dbForSalience;
1662
- try {
1663
- dbForSalience = persist
1664
- ? openExistingDatabase()
1665
- : openReadonlyExistingDatabase(undefined, { isolatedSnapshot: true });
1666
- if (dbForSalience) {
1667
- lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs);
1668
- }
859
+ function selectHighSalienceLane(options, improveProfile, eventsCtx, candidates, lastReflectAttemptAt, persist) {
860
+ const threshold = (options.config ?? loadConfig()).improve?.salience?.salienceThreshold ?? 0.75;
861
+ const effectiveLimit = options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit ?? 10;
862
+ const selected = withRunState(eventsCtx, persist, (db) => candidates
863
+ .flatMap((r) => {
864
+ const row = getAssetSalience(db, keyOf(r));
865
+ return row &&
866
+ isContentEncodingRow(row) &&
867
+ row.encoding_salience >= threshold &&
868
+ !lastReflectAttemptAt.has(r.ref)
869
+ ? [{ ref: r, score: row.encoding_salience }]
870
+ : [];
871
+ })
872
+ .sort((a, b) => b.score - a.score)
873
+ .slice(0, Math.max(1, Math.floor(effectiveLimit * 0.1)))
874
+ .map((q) => q.ref)) ?? [];
875
+ if (selected.length > 0) {
876
+ info(`[improve] high-salience lane admitted ${selected.length} content-scored ref(s) ` +
877
+ `(threshold=${threshold}, requires content-derived encoding_source)`);
1669
878
  }
1670
- catch (err) {
1671
- rethrowIfTestIsolationError(err);
1672
- // best-effort: if DB unavailable, recency term stays at floor (lastUseMs=0)
1673
- }
1674
- finally {
1675
- if (dbForSalience)
1676
- closeDatabase(dbForSalience);
1677
- }
1678
- const outcomeSalienceByRef = updateOutcomeScores({
1679
- mergedRefs,
1680
- itemRefByRef,
1681
- feedbackSummary,
1682
- retrievalCounts,
1683
- lastUseMsByRef,
1684
- utilityMap,
1685
- primaryStashDir,
1686
- eventsCtx,
1687
- persist,
1688
- });
1689
- const { salienceMap, nowForSalience } = computeSalienceVectors({
879
+ return selected;
880
+ }
881
+ /**
882
+ * Score the merged refs: update `asset_outcome` (projected on a plan-only
883
+ * run), compute each salience vector (keeping a stored content-derived
884
+ * encoding score), then persist it.
885
+ */
886
+ function scoreSalience(args, mergedRefs, feedback, retrievalCounts, persist) {
887
+ const { options, eventsCtx } = args;
888
+ const utilityMap = buildUtilityMap(mergedRefs, !persist);
889
+ const lastUseMsByRef = withIndexDb(!persist, (db) => getLastUseMsByRef(db, mergedRefs)) ?? new Map();
890
+ const outcomeSalience = updateOutcomeScores({
1690
891
  mergedRefs,
1691
- itemRefByRef,
1692
- options,
1693
- eventsCtx,
892
+ feedback,
1694
893
  retrievalCounts,
1695
894
  lastUseMsByRef,
1696
895
  utilityMap,
1697
- outcomeSalienceByRef,
1698
- persist,
1699
- });
1700
- const pendingForgettingRefs = persistSalienceAndReportRanks({
1701
- salienceMap,
1702
- itemRefByRef,
1703
- utilityMap,
1704
- feedbackSummary,
1705
- options,
896
+ primaryStashDir: args.primaryStashDir,
1706
897
  eventsCtx,
1707
- nowForSalience,
1708
898
  persist,
1709
899
  });
1710
- const finalMergedRefs = applyForgettingSafety({
1711
- pendingForgettingRefs,
1712
- scope,
1713
- mergedRefs,
1714
- eligibleRefs: forgettingEligibleRefs,
1715
- allowFallbacks: options.requireFeedbackSignal !== true,
1716
- eligibilitySourceByRef,
1717
- highSalienceRefs,
1718
- proactiveRefs,
1719
- signalFiltered,
1720
- });
1721
- return { mergedRefs: finalMergedRefs, utilityMap, lastUseMsByRef, salienceMap, nowForSalience };
1722
- }
1723
- /** WS-2 — update asset_outcome for the merged set; returns outcomeSalience by ref. */
1724
- function updateOutcomeScores(args) {
1725
- const { mergedRefs, itemRefByRef, feedbackSummary, retrievalCounts, lastUseMsByRef, utilityMap, primaryStashDir, eventsCtx, persist, } = args;
1726
- // ── WS-2 Outcome loop ─────────────────────────────────────────────────────
1727
- //
1728
- // Update asset_outcome for every ref in the merged set BEFORE computing the
1729
- // salience vector so the updated outcome_score feeds outcomeSalience this run.
1730
- //
1731
- // Inputs per ref:
1732
- // - currentRetrievalCount: from retrievalCounts (index DB)
1733
- // - lastRetrievedAt: from lastUseMsByRef (utility_scores.last_used_at)
1734
- // - negativeFeedbackCount: cumulative negatives from feedbackSummary
1735
- // - acceptedChangeCount: accepted proposals for this ref (state.db)
1736
- // - valence: net valence from computeValenceScore(feedbackSummary.get(ref))
1737
- // - utilityScore: from utilityMap (for warm-start seed on new rows)
1738
- //
1739
- // Best-effort: outcome failures never block the salience or ranking pass.
1740
- const outcomeSalienceByRef = new Map();
1741
- // Missing state.db is itself a complete snapshot: no prior outcome rows and
1742
- // no accepted proposals. Project the same warm-start values a live run would
1743
- // insert, without creating the database merely to represent empty tables.
1744
- if (!persist && !eventsCtx?.db) {
1745
- const projectedScores = new Map();
1746
- const nowForOutcome = Date.now();
1747
- for (const ref of mergedRefs) {
1748
- const feedback = feedbackSummary.get(ref.ref) ?? { positive: 0, negative: 0 };
1749
- const result = projectAssetOutcome(undefined, {
1750
- ref: outcomeWriteKey(ref.ref, itemRefByRef),
1751
- currentRetrievalCount: retrievalCounts.get(ref.ref) ?? 0,
1752
- lastRetrievedAt: lastUseMsByRef.get(ref.ref) ?? 0,
1753
- acceptedChangeCount: 0,
1754
- negativeFeedbackCount: feedback.negative,
1755
- valence: computeValenceScore(feedback).valence,
1756
- utilityScore: utilityMap.get(ref.ref),
1757
- now: nowForOutcome,
1758
- });
1759
- projectedScores.set(ref.ref, result.outcomeScore);
1760
- }
1761
- const maxOutcomeScore = Math.min(OUTCOME_SCORE_MAX, Math.max(0, ...projectedScores.values()));
1762
- for (const [ref, score] of projectedScores) {
1763
- outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
900
+ const outcomeWeightEnabled = (options.config ?? loadConfig()).improve?.salience?.outcomeWeightEnabled !== false;
901
+ const storedEncoding = new Map();
902
+ withRunState(eventsCtx, persist, (db) => {
903
+ for (const r of mergedRefs) {
904
+ const row = getAssetSalience(db, keyOf(r));
905
+ if (row && isContentEncodingRow(row))
906
+ storedEncoding.set(r.ref, row.encoding_salience);
1764
907
  }
1765
- return outcomeSalienceByRef;
1766
- }
1767
- try {
1768
- withStateDb((outcomeDb) => {
1769
- // Count accepted proposals per ref in one pass (avoid N separate queries).
1770
- // Scoped to primaryStashDir when available so multi-stash installs don't
1771
- // inflate counts with proposals from other stashes.
1772
- const acceptedCountByRef = new Map();
1773
- try {
1774
- // #858/#859: listStateProposals() now skips-and-warns on individual
1775
- // unparseable rows (including legacy pre-#578 rows with no
1776
- // persisted `changes`, which it tolerates directly) instead of
1777
- // throwing, so this no longer silently zeroes out every ref's
1778
- // count on a single bad row. The outer try/catch stays as a
1779
- // defense-in-depth fallback for unexpected failures (e.g. a query
1780
- // error), not the primary safeguard it used to be.
1781
- const acceptedProposals = listStateProposals(outcomeDb, {
1782
- status: "accepted",
1783
- ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
1784
- });
1785
- for (const p of acceptedProposals) {
1786
- acceptedCountByRef.set(p.ref, (acceptedCountByRef.get(p.ref) ?? 0) + 1);
1787
- }
1788
- }
1789
- catch {
1790
- // best-effort: if the query itself fails, accepted counts stay at 0
1791
- }
1792
- // Update each ref's outcome row and collect the resulting outcome scores.
1793
- const rawOutcomeScores = new Map();
1794
- const projectedByWriteKey = new Map();
1795
- const nowForOutcome = Date.now();
1796
- for (const r of mergedRefs) {
1797
- const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
1798
- const valenceResult = computeValenceScore(fb);
1799
- try {
1800
- const writeKey = outcomeWriteKey(r.ref, itemRefByRef);
1801
- const inputs = {
1802
- // Key by item_ref when resolved, else by the conceptId. Keep
1803
- // rawOutcomeScores keyed by r.ref, its in-memory identity.
1804
- ref: writeKey,
1805
- currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
1806
- lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
1807
- acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
1808
- negativeFeedbackCount: fb.negative,
1809
- valence: valenceResult.valence,
1810
- utilityScore: utilityMap.get(r.ref),
1811
- now: nowForOutcome,
1812
- };
1813
- const result = persist
1814
- ? updateAssetOutcome(outcomeDb, inputs)
1815
- : projectAssetOutcome(getAssetOutcome(outcomeDb, writeKey), inputs);
1816
- rawOutcomeScores.set(r.ref, result.outcomeScore);
1817
- projectedByWriteKey.set(writeKey, result.outcomeScore);
1818
- }
1819
- catch {
1820
- // best-effort per-ref: skip this ref's outcome update on failure
1821
- }
1822
- }
1823
- // Compute stash-wide max outcome_score for normalisation (diversity floor).
1824
- // Read ALL rows (not just this run's batch) so the normalisation is
1825
- // stash-relative, not pool-relative.
1826
- let maxOutcomeScore = 0;
1827
- try {
1828
- const allOutcomes = getAllAssetOutcomes(outcomeDb);
1829
- const scoreByRef = new Map(allOutcomes.map((row) => [row.asset_ref, row.outcome_score]));
1830
- for (const [ref, score] of projectedByWriteKey)
1831
- scoreByRef.set(ref, score);
1832
- for (const score of scoreByRef.values()) {
1833
- if (score > maxOutcomeScore)
1834
- maxOutcomeScore = score;
1835
- }
1836
- // Keep the normalization denominator within the writer's score bounds.
1837
- maxOutcomeScore = Math.min(maxOutcomeScore, OUTCOME_SCORE_MAX);
1838
- // Proxy-adequacy tripwire (two-tailed): inverted (corr < −0.3) and
1839
- // dead (|corr| < 0.1 at n ≥ 500) both emit health events.
1840
- const adequacy = persist ? computeProxyAdequacy(allOutcomes) : undefined;
1841
- if (adequacy?.isInverted) {
1842
- appendEvent({
1843
- eventType: "outcome_proxy_inverted",
1844
- ref: undefined,
1845
- metadata: {
1846
- correlation: adequacy.correlation,
1847
- n: adequacy.n,
1848
- note: "corr(outcome_score, accepted_change_rate) < −0.3: high-outcome_score assets have LOW accepted-change rates — the proxy's 'doing well' signal is inverted, so the coarse retrieval-delta signal is no longer trustworthy and the 0.10+ rich in-session signal is no longer deferrable. See plan §WS-2 proxy-adequacy tripwire.",
1849
- },
1850
- }, eventsCtx);
1851
- }
1852
- if (adequacy?.isDead) {
1853
- appendEvent({
1854
- eventType: "outcome_proxy_dead",
1855
- ref: undefined,
1856
- metadata: {
1857
- correlation: adequacy.correlation,
1858
- n: adequacy.n,
1859
- note: "|corr(outcome_score, accepted_change_rate)| < 0.1 at n ≥ 500: outcome_score is statistically unrelated to improvement outcomes — the proxy is noise, not signal. Rank contributions derived from it are not currently informative.",
1860
- },
1861
- }, eventsCtx);
1862
- }
1863
- }
1864
- catch {
1865
- // best-effort: tripwire failure never blocks ranking
1866
- }
1867
- // Convert raw outcome scores → normalised outcomeSalience values in [0,1].
1868
- for (const [ref, score] of rawOutcomeScores) {
1869
- const normalised = outcomeScoreToSalience(score, maxOutcomeScore);
1870
- outcomeSalienceByRef.set(ref, normalised);
1871
- }
1872
- // Also fetch outcome scores for refs NOT updated this run (stale or absent)
1873
- // so the outcomeSalience read path works for all refs in the batch.
1874
- // Chunk-5 flip F5e — query by each missing ref's WRITE key (item_ref,
1875
- // else bare) and map the stored-key result back to the bare `r.ref`
1876
- // identity that outcomeSalienceByRef is keyed on.
1877
- const missingRefs = mergedRefs.map((r) => r.ref).filter((ref) => !rawOutcomeScores.has(ref));
1878
- if (missingRefs.length > 0) {
1879
- const refByWriteKey = new Map();
1880
- for (const ref of missingRefs)
1881
- refByWriteKey.set(outcomeWriteKey(ref, itemRefByRef), ref);
1882
- const storedScores = getOutcomeScoresByRef(outcomeDb, [...refByWriteKey.keys()]);
1883
- for (const [writeKey, score] of storedScores) {
1884
- const bareRef = refByWriteKey.get(writeKey) ?? writeKey;
1885
- outcomeSalienceByRef.set(bareRef, outcomeScoreToSalience(score, maxOutcomeScore));
1886
- }
1887
- }
1888
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1889
- }
1890
- catch (err) {
1891
- rethrowIfTestIsolationError(err);
1892
- // best-effort: outcome failures never block salience computation
1893
- }
1894
- return outcomeSalienceByRef;
1895
- }
1896
- /** WS-1 — compute the salience vector per ref (#644 provenance preserved). */
1897
- function computeSalienceVectors(args) {
1898
- const { mergedRefs, itemRefByRef, options, eventsCtx, retrievalCounts, lastUseMsByRef, utilityMap, outcomeSalienceByRef, persist, } = args;
1899
- // Compute the salience vector for every ref in the merged set.
1900
- // retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
1901
- // so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
1902
- // outcomeSalienceByRef is populated by WS-2 above (or empty on first run).
1903
- //
1904
- // R1 loop closure: the outcome weight is ON by default (the G2 saturation
1905
- // cap makes it safe). Operators opt out with
1906
- // improve.salience.outcomeWeightEnabled: false in the config.
1907
- const salienceConfig = (options.config ?? loadConfig()).improve?.salience;
1908
- const outcomeWeightEnabled = salienceConfig?.outcomeWeightEnabled !== false;
908
+ });
909
+ const now = Date.now();
1909
910
  const salienceMap = new Map();
1910
- const nowForSalience = Date.now();
1911
- // #644 — preserve content-derived encoding scores across runs.
1912
- //
1913
- // Before computing the salience vector, load each ref's stored encoding score
1914
- // and its provenance. When the stored row carries a genuine content-derived
1915
- // score (written by the distill path via `scoreEncodingSalience`), pass that
1916
- // value back in as `inputs.encodingSalience` so `computeSalience` does NOT fall
1917
- // back to the type-weight stub — keeping both the persisted `encoding_salience`
1918
- // AND the derived `rank_score` keyed on real novelty/magnitude/prediction-error.
1919
- // Refs that have never been content-scored keep the type-weight stub fallback.
1920
- const storedEncodingByRef = new Map();
1921
- try {
1922
- if (persist || eventsCtx?.db) {
1923
- withStateDb((dbForStoredEncoding) => {
1924
- for (const r of mergedRefs) {
1925
- const row = readAssetSalienceForImproveRef(dbForStoredEncoding, r.ref, itemRefByRef.get(r.ref));
1926
- if (row && isContentEncodingRow(row)) {
1927
- storedEncodingByRef.set(r.ref, row.encoding_salience);
1928
- }
1929
- }
1930
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
1931
- }
1932
- }
1933
- catch (err) {
1934
- rethrowIfTestIsolationError(err);
1935
- // best-effort: if DB unavailable, fall back to type-weight stub (prior behaviour)
1936
- }
1937
911
  for (const r of mergedRefs) {
1938
- const type = assetTypeOf(r.ref);
1939
- const sizeBytes = (() => {
1940
- const fp = r.filePath;
1941
- if (!fp)
1942
- return undefined;
1943
- try {
1944
- return fs.statSync(fp).size;
1945
- }
1946
- catch {
1947
- return undefined;
1948
- }
1949
- })();
1950
- const storedEncoding = storedEncodingByRef.get(r.ref);
1951
- const vector = computeSalience({
912
+ const encoding = storedEncoding.get(r.ref);
913
+ salienceMap.set(r.ref, computeSalience({
1952
914
  ref: r.ref,
1953
- type,
1954
- // #644: pass the stored content-derived score (if any) so the type-weight
1955
- // stub is NOT re-asserted over a real distill-written encoding score.
1956
- ...(storedEncoding !== undefined ? { encodingSalience: storedEncoding } : {}),
915
+ type: assetTypeOf(r.ref),
916
+ ...(encoding !== undefined ? { encodingSalience: encoding } : {}),
1957
917
  retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
1958
918
  lastUseMs: lastUseMsByRef.get(r.ref),
1959
919
  utilityScore: utilityMap.get(r.ref),
1960
- outcomeSalience: outcomeSalienceByRef.get(r.ref),
1961
- sizeBytes,
1962
- now: nowForSalience,
920
+ outcomeSalience: outcomeSalience.get(r.ref),
921
+ sizeBytes: fileSize(r.filePath),
922
+ now,
1963
923
  outcomeWeightEnabled,
924
+ }));
925
+ }
926
+ if (persist) {
927
+ withRunState(eventsCtx, persist, (db) => {
928
+ for (const r of mergedRefs)
929
+ upsertAssetSalience(db, keyOf(r), salienceMap.get(r.ref), now);
1964
930
  });
1965
- salienceMap.set(r.ref, vector);
1966
931
  }
1967
- return { salienceMap, nowForSalience };
1968
- }
1969
- /** 1-indexed rank positions sorted by score desc (deterministic ref-asc tie-break). */
1970
- function toRankPositions(scores) {
1971
- const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
1972
- return new Map(sorted.map(([ref], i) => [ref, i + 1]));
932
+ return salienceMap;
1973
933
  }
1974
934
  /**
1975
- * Chunk-5 flip F5e — the durable salience write-key maps for one improve pass.
1976
- * `wk(ref)` is a pool asset's write key (item_ref, else conceptId);
1977
- * `normalizeStoredKey` maps each in-pool asset's durable spelling to its write
1978
- * key (no stashSize double-count); and
1979
- * `refByWriteKey` reverses a write key back to its filesystem-facing bare ref.
935
+ * Update each ref's outcome row and return its outcome salience, normalized
936
+ * against the stash-wide maximum. Without state.db on a plan-only run, the
937
+ * values a live run would insert are projected instead.
1980
938
  */
1981
- function buildSalienceWriteKeyMaps(itemRefByRef) {
1982
- const wk = (ref) => salienceWriteKey(ref, itemRefByRef);
1983
- const normalizeStoredKey = new Map();
1984
- const refByWriteKey = new Map();
1985
- for (const [ref, itemRef] of itemRefByRef) {
1986
- const writeKey = wk(ref);
1987
- refByWriteKey.set(writeKey, ref);
1988
- for (const spelling of improveStateReadRefs(ref, itemRef)) {
1989
- normalizeStoredKey.set(spelling, writeKey);
1990
- }
1991
- }
1992
- return { wk, normalizeStoredKey, refByWriteKey };
1993
- }
1994
- /** Persist salience vectors + the WS-1 step-7 rank-change/forgetting report. */
1995
- function persistSalienceAndReportRanks(args) {
1996
- const { salienceMap, itemRefByRef, utilityMap, feedbackSummary, options, eventsCtx, nowForSalience, persist } = args;
1997
- // Chunk-5 flip F5e — the WRITE-key space. salienceMap stays keyed by each
1998
- // candidate's own short `r.ref`; the state.db boundary keys by item_ref when
1999
- // available and otherwise by conceptId.
2000
- const { wk, normalizeStoredKey, refByWriteKey } = buildSalienceWriteKeyMaps(itemRefByRef);
2001
- // Persist salience vectors to state.db (best-effort, non-blocking).
2002
- // The canonical store enables WS-3 homeostatic demotion and WS-2 outcome reads.
2003
- //
2004
- // Forgetting-safety report (plan §WS-1 step 7) — stash-wide rank comparison:
2005
- //
2006
- // BEFORE persisting the new rankScores, read ALL existing rows from state.db
2007
- // (not just the per-run candidate pool). This gives stash-wide rank positions so
2008
- // the top-200/below-500 thresholds are meaningful.
2009
- //
2010
- // Two distinct scenarios:
2011
- //
2012
- // A. First WS-1 run (table empty): the old stash-wide combinedEligibilityScore
2013
- // ordering was never persisted in state.db (asset_salience is a new WS-1 table).
2014
- // However, the old formula's inputs are available in-scope for every candidate
2015
- // in the current pool: utility comes from utilityMap and the attention term
2016
- // from feedbackSummary (positive/negative counts). We reconstruct the old
2017
- // combinedEligibilityScore = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT
2018
- // for every ref in salienceMap and rank them, giving a candidate-pool-scoped
2019
- // old ordering. This is a partial reconstruction (only current-pool refs, not
2020
- // stash-wide), but it is the most faithful comparison possible at cutover and
2021
- // allows the top-200→below-500 forgetting guard to fire if the formula change
2022
- // dramatically reorders the candidate pool.
2023
- // WS-1 step 7 — the stash-wide
2024
- // ordering was unreconstructable (no prior state.db snapshot), so this candidate-
2025
- // pool partial reconstruction is the documented resolution for the first-run case.
2026
- // Emit `improve_salience_first_run` to mark the cutover moment and include the
2027
- // reconstructed comparison result in the metadata.
2028
- //
2029
- // B. Subsequent runs (table has rows): use ALL existing rows as old ranks, merge
2030
- // them with the current run's salienceMap updates for new ranks, and call
2031
- // buildRankChangeReport with stash-wide positions. This detects real rank drift
2032
- // — e.g. a retrieval-pattern shift causing a previously top-200 asset to slip
2033
- // below position 500.
2034
- //
2035
- // Measurement-protocol deferral (plan §269, Part-V):
2036
- // The Part-V T0 baseline (scripts/akm-eval + health report) and the throughput/
2037
- // quality gate are deferred pending owner sign-off. Full measurement requires a
2038
- // before/after `akm health` report. Owner-acknowledged deferral: WS-2 landing
2039
- // will re-introduce outcome salience and trigger the full re-tuning pass at that
2040
- // time. salience.ts already accepts outcomeSalience directly as an input
2041
- // (see SalienceInputs.outcomeSalience); no separate hook is needed.
2042
- //
2043
- // Forgetting-safety collection: populated inside scenario B below, consumed
2044
- // after the try/catch to union candidates into mergedRefs before the sort.
2045
- // Only refs from a real pre-existing ordering (scenario B) are collected;
2046
- // empty on scenario A or when no candidates dropped below the threshold.
2047
- let pendingForgettingRefs = [];
2048
- try {
2049
- if (!persist && !eventsCtx?.db)
2050
- return pendingForgettingRefs;
2051
- withStateDb((stateDb) => {
2052
- // Step 7: stash-wide rank-change report BEFORE overwriting the table.
2053
- //
2054
- // Load ALL existing rows so rank positions are stash-relative, not pool-relative.
2055
- // Source-scope by the `<bundle>//` prefix and fold each in-pool asset's
2056
- // stored spelling onto its single write key so
2057
- // the merge below never double-counts one asset across two spellings.
2058
- const allStoredScores = getAllRankScores(stateDb);
2059
- const existingAllScores = new Map();
2060
- for (const [ref, score] of allStoredScores) {
2061
- if (options.sourceName) {
2062
- const boundary = ref.indexOf("//");
2063
- const prefix = boundary >= 0 ? ref.slice(0, boundary) : undefined;
2064
- const belongs = prefix === options.sourceName;
2065
- if (!belongs)
2066
- continue;
2067
- }
2068
- existingAllScores.set(normalizeStoredKey.get(ref) ?? ref, score);
2069
- }
2070
- if (existingAllScores.size === 0) {
2071
- // Scenario A: first WS-1 run — table empty.
2072
- //
2073
- // Reconstruct the old combinedEligibilityScore ordering for the current
2074
- // candidate pool using inputs that are already in-scope: utility from
2075
- // utilityMap and the attention term from feedbackSummary (positive/negative
2076
- // counts). Old formula: score = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT.
2077
- //
2078
- // Limitation: this covers only the current-run candidate pool, not the full
2079
- // stash. The stash-wide ordering was never persisted (asset_salience is a new
2080
- // WS-1 table), so this is the most faithful comparison possible at cutover.
2081
- // WS-1 step 7.
2082
- const reconstructedOldScores = new Map();
2083
- for (const ref of salienceMap.keys()) {
2084
- const utility = utilityMap.get(ref) ?? 0;
2085
- const fb = feedbackSummary.get(ref) ?? { positive: 0, negative: 0 };
2086
- const attention = computeValenceScore(fb).attention;
2087
- reconstructedOldScores.set(ref, utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT);
2088
- }
2089
- // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2090
- const oldRanks = toRankPositions(reconstructedOldScores);
2091
- const newRanks = toRankPositions(new Map([...salienceMap.entries()].map(([ref, v]) => [ref, v.rankScore])));
2092
- const firstRunReport = buildRankChangeReport(oldRanks, newRanks);
2093
- if (firstRunReport.forgettingCandidates.length > 0) {
2094
- warn(`[improve/salience] WS-1 first-run rank-change report: ${firstRunReport.forgettingCandidates.length} asset(s) fell from top-200 to below position 500 (cutover formula change). ` +
2095
- `Top drops: ${firstRunReport.forgettingCandidates
2096
- .slice(0, 5)
2097
- .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2098
- .join(", ")}`);
2099
- pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
2100
- }
2101
- if (persist) {
2102
- appendEvent({
2103
- eventType: "improve_salience_first_run",
2104
- ref: undefined,
2105
- metadata: {
2106
- candidateCount: salienceMap.size,
2107
- note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); WS-1 step 7",
2108
- forgettingCandidates: firstRunReport.forgettingCandidates.length,
2109
- topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
2110
- ref: e.ref,
2111
- oldRank: e.oldRank,
2112
- newRank: e.newRank,
2113
- })),
2114
- },
2115
- }, eventsCtx);
2116
- }
2117
- }
2118
- else {
2119
- // Scenario B: subsequent run — compare stash-wide old vs. new ranks.
2120
- //
2121
- // Build new scores by merging the full table with this run's updates.
2122
- // Refs in salienceMap override their stored value; refs not in this run
2123
- // retain their stored value unchanged. This gives a complete stash-wide
2124
- // picture of what the new ordering looks like after this run.
2125
- const mergedNewScores = new Map(existingAllScores);
2126
- for (const [ref, vector] of salienceMap) {
2127
- // Chunk-5 flip F5e — key this run's fresh scores by the WRITE key so
2128
- // they overwrite (never duplicate) the same asset's normalized stored row.
2129
- mergedNewScores.set(wk(ref), vector.rankScore);
2130
- }
2131
- // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2132
- const oldRanks = toRankPositions(existingAllScores);
2133
- const newRanks = toRankPositions(mergedNewScores);
2134
- const report = buildRankChangeReport(oldRanks, newRanks);
2135
- if (report.forgettingCandidates.length > 0) {
2136
- warn(`[improve/salience] WS-1 rank-change report: ${report.forgettingCandidates.length} asset(s) fell from top-200 to below position 500. ` +
2137
- `Top drops: ${report.forgettingCandidates
2138
- .slice(0, 5)
2139
- .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2140
- .join(", ")}`);
2141
- // Collect refs for protective consolidation pass (plan §WS-1 step 7).
2142
- // These are force-included in the candidate pool (mergedRefs) after
2143
- // this try block, bypassing cooldown/signal-delta gating.
2144
- // Chunk-5 flip F5e — map an in-pool candidate's write-key spelling
2145
- // back to its bare `r.ref` so applyForgettingSafety re-stamps the
2146
- // existing pool ref. Other stored spellings stay qualified here;
2147
- // the downstream admission boundary resolves them only when they
2148
- // match an exact current-plan item_ref.
2149
- pendingForgettingRefs = report.forgettingCandidates.map((e) => refByWriteKey.get(e.ref) ?? e.ref);
2150
- }
2151
- if (persist) {
2152
- appendEvent({
2153
- eventType: "improve_salience_rank_change",
2154
- ref: undefined,
2155
- metadata: {
2156
- stashSize: existingAllScores.size,
2157
- totalChanged: report.allChanges.length,
2158
- forgettingCandidates: report.forgettingCandidates.length,
2159
- topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
2160
- ref: e.ref,
2161
- oldRank: e.oldRank,
2162
- newRank: e.newRank,
2163
- })),
2164
- },
2165
- }, eventsCtx);
2166
- }
2167
- }
2168
- if (persist) {
2169
- for (const [ref, vector] of salienceMap) {
2170
- // Persist salience under item_ref when resolved, else the conceptId.
2171
- upsertAssetSalience(stateDb, wk(ref), vector, nowForSalience);
2172
- }
2173
- }
2174
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
2175
- }
2176
- catch (err) {
2177
- rethrowIfTestIsolationError(err);
2178
- // best-effort: salience persistence failure never blocks ranking
939
+ function updateOutcomeScores(args) {
940
+ const { mergedRefs, feedback, eventsCtx, persist } = args;
941
+ const now = Date.now();
942
+ const inputsFor = (r, accepted) => {
943
+ const fb = feedback.get(r.ref) ?? { positive: 0, negative: 0 };
944
+ return {
945
+ ref: keyOf(r),
946
+ currentRetrievalCount: args.retrievalCounts.get(r.ref) ?? 0,
947
+ lastRetrievedAt: args.lastUseMsByRef.get(r.ref) ?? 0,
948
+ acceptedChangeCount: accepted,
949
+ negativeFeedbackCount: fb.negative,
950
+ valence: computeValenceScore(fb).valence,
951
+ utilityScore: args.utilityMap.get(r.ref),
952
+ now,
953
+ };
954
+ };
955
+ const out = new Map();
956
+ if (!persist && !eventsCtx?.db) {
957
+ const projected = new Map(mergedRefs.map((r) => [r.ref, projectAssetOutcome(undefined, inputsFor(r, 0)).outcomeScore]));
958
+ const max = Math.min(OUTCOME_SCORE_MAX, Math.max(0, ...projected.values()));
959
+ for (const [ref, score] of projected)
960
+ out.set(ref, outcomeScoreToSalience(score, max));
961
+ return out;
2179
962
  }
2180
- return pendingForgettingRefs;
2181
- }
2182
- /**
2183
- * The protective forgetting-safety injection (plan §WS-1 step 7). Returns the
2184
- * (possibly extended) mergedRefs; re-stamps lane attribution in precedence
2185
- * order on the SHARED ref objects and eligibilitySourceByRef map.
2186
- */
2187
- export function applyForgettingSafety(args) {
2188
- const { pendingForgettingRefs, scope, eligibleRefs, allowFallbacks, eligibilitySourceByRef, highSalienceRefs, proactiveRefs, signalFiltered, } = args;
2189
- let mergedRefs = args.mergedRefs;
2190
- // ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
2191
- // Forgetting candidates detected in scenario B are force-injected into
2192
- // mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
2193
- // signal-delta gating. The lane may only reuse exact objects from this
2194
- // invocation's post-cleanup/post-validation plan. Stale, out-of-scope, and
2195
- // differently-qualified durable state must never synthesize executable work.
2196
- if (pendingForgettingRefs.length > 0 && scope.mode !== "ref" && allowFallbacks) {
2197
- const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
2198
- const eligibleByRef = new Map(eligibleRefs.map((candidate) => [candidate.ref, candidate]));
2199
- const eligibleByItemRef = new Map();
2200
- for (const candidate of eligibleRefs) {
2201
- if (candidate.itemRef)
2202
- eligibleByItemRef.set(candidate.itemRef, candidate);
2203
- }
2204
- const newForgettingRefs = [];
2205
- const forgettingRefSet = new Set();
2206
- for (const stateRef of pendingForgettingRefs) {
2207
- const boundary = stateRef.indexOf("//");
2208
- const candidate = eligibleByItemRef.get(stateRef) ?? (boundary < 0 ? eligibleByRef.get(bareImproveRef(stateRef)) : undefined);
2209
- if (!candidate || forgettingRefSet.has(candidate.ref))
2210
- continue;
2211
- forgettingRefSet.add(candidate.ref);
2212
- if (!existingRefSet.has(candidate.ref)) {
2213
- newForgettingRefs.push(candidate);
2214
- existingRefSet.add(candidate.ref);
2215
- }
2216
- // Always stamp the lane in the attribution map (overwrites weaker lanes;
2217
- // stronger reactive signals — scope/signal-delta/proactive — are written
2218
- // after this block so they take precedence).
2219
- eligibilitySourceByRef.set(candidate.ref, "forgetting-safety");
2220
- }
2221
- if (newForgettingRefs.length > 0) {
2222
- mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
963
+ withRunState(eventsCtx, persist, (db) => {
964
+ const accepted = new Map();
965
+ try {
966
+ const rows = listStateProposals(db, {
967
+ status: "accepted",
968
+ ...(args.primaryStashDir ? { stashDir: args.primaryStashDir } : {}),
969
+ });
970
+ for (const p of rows)
971
+ accepted.set(p.ref, (accepted.get(p.ref) ?? 0) + 1);
2223
972
  }
2224
- // Re-stamp attribution for any refs whose lane needs updating.
2225
- // Precedence (weakest → strongest, each overwrites the previous):
2226
- // proactive < forgetting-safety < signal-delta
2227
- // Scope mode is already excluded by the outer guard (`scope.mode !== "ref"`).
2228
- // forgetting-safety sits above proactive so that a ref flagged as a
2229
- // forgetting candidate is always visible to S5/WS-5 as such, even when it
2230
- // was also due for a proactive maintenance run. signal-delta overrides
2231
- // forgetting-safety because a ref with fresh feedback is reactive and
2232
- // doesn't need the protective pass label for measurement purposes.
2233
- for (const r of highSalienceRefs)
2234
- eligibilitySourceByRef.set(r.ref, "high-salience");
2235
- for (const r of proactiveRefs)
2236
- eligibilitySourceByRef.set(r.ref, "proactive");
2237
- // Apply forgetting-safety OVER proactive and high-salience (already
2238
- // stamped in the loop above via
2239
- // `eligibilitySourceByRef.set(ref, "forgetting-safety")`). No-op here: the
2240
- // set() calls above for proactive/high-salience overwrite the earlier
2241
- // forgetting-safety stamp — so we re-apply forgetting-safety now for those
2242
- // refs that are both forgetting candidates AND in another fallback lane.
2243
- for (const ref of forgettingRefSet) {
2244
- eligibilitySourceByRef.set(ref, "forgetting-safety");
973
+ catch {
974
+ // Accepted counts stay 0.
2245
975
  }
2246
- // signal-delta is the strongest reactive signal and overrides forgetting-safety.
2247
- for (const r of signalFiltered)
2248
- eligibilitySourceByRef.set(r.ref, "signal-delta");
2249
- // Update eligibilitySource on the ref objects themselves for any refs whose
2250
- // lane changed (covers both new stubs and pre-existing refs).
976
+ const raw = new Map();
977
+ const byKey = new Map();
2251
978
  for (const r of mergedRefs) {
2252
- r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2253
- }
2254
- }
2255
- return mergedRefs;
2256
- }
2257
- /**
2258
- * Pass: eligibility-filter — replay selection (#610), the no-op dampener sort,
2259
- * coverage gaps, the disk-existence guard, the --limit slice, and the summary
2260
- * info emits.
2261
- */
2262
- async function filterEligibility(args) {
2263
- const { scope, options, replayEligibleRefs, eventsCtx, salienceMap, eligibilitySourceByRef, distillOnlyRefs, persist, } = args;
2264
- const { signalAndRetrievalRefs, signalFiltered } = args.summary;
2265
- const validationFailureRefs = args.validationFailureRefs;
2266
- const replay = applyReplaySelection({
2267
- scope,
2268
- options,
2269
- plannedRefs: replayEligibleRefs,
2270
- eventsCtx,
2271
- mergedRefs: args.mergedRefs,
2272
- salienceMap,
2273
- eligibilitySourceByRef,
2274
- persist,
2275
- });
2276
- const mergedRefs = replay.mergedRefs;
2277
- const { replayRefSet, replayBudget } = replay;
2278
- // Build no-op map for consolidation-selection dampener (plan §WS-1 step 8).
2279
- // Reads consecutive_no_ops from the SAME pinned db handle used elsewhere in
2280
- // this function. The effective score is used ONLY for processing/selection
2281
- // order — the persisted rank_score in asset_salience is never mutated here.
2282
- const noOpMap = new Map();
2283
- try {
2284
- const noOpDb = eventsCtx?.db ?? (persist && eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
2285
- if (noOpDb) {
2286
- const ownsNoOpDb = !eventsCtx?.db;
2287
979
  try {
2288
- for (const r of mergedRefs) {
2289
- noOpMap.set(r.ref, readConsecutiveNoOpsForImproveRef(noOpDb, r.ref, r.itemRef));
2290
- }
980
+ const inputs = inputsFor(r, accepted.get(r.ref) ?? 0);
981
+ const result = persist
982
+ ? updateAssetOutcome(db, inputs)
983
+ : projectAssetOutcome(getAssetOutcome(db, inputs.ref), inputs);
984
+ raw.set(r.ref, result.outcomeScore);
985
+ byKey.set(inputs.ref, result.outcomeScore);
2291
986
  }
2292
- finally {
2293
- if (ownsNoOpDb)
2294
- noOpDb.close();
2295
- }
2296
- }
2297
- }
2298
- catch {
2299
- // best-effort: dampener failure never blocks selection
2300
- }
2301
- // Sort by effective selection score (desc), with explicit ref-string tie-break
2302
- // for determinism. The effective score applies the consolidation-selection
2303
- // dampener: assets that have been repeatedly skipped (consecutive_no_ops >=
2304
- // THRESHOLD) are penalised by FACTOR so they sort after peers with similar
2305
- // rankScore. The persisted rank_score is left unchanged — this is the whole
2306
- // point of the dampener (stable assets stay fully retrievable).
2307
- //
2308
- // WIRING NOTE (plan §WS-1 step 8 / "consolidation-selection" disambiguation):
2309
- // "consolidation-selection" in the plan refers to THIS reflect/distill
2310
- // eligibility ordering — i.e. which assets are chosen for the reflect/distill
2311
- // LLM pass — NOT to akmConsolidate (the cluster-merge phase at ~line 1994,
2312
- // which runs earlier and never reads noOpMap). The no-op counter originates
2313
- // from no-change reflect / quality-rejected distill outcomes; the dampener
2314
- // suppresses repeated LLM attempts on those same assets without touching their
2315
- // persisted rank_score (so they remain fully retrievable).
2316
- //
2317
- // This is the only ranking path. The eligibilitySource lanes (signal-delta /
2318
- // proactive / high-salience) survive as labels set above.
2319
- const effectiveScore = (ref) => {
2320
- const rankScore = salienceMap.get(ref)?.rankScore ?? 0;
2321
- const noOps = noOpMap.get(ref) ?? 0;
2322
- return noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rankScore * SALIENCE_NO_OP_DAMPEN_FACTOR : rankScore;
2323
- };
2324
- const sorted = [...mergedRefs].sort((a, b) => {
2325
- const scoreA = effectiveScore(a.ref);
2326
- const scoreB = effectiveScore(b.ref);
2327
- if (scoreB !== scoreA)
2328
- return scoreB - scoreA;
2329
- // Stable tie-break: deterministic regardless of input ordering.
2330
- return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
2331
- });
2332
- // Phase 0: surface coverage gaps from zero-result search queries
2333
- let coverageGaps = [];
2334
- try {
2335
- const dbForGaps = persist
2336
- ? openExistingDatabase()
2337
- : openReadonlyExistingDatabase(undefined, { isolatedSnapshot: true });
2338
- if (dbForGaps) {
2339
- try {
2340
- coverageGaps = getZeroResultSearches(dbForGaps);
2341
- }
2342
- finally {
2343
- closeDatabase(dbForGaps);
987
+ catch {
988
+ // This ref keeps its stored score.
2344
989
  }
2345
990
  }
2346
- }
2347
- catch (err) {
2348
- rethrowIfTestIsolationError(err);
2349
- // best-effort
2350
- }
2351
- const diskCheck = await dropRefsMissingOnDisk({ sorted, options, eventsCtx, persist });
2352
- const assetMissingOnDisk = diskCheck.assetMissingOnDisk;
2353
- const actionableRefs = diskCheck.actionableRefs;
2354
- // Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
2355
- // preserving sort order. distillOnlyRefs participate in the sort so --limit
2356
- // picks them by score, not by arbitrary position.
2357
- // ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
2358
- //
2359
- // #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
2360
- // fresh slice. Without this split, a high-rankScore replay ref could sort above
2361
- // a fresh ref in the single combined slice and STEAL its slot (violating AC2).
2362
- // We partition into the replay lane vs the rest, apply --limit to the
2363
- // non-replay (fresh) refs only, then APPEND up to `replayBudget` replay refs
2364
- // after the fresh slice. Sort order within each partition is preserved.
2365
- //
2366
- // Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
2367
- // replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
2368
- // `replayLoop.slice(0, 0) === []` — byte-identical.
2369
- const selection = selectEffectiveImproveRefs({
2370
- rankedRefs: actionableRefs,
2371
- distillOnlyRefs,
2372
- limit: options.limit,
2373
- replayBudget,
2374
- });
2375
- const loopRefs = selection.loopRefs;
2376
- const distillOnlyRefsResult = selection.distillOnlyRefs;
2377
- if (signalAndRetrievalRefs.length > 0) {
2378
- info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
2379
- }
2380
- if (validationFailureRefs.size > 0) {
2381
- info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
2382
- }
2383
- if (persist && assetMissingOnDisk.length > 0) {
2384
- info(`[improve] ${assetMissingOnDisk.length} candidates dropped — file not on disk`);
2385
- }
2386
- const deferredCount = actionableRefs.length - loopRefs.length;
2387
- info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
2388
- (options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
2389
- return {
2390
- loopRefs,
2391
- actionableRefs,
2392
- distillOnlyRefs: distillOnlyRefsResult,
2393
- coverageGaps,
2394
- limitRemoved: selection.limitRemoved,
2395
- missingDiskCount: assetMissingOnDisk.length,
2396
- replayBudget,
2397
- preDiskRefs: sorted,
2398
- };
2399
- }
2400
- /** The #610 bounded, additive replay-selection lane (eligibility-filter). */
2401
- function applyReplaySelection(args) {
2402
- const { scope, options, plannedRefs, eventsCtx, salienceMap, eligibilitySourceByRef, persist } = args;
2403
- let mergedRefs = args.mergedRefs;
2404
- // ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
2405
- // Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
2406
- // revisited even with zero reactive signal (no feedback, no retrieval) and
2407
- // regardless of cooldown — exactly like the forgetting-safety lane, replay is
2408
- // injected AFTER cooldown/signal-delta partitioning so it bypasses those gates.
2409
- //
2410
- // Strictly additive: the replay slice is appended AFTER the --limit fresh slice
2411
- // (see the loopRefs partition below), so it can never shrink the fresh-ref set.
2412
- // Replay is the WEAKEST lane — it only stamps refs no other lane already claimed,
2413
- // and budget is spent only on refs not already in mergedRefs (so a stronger lane
2414
- // never has its budget wasted or its label overwritten).
2415
- //
2416
- // Default replayBudget=0 ⇒ this whole block is a no-op (no DB open, no event,
2417
- // no mergedRefs mutation), preserving byte-identical pre-#610 selection behavior.
2418
- const replayBudget = (options.config ?? loadConfig()).improve?.salience?.replayBudget ?? 0;
2419
- const replayRefSet = new Set();
2420
- if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
991
+ // Normalize stash-wide (every row, this run's overlaid), within the writer's bound.
992
+ let max = 0;
2421
993
  try {
2422
- if (!persist && !eventsCtx?.db)
2423
- return { mergedRefs, replayRefSet, replayBudget };
2424
- withStateDb((replayDb) => {
2425
- const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
2426
- const storedRankScores = getAllRankScores(replayDb);
2427
- const plannedByRef = new Map(plannedRefs.map((planned) => [planned.ref, planned]));
2428
- const plannedByItemRef = new Map();
2429
- for (const planned of plannedRefs) {
2430
- if (planned.itemRef)
2431
- plannedByItemRef.set(planned.itemRef, planned);
2432
- }
2433
- // Replay can only revisit an entry selected into THIS invocation's
2434
- // source/type plan. Match durable item_ref rows by exact provenance;
2435
- // legacy bare rows may match the current plan's concept ref. Folding
2436
- // both spellings onto the planned ref also prevents duplicate budget
2437
- // spend when old and current state rows coexist.
2438
- const allRankScores = new Map();
2439
- for (const [stateRef, score] of storedRankScores) {
2440
- const boundary = stateRef.indexOf("//");
2441
- if (options.sourceName && (boundary < 0 || stateRef.slice(0, boundary) !== options.sourceName))
2442
- continue;
2443
- const planned = plannedByItemRef.get(stateRef) ?? (boundary < 0 ? plannedByRef.get(bareImproveRef(stateRef)) : undefined);
2444
- if (!planned)
2445
- continue;
2446
- const previous = allRankScores.get(planned.ref);
2447
- if (previous === undefined || score > previous)
2448
- allRankScores.set(planned.ref, score);
2449
- }
2450
- // Candidate universe = every current-plan salience match NOT already in the
2451
- // pool, ordered by rank_score desc with a deterministic ref-string tie-break
2452
- // (mirrors the main sort). Converged refs (consecutive_no_ops >= dampener
2453
- // threshold) are fully EXCLUDED — a stronger skip than the dampener (which
2454
- // only halves order).
2455
- let convergedSkipped = 0;
2456
- const candidates = [];
2457
- for (const [ref, rankScore] of allRankScores) {
2458
- if (alreadyInPool.has(ref))
2459
- continue;
2460
- const planned = plannedByRef.get(ref);
2461
- if (!planned)
2462
- continue;
2463
- const noOps = readConsecutiveNoOpsForImproveRef(replayDb, ref, planned.itemRef);
2464
- if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
2465
- convergedSkipped++;
2466
- continue;
2467
- }
2468
- candidates.push({ planned, rankScore });
2469
- }
2470
- candidates.sort((a, b) => b.rankScore !== a.rankScore
2471
- ? b.rankScore - a.rankScore
2472
- : a.planned.ref < b.planned.ref
2473
- ? -1
2474
- : a.planned.ref > b.planned.ref
2475
- ? 1
2476
- : 0);
2477
- const candidatePool = candidates.length;
2478
- const selected = candidates.slice(0, replayBudget);
2479
- const newReplayRefs = [];
2480
- for (const { planned } of selected) {
2481
- const ref = planned.ref;
2482
- replayRefSet.add(ref);
2483
- newReplayRefs.push({
2484
- ...planned,
2485
- eligibilitySource: "replay",
2486
- });
2487
- // Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
2488
- if (!salienceMap.has(ref)) {
2489
- salienceMap.set(ref, {
2490
- encoding: 0,
2491
- outcome: 0,
2492
- retrieval: 0,
2493
- rankScore: allRankScores.get(ref) ?? 0,
2494
- });
2495
- }
2496
- }
2497
- if (newReplayRefs.length > 0) {
2498
- mergedRefs = dedupeRefs([...mergedRefs, ...newReplayRefs]);
2499
- // Replay is the WEAKEST lane: stamp 'replay' ONLY for refs not already
2500
- // keyed by a stronger lane.
2501
- for (const ref of replayRefSet) {
2502
- if (!eligibilitySourceByRef.has(ref))
2503
- eligibilitySourceByRef.set(ref, "replay");
2504
- }
2505
- for (const r of mergedRefs) {
2506
- r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2507
- }
2508
- }
2509
- // Aggregated observability event (never per-ref).
2510
- if (persist) {
2511
- appendEvent({
2512
- eventType: "improve_replay_selected",
2513
- ref: undefined,
2514
- metadata: {
2515
- count: newReplayRefs.length,
2516
- budget: replayBudget,
2517
- convergedSkipped,
2518
- candidatePool,
2519
- },
2520
- }, eventsCtx);
2521
- }
2522
- }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
994
+ const scores = new Map(getAllAssetOutcomes(db).map((row) => [row.asset_ref, row.outcome_score]));
995
+ for (const [key, score] of byKey)
996
+ scores.set(key, score);
997
+ for (const score of scores.values())
998
+ if (score > max)
999
+ max = score;
1000
+ max = Math.min(max, OUTCOME_SCORE_MAX);
2523
1001
  }
2524
- catch (err) {
2525
- rethrowIfTestIsolationError(err);
2526
- // best-effort: if DB unavailable, replayRefSet stays empty
1002
+ catch {
1003
+ max = 0;
2527
1004
  }
2528
- }
2529
- return { mergedRefs, replayRefSet, replayBudget };
1005
+ for (const [ref, score] of raw)
1006
+ out.set(ref, outcomeScoreToSalience(score, max));
1007
+ const missing = mergedRefs.filter((r) => !raw.has(r.ref));
1008
+ if (missing.length > 0) {
1009
+ const refByKey = new Map(missing.map((r) => [keyOf(r), r.ref]));
1010
+ for (const [key, score] of getOutcomeScoresByRef(db, [...refByKey.keys()])) {
1011
+ out.set(refByKey.get(key) ?? key, outcomeScoreToSalience(score, max));
1012
+ }
1013
+ }
1014
+ });
1015
+ return out;
2530
1016
  }
2531
- /** The final disk-existence guard + its aggregated audit event (eligibility-filter). */
2532
- async function dropRefsMissingOnDisk(args) {
2533
- const { sorted, options, eventsCtx, persist } = args;
2534
- // actionableRefs is the post-cooldown, post-validation, post-signal, post-sort
2535
- // set — i.e. the genuinely processable refs in priority order. Note: this is
2536
- // a semantic shift from earlier code where actionableRefs was the pre-cooldown
2537
- // sorted set; the new meaning matches reality and is documented on
2538
- // ImprovePreparationResult.actionableRefs.
2539
- //
2540
- // Final guard: drop any candidate whose backing file is no longer on disk.
2541
- // Phase 1 validation captures missing files at the start of preparation, but
2542
- // the gap between that check and dispatch can be minutes on large stashes —
2543
- // long enough for a checkpoint / git checkout / external cleanup to delete
2544
- // the asset. Empirically (improve-critical-review 2026-05-20) the single
2545
- // biggest reject category was "Asset no longer exists on disk" (604/1407 =
2546
- // 43%), meaning reflect/distill was producing proposals against deleted refs.
2547
- // A cheap existsSync per surviving candidate eliminates that wasted work.
2548
- const assetMissingOnDisk = [];
2549
- const existsCheckedActionable = [];
1017
+ /** Drop candidates whose file vanished since planning, with one aggregate event. */
1018
+ async function dropRefsMissingOnDisk(sorted, options, eventsCtx, persist) {
1019
+ const actionableRefs = [];
1020
+ const missing = [];
2550
1021
  for (const candidate of sorted) {
2551
- // #591: prefer the path pre-resolved at planning time (synchronous
2552
- // existsSync) over a serial async DB lookup per ref.
2553
1022
  const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
2554
1023
  ? candidate.filePath
2555
1024
  : await findAssetFilePath(candidate.ref, options.stashDir);
2556
- if (filePath && fs.existsSync(filePath)) {
2557
- existsCheckedActionable.push(candidate);
2558
- }
2559
- else {
2560
- assetMissingOnDisk.push(candidate.ref);
2561
- }
1025
+ if (filePath && fs.existsSync(filePath))
1026
+ actionableRefs.push(candidate);
1027
+ else
1028
+ missing.push(candidate.ref);
2562
1029
  }
2563
- // #592 audit: one summary event instead of one per missing ref. Normally
2564
- // tiny, but a stash deletion racing the run could make this O(n) sequential
2565
- // state.db writes. `refs` is capped so the metadata row stays bounded.
2566
- if (persist && assetMissingOnDisk.length > 0) {
2567
- appendEvent({
2568
- eventType: "improve_skipped",
2569
- ref: undefined,
2570
- metadata: {
2571
- reason: "asset_missing_on_disk",
2572
- count: assetMissingOnDisk.length,
2573
- refs: assetMissingOnDisk.slice(0, 50),
2574
- },
2575
- }, eventsCtx);
1030
+ if (persist && missing.length > 0) {
1031
+ recordImproveSkip(eventsCtx, undefined, {
1032
+ reason: "asset_missing_on_disk",
1033
+ count: missing.length,
1034
+ refs: missing.slice(0, 50),
1035
+ });
2576
1036
  }
2577
- return { actionableRefs: existsCheckedActionable, assetMissingOnDisk };
1037
+ return { actionableRefs, missing };
2578
1038
  }