akm-cli 0.7.5 → 0.8.0-rc.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. package/{.github/CHANGELOG.md → CHANGELOG.md} +192 -2
  2. package/README.md +22 -6
  3. package/SECURITY.md +93 -0
  4. package/dist/cli/config-migrate.js +144 -0
  5. package/dist/cli/config-validate.js +39 -0
  6. package/dist/cli/confirm.js +73 -0
  7. package/dist/cli/parse-args.js +133 -0
  8. package/dist/cli/shared.js +129 -0
  9. package/dist/cli.js +2569 -1449
  10. package/dist/commands/add-cli.js +279 -0
  11. package/dist/commands/agent-dispatch.js +110 -0
  12. package/dist/commands/agent-support.js +68 -0
  13. package/dist/commands/completions.js +3 -0
  14. package/dist/commands/config-cli.js +130 -534
  15. package/dist/commands/consolidate.js +2122 -0
  16. package/dist/commands/curate.js +44 -3
  17. package/dist/commands/db-cli.js +23 -0
  18. package/dist/commands/distill-promotion-policy.js +660 -0
  19. package/dist/commands/distill.js +1075 -77
  20. package/dist/commands/env.js +213 -0
  21. package/dist/commands/eval-cases.js +43 -0
  22. package/dist/commands/events.js +5 -23
  23. package/dist/commands/extract-cli.js +127 -0
  24. package/dist/commands/extract-prompt.js +204 -0
  25. package/dist/commands/extract.js +477 -0
  26. package/dist/commands/feedback-cli.js +331 -0
  27. package/dist/commands/graph.js +477 -0
  28. package/dist/commands/health.js +1302 -0
  29. package/dist/commands/help/help-accept.md +12 -0
  30. package/dist/commands/help/help-improve.md +69 -0
  31. package/dist/commands/help/help-proposals.md +18 -0
  32. package/dist/commands/help/help-propose.md +17 -0
  33. package/dist/commands/help/help-reject.md +11 -0
  34. package/dist/commands/history.js +54 -46
  35. package/dist/commands/improve-auto-accept.js +97 -0
  36. package/dist/commands/improve-cli.js +217 -0
  37. package/dist/commands/improve-profiles.js +166 -0
  38. package/dist/commands/improve-result-file.js +167 -0
  39. package/dist/commands/improve.js +2373 -0
  40. package/dist/commands/info.js +5 -2
  41. package/dist/commands/init.js +50 -2
  42. package/dist/commands/installed-stashes.js +102 -139
  43. package/dist/commands/knowledge.js +136 -0
  44. package/dist/commands/lint/agent-linter.js +49 -0
  45. package/dist/commands/lint/base-linter.js +479 -0
  46. package/dist/commands/lint/command-linter.js +49 -0
  47. package/dist/commands/lint/default-linter.js +16 -0
  48. package/dist/commands/lint/env-key-rules.js +154 -0
  49. package/dist/commands/lint/index.js +196 -0
  50. package/dist/commands/lint/knowledge-linter.js +16 -0
  51. package/dist/commands/lint/markdown-insertion.js +343 -0
  52. package/dist/commands/lint/memory-linter.js +61 -0
  53. package/dist/commands/lint/registry.js +36 -0
  54. package/dist/commands/lint/skill-linter.js +45 -0
  55. package/dist/commands/lint/task-linter.js +50 -0
  56. package/dist/commands/lint/types.js +4 -0
  57. package/dist/commands/lint/workflow-linter.js +56 -0
  58. package/dist/commands/lint.js +4 -0
  59. package/dist/commands/migration-help.js +5 -2
  60. package/dist/commands/proposal.js +67 -12
  61. package/dist/commands/propose.js +86 -31
  62. package/dist/commands/reflect.js +1091 -73
  63. package/dist/commands/registry-cli.js +150 -0
  64. package/dist/commands/registry-search.js +5 -2
  65. package/dist/commands/remember-cli.js +257 -0
  66. package/dist/commands/remember.js +69 -6
  67. package/dist/commands/schema-repair.js +203 -0
  68. package/dist/commands/search.js +115 -14
  69. package/dist/commands/secret.js +173 -0
  70. package/dist/commands/self-update.js +3 -0
  71. package/dist/commands/show.js +148 -25
  72. package/dist/commands/source-add.js +17 -45
  73. package/dist/commands/source-clone.js +3 -0
  74. package/dist/commands/source-manage.js +14 -19
  75. package/dist/commands/tasks.js +437 -0
  76. package/dist/commands/url-checker.js +42 -0
  77. package/dist/core/action-contributors.js +28 -0
  78. package/dist/core/asset-ref.js +17 -2
  79. package/dist/core/asset-registry.js +12 -17
  80. package/dist/core/asset-serialize.js +88 -0
  81. package/dist/core/asset-spec.js +67 -1
  82. package/dist/core/common.js +182 -0
  83. package/dist/core/concurrent.js +25 -0
  84. package/dist/core/config-io.js +347 -0
  85. package/dist/core/config-migration.js +622 -0
  86. package/dist/core/config-schema.js +534 -0
  87. package/dist/core/config-sources.js +108 -0
  88. package/dist/core/config-types.js +4 -0
  89. package/dist/core/config-walker.js +337 -0
  90. package/dist/core/config.js +364 -981
  91. package/dist/core/errors.js +42 -20
  92. package/dist/core/events.js +91 -138
  93. package/dist/core/file-lock.js +104 -0
  94. package/dist/core/frontmatter.js +75 -8
  95. package/dist/core/lesson-lint.js +3 -0
  96. package/dist/core/markdown.js +20 -0
  97. package/dist/core/memory-belief.js +62 -0
  98. package/dist/core/memory-contradiction-detect.js +274 -0
  99. package/dist/core/memory-improve.js +806 -0
  100. package/dist/core/parse.js +158 -0
  101. package/dist/core/paths.js +280 -14
  102. package/dist/core/proposal-quality-validators.js +380 -0
  103. package/dist/core/proposal-validators.js +69 -0
  104. package/dist/core/proposals.js +512 -42
  105. package/dist/core/state-db.js +1068 -0
  106. package/dist/core/text-truncation.js +107 -0
  107. package/dist/core/time.js +54 -0
  108. package/dist/core/tty.js +59 -0
  109. package/dist/core/warn.js +64 -1
  110. package/dist/core/write-source.js +3 -0
  111. package/dist/indexer/db-backup.js +391 -0
  112. package/dist/indexer/db-search.js +178 -256
  113. package/dist/indexer/db.js +975 -103
  114. package/dist/indexer/ensure-index.js +64 -0
  115. package/dist/indexer/file-context.js +3 -0
  116. package/dist/indexer/graph-boost.js +376 -101
  117. package/dist/indexer/graph-db.js +391 -0
  118. package/dist/indexer/graph-dedup.js +95 -0
  119. package/dist/indexer/graph-extraction.js +550 -124
  120. package/dist/indexer/index-context.js +4 -0
  121. package/dist/indexer/indexer.js +523 -301
  122. package/dist/indexer/llm-cache.js +52 -0
  123. package/dist/indexer/manifest.js +3 -0
  124. package/dist/indexer/matchers.js +167 -160
  125. package/dist/indexer/memory-inference.js +152 -74
  126. package/dist/indexer/metadata-contributors.js +29 -0
  127. package/dist/indexer/metadata.js +275 -196
  128. package/dist/indexer/path-resolver.js +92 -0
  129. package/dist/indexer/project-context.js +192 -0
  130. package/dist/indexer/ranking-contributors.js +331 -0
  131. package/dist/indexer/ranking.js +81 -0
  132. package/dist/indexer/search-fields.js +5 -9
  133. package/dist/indexer/search-hit-enrichers.js +111 -0
  134. package/dist/indexer/search-source.js +44 -10
  135. package/dist/indexer/semantic-status.js +6 -17
  136. package/dist/indexer/staleness-detect.js +447 -0
  137. package/dist/indexer/usage-events.js +12 -9
  138. package/dist/indexer/walker.js +28 -0
  139. package/dist/integrations/agent/builders.js +135 -0
  140. package/dist/integrations/agent/config.js +122 -230
  141. package/dist/integrations/agent/detect.js +3 -0
  142. package/dist/integrations/agent/index.js +7 -13
  143. package/dist/integrations/agent/model-aliases.js +55 -0
  144. package/dist/integrations/agent/profiles.js +70 -5
  145. package/dist/integrations/agent/prompts.js +214 -80
  146. package/dist/integrations/agent/runner.js +151 -0
  147. package/dist/integrations/agent/sdk-runner.js +126 -0
  148. package/dist/integrations/agent/spawn.js +118 -23
  149. package/dist/integrations/github.js +3 -0
  150. package/dist/integrations/lockfile.js +32 -69
  151. package/dist/integrations/session-logs/index.js +69 -0
  152. package/dist/integrations/session-logs/inline-refs.js +35 -0
  153. package/dist/integrations/session-logs/pre-filter.js +152 -0
  154. package/dist/integrations/session-logs/providers/claude-code.js +282 -0
  155. package/dist/integrations/session-logs/providers/opencode.js +258 -0
  156. package/dist/integrations/session-logs/types.js +4 -0
  157. package/dist/llm/call-ai.js +62 -0
  158. package/dist/llm/client.js +77 -124
  159. package/dist/llm/embedder.js +20 -29
  160. package/dist/llm/embedders/cache.js +3 -7
  161. package/dist/llm/embedders/local.js +42 -1
  162. package/dist/llm/embedders/remote.js +20 -8
  163. package/dist/llm/embedders/types.js +3 -7
  164. package/dist/llm/feature-gate.js +95 -48
  165. package/dist/llm/graph-extract.js +676 -70
  166. package/dist/llm/index-passes.js +44 -29
  167. package/dist/llm/memory-infer.js +77 -71
  168. package/dist/llm/metadata-enhance.js +42 -29
  169. package/dist/llm/prompts/extract-session.md +80 -0
  170. package/dist/llm/prompts/graph-extract-user-prompt.md +35 -0
  171. package/dist/output/cli-hints-full.md +292 -0
  172. package/dist/output/cli-hints-short.md +66 -0
  173. package/dist/output/cli-hints.js +7 -320
  174. package/dist/output/context.js +60 -8
  175. package/dist/output/renderers.js +300 -257
  176. package/dist/output/shapes/curate.js +56 -0
  177. package/dist/output/shapes/distill.js +10 -0
  178. package/dist/output/shapes/env-list.js +19 -0
  179. package/dist/output/shapes/events.js +11 -0
  180. package/dist/output/shapes/helpers.js +424 -0
  181. package/dist/output/shapes/history.js +7 -0
  182. package/dist/output/shapes/passthrough.js +102 -0
  183. package/dist/output/shapes/proposal-accept.js +7 -0
  184. package/dist/output/shapes/proposal-diff.js +7 -0
  185. package/dist/output/shapes/proposal-list.js +7 -0
  186. package/dist/output/shapes/proposal-producer.js +11 -0
  187. package/dist/output/shapes/proposal-reject.js +7 -0
  188. package/dist/output/shapes/proposal-show.js +7 -0
  189. package/dist/output/shapes/registry-search.js +6 -0
  190. package/dist/output/shapes/registry.js +30 -0
  191. package/dist/output/shapes/search.js +6 -0
  192. package/dist/output/shapes/secret-list.js +19 -0
  193. package/dist/output/shapes/show.js +6 -0
  194. package/dist/output/shapes/vault-list.js +19 -0
  195. package/dist/output/shapes.js +51 -516
  196. package/dist/output/text/add.js +6 -0
  197. package/dist/output/text/clone.js +6 -0
  198. package/dist/output/text/config.js +6 -0
  199. package/dist/output/text/curate.js +6 -0
  200. package/dist/output/text/distill.js +7 -0
  201. package/dist/output/text/enable-disable.js +7 -0
  202. package/dist/output/text/events.js +10 -0
  203. package/dist/output/text/feedback.js +6 -0
  204. package/dist/output/text/helpers.js +1039 -0
  205. package/dist/output/text/history.js +7 -0
  206. package/dist/output/text/import.js +6 -0
  207. package/dist/output/text/index.js +6 -0
  208. package/dist/output/text/info.js +6 -0
  209. package/dist/output/text/init.js +6 -0
  210. package/dist/output/text/list.js +6 -0
  211. package/dist/output/text/proposal-producer.js +8 -0
  212. package/dist/output/text/proposal.js +11 -0
  213. package/dist/output/text/registry-commands.js +11 -0
  214. package/dist/output/text/registry.js +30 -0
  215. package/dist/output/text/remember.js +6 -0
  216. package/dist/output/text/remove.js +6 -0
  217. package/dist/output/text/save.js +6 -0
  218. package/dist/output/text/search.js +6 -0
  219. package/dist/output/text/show.js +6 -0
  220. package/dist/output/text/update.js +6 -0
  221. package/dist/output/text/upgrade.js +6 -0
  222. package/dist/output/text/vault.js +16 -0
  223. package/dist/output/text/wiki.js +15 -0
  224. package/dist/output/text/workflow.js +14 -0
  225. package/dist/output/text.js +44 -1092
  226. package/dist/registry/build-index.js +3 -0
  227. package/dist/registry/create-provider-registry.js +3 -0
  228. package/dist/registry/factory.js +4 -1
  229. package/dist/registry/origin-resolve.js +3 -0
  230. package/dist/registry/providers/index.js +3 -0
  231. package/dist/registry/providers/skills-sh.js +71 -50
  232. package/dist/registry/providers/static-index.js +53 -48
  233. package/dist/registry/providers/types.js +3 -24
  234. package/dist/registry/resolve.js +11 -16
  235. package/dist/registry/types.js +3 -0
  236. package/dist/scripts/migrate-storage.js +17750 -0
  237. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +9031 -0
  238. package/dist/scripts/migrations/v16-to-v17.js +141 -0
  239. package/dist/setup/detect.js +3 -0
  240. package/dist/setup/ripgrep-install.js +3 -0
  241. package/dist/setup/ripgrep-resolve.js +3 -0
  242. package/dist/setup/setup.js +775 -37
  243. package/dist/setup/steps.js +3 -15
  244. package/dist/sources/include.js +3 -0
  245. package/dist/sources/provider-factory.js +5 -12
  246. package/dist/sources/provider.js +3 -20
  247. package/dist/sources/providers/filesystem.js +19 -23
  248. package/dist/sources/providers/git.js +138 -21
  249. package/dist/sources/providers/index.js +3 -0
  250. package/dist/sources/providers/install-types.js +3 -13
  251. package/dist/sources/providers/npm.js +3 -4
  252. package/dist/sources/providers/provider-utils.js +3 -0
  253. package/dist/sources/providers/sync-from-ref.js +3 -11
  254. package/dist/sources/providers/tar-utils.js +3 -0
  255. package/dist/sources/providers/website.js +18 -22
  256. package/dist/sources/resolve.js +3 -0
  257. package/dist/sources/types.js +3 -0
  258. package/dist/sources/website-ingest.js +7 -0
  259. package/dist/tasks/backends/cron.js +203 -0
  260. package/dist/tasks/backends/exec-utils.js +28 -0
  261. package/dist/tasks/backends/index.js +24 -0
  262. package/dist/tasks/backends/launchd-template.xml +19 -0
  263. package/dist/tasks/backends/launchd.js +187 -0
  264. package/dist/tasks/backends/schtasks-template.xml +29 -0
  265. package/dist/tasks/backends/schtasks.js +215 -0
  266. package/dist/tasks/parser.js +211 -0
  267. package/dist/tasks/resolveAkmBin.js +87 -0
  268. package/dist/tasks/runner.js +458 -0
  269. package/dist/tasks/schedule.js +227 -0
  270. package/dist/tasks/schema.js +15 -0
  271. package/dist/tasks/validator.js +62 -0
  272. package/dist/version.js +3 -0
  273. package/dist/wiki/index-template.md +12 -0
  274. package/dist/wiki/ingest-workflow-template.md +54 -0
  275. package/dist/wiki/log-template.md +8 -0
  276. package/dist/wiki/schema-template.md +61 -0
  277. package/dist/wiki/wiki-templates.js +15 -0
  278. package/dist/wiki/wiki.js +13 -61
  279. package/dist/workflows/authoring.js +8 -25
  280. package/dist/workflows/cli.js +3 -0
  281. package/dist/workflows/db.js +140 -10
  282. package/dist/workflows/document-cache.js +3 -10
  283. package/dist/workflows/parser.js +3 -0
  284. package/dist/workflows/renderer.js +11 -3
  285. package/dist/workflows/runs.js +77 -92
  286. package/dist/workflows/schema.js +3 -0
  287. package/dist/workflows/scope-key.js +3 -0
  288. package/dist/workflows/validator.js +4 -8
  289. package/dist/workflows/workflow-template.md +24 -0
  290. package/docs/README.md +10 -2
  291. package/docs/data-and-telemetry.md +225 -0
  292. package/docs/migration/release-notes/0.7.0.md +1 -1
  293. package/docs/migration/release-notes/0.7.5.md +2 -2
  294. package/docs/migration/release-notes/0.8.0.md +48 -0
  295. package/docs/migration/v0.7-to-v0.8.md +1307 -0
  296. package/package.json +30 -12
  297. package/.github/LICENSE +0 -374
  298. package/dist/commands/install-audit.js +0 -381
  299. package/dist/commands/vault.js +0 -328
  300. package/dist/templates/wiki-templates.js +0 -100
@@ -0,0 +1,2373 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { makeAssetRef, parseAssetRef } from "../core/asset-ref";
7
+ import { daysToMs, isAssetType } from "../core/common";
8
+ import { getDefaultLlmConfig, loadConfig } from "../core/config";
9
+ import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../core/errors";
10
+ import { appendEvent, readEvents } from "../core/events";
11
+ import { probeLock, releaseLock, tryAcquireLockSync } from "../core/file-lock";
12
+ import { parseFrontmatter } from "../core/frontmatter";
13
+ import { detectAndWriteContradictions } from "../core/memory-contradiction-detect";
14
+ import { analyzeMemoryCleanup, applyMemoryCleanup, } from "../core/memory-improve";
15
+ import { getDbPath } from "../core/paths";
16
+ import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../core/proposals";
17
+ import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../core/state-db";
18
+ import { info, warn } from "../core/warn";
19
+ import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../indexer/db";
20
+ import { ensureIndex } from "../indexer/ensure-index";
21
+ import { runGraphExtractionPass } from "../indexer/graph-extraction";
22
+ import { akmIndex } from "../indexer/indexer";
23
+ import { runMemoryInferencePass, } from "../indexer/memory-inference";
24
+ import { resolveAssetPath } from "../indexer/path-resolver";
25
+ import { getWritableStashDirs, resolveSourceEntries } from "../indexer/search-source";
26
+ import { runStalenessDetectionPass } from "../indexer/staleness-detect";
27
+ import { resolveImproveProcessRunnerFromProfile } from "../integrations/agent/runner";
28
+ import { getAvailableHarnesses, getExecutionLogCandidates } from "../integrations/session-logs";
29
+ import { isLlmFeatureEnabled, isProcessEnabled } from "../llm/feature-gate";
30
+ import { akmConsolidate } from "./consolidate";
31
+ import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill";
32
+ import { deriveKnowledgeRef } from "./distill-promotion-policy";
33
+ import { countEvalCases, writeEvalCase } from "./eval-cases";
34
+ import { akmExtract } from "./extract";
35
+ import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept";
36
+ import { isProfileFilteredForAllPasses, resolveImproveProfile, shouldSkipRef } from "./improve-profiles";
37
+ import { akmLint } from "./lint/index";
38
+ import { akmReflect } from "./reflect";
39
+ import { runSchemaRepairPass } from "./schema-repair";
40
+ import { checkDeadUrls } from "./url-checker";
41
+ function resolveImproveScope(scope) {
42
+ const trimmed = scope?.trim();
43
+ if (!trimmed)
44
+ return { mode: "all" };
45
+ try {
46
+ parseAssetRef(trimmed);
47
+ return { mode: "ref", value: trimmed };
48
+ }
49
+ catch {
50
+ if (!isAssetType(trimmed)) {
51
+ throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, vault, task.\n` +
52
+ `If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
53
+ }
54
+ return { mode: "type", value: trimmed };
55
+ }
56
+ }
57
+ async function collectEligibleRefs(scope, stashDir, improveProfile) {
58
+ if (scope.mode === "ref" && scope.value) {
59
+ const parsed = parseAssetRef(scope.value);
60
+ const writableDirs = new Set(getWritableStashDirs(stashDir).map((dir) => path.resolve(dir)));
61
+ const filePath = await findAssetFilePath(scope.value, stashDir, writableDirs);
62
+ if (!filePath) {
63
+ return {
64
+ plannedRefs: [],
65
+ memorySummary: { eligible: 0, derived: 0 },
66
+ profileFilteredRefs: [],
67
+ };
68
+ }
69
+ return {
70
+ plannedRefs: [{ ref: scope.value, reason: "scope-ref" }],
71
+ memorySummary: {
72
+ eligible: parsed.type === "memory" ? 1 : 0,
73
+ derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
74
+ },
75
+ profileFilteredRefs: [],
76
+ };
77
+ }
78
+ let sources;
79
+ try {
80
+ sources = resolveSourceEntries(stashDir);
81
+ }
82
+ catch {
83
+ return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
84
+ }
85
+ if (sources.length === 0) {
86
+ return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
87
+ }
88
+ // Only operate on writable sources — never mutate read-only registry caches
89
+ // or remote stashes that the user did not mark writable.
90
+ let writableDirs;
91
+ try {
92
+ writableDirs = getWritableStashDirs(stashDir);
93
+ }
94
+ catch {
95
+ writableDirs = sources.slice(0, 1).map((s) => s.path); // fallback: primary only
96
+ }
97
+ const writableDirSet = new Set(writableDirs.map((d) => path.resolve(d)));
98
+ let db;
99
+ try {
100
+ db = openExistingDatabase();
101
+ const entries = getAllEntries(db, scope.mode === "type" ? scope.value : undefined).filter((indexed) => {
102
+ // First apply the existing stashDir-scope filter (no-op when stashDir is unset).
103
+ if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir))
104
+ return false;
105
+ // Then restrict to writable sources only.
106
+ return isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirSet);
107
+ });
108
+ const planned = new Map();
109
+ const profileFiltered = new Map();
110
+ let memoryEligible = 0;
111
+ let memoryDerived = 0;
112
+ for (const indexed of entries) {
113
+ const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
114
+ const isDerived = indexed.entry.name.endsWith(".derived");
115
+ // `.derived` memories are LLM-inferred and intentionally skip reflect
116
+ // (see the synthetic `derived-memory-reflect-skipped` branch in the
117
+ // improve loop). Enqueueing them here just produced one synthetic skip
118
+ // per derived memory per hour with no real work — pure churn observed
119
+ // 2026-05-21: 11 derived refs re-planned every hour during idle periods.
120
+ // The cleanup phase (analyzeMemoryCleanup) inspects derived memories
121
+ // independently of `plannedRefs`, so dropping them here loses nothing.
122
+ if (!isDerived && !planned.has(ref) && !profileFiltered.has(ref)) {
123
+ // 2026-05-27: extend the .derived precedent to profile-incompatible
124
+ // refs. If every per-ref pass (reflect + distill) on the active
125
+ // profile would refuse this ref, drop it from `plannedRefs`. The
126
+ // caller emits `improve_skipped { reason: profile_filtered_all_passes }`
127
+ // once `eventsCtx` is available so the audit trail is preserved in a
128
+ // single event per ref instead of 2× synthetic actions per run.
129
+ // Background: see /tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md
130
+ if (improveProfile && isProfileFilteredForAllPasses(ref, improveProfile)) {
131
+ profileFiltered.set(ref, {
132
+ ref,
133
+ reason: "profile_filtered_all_passes",
134
+ });
135
+ }
136
+ else {
137
+ planned.set(ref, {
138
+ ref,
139
+ reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
140
+ });
141
+ }
142
+ }
143
+ if (indexed.entry.type === "memory") {
144
+ memoryEligible += 1;
145
+ if (isDerived)
146
+ memoryDerived += 1;
147
+ }
148
+ }
149
+ return {
150
+ plannedRefs: [...planned.values()],
151
+ memorySummary: { eligible: memoryEligible, derived: memoryDerived },
152
+ profileFilteredRefs: [...profileFiltered.values()],
153
+ };
154
+ }
155
+ catch (error) {
156
+ // The bun-test isolation guard must never be downgraded to "empty plan".
157
+ rethrowIfTestIsolationError(error);
158
+ if (error instanceof NotFoundError || error instanceof Error) {
159
+ return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
160
+ }
161
+ throw error;
162
+ }
163
+ finally {
164
+ if (db)
165
+ closeDatabase(db);
166
+ }
167
+ }
168
+ function isEntryInScope(entryStashDir, filePath, stashDir) {
169
+ if (!stashDir)
170
+ return true;
171
+ const resolvedEntryStashDir = path.resolve(entryStashDir);
172
+ const resolvedFilePath = path.resolve(filePath);
173
+ const resolvedScopeStashDir = path.resolve(stashDir);
174
+ return (resolvedEntryStashDir === resolvedScopeStashDir ||
175
+ resolvedEntryStashDir.startsWith(`${resolvedScopeStashDir}${path.sep}`) ||
176
+ resolvedFilePath.startsWith(`${resolvedScopeStashDir}${path.sep}`));
177
+ }
178
+ /**
179
+ * Return true when the indexed entry belongs to one of the writable source
180
+ * directories. Entries from read-only registry caches or remote stashes that
181
+ * the user has not marked writable must never enter the improve/distill loop.
182
+ */
183
+ function isEntryInWritableSource(entryStashDir, filePath, writableDirSet) {
184
+ const resolvedEntryStashDir = path.resolve(entryStashDir);
185
+ const resolvedFilePath = path.resolve(filePath);
186
+ for (const writableDir of writableDirSet) {
187
+ if (resolvedEntryStashDir === writableDir ||
188
+ resolvedEntryStashDir.startsWith(`${writableDir}${path.sep}`) ||
189
+ resolvedFilePath.startsWith(`${writableDir}${path.sep}`)) {
190
+ return true;
191
+ }
192
+ }
193
+ return false;
194
+ }
195
+ function memoryCleanupParentRef(scope, stashDir) {
196
+ if (scope.mode !== "ref" || !scope.value)
197
+ return undefined;
198
+ const parsed = parseAssetRef(scope.value);
199
+ if (parsed.type !== "memory")
200
+ return undefined;
201
+ if (!parsed.name.endsWith(".derived"))
202
+ return scope.value;
203
+ const sources = resolveSourceEntries(stashDir);
204
+ for (const source of sources) {
205
+ const candidate = path.join(source.path, "memories", `${parsed.name}.md`);
206
+ if (!fs.existsSync(candidate))
207
+ continue;
208
+ const raw = fs.readFileSync(candidate, "utf8");
209
+ const fm = parseFrontmatter(raw).data;
210
+ const sourceRef = typeof fm.source === "string" ? fm.source : undefined;
211
+ if (sourceRef) {
212
+ try {
213
+ const parent = parseAssetRef(sourceRef.trim());
214
+ if (parent.type === "memory")
215
+ return makeAssetRef(parent.type, parent.name);
216
+ }
217
+ catch { }
218
+ }
219
+ }
220
+ return makeAssetRef("memory", parsed.name.slice(0, -".derived".length));
221
+ }
222
+ function isLessonCandidate(ref) {
223
+ // Only lesson assets need lesson-schema validation (description + when_to_use).
224
+ // Memories have their own distill path via shouldDistillMemoryRef.
225
+ // All other types go through reflect, not distill.
226
+ return parseAssetRef(ref).type === "lesson";
227
+ }
228
+ /**
229
+ * Planner-side check: should this ref enter the distill queue?
230
+ *
231
+ * Distill produces lessons from non-lesson sources. Two cases are eligible:
232
+ *
233
+ * 1. Memory refs that pass {@link shouldDistillMemoryRef} (the existing
234
+ * memory→lesson/knowledge promotion path).
235
+ *
236
+ * Refs whose `type` is in {@link DISTILL_REFUSED_INPUT_TYPES} (currently
237
+ * `lesson:*`) are explicitly excluded — distill refuses them at runtime and
238
+ * queuing them just produces a no-op `skipped` outcome per ref per hour. That
239
+ * planner waste was the bug fixed in commit
240
+ * fix(improve): drop distill-refused types from planner.
241
+ *
242
+ * Note: prior to this fix the gate used `isLessonCandidate(ref)` directly,
243
+ * which was true *only* for `lesson:*` refs — exactly the set distill refuses.
244
+ * The result: every hourly run re-queued the same lesson refs, the same skip
245
+ * message returned, and no work was ever done. See
246
+ * `tests/commands/improve-distill-planner-skip-lessons.test.ts`.
247
+ */
248
+ function isDistillCandidateRef(ref, stashDir) {
249
+ const parsed = parseAssetRef(ref);
250
+ if (isDistillRefusedInputType(parsed.type))
251
+ return false;
252
+ return shouldDistillMemoryRef(ref, stashDir);
253
+ }
254
+ function shouldDistillMemoryRef(ref, stashDir) {
255
+ const parsed = parseAssetRef(ref);
256
+ if (parsed.type !== "memory")
257
+ return false;
258
+ const sources = resolveSourceEntries(stashDir);
259
+ for (const source of sources) {
260
+ const candidate = `${source.path}/memories/${parsed.name}.md`;
261
+ if (!fs.existsSync(candidate))
262
+ continue;
263
+ const raw = fs.readFileSync(candidate, "utf8");
264
+ const fm = parseFrontmatter(raw).data;
265
+ const quality = typeof fm.quality === "string" ? fm.quality : undefined;
266
+ if (quality === "proposed")
267
+ return false;
268
+ return !parsed.name.endsWith(".derived");
269
+ }
270
+ return !parsed.name.endsWith(".derived");
271
+ }
272
+ // ── Signal-delta eligibility helpers (0.8.0) ────────────────────────────────
273
+ //
274
+ // The 0.8.0 redesign replaced flat time-based cooldowns for reflect/distill
275
+ // with a *signal-delta* gate: a ref is re-eligible iff new feedback has
276
+ // landed since the last proposal was generated for it. These helpers build
277
+ // the two timestamp maps the gate needs in bulk, so the planner avoids
278
+ // N+1 queries across the full postCleanupRefs set.
279
+ /**
280
+ * Latest feedback event timestamp per ref in the active window. Reads all
281
+ * `feedback` events newer than `sinceIso` in one query and indexes by ref,
282
+ * keeping the maximum `ts` per ref.
283
+ *
284
+ * Only events with a meaningful payload count as "signal" — `metadata.signal`
285
+ * (positive/negative) OR `metadata.note` (a free-form annotation). Empty
286
+ * metadata events are ignored so a stray `akm feedback <ref>` invocation
287
+ * without a flag doesn't trigger downstream re-processing.
288
+ */
289
+ function buildLatestFeedbackTsMap(refs, sinceIso) {
290
+ const out = new Map();
291
+ if (refs.length === 0)
292
+ return out;
293
+ const refSet = new Set(refs);
294
+ const { events } = readEvents({ type: "feedback", since: sinceIso });
295
+ for (const e of events) {
296
+ const ref = e.ref;
297
+ if (!ref || !refSet.has(ref))
298
+ continue;
299
+ const meta = e.metadata;
300
+ const hasSignal = meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
301
+ if (!hasSignal)
302
+ continue;
303
+ const ts = e.ts ?? "";
304
+ if (ts > (out.get(ref) ?? ""))
305
+ out.set(ref, ts);
306
+ }
307
+ return out;
308
+ }
309
+ /**
310
+ * Latest proposal timestamp per input-ref, filtered by source ('reflect' or
311
+ * 'distill'). Reads the corresponding `*_invoked` events from state.db —
312
+ * these events are emitted at proposal creation time and carry the *input*
313
+ * asset ref (memory:foo, skill:bar, etc.) directly. We use them rather than
314
+ * `listProposals` because distill proposals are keyed by the derived
315
+ * lesson/knowledge ref, not the source memory — joining back through the
316
+ * payload would be fragile.
317
+ */
318
+ function buildLatestProposalTsMap(refs, source) {
319
+ const out = new Map();
320
+ if (refs.length === 0)
321
+ return out;
322
+ const refSet = new Set(refs);
323
+ const eventType = source === "reflect" ? "reflect_invoked" : "distill_invoked";
324
+ const { events } = readEvents({ type: eventType });
325
+ for (const e of events) {
326
+ const ref = e.ref;
327
+ if (!ref || !refSet.has(ref))
328
+ continue;
329
+ // For distill_invoked we only count attempts that produced (or attempted
330
+ // to produce) a real proposal — config_disabled / parse-error outcomes
331
+ // should not move the signal-delta cursor forward.
332
+ if (eventType === "distill_invoked") {
333
+ const outcome = e.metadata?.outcome;
334
+ if (outcome !== "queued" && outcome !== "skipped" && outcome !== "validation_failed")
335
+ continue;
336
+ }
337
+ const ts = e.ts ?? "";
338
+ if (ts > (out.get(ref) ?? ""))
339
+ out.set(ref, ts);
340
+ }
341
+ return out;
342
+ }
343
+ /**
344
+ * Signal-delta eligibility predicate.
345
+ *
346
+ * True iff `latestFeedback[ref]` is defined AND either no prior proposal
347
+ * exists for this (ref, source) OR `latestFeedback[ref] > lastProposal[ref]`.
348
+ *
349
+ * Refs with no feedback signal at all are ineligible by definition — the
350
+ * high-retrieval fallback path (see `noFeedbackCandidates` later in the
351
+ * planner) handles never-touched-but-frequently-read assets separately.
352
+ */
353
+ function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
354
+ const fb = latestFeedback.get(ref);
355
+ if (!fb)
356
+ return false;
357
+ const lp = lastProposal.get(ref);
358
+ if (!lp)
359
+ return true;
360
+ return fb > lp;
361
+ }
362
+ export async function akmImprove(options = {}) {
363
+ const scope = resolveImproveScope(options.scope);
364
+ const reflectFn = options.reflectFn ?? akmReflect;
365
+ const distillFn = options.distillFn ?? akmDistill;
366
+ const ensureIndexFn = options.ensureIndexFn ?? ensureIndex;
367
+ const reindexFn = options.reindexFn ?? akmIndex;
368
+ // Resolve the improve profile for this run. Profile drives type filtering,
369
+ // process gating, and default autoAccept/limit values.
370
+ const _earlyConfig = options.config ?? loadConfig();
371
+ const improveProfile = resolveImproveProfile(options.profile, _earlyConfig);
372
+ // Apply profile defaults — CLI flags take precedence over profile defaults.
373
+ // Rebuild options with effective values so all downstream stage functions
374
+ // automatically pick up the profile-driven defaults.
375
+ options = {
376
+ ...options,
377
+ autoAccept: options.autoAccept ?? improveProfile.autoAccept,
378
+ limit: options.limit ?? improveProfile.limit,
379
+ };
380
+ let primaryStashDir;
381
+ try {
382
+ primaryStashDir = resolveSourceEntries(options.stashDir)[0]?.path;
383
+ }
384
+ catch {
385
+ primaryStashDir = undefined;
386
+ }
387
+ // #339 fix: ensureIndex MUST run BEFORE collectEligibleRefs. The eligible-ref
388
+ // query reads the `entries` table; if a DB version upgrade just dropped that
389
+ // table (or the index is otherwise empty), the prior run order silently
390
+ // returned plannedRefs=[] and the improve loop no-op'd. Hoisting the call
391
+ // here repopulates the index first so the subsequent query sees fresh data.
392
+ const preEnsureCleanupWarnings = [];
393
+ if (primaryStashDir) {
394
+ // Probe pre-ensureIndex entry count to drive the loud-fail warning below.
395
+ // Best-effort: a missing DB / unreadable schema is the fresh-install case
396
+ // and not a bug — we silently skip the probe.
397
+ let preEnsureEntryCount;
398
+ try {
399
+ const dbPath = getDbPath();
400
+ if (fs.existsSync(dbPath)) {
401
+ const probeDb = openExistingDatabase();
402
+ try {
403
+ preEnsureEntryCount = getEntryCount(probeDb);
404
+ }
405
+ finally {
406
+ closeDatabase(probeDb);
407
+ }
408
+ }
409
+ }
410
+ catch (err) {
411
+ rethrowIfTestIsolationError(err);
412
+ // best-effort; leave preEnsureEntryCount undefined
413
+ }
414
+ try {
415
+ await ensureIndexFn(primaryStashDir);
416
+ }
417
+ catch (err) {
418
+ preEnsureCleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
419
+ }
420
+ // #339 loud-fail: if the index was empty pre-ensureIndex but is now
421
+ // populated, a version-upgrade-triggered rebuild just happened. Surface
422
+ // that on stderr so the improve run is not silently masked by stale
423
+ // index state. Zero-before AND zero-after is the empty-stash case and
424
+ // is intentionally not warned (not a bug).
425
+ if (preEnsureEntryCount === 0) {
426
+ try {
427
+ const probeDb = openExistingDatabase();
428
+ let postCount = 0;
429
+ try {
430
+ postCount = getEntryCount(probeDb);
431
+ }
432
+ finally {
433
+ closeDatabase(probeDb);
434
+ }
435
+ if (postCount > 0) {
436
+ warn("[improve] index was empty after DB version upgrade — repopulating before continuing");
437
+ }
438
+ }
439
+ catch (err) {
440
+ rethrowIfTestIsolationError(err);
441
+ // best-effort
442
+ }
443
+ }
444
+ }
445
+ const { plannedRefs, memorySummary, profileFilteredRefs } = await collectEligibleRefs(scope, options.stashDir, improveProfile);
446
+ const cleanupParentRef = memoryCleanupParentRef(scope, options.stashDir);
447
+ // M-1 (#367): Run contradiction-detection BEFORE analyzeMemoryCleanup so
448
+ // the SCC resolver in resolveFamilyContradictions has edges to work on.
449
+ // Best-effort: failures are warnings, never fatal.
450
+ if (primaryStashDir && shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)) {
451
+ try {
452
+ const config = options.config ?? loadConfig();
453
+ await detectAndWriteContradictions(primaryStashDir, config);
454
+ }
455
+ catch (err) {
456
+ // Non-fatal: contradiction detection is a best-effort pass.
457
+ warn(`[improve] contradiction detection failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
458
+ }
459
+ }
460
+ const memoryCleanupPlan = shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)
461
+ ? analyzeMemoryCleanup(primaryStashDir, cleanupParentRef ? { parentRef: cleanupParentRef } : undefined)
462
+ : undefined;
463
+ const guidance = memorySummary.eligible > 0
464
+ ? "Improve folds memory cleanup into the same proposal queue: speculative promotions still go through reflect/distill proposals, while high-confidence redundant derived memories are moved into a recoverable cleanup archive instead of being left active in the stash."
465
+ : undefined;
466
+ if (options.dryRun) {
467
+ const result = {
468
+ schemaVersion: 1,
469
+ ok: true,
470
+ scope,
471
+ dryRun: true,
472
+ ...(guidance ? { guidance } : {}),
473
+ memorySummary,
474
+ ...(memoryCleanupPlan ? { memoryCleanup: shapeMemoryCleanup(memoryCleanupPlan) } : {}),
475
+ plannedRefs,
476
+ ...(profileFilteredRefs.length > 0 ? { profileFilteredRefs } : {}),
477
+ };
478
+ return result;
479
+ }
480
+ const resolvedLockPath = primaryStashDir
481
+ ? path.join(primaryStashDir, ".akm", "improve.lock")
482
+ : path.join(options.stashDir ?? ".", ".akm", "improve.lock");
483
+ const MAX_LOCK_AGE_MS = 4 * 60 * 60 * 1000; // 4 hours
484
+ fs.mkdirSync(path.dirname(resolvedLockPath), { recursive: true });
485
+ const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
486
+ const acquireLock = () => {
487
+ if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
488
+ return;
489
+ // Lock file already exists — probe to determine whether it's still held
490
+ // or whether the prior run died without cleaning up.
491
+ const probe = probeLock(resolvedLockPath, { staleAfterMs: MAX_LOCK_AGE_MS });
492
+ const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
493
+ const lock = rawContent
494
+ ? (() => {
495
+ try {
496
+ return JSON.parse(rawContent);
497
+ }
498
+ catch {
499
+ return null;
500
+ }
501
+ })()
502
+ : null;
503
+ if (probe.state === "stale") {
504
+ // O-7 / #394: Emit improve_lock_recovered event before recovery so the
505
+ // audit trail records the abnormal prior-run exit (Temporal/Airflow pattern).
506
+ try {
507
+ appendEvent({
508
+ eventType: "improve_lock_recovered",
509
+ metadata: {
510
+ stalePid: lock?.pid ?? null,
511
+ lockedAt: lock?.startedAt ?? null,
512
+ recoveredAt: new Date().toISOString(),
513
+ lockAgeMs: probe.ageMs ?? null,
514
+ reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
515
+ },
516
+ });
517
+ }
518
+ catch {
519
+ /* event emission is best-effort; never block lock recovery */
520
+ }
521
+ releaseLock(resolvedLockPath);
522
+ if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
523
+ return;
524
+ throw new ConfigError(`akm improve is already running. Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
525
+ }
526
+ throw new ConfigError(`akm improve is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
527
+ };
528
+ acquireLock();
529
+ const budgetMs = options.timeoutMs ?? 2 * 60 * 60 * 1000; // default 2 hours
530
+ const startMs = Date.now();
531
+ // O-1 (#364): Create a shared AbortController derived from startMs + budgetMs.
532
+ // Every async seam receives this signal so a hung sub-call cannot extend the
533
+ // run past the declared budget.
534
+ // References: Anthropic *Building Effective Agents* (2024); CoALA §5 (arXiv:2309.02427).
535
+ const budgetAbortController = new AbortController();
536
+ const budgetTimer = setTimeout(() => budgetAbortController.abort("improve budget exhausted"), budgetMs);
537
+ // Clear the timer when the run ends to avoid keeping the event loop alive.
538
+ const clearBudgetTimer = () => clearTimeout(budgetTimer);
539
+ // I1: open a single state.db connection for the entire improve run so all
540
+ // appendEvent calls reuse one handle instead of open/migrate/close per call.
541
+ let eventsDb;
542
+ let eventsCtx;
543
+ try {
544
+ eventsDb = openStateDatabase();
545
+ eventsCtx = { db: eventsDb };
546
+ }
547
+ catch (err) {
548
+ rethrowIfTestIsolationError(err);
549
+ // If we cannot open state.db up-front, fall back to per-call opens.
550
+ eventsCtx = {};
551
+ }
552
+ // 2026-05-27: emit `improve_skipped` audit events for refs the planner
553
+ // pre-filtered (reflect AND distill both refuse them under the active
554
+ // profile). One event per ref so the existing improve_skipped histogram in
555
+ // `health.ts#improveSummary.skipReasons` accumulates the right count under
556
+ // the new `profile_filtered_all_passes` reason code. See
557
+ // `/tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md`.
558
+ for (const filtered of profileFilteredRefs) {
559
+ appendEvent({
560
+ eventType: "improve_skipped",
561
+ ref: filtered.ref,
562
+ metadata: { reason: "profile_filtered_all_passes" },
563
+ }, eventsCtx);
564
+ }
565
+ try {
566
+ const preparation = await runImprovePreparationStage({
567
+ scope,
568
+ options,
569
+ plannedRefs,
570
+ memoryCleanupPlan,
571
+ primaryStashDir,
572
+ memorySummary,
573
+ reindexFn,
574
+ startMs,
575
+ budgetMs,
576
+ eventsCtx,
577
+ initialCleanupWarnings: preEnsureCleanupWarnings,
578
+ improveProfile,
579
+ });
580
+ // D6: pre-load all proposal_rejected events from the last 30 days once,
581
+ // so the per-asset loop can use a Map lookup instead of N DB round trips.
582
+ const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
583
+ const rejectedProposalSince = new Date(Date.now() - REJECTED_PROPOSAL_WINDOW_MS).toISOString();
584
+ const allRejectedProposalEvents = readEvents({ type: "proposal_rejected", since: rejectedProposalSince }).events;
585
+ const rejectedProposalsByRef = new Map();
586
+ for (const e of allRejectedProposalEvents) {
587
+ if (e.ref && (!rejectedProposalsByRef.has(e.ref) || e.ts > (rejectedProposalsByRef.get(e.ref)?.ts ?? ""))) {
588
+ rejectedProposalsByRef.set(e.ref, e);
589
+ }
590
+ }
591
+ const { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount: loopGateCount, } = await runImproveLoopStage({
592
+ scope,
593
+ options,
594
+ primaryStashDir,
595
+ reflectFn,
596
+ distillFn,
597
+ loopRefs: preparation.loopRefs,
598
+ actions: preparation.actions,
599
+ signalBearingSet: preparation.signalBearingSet,
600
+ distillCooledRefs: preparation.distillCooledRefs,
601
+ distillOnlyRefs: preparation.distillOnlyRefs,
602
+ recentErrors: preparation.recentErrors,
603
+ rejectedProposalsByRef,
604
+ utilityMap: preparation.utilityMap,
605
+ startMs,
606
+ budgetMs,
607
+ eventsCtx,
608
+ improveProfile,
609
+ });
610
+ const { allWarnings, consolidation, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, } = await runImprovePostLoopStage({
611
+ scope,
612
+ options,
613
+ primaryStashDir,
614
+ actionableRefs: preparation.actionableRefs,
615
+ appliedCleanup: preparation.appliedCleanup,
616
+ cleanupWarnings: preparation.cleanupWarnings,
617
+ memorySummary,
618
+ memoryRefsForInference,
619
+ reindexFn,
620
+ eventsCtx,
621
+ // O-1 (#364): propagate wall-clock budget signal to post-loop maintenance.
622
+ budgetSignal: budgetAbortController.signal,
623
+ improveProfile,
624
+ });
625
+ const finalActions = maintenanceActions && maintenanceActions.length > 0
626
+ ? [...preparation.actions, ...maintenanceActions]
627
+ : preparation.actions;
628
+ const result = {
629
+ schemaVersion: 1,
630
+ ok: true,
631
+ scope,
632
+ dryRun: false,
633
+ ...(guidance ? { guidance } : {}),
634
+ memorySummary,
635
+ ...(memoryCleanupPlan
636
+ ? {
637
+ memoryCleanup: {
638
+ ...shapeMemoryCleanup(memoryCleanupPlan),
639
+ ...(preparation.appliedCleanup
640
+ ? {
641
+ archived: preparation.appliedCleanup.archived,
642
+ ...(preparation.appliedCleanup.transitionLogPath
643
+ ? { transitionLogPath: preparation.appliedCleanup.transitionLogPath }
644
+ : {}),
645
+ ...(preparation.appliedCleanup.transitionLogEntries !== undefined
646
+ ? { transitionLogEntries: preparation.appliedCleanup.transitionLogEntries }
647
+ : {}),
648
+ ...(allWarnings.length > 0 ? { warnings: allWarnings } : {}),
649
+ }
650
+ : preparation.cleanupWarnings.length > 0
651
+ ? { warnings: preparation.cleanupWarnings }
652
+ : {}),
653
+ },
654
+ }
655
+ : {}),
656
+ plannedRefs: preparation.actionableRefs,
657
+ ...(profileFilteredRefs.length > 0 ? { profileFilteredRefs } : {}),
658
+ actions: finalActions,
659
+ ...(preparation.validationFailures.length > 0 ? { validationFailures: preparation.validationFailures } : {}),
660
+ ...(preparation.schemaRepairs.length > 0 ? { schemaRepairs: preparation.schemaRepairs } : {}),
661
+ ...(consolidation.processed > 0 || consolidation.warnings.length > 0 ? { consolidation } : {}),
662
+ ...(preparation.lintSummary !== undefined ? { lintSummary: preparation.lintSummary } : {}),
663
+ ...(preparation.memoryIndexHealth !== undefined ? { memoryIndexHealth: preparation.memoryIndexHealth } : {}),
664
+ ...(preparation.coverageGaps.length > 0 ? { coverageGaps: preparation.coverageGaps } : {}),
665
+ ...(preparation.executionLogCandidates.length > 0
666
+ ? { executionLogCandidates: preparation.executionLogCandidates }
667
+ : {}),
668
+ ...(preparation.extract && preparation.extract.length > 0 ? { extract: preparation.extract } : {}),
669
+ ...(primaryStashDir !== undefined ? { evalCasesWritten: countEvalCases(primaryStashDir) } : {}),
670
+ ...(deadUrls !== undefined && deadUrls.length > 0 ? { deadUrls } : {}),
671
+ ...(reflectsWithErrorContext > 0 ? { reflectsWithErrorContext } : {}),
672
+ ...(memoryInference ? { memoryInference } : {}),
673
+ ...(graphExtraction ? { graphExtraction } : {}),
674
+ // Per-phase wall-clock durations. Surfaced at the top level of the
675
+ // envelope (not nested) because `health.ts`'s `wallTime.byPhase`
676
+ // aggregator and the existing `memoryInference.durationMs` /
677
+ // `graphExtraction.durationMs` health buckets all read
678
+ // `result.{memoryInferenceDurationMs,graphExtractionDurationMs}`
679
+ // directly. Mirrors how `consolidation.durationMs` is surfaced inside
680
+ // the consolidation sub-object (different convention because the
681
+ // consolidation result type already owns that field). Phases that did
682
+ // not run (zero duration) are omitted so the aggregator's
683
+ // "phase actually ran" filter (`> 0`) excludes them from the median/p95
684
+ // sample. Plumbed in d1273d0's follow-up — see
685
+ // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1k / §3.
686
+ ...(memoryInferenceDurationMs > 0 ? { memoryInferenceDurationMs } : {}),
687
+ ...(graphExtractionDurationMs > 0 ? { graphExtractionDurationMs } : {}),
688
+ ...(stalenessDetection ? { stalenessDetection } : {}),
689
+ ...(orphansPurged !== undefined ? { orphansPurged } : {}),
690
+ ...(proposalsExpired !== undefined && proposalsExpired > 0 ? { proposalsExpired } : {}),
691
+ reflectCooldownActions: finalActions.filter((a) => a.mode === "reflect-cooldown").length,
692
+ reflectSkippedActions: finalActions.filter((a) => a.mode === "reflect-skipped").length,
693
+ reflectGuardRejectedActions: finalActions.filter((a) => a.mode === "reflect-guard-rejected").length,
694
+ ...(() => {
695
+ const t = preparation.gateAutoAcceptedCount + loopGateCount + postLoopGateCount;
696
+ return t > 0 ? { gateAutoAcceptedCount: t } : {};
697
+ })(),
698
+ };
699
+ if (!result.dryRun)
700
+ emitImproveCompletedEvent(result, {
701
+ memoryInferenceDurationMs,
702
+ graphExtractionDurationMs,
703
+ totalDurationMs: Date.now() - startMs,
704
+ warningCount: allWarnings.length,
705
+ orphansPurged: orphansPurged ?? 0,
706
+ }, eventsCtx);
707
+ return result;
708
+ }
709
+ catch (err) {
710
+ // D3: emit improve_failed on unexpected crash so dashboards can detect failures.
711
+ appendEvent({
712
+ eventType: "improve_failed",
713
+ ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
714
+ metadata: {
715
+ error: err instanceof Error ? err.message : String(err),
716
+ durationMs: Date.now() - startMs,
717
+ },
718
+ }, eventsCtx);
719
+ throw err;
720
+ }
721
+ finally {
722
+ // O-1 (#364): Clear the budget abort timer so it does not keep the event
723
+ // loop alive after the run completes.
724
+ clearBudgetTimer();
725
+ try {
726
+ fs.unlinkSync(resolvedLockPath);
727
+ }
728
+ catch {
729
+ // ignore
730
+ }
731
+ // I1: close the long-lived state.db connection opened at the top of the run.
732
+ try {
733
+ eventsDb?.close();
734
+ }
735
+ catch {
736
+ // ignore — DB may already be closed
737
+ }
738
+ }
739
+ }
740
+ function emitImproveCompletedEvent(result, durations, eventsCtx) {
741
+ const actionCounts = {
742
+ reflect: 0,
743
+ reflectFailed: 0,
744
+ reflectCooldown: 0,
745
+ reflectSkipped: 0,
746
+ distill: 0,
747
+ distillSkipped: 0,
748
+ memoryPrune: 0,
749
+ memoryInference: 0,
750
+ graphExtraction: 0,
751
+ error: 0,
752
+ };
753
+ for (const action of result.actions ?? []) {
754
+ switch (action.mode) {
755
+ case "reflect":
756
+ actionCounts.reflect += 1;
757
+ break;
758
+ case "reflect-failed":
759
+ actionCounts.reflectFailed += 1;
760
+ break;
761
+ case "reflect-cooldown":
762
+ actionCounts.reflectCooldown += 1;
763
+ break;
764
+ case "reflect-skipped":
765
+ actionCounts.reflectSkipped += 1;
766
+ break;
767
+ case "distill":
768
+ actionCounts.distill += 1;
769
+ break;
770
+ case "distill-skipped":
771
+ actionCounts.distillSkipped += 1;
772
+ break;
773
+ case "memory-prune":
774
+ actionCounts.memoryPrune += 1;
775
+ break;
776
+ case "memory-inference":
777
+ actionCounts.memoryInference += 1;
778
+ break;
779
+ case "graph-extraction":
780
+ actionCounts.graphExtraction += 1;
781
+ break;
782
+ case "error":
783
+ actionCounts.error += 1;
784
+ break;
785
+ }
786
+ }
787
+ appendEvent({
788
+ eventType: "improve_completed",
789
+ ref: result.scope.mode === "ref"
790
+ ? result.scope.value
791
+ : `improve:${result.scope.mode}:${result.scope.value ?? "all"}`,
792
+ metadata: {
793
+ plannedRefs: result.plannedRefs.length,
794
+ reflectActions: actionCounts.reflect,
795
+ distillActions: actionCounts.distill,
796
+ distillSkippedActions: actionCounts.distillSkipped,
797
+ memoryPruneActions: actionCounts.memoryPrune,
798
+ memoryInferenceActions: actionCounts.memoryInference,
799
+ graphExtractionActions: actionCounts.graphExtraction,
800
+ errorActions: actionCounts.error,
801
+ reflectFailedActions: actionCounts.reflectFailed,
802
+ reflectCooldownActions: actionCounts.reflectCooldown,
803
+ reflectSkippedActions: actionCounts.reflectSkipped,
804
+ reflectsWithErrorContext: result.reflectsWithErrorContext ?? 0,
805
+ coverageGapCount: result.coverageGaps?.length ?? 0,
806
+ executionLogCandidateCount: result.executionLogCandidates?.length ?? 0,
807
+ evalCasesWritten: result.evalCasesWritten ?? 0,
808
+ deadUrlCount: result.deadUrls?.length ?? 0,
809
+ memoryEligible: result.memorySummary.eligible,
810
+ memoryDerived: result.memorySummary.derived,
811
+ memoryCleanupPruneCandidates: result.memoryCleanup?.pruneCandidates.length ?? 0,
812
+ memoryCleanupContradictionCandidates: result.memoryCleanup?.contradictionCandidates.length ?? 0,
813
+ memoryCleanupBeliefStateTransitions: result.memoryCleanup?.beliefStateTransitions.length ?? 0,
814
+ memoryCleanupConsolidationCandidates: result.memoryCleanup?.consolidationCandidates.length ?? 0,
815
+ memoryCleanupArchived: result.memoryCleanup?.archived?.length ?? 0,
816
+ memoryCleanupWarnings: result.memoryCleanup?.warnings?.length ?? 0,
817
+ consolidationProcessed: result.consolidation?.processed ?? 0,
818
+ consolidationDurationMs: result.consolidation?.durationMs ?? 0,
819
+ memoryInferenceWrites: result.memoryInference?.writtenFacts ?? 0,
820
+ memoryInferenceDurationMs: durations.memoryInferenceDurationMs,
821
+ graphExtractionExtractedFiles: result.graphExtraction?.quality.extractedFiles ?? 0,
822
+ graphExtractionDurationMs: durations.graphExtractionDurationMs,
823
+ // New metrics for tuning the improve loop.
824
+ ...(durations.totalDurationMs !== undefined ? { durationMs: durations.totalDurationMs } : {}),
825
+ ...(durations.warningCount !== undefined ? { warningCount: durations.warningCount } : {}),
826
+ ...(durations.orphansPurged !== undefined ? { orphansPurged: durations.orphansPurged } : {}),
827
+ ...(result.graphExtraction?.quality
828
+ ? {
829
+ graphCoverage: result.graphExtraction.quality.extractionCoverage,
830
+ graphDensity: result.graphExtraction.quality.density,
831
+ graphEntities: result.graphExtraction.quality.entityCount,
832
+ }
833
+ : {}),
834
+ },
835
+ }, eventsCtx);
836
+ }
837
+ async function runImprovePreparationStage(args) {
838
+ const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings,
839
+ // improveProfile is part of the preparation-stage signature for future use
840
+ // (per-process gating moved into the in-loop stage). Kept here so the
841
+ // signature does not drift away from the rest of the planner stack.
842
+ improveProfile: _improveProfile, } = args;
843
+ const actions = [];
844
+ const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
845
+ // Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
846
+ let memoryIndexHealth;
847
+ if (primaryStashDir) {
848
+ const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
849
+ if (fs.existsSync(memoryMdPath)) {
850
+ try {
851
+ const lines = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
852
+ const overBudget = lines >= 180;
853
+ memoryIndexHealth = { lineCount: lines, overBudget };
854
+ if (overBudget) {
855
+ cleanupWarnings.push(`MEMORY.md has ${lines} lines (budget: 200). Consolidation strongly recommended.`);
856
+ }
857
+ }
858
+ catch {
859
+ // best-effort
860
+ }
861
+ }
862
+ }
863
+ // Phase 0 — execution log synthesis
864
+ let executionLogCandidates = [];
865
+ try {
866
+ const logEntries = getExecutionLogCandidates(7);
867
+ executionLogCandidates = logEntries.filter((e) => e.isFailurePattern).map((e) => e.topic);
868
+ }
869
+ catch {
870
+ // best-effort
871
+ }
872
+ // Phase 0.4 — session-extract pass.
873
+ //
874
+ // Reads native session files (claude-code JSONL, opencode storage tree)
875
+ // through the SessionLogHarness registry, pre-filters noise, and asks a
876
+ // bounded in-tree LLM to produce candidate memory/lesson/knowledge
877
+ // proposals for content the agent did NOT preserve via inline `akm remember`
878
+ // / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
879
+ // hook with an on-demand pull pipeline.
880
+ //
881
+ // Default-on; opt out via `profiles.improve.default.processes.extract.enabled: false`.
882
+ // Each available harness gets one call with the default --since window;
883
+ // already-seen sessions (tracked in state.db.extract_sessions_seen) are
884
+ // skipped automatically so re-runs don't burn LLM calls on unchanged data.
885
+ //
886
+ // Failures are non-fatal — one harness throwing doesn't abort improve.
887
+ // The extract envelope's own `warnings` field surfaces what went wrong.
888
+ let extractResults;
889
+ let gateAutoAcceptedCount = 0;
890
+ const extractConfig = options.config ?? loadConfig();
891
+ const extractGateCfg = makeGateConfig("extract", {
892
+ globalThreshold: options.autoAccept,
893
+ dryRun: options.dryRun ?? false,
894
+ stashDir: primaryStashDir,
895
+ config: extractConfig,
896
+ eventsCtx,
897
+ });
898
+ if (isLlmFeatureEnabled(extractConfig, "session_extraction")) {
899
+ const availableHarnesses = getAvailableHarnesses();
900
+ if (availableHarnesses.length > 0) {
901
+ extractResults = [];
902
+ for (const h of availableHarnesses) {
903
+ try {
904
+ const result = await akmExtract({
905
+ type: h.name,
906
+ ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
907
+ config: extractConfig,
908
+ dryRun: options.dryRun ?? false,
909
+ });
910
+ extractResults.push(result);
911
+ gateAutoAcceptedCount += (await runAutoAcceptGate(primaryStashDir
912
+ ? result.proposals.map((proposalId) => {
913
+ const proposal = getProposal(primaryStashDir, proposalId);
914
+ return { proposalId, confidence: resolveExtractConfidence(proposal) };
915
+ })
916
+ : [], extractGateCfg)).promoted.length;
917
+ }
918
+ catch (err) {
919
+ const msg = err instanceof Error ? err.message : String(err);
920
+ cleanupWarnings.push(`extract(${h.name}) failed: ${msg}`);
921
+ }
922
+ }
923
+ if (extractResults.length === 0) {
924
+ // All harnesses threw — clear so the envelope's `extract` field is
925
+ // absent rather than misleadingly empty.
926
+ extractResults = undefined;
927
+ }
928
+ }
929
+ }
930
+ // Backlog drain: gate any pending extract proposals that weren't created in
931
+ // this run (i.e. pre-date the gate or were produced by a run that timed out
932
+ // before the gate fired). Without this, eligible proposals accumulate
933
+ // indefinitely — the fresh-gate only covers the current run's output.
934
+ if (primaryStashDir && !options.dryRun && options.autoAccept !== undefined) {
935
+ const freshIds = new Set((extractResults ?? []).flatMap((r) => r.proposals));
936
+ const backlog = listProposals(primaryStashDir, { status: "pending" }).filter((p) => p.source === "extract" && !freshIds.has(p.id));
937
+ if (backlog.length > 0) {
938
+ const backlogCandidates = backlog.map((p) => ({
939
+ proposalId: p.id,
940
+ confidence: resolveExtractConfidence(p),
941
+ }));
942
+ gateAutoAcceptedCount += (await runAutoAcceptGate(backlogCandidates, extractGateCfg)).promoted.length;
943
+ }
944
+ }
945
+ // eligibleCount = raw pre-filter count (before cooldown/signal/cleanup filters).
946
+ // improve_completed.plannedRefs = post-filter count of refs that actually entered the loop.
947
+ appendEvent({
948
+ eventType: "improve_invoked",
949
+ ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
950
+ metadata: { scope, dryRun: options.dryRun ?? false, eligibleCount: plannedRefs.length },
951
+ }, eventsCtx);
952
+ // ensureIndex now runs in akmImprove() BEFORE collectEligibleRefs so the
953
+ // eligible-ref query sees a populated `entries` table on the very first
954
+ // pass after a DB version upgrade (#339). Any failure messages from that
955
+ // earlier call were threaded in via args.initialCleanupWarnings.
956
+ let appliedCleanup;
957
+ try {
958
+ appliedCleanup =
959
+ primaryStashDir && memoryCleanupPlan ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan) : undefined;
960
+ }
961
+ catch (err) {
962
+ cleanupWarnings.push(`applyMemoryCleanup failed: ${err instanceof Error ? err.message : String(err)}`);
963
+ }
964
+ const archivedRefs = appliedCleanup?.archived.map((record) => record.ref) ?? [];
965
+ const removed = new Set(archivedRefs);
966
+ const postCleanupRefs = archivedRefs.length === 0 ? plannedRefs : plannedRefs.filter((r) => !removed.has(r.ref));
967
+ // ── Phase 1: validation pass + schema repair (run on full postCleanupRefs) ──
968
+ // Identifies refs whose on-disk asset has structural problems. Validation
969
+ // failures are excluded from every downstream bucket. Run early so the
970
+ // cooldown partition operates on a clean set.
971
+ if (appliedCleanup) {
972
+ for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
973
+ const archived = appliedCleanup.archived.find((record) => record.ref === candidate.ref);
974
+ if (!archived)
975
+ continue;
976
+ actions.push({
977
+ ref: candidate.ref,
978
+ mode: "memory-prune",
979
+ result: { ok: true, pruned: true, reason: candidate.reason },
980
+ });
981
+ }
982
+ if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
983
+ try {
984
+ await reindexFn({ stashDir: primaryStashDir });
985
+ }
986
+ catch (err) {
987
+ cleanupWarnings.push(`reindex after cleanup failed: ${err instanceof Error ? err.message : String(err)}`);
988
+ }
989
+ }
990
+ }
991
+ const validationFailures = [];
992
+ for (const candidate of postCleanupRefs) {
993
+ try {
994
+ const filePath = await findAssetFilePath(candidate.ref, options.stashDir);
995
+ if (!filePath) {
996
+ validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
997
+ continue;
998
+ }
999
+ if (path.extname(filePath).toLowerCase() !== ".md") {
1000
+ continue;
1001
+ }
1002
+ if (isLessonCandidate(candidate.ref)) {
1003
+ const raw = fs.readFileSync(filePath, "utf8");
1004
+ const fm = parseFrontmatter(raw).data;
1005
+ if (!fm.description)
1006
+ validationFailures.push({ ref: candidate.ref, reason: "missing description" });
1007
+ }
1008
+ }
1009
+ catch (e) {
1010
+ validationFailures.push({ ref: candidate.ref, reason: String(e) });
1011
+ }
1012
+ }
1013
+ if (validationFailures.length > 0) {
1014
+ info(`[improve] ${validationFailures.length} assets have validation issues (will attempt schema repair):`);
1015
+ for (const f of validationFailures)
1016
+ info(` ${f.ref}: ${f.reason}`);
1017
+ }
1018
+ let schemaRepairs = [];
1019
+ let repairedRefs = new Set();
1020
+ // Schema repair pass: attempt to fix validation failures via LLM before skipping.
1021
+ if (validationFailures.length > 0 && options.repairValidationFailures !== false) {
1022
+ const baseConfigForRepair = options.config ?? loadConfig();
1023
+ const llmCfg = getDefaultLlmConfig(baseConfigForRepair);
1024
+ if (llmCfg) {
1025
+ const result = await runSchemaRepairPass(validationFailures, {
1026
+ startMs,
1027
+ budgetMs,
1028
+ llmConfig: llmCfg,
1029
+ stashDir: options.stashDir,
1030
+ findFilePath: findAssetFilePath,
1031
+ isLessonCandidateFn: isLessonCandidate,
1032
+ });
1033
+ schemaRepairs = result.repairs;
1034
+ repairedRefs = result.repairedRefs;
1035
+ }
1036
+ }
1037
+ const validationFailureRefs = new Set(validationFailures.filter((f) => !repairedRefs.has(f.ref)).map((f) => f.ref));
1038
+ if (repairedRefs.size > 0) {
1039
+ info(`[improve] schema repair fixed ${repairedRefs.size}/${validationFailures.length} validation failures; ${validationFailureRefs.size} remain`);
1040
+ }
1041
+ // Phase 0.5 — structural hygiene pass
1042
+ let lintSummary;
1043
+ if (primaryStashDir) {
1044
+ try {
1045
+ const lintResult = akmLint({ fix: true, dir: primaryStashDir });
1046
+ lintSummary = { fixed: lintResult.summary.fixed, flagged: lintResult.summary.flagged };
1047
+ }
1048
+ catch {
1049
+ // lint is best-effort; never block improve
1050
+ }
1051
+ }
1052
+ // O-5 / #378: Per-originator rolling error windows.
1053
+ // Reflexion (arXiv:2303.11366) warns that cross-task verbal critique
1054
+ // contamination degrades below single-shot baseline. Each originator key
1055
+ // ("schema-repair", "reflect") maintains its own rolling window so that
1056
+ // schema-repair failures are not injected as avoidPatterns into reflect calls.
1057
+ const recentErrors = {};
1058
+ const RECENT_ERRORS_CAP = 3;
1059
+ // Helper: push an error onto an originator's rolling window.
1060
+ function pushRecentError(originator, msg) {
1061
+ if (!recentErrors[originator])
1062
+ recentErrors[originator] = [];
1063
+ recentErrors[originator].push(msg);
1064
+ if (recentErrors[originator].length > RECENT_ERRORS_CAP)
1065
+ recentErrors[originator].shift();
1066
+ }
1067
+ // Seed schema-repair originator window from any schema-repair errors.
1068
+ for (const repair of schemaRepairs) {
1069
+ if (repair.outcome === "error") {
1070
+ const errMsg = repair.error ?? `schema repair error: ${repair.reason}`;
1071
+ pushRecentError("schema-repair", errMsg);
1072
+ }
1073
+ }
1074
+ // ── Phase 2: signal-delta eligibility sets built EARLY ────────────────────
1075
+ // 0.8.0 replaces the flat time-based cooldowns (which produced synchronised
1076
+ // waves whenever many refs cooled at the same instant — see the 2026-05-26
1077
+ // 54-ref simultaneous-reflect incident) with a *signal-delta* gate:
1078
+ //
1079
+ // reflectEligible(ref) ≡ latestFeedbackTs(ref) > lastReflectProposalTs(ref)
1080
+ // distillEligible(ref) ≡ latestFeedbackTs(ref) > lastDistillProposalTs(ref)
1081
+ //
1082
+ // i.e. a ref is re-eligible iff new feedback has landed since the last
1083
+ // proposal was generated for it. Stable content with no new signal stays
1084
+ // out of the queue regardless of clock time; a sudden burst of feedback
1085
+ // surfaces only the refs that the burst actually touches.
1086
+ //
1087
+ // The 30-day FEEDBACK_SIGNAL_WINDOW_DAYS bound still applies — only feedback
1088
+ // events newer than that count as "current signal". Ancient one-off
1089
+ // negatives don't permanently lock a ref into every run.
1090
+ //
1091
+ // High-retrieval refs (P0-A path) use a simpler "eligible once" rule: a
1092
+ // ref with no feedback signal but retrievalCount ≥ threshold is eligible
1093
+ // exactly once (no prior reflect proposal). Subsequent re-eligibility for
1094
+ // those refs requires either a new feedback event (then the normal
1095
+ // signal-delta gate applies) or human action. Documented limitation: this
1096
+ // path does not re-fire on retrieval-count growth alone in 0.8.0; storing
1097
+ // the retrieval count in proposal metadata for proper delta-tracking is
1098
+ // captured as future work.
1099
+ const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
1100
+ const feedbackSinceCutoff = new Date(Date.now() - daysToMs(FEEDBACK_SIGNAL_WINDOW_DAYS)).toISOString();
1101
+ // Build the three timestamp maps once across the entire postCleanupRefs set.
1102
+ // Per-ref queries would be N+1 and the planner is already the hottest path
1103
+ // in `akm improve`.
1104
+ const candidateRefs = postCleanupRefs.filter((r) => !validationFailureRefs.has(r.ref)).map((r) => r.ref);
1105
+ const latestFeedbackTs = buildLatestFeedbackTsMap(candidateRefs, feedbackSinceCutoff);
1106
+ const lastReflectProposalTs = buildLatestProposalTsMap(candidateRefs, "reflect");
1107
+ const lastDistillProposalTs = buildLatestProposalTsMap(candidateRefs, "distill");
1108
+ // Refs the distill signal-delta gate rejected at planning time. The main
1109
+ // loop reads this to skip distill for these refs without re-checking
1110
+ // eligibility per iteration.
1111
+ const distillCooledRefs = new Set();
1112
+ const preCooldownCount = postCleanupRefs.length;
1113
+ // ── Phase 3: partition postCleanupRefs by signal-delta eligibility ────────
1114
+ // Three buckets (validation failures are excluded entirely):
1115
+ // eligibleRefs — reflect signal-delta passes (full reflect+distill
1116
+ // loop path; distill guard remains in the loop for
1117
+ // refs that fail the distill signal-delta gate).
1118
+ // distillOnlyRefs — reflect blocked but distill signal-delta passes
1119
+ // AND ref is a distill candidate.
1120
+ // fullySkippedCount — neither gate passes → synthetic skip action
1121
+ // + improve_skipped event, excluded from sort.
1122
+ const eligibleRefs = [];
1123
+ const distillOnlyRefs = [];
1124
+ let fullySkippedCount = 0;
1125
+ // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
1126
+ const scopeRefBypass = scope.mode === "ref";
1127
+ for (const r of postCleanupRefs) {
1128
+ if (validationFailureRefs.has(r.ref))
1129
+ continue;
1130
+ if (scopeRefBypass) {
1131
+ eligibleRefs.push(r);
1132
+ continue;
1133
+ }
1134
+ const reflectOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastReflectProposalTs);
1135
+ const distillOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastDistillProposalTs);
1136
+ const isDistillCandidate = isDistillCandidateRef(r.ref, options.stashDir);
1137
+ if (reflectOk) {
1138
+ if (!distillOk && isDistillCandidate) {
1139
+ // Reflect passes the gate, distill does not — emit the synthetic
1140
+ // distill-skipped action and event up-front so the in-loop guard
1141
+ // does not have to re-derive eligibility.
1142
+ distillCooledRefs.add(r.ref);
1143
+ actions.push({ ref: r.ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
1144
+ appendEvent({
1145
+ eventType: "improve_skipped",
1146
+ ref: r.ref,
1147
+ metadata: { reason: "distill_no_new_signal" },
1148
+ }, eventsCtx);
1149
+ }
1150
+ else if (!distillOk) {
1151
+ // Not a distill candidate AND distill gate doesn't pass — just mark
1152
+ // distillCooled so the loop's distill section is a no-op.
1153
+ distillCooledRefs.add(r.ref);
1154
+ }
1155
+ eligibleRefs.push(r);
1156
+ }
1157
+ else if (distillOk && isDistillCandidate) {
1158
+ // Reflect blocked but distill passes → distill-only bucket.
1159
+ distillOnlyRefs.push(r);
1160
+ }
1161
+ else {
1162
+ // Neither gate passes — fully skipped.
1163
+ fullySkippedCount++;
1164
+ actions.push({
1165
+ ref: r.ref,
1166
+ mode: "distill-skipped",
1167
+ result: { ok: true, reason: "no new signal since last proposal" },
1168
+ });
1169
+ appendEvent({ eventType: "improve_skipped", ref: r.ref, metadata: { reason: "no_new_signal" } }, eventsCtx);
1170
+ }
1171
+ }
1172
+ // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
1173
+ // Everything from here works only on (eligibleRefs ∪ distillOnlyRefs). The
1174
+ // fully-skipped bucket has already been routed and emitted; we deliberately
1175
+ // avoid spending DB/CPU on refs that cannot enter the loop.
1176
+ const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
1177
+ // Gap 6: only surface feedback signals from the last 30 days so that
1178
+ // ancient one-off feedback events don't permanently lock an asset into
1179
+ // every improve run. Assets with only stale signals fall through to the
1180
+ // high-retrieval path (P0-A) or are skipped until new signals arrive.
1181
+ // (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
1182
+ // Phase 2 above for the signal-delta gate; we reuse them here.)
1183
+ // Pre-compute feedback summary per ref in a single pass so we don't issue
1184
+ // two readEvents({type:"feedback", ref}) per asset (one for signal filtering,
1185
+ // one for ratio computation).
1186
+ const feedbackSummary = new Map();
1187
+ for (const candidate of processableRefs) {
1188
+ const { events } = readEvents({ type: "feedback", ref: candidate.ref });
1189
+ let hasSignal = false;
1190
+ let positive = 0;
1191
+ let negative = 0;
1192
+ for (const e of events) {
1193
+ if (!hasSignal &&
1194
+ (e.ts ?? "") >= feedbackSinceCutoff &&
1195
+ e.metadata !== undefined &&
1196
+ (typeof e.metadata.signal === "string" || typeof e.metadata.note === "string")) {
1197
+ hasSignal = true;
1198
+ }
1199
+ if (e.metadata?.signal === "positive")
1200
+ positive++;
1201
+ else if (e.metadata?.signal === "negative")
1202
+ negative++;
1203
+ }
1204
+ feedbackSummary.set(candidate.ref, { hasSignal, positive, negative });
1205
+ }
1206
+ const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
1207
+ // P0-A: also surface zero-feedback assets that have been retrieved many times.
1208
+ const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
1209
+ const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
1210
+ const noFeedbackCandidates = processableRefs.filter((r) => !signalBearingSet.has(r.ref));
1211
+ let highRetrievalRefs = [];
1212
+ let dbForRetrieval;
1213
+ try {
1214
+ dbForRetrieval = openExistingDatabase();
1215
+ const showEventCount = dbForRetrieval.prepare("SELECT COUNT(*) AS cnt FROM usage_events WHERE event_type = 'show'").get().cnt;
1216
+ if (showEventCount === 0) {
1217
+ warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
1218
+ }
1219
+ const retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
1220
+ // High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
1221
+ // ref qualifies exactly once — when retrievalCount ≥ threshold AND no
1222
+ // prior reflect proposal exists for it. Once a reflect proposal is on
1223
+ // record, subsequent re-eligibility requires explicit feedback (which
1224
+ // flows through the normal signal-delta gate above). Tracking growth in
1225
+ // retrieval count would require persisting the count in proposal
1226
+ // metadata; deferred to a follow-up.
1227
+ highRetrievalRefs = noFeedbackCandidates.filter((r) => (retrievalCounts.get(r.ref) ?? 0) >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref));
1228
+ }
1229
+ catch (err) {
1230
+ rethrowIfTestIsolationError(err);
1231
+ // best-effort: if DB unavailable, highRetrievalRefs stays empty
1232
+ }
1233
+ finally {
1234
+ if (dbForRetrieval)
1235
+ closeDatabase(dbForRetrieval);
1236
+ }
1237
+ // If the user explicitly scoped to a single ref, always act on it —
1238
+ // skip the signal/retrieval filter entirely. The filter exists to avoid
1239
+ // noisy "improve everything" runs; it should not gate an intentional
1240
+ // per-ref invocation where the user's explicit choice is the signal.
1241
+ //
1242
+ // For type/all scope: only process refs with usage signals (recent feedback
1243
+ // or sufficient retrievals). A stash with no signals has 0 eligible refs —
1244
+ // usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
1245
+ // to bring them into the eligible pool.
1246
+ const signalAndRetrievalRefs = [...signalFiltered, ...highRetrievalRefs];
1247
+ const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
1248
+ const utilityMap = buildUtilityMap(mergedRefs);
1249
+ // Load feedback ratio per ref from the pre-computed summary (no extra DB pass).
1250
+ const feedbackRatios = new Map();
1251
+ for (const ref of mergedRefs) {
1252
+ const summary = feedbackSummary.get(ref.ref);
1253
+ const positive = summary?.positive ?? 0;
1254
+ const negative = summary?.negative ?? 0;
1255
+ const total = positive + negative;
1256
+ // ratio = negative proportion (high = needs more improvement)
1257
+ feedbackRatios.set(ref.ref, total > 0 ? negative / total : 0);
1258
+ }
1259
+ // Sort: combine utility (desc) with feedback negativity (desc) — high-negative assets rank higher
1260
+ const sorted = [...mergedRefs].sort((a, b) => {
1261
+ const utilA = utilityMap.get(a.ref) ?? 0;
1262
+ const utilB = utilityMap.get(b.ref) ?? 0;
1263
+ const ratioA = feedbackRatios.get(a.ref) ?? 0;
1264
+ const ratioB = feedbackRatios.get(b.ref) ?? 0;
1265
+ // Combined score: 70% utility, 30% negative ratio
1266
+ const scoreA = utilA * 0.7 + ratioA * 0.3;
1267
+ const scoreB = utilB * 0.7 + ratioB * 0.3;
1268
+ return scoreB - scoreA;
1269
+ });
1270
+ // Phase 0: surface coverage gaps from zero-result search queries
1271
+ let coverageGaps = [];
1272
+ try {
1273
+ const dbForGaps = openExistingDatabase();
1274
+ try {
1275
+ coverageGaps = getZeroResultSearches(dbForGaps);
1276
+ }
1277
+ finally {
1278
+ closeDatabase(dbForGaps);
1279
+ }
1280
+ }
1281
+ catch (err) {
1282
+ rethrowIfTestIsolationError(err);
1283
+ // best-effort
1284
+ }
1285
+ // actionableRefs is the post-cooldown, post-validation, post-signal, post-sort
1286
+ // set — i.e. the genuinely processable refs in priority order. Note: this is
1287
+ // a semantic shift from earlier code where actionableRefs was the pre-cooldown
1288
+ // sorted set; the new meaning matches reality and is documented on
1289
+ // ImprovePreparationResult.actionableRefs.
1290
+ //
1291
+ // Final guard: drop any candidate whose backing file is no longer on disk.
1292
+ // Phase 1 validation captures missing files at the start of preparation, but
1293
+ // the gap between that check and dispatch can be minutes on large stashes —
1294
+ // long enough for a checkpoint / git checkout / external cleanup to delete
1295
+ // the asset. Empirically (improve-critical-review 2026-05-20) the single
1296
+ // biggest reject category was "Asset no longer exists on disk" (604/1407 =
1297
+ // 43%), meaning reflect/distill was producing proposals against deleted refs.
1298
+ // A cheap existsSync per surviving candidate eliminates that wasted work.
1299
+ const assetMissingOnDisk = [];
1300
+ const existsCheckedActionable = [];
1301
+ for (const candidate of sorted) {
1302
+ const filePath = await findAssetFilePath(candidate.ref, options.stashDir);
1303
+ if (filePath && fs.existsSync(filePath)) {
1304
+ existsCheckedActionable.push(candidate);
1305
+ }
1306
+ else {
1307
+ assetMissingOnDisk.push(candidate.ref);
1308
+ appendEvent({ eventType: "improve_skipped", ref: candidate.ref, metadata: { reason: "asset_missing_on_disk" } }, eventsCtx);
1309
+ }
1310
+ }
1311
+ const actionableRefs = existsCheckedActionable;
1312
+ // Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
1313
+ // preserving sort order. distillOnlyRefs participate in the sort so --limit
1314
+ // picks them by score, not by arbitrary position.
1315
+ const distillOnlyRefSetForSort = new Set(distillOnlyRefs.map((r) => r.ref));
1316
+ const reflectAndDistillRefsAfterSort = [];
1317
+ const distillOnlyRefsAfterSort = [];
1318
+ for (const r of actionableRefs) {
1319
+ if (distillOnlyRefSetForSort.has(r.ref)) {
1320
+ distillOnlyRefsAfterSort.push(r);
1321
+ }
1322
+ else {
1323
+ reflectAndDistillRefsAfterSort.push(r);
1324
+ }
1325
+ }
1326
+ // ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
1327
+ const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
1328
+ const loopRefs = options.limit ? allLoopRefs.slice(0, options.limit) : allLoopRefs;
1329
+ // Update the returned distillOnlyRefs to the sorted order so callers see the
1330
+ // ranked view (loop stage uses it as a Set so order is irrelevant, but the
1331
+ // shape change keeps downstream consumers consistent).
1332
+ const distillOnlyRefsResult = distillOnlyRefsAfterSort;
1333
+ const totalReflectBlocked = fullySkippedCount + distillOnlyRefs.length;
1334
+ if (totalReflectBlocked > 0) {
1335
+ info(`[improve] ${totalReflectBlocked} of ${preCooldownCount} indexed refs blocked by reflect signal-delta ` +
1336
+ `(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
1337
+ }
1338
+ if (signalAndRetrievalRefs.length > 0) {
1339
+ info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval)`);
1340
+ }
1341
+ if (validationFailureRefs.size > 0) {
1342
+ info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
1343
+ }
1344
+ if (assetMissingOnDisk.length > 0) {
1345
+ info(`[improve] ${assetMissingOnDisk.length} candidates dropped — file not on disk`);
1346
+ }
1347
+ const deferredCount = actionableRefs.length - loopRefs.length;
1348
+ info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
1349
+ (options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
1350
+ return {
1351
+ actions,
1352
+ cleanupWarnings,
1353
+ appliedCleanup,
1354
+ memoryIndexHealth,
1355
+ executionLogCandidates,
1356
+ extract: extractResults,
1357
+ actionableRefs,
1358
+ signalBearingSet,
1359
+ validationFailures,
1360
+ schemaRepairs,
1361
+ lintSummary,
1362
+ loopRefs,
1363
+ distillCooledRefs,
1364
+ distillOnlyRefs: distillOnlyRefsResult,
1365
+ coverageGaps,
1366
+ recentErrors,
1367
+ utilityMap,
1368
+ gateAutoAcceptedCount,
1369
+ };
1370
+ }
1371
+ // TODO(refactor): 13 args including `actions`/`recentErrors` mutation channels. Restructure into immutable plan + mutable context objects — deferred to dedicated refactor with isolated testing.
1372
+ async function runImproveLoopStage(args) {
1373
+ const { scope, options, primaryStashDir, reflectFn, distillFn, loopRefs, actions, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, utilityMap, startMs, budgetMs, eventsCtx, improveProfile, } = args;
1374
+ // O-1 (#364): compute remaining budget at call time so each sub-call
1375
+ // receives only its fair share of the wall-clock budget.
1376
+ const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
1377
+ const RECENT_ERRORS_CAP = 3;
1378
+ // R-2 / #389: Self-Consistency multi-sample voting helpers.
1379
+ // Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
1380
+ const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
1381
+ const SC_N = Math.min(Math.max(2, options.selfConsistencyN ?? 3), 5);
1382
+ /**
1383
+ * Compute Jaccard token overlap between two strings.
1384
+ * Tokenizes by whitespace; returns 0 when both are empty.
1385
+ */
1386
+ function jaccardSimilarity(a, b) {
1387
+ const tokensA = new Set(a.split(/\s+/).filter(Boolean));
1388
+ const tokensB = new Set(b.split(/\s+/).filter(Boolean));
1389
+ if (tokensA.size === 0 && tokensB.size === 0)
1390
+ return 1;
1391
+ let intersection = 0;
1392
+ for (const t of tokensA) {
1393
+ if (tokensB.has(t))
1394
+ intersection++;
1395
+ }
1396
+ const union = tokensA.size + tokensB.size - intersection;
1397
+ return union > 0 ? intersection / union : 0;
1398
+ }
1399
+ /**
1400
+ * Given N reflect results, return the one with the highest average Jaccard
1401
+ * similarity to all other successful results (majority-vote winner).
1402
+ * Falls back to the first successful result when N < 2.
1403
+ */
1404
+ function pickMajorityVote(results) {
1405
+ const successful = results.filter((r) => r.ok);
1406
+ if (successful.length === 0)
1407
+ return (results[0] ?? {
1408
+ schemaVersion: 1,
1409
+ ok: false,
1410
+ reason: "non_zero_exit",
1411
+ error: "all samples failed",
1412
+ exitCode: null,
1413
+ });
1414
+ if (successful.length === 1)
1415
+ return successful[0];
1416
+ let bestIdx = 0;
1417
+ let bestScore = -1;
1418
+ for (let i = 0; i < successful.length; i++) {
1419
+ let totalSim = 0;
1420
+ for (let j = 0; j < successful.length; j++) {
1421
+ if (i === j)
1422
+ continue;
1423
+ totalSim += jaccardSimilarity(successful[i].proposal.payload.content ?? "", successful[j].proposal.payload.content ?? "");
1424
+ }
1425
+ const avgSim = totalSim / (successful.length - 1);
1426
+ if (avgSim > bestScore) {
1427
+ bestScore = avgSim;
1428
+ bestIdx = i;
1429
+ }
1430
+ }
1431
+ return successful[bestIdx] ?? successful[0];
1432
+ }
1433
+ // O-5 / #378: helper to push per-originator errors into the rolling window.
1434
+ function pushRecentError(originator, msg) {
1435
+ if (!recentErrors[originator])
1436
+ recentErrors[originator] = [];
1437
+ recentErrors[originator].push(msg);
1438
+ if (recentErrors[originator].length > RECENT_ERRORS_CAP)
1439
+ recentErrors[originator].shift();
1440
+ }
1441
+ // Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
1442
+ const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
1443
+ let completedCount = 0;
1444
+ let reflectsWithErrorContext = 0;
1445
+ const memoryRefsForInference = new Set();
1446
+ // Pre-load all pending proposals once instead of querying per asset in the loop.
1447
+ const dedupeStashDirForProposals = primaryStashDir ?? options.stashDir;
1448
+ const pendingProposalRefSet = new Set(dedupeStashDirForProposals
1449
+ ? listProposals(dedupeStashDirForProposals, { status: "pending" }).map((p) => p.ref)
1450
+ : []);
1451
+ let gateAutoAcceptedCount = 0;
1452
+ const reflectGateCfg = makeGateConfig("reflect", {
1453
+ globalThreshold: options.autoAccept,
1454
+ dryRun: options.dryRun ?? false,
1455
+ stashDir: primaryStashDir,
1456
+ config: options.config ?? loadConfig(),
1457
+ eventsCtx,
1458
+ });
1459
+ const distillGateCfg = makeGateConfig("distill", {
1460
+ globalThreshold: options.autoAccept,
1461
+ dryRun: options.dryRun ?? false,
1462
+ stashDir: primaryStashDir,
1463
+ config: options.config ?? loadConfig(),
1464
+ eventsCtx,
1465
+ });
1466
+ for (const planned of loopRefs) {
1467
+ if (Date.now() - startMs >= budgetMs) {
1468
+ const remaining = loopRefs.length - completedCount;
1469
+ info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
1470
+ appendEvent({
1471
+ eventType: "improve_skipped",
1472
+ ref: planned.ref,
1473
+ metadata: {
1474
+ reason: "budget_exhausted",
1475
+ remaining,
1476
+ },
1477
+ }, eventsCtx);
1478
+ // B11: Emit improve_skipped for all remaining assets that will not be processed.
1479
+ for (const remainingRef of loopRefs.slice(completedCount + 1)) {
1480
+ appendEvent({
1481
+ eventType: "improve_skipped",
1482
+ ref: remainingRef.ref,
1483
+ metadata: { reason: "budget_exhausted_batch", remaining: loopRefs.length - completedCount - 1 },
1484
+ }, eventsCtx);
1485
+ }
1486
+ actions.push({
1487
+ ref: planned.ref,
1488
+ mode: "error",
1489
+ result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
1490
+ });
1491
+ break;
1492
+ }
1493
+ try {
1494
+ // Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
1495
+ // Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
1496
+ // have their synthetic actions emitted in runImprovePreparationStage.
1497
+ const isDistillOnly = distillOnlyRefSet.has(planned.ref);
1498
+ const parsedPlannedRef = parseAssetRef(planned.ref);
1499
+ // B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
1500
+ // shouldDistillMemoryRef already returns false for .derived refs, so the distill
1501
+ // path is also a no-op for them — we just avoid unnecessary agent spawns.
1502
+ // D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
1503
+ if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
1504
+ // Type guard: skip reflect for unsupported types (script, vault, task, etc.)
1505
+ // and raw wiki directories, driven by the active improve profile.
1506
+ const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
1507
+ if (reflectSkip.skip) {
1508
+ actions.push({
1509
+ ref: planned.ref,
1510
+ mode: "reflect-skipped",
1511
+ result: { ok: true, reason: reflectSkip.reason },
1512
+ });
1513
+ }
1514
+ else {
1515
+ // O-5 / #378: only inject reflect-originator errors into the reflect call.
1516
+ // Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
1517
+ const reflectErrors = recentErrors.reflect ?? [];
1518
+ if (reflectErrors.length > 0)
1519
+ reflectsWithErrorContext++;
1520
+ // O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
1521
+ // bounded by the wall-clock deadline rather than the default per-profile timeout.
1522
+ const reflectBudgetMs = remainingBudgetMs();
1523
+ // Wire profile.processes.reflect.{mode, profile, timeoutMs} into the reflect
1524
+ // dispatch when present. Falls back to akmReflect's own config-based resolution
1525
+ // (profiles.improve.<name>.processes.reflect → defaults.llm) when the profile
1526
+ // does not specify.
1527
+ const reflectProfileRunner = resolveImproveProcessRunnerFromProfile(improveProfile.processes?.reflect, options.config ?? loadConfig());
1528
+ const reflectCallArgs = {
1529
+ ref: planned.ref,
1530
+ task: options.task,
1531
+ ...(options.stashDir ? { stashDir: options.stashDir } : {}),
1532
+ ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
1533
+ agentProcess: options.agentProcess ?? "reflect",
1534
+ eventSource: "improve",
1535
+ ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
1536
+ ...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
1537
+ };
1538
+ // R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
1539
+ // Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
1540
+ const refUtility = utilityMap.get(planned.ref) ?? 0;
1541
+ const useConsistency = refUtility >= SC_THRESHOLD && SC_N >= 2;
1542
+ let reflectResult;
1543
+ if (useConsistency) {
1544
+ const samples = [];
1545
+ for (let s = 0; s < SC_N; s++) {
1546
+ if (remainingBudgetMs() <= 0)
1547
+ break;
1548
+ // draftMode: skip DB write so each sample doesn't create a proposal.
1549
+ samples.push(await reflectFn({ ...reflectCallArgs, draftMode: true }));
1550
+ }
1551
+ const winner = pickMajorityVote(samples.length > 0 ? samples : [await reflectFn({ ...reflectCallArgs, draftMode: true })]);
1552
+ // Persist only the majority-vote winner as a single real proposal.
1553
+ if (winner.ok && primaryStashDir) {
1554
+ const persistResult = createProposal(primaryStashDir, {
1555
+ ref: winner.proposal.ref,
1556
+ source: "reflect",
1557
+ sourceRun: `reflect-sc-${Date.now()}`,
1558
+ payload: winner.proposal.payload,
1559
+ });
1560
+ reflectResult = isProposalSkipped(persistResult)
1561
+ ? {
1562
+ schemaVersion: 1,
1563
+ ok: false,
1564
+ reason: "cooldown",
1565
+ error: `SC proposal skipped: ${persistResult.message}`,
1566
+ ref: winner.ref,
1567
+ exitCode: null,
1568
+ }
1569
+ : { ...winner, proposal: persistResult };
1570
+ }
1571
+ else {
1572
+ reflectResult = winner;
1573
+ }
1574
+ }
1575
+ else {
1576
+ reflectResult = await reflectFn(reflectCallArgs);
1577
+ }
1578
+ const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
1579
+ // Content-policy guard hits (reflect size-rail rejections) are NOT
1580
+ // LLM faults — the agent responded fine, the downstream guard
1581
+ // blocked the output. Route them to a distinct `reflect-guard-rejected`
1582
+ // mode so health metrics can split deterministic guard hits out of
1583
+ // true LLM failures. See
1584
+ // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
1585
+ const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
1586
+ // Type-guard rejection (reflect refused a script/vault/task ref) is
1587
+ // also NOT an LLM failure — the LLM is never invoked. Route to the
1588
+ // existing `reflect-skipped` bucket so it does not inflate the
1589
+ // failure-rate numerator. ~9% of `reflect-failed` events in the
1590
+ // user's stack were this case; see review §1a row "Reflect refused
1591
+ // asset type".
1592
+ const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
1593
+ actions.push({
1594
+ ref: planned.ref,
1595
+ mode: reflectResult.ok
1596
+ ? "reflect"
1597
+ : isCooldown
1598
+ ? "reflect-cooldown"
1599
+ : isGuardReject
1600
+ ? "reflect-guard-rejected"
1601
+ : isTypeRefused
1602
+ ? "reflect-skipped"
1603
+ : "reflect-failed",
1604
+ result: reflectResult,
1605
+ });
1606
+ // Cooldown skips, guard rejects, and type-refused skips are not
1607
+ // failures — do not pollute recentErrors with them (those get
1608
+ // injected as `avoidPatterns` into the next reflect prompt). Guard
1609
+ // rejects ARE worth showing the LLM as a learn-signal so the next
1610
+ // iteration sees "your last expansion was too large"; type-refused
1611
+ // is deterministic and adds no learning signal.
1612
+ if (!reflectResult.ok && !isCooldown && !isTypeRefused) {
1613
+ const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
1614
+ pushRecentError("reflect", errMsg);
1615
+ }
1616
+ // improve_reflect_outcome — per-asset metric for tuning the reflect path.
1617
+ appendEvent({
1618
+ eventType: "improve_reflect_outcome",
1619
+ ref: planned.ref,
1620
+ metadata: {
1621
+ ok: reflectResult.ok,
1622
+ durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
1623
+ agentProfile: reflectResult.ok ? reflectResult.agentProfile : undefined,
1624
+ reason: reflectResult.ok ? undefined : reflectResult.reason,
1625
+ },
1626
+ }, eventsCtx);
1627
+ if (reflectResult.ok) {
1628
+ gateAutoAcceptedCount += (await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg)).promoted.length;
1629
+ }
1630
+ } // end else (reflect type/profile check)
1631
+ }
1632
+ else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
1633
+ // B6: .derived refs skip reflect; record synthetic skip action.
1634
+ actions.push({
1635
+ ref: planned.ref,
1636
+ mode: "distill-skipped",
1637
+ result: { ok: true, reason: "derived-memory-reflect-skipped" },
1638
+ });
1639
+ appendEvent({
1640
+ eventType: "improve_skipped",
1641
+ ref: planned.ref,
1642
+ metadata: { reason: "derived_memory_reflect_skipped" },
1643
+ }, eventsCtx);
1644
+ }
1645
+ // isDistillOnly refs: no reflect action emitted — proceed directly to distill path below.
1646
+ const hasRecentFeedbackSignal = signalBearingSet.has(planned.ref);
1647
+ const explicitRefScope = scope.mode === "ref";
1648
+ // Profile gate: apply the full type-filter / raw-wiki / disabled rules to
1649
+ // distill so callers who configure `profile.processes.distill.allowedTypes`
1650
+ // or land on raw-wiki refs get a recorded skip action instead of silently
1651
+ // proceeding.
1652
+ const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
1653
+ if (distillSkip.skip) {
1654
+ actions.push({
1655
+ ref: planned.ref,
1656
+ mode: "distill-skipped",
1657
+ result: { ok: true, reason: distillSkip.reason },
1658
+ });
1659
+ completedCount++;
1660
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
1661
+ continue;
1662
+ }
1663
+ // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
1664
+ // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
1665
+ // it will refuse.
1666
+ const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
1667
+ const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
1668
+ // distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
1669
+ // refs; non-candidate refs in the set are blocked here.
1670
+ // O-2 (#365): bypass the distill cooldown when the user explicitly targeted
1671
+ // this ref via --scope — their intent overrides unattended-run policies.
1672
+ if (shouldAttemptDistill &&
1673
+ !skipMemoryDistillForWeakSignal &&
1674
+ (!distillCooledRefs.has(planned.ref) || explicitRefScope)) {
1675
+ // TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
1676
+ const lessonRef = deriveLessonRef(planned.ref);
1677
+ const knowledgeRef = deriveKnowledgeRef(planned.ref);
1678
+ const dedupeStashDir = primaryStashDir ?? options.stashDir;
1679
+ if (dedupeStashDir) {
1680
+ // B2: check both lesson ref and knowledge ref since auto-promoted memories
1681
+ // create knowledge: proposals, not lesson: proposals.
1682
+ const hasExistingPending = pendingProposalRefSet.has(lessonRef) || pendingProposalRefSet.has(knowledgeRef);
1683
+ if (hasExistingPending) {
1684
+ actions.push({
1685
+ ref: planned.ref,
1686
+ mode: "distill-skipped",
1687
+ result: { ok: true, reason: "pending proposal exists" },
1688
+ });
1689
+ appendEvent({
1690
+ eventType: "improve_skipped",
1691
+ ref: planned.ref,
1692
+ metadata: { reason: "pending_proposal_exists" },
1693
+ }, eventsCtx);
1694
+ completedCount++;
1695
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
1696
+ continue;
1697
+ }
1698
+ // D-2 (#370): reject-aware cooldown for distill. When the reviewer
1699
+ // recently rejected a distilled lesson or knowledge proposal for this
1700
+ // asset, skip re-distillation for a 1-day grace window. Prevents the
1701
+ // same rejected proposal from being regenerated immediately. The
1702
+ // window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
1703
+ // signal-delta gates and dropped --distill-cooldown-days; a short
1704
+ // reject grace is preserved here so a fresh rejection isn't
1705
+ // overridden by the same run).
1706
+ // References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
1707
+ const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
1708
+ const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
1709
+ (rejectedProposalsByRef.has(lessonRef) || rejectedProposalsByRef.has(knowledgeRef));
1710
+ if (recentlyRejectedLesson) {
1711
+ const rejectedEntry = rejectedProposalsByRef.get(lessonRef) ?? rejectedProposalsByRef.get(knowledgeRef);
1712
+ const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
1713
+ if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
1714
+ actions.push({
1715
+ ref: planned.ref,
1716
+ mode: "distill-skipped",
1717
+ result: { ok: true, reason: "distill reject grace window" },
1718
+ });
1719
+ appendEvent({
1720
+ eventType: "improve_skipped",
1721
+ ref: planned.ref,
1722
+ metadata: {
1723
+ reason: "distill_reject_grace_window",
1724
+ },
1725
+ }, eventsCtx);
1726
+ completedCount++;
1727
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
1728
+ continue;
1729
+ }
1730
+ }
1731
+ }
1732
+ const distillResult = await distillFn({
1733
+ ref: planned.ref,
1734
+ ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
1735
+ ...(options.stashDir ? { stashDir: options.stashDir } : {}),
1736
+ });
1737
+ actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
1738
+ if (distillResult.outcome === "queued" && distillResult.proposal) {
1739
+ gateAutoAcceptedCount += (await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg)).promoted.length;
1740
+ }
1741
+ if (parsedPlannedRef.type === "memory") {
1742
+ const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
1743
+ if (!promotedToKnowledge)
1744
+ memoryRefsForInference.add(planned.ref);
1745
+ }
1746
+ if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
1747
+ const slug = planned.ref
1748
+ .replace(/[^a-z0-9]/gi, "-")
1749
+ .toLowerCase()
1750
+ .slice(0, 60);
1751
+ writeEvalCase(primaryStashDir, {
1752
+ ref: planned.ref,
1753
+ failureReason: distillResult.reason ?? "quality gate rejected",
1754
+ assetType: parseAssetRef(planned.ref).type ?? "unknown",
1755
+ rejectedAt: Date.now(),
1756
+ source: "distill_quality_rejected",
1757
+ slug: `${slug}-${Date.now()}`,
1758
+ });
1759
+ }
1760
+ // D6: use pre-loaded map instead of per-iteration DB query
1761
+ const rejectedProposalEvent = rejectedProposalsByRef.get(planned.ref);
1762
+ if (rejectedProposalEvent && primaryStashDir) {
1763
+ const slug = planned.ref
1764
+ .replace(/[^a-z0-9]/gi, "-")
1765
+ .toLowerCase()
1766
+ .slice(0, 60);
1767
+ writeEvalCase(primaryStashDir, {
1768
+ ref: planned.ref,
1769
+ failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
1770
+ assetType: parseAssetRef(planned.ref).type ?? "unknown",
1771
+ rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
1772
+ source: "proposal_rejected",
1773
+ slug: `${slug}-rejected`,
1774
+ });
1775
+ }
1776
+ }
1777
+ else if (skipMemoryDistillForWeakSignal) {
1778
+ actions.push({
1779
+ ref: planned.ref,
1780
+ mode: "distill-skipped",
1781
+ result: { ok: true, reason: "memory requires recent feedback signal" },
1782
+ });
1783
+ appendEvent({
1784
+ eventType: "improve_skipped",
1785
+ ref: planned.ref,
1786
+ metadata: { reason: "memory_distill_requires_feedback" },
1787
+ }, eventsCtx);
1788
+ }
1789
+ }
1790
+ catch (err) {
1791
+ // B7: UsageError thrown by akmDistill on validation_failed should be recorded
1792
+ // as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
1793
+ // The distill_invoked event was already emitted inside akmDistill before the throw.
1794
+ if (err instanceof UsageError) {
1795
+ actions.push({
1796
+ ref: planned.ref,
1797
+ mode: "distill",
1798
+ result: { ok: false, outcome: "validation_failed", error: err.message },
1799
+ });
1800
+ }
1801
+ else {
1802
+ actions.push({
1803
+ ref: planned.ref,
1804
+ mode: "error",
1805
+ result: { ok: false, error: err instanceof Error ? err.message : String(err) },
1806
+ });
1807
+ }
1808
+ }
1809
+ completedCount++;
1810
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
1811
+ }
1812
+ return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount };
1813
+ }
1814
+ async function runImprovePostLoopStage(args) {
1815
+ const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memorySummary, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, } = args;
1816
+ const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
1817
+ const baseConfig = options.config ?? loadConfig();
1818
+ const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
1819
+ const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
1820
+ const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
1821
+ // When volume triggers a consolidation pass, force-enable the consolidate
1822
+ // process on the default improve profile so the gate accepts the run even
1823
+ // if the user's config disabled it. We synthesise a new profile override
1824
+ // rather than mutating connection settings.
1825
+ const consolidationConfig = volumeTriggered
1826
+ ? {
1827
+ ...baseConfig,
1828
+ profiles: {
1829
+ ...(baseConfig.profiles ?? {}),
1830
+ improve: {
1831
+ ...(baseConfig.profiles?.improve ?? {}),
1832
+ default: {
1833
+ ...(baseConfig.profiles?.improve?.default ?? {}),
1834
+ processes: {
1835
+ ...(baseConfig.profiles?.improve?.default?.processes ?? {}),
1836
+ consolidate: {
1837
+ ...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
1838
+ enabled: true,
1839
+ },
1840
+ },
1841
+ },
1842
+ },
1843
+ },
1844
+ }
1845
+ : baseConfig;
1846
+ // 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
1847
+ // memory file has been updated since the most recent successful
1848
+ // consolidate_completed event. Time-based cooldowns produced the same
1849
+ // synchronised-wave failure mode the reflect/distill cooldowns did; the
1850
+ // pool-delta gate ties consolidation to actual work-to-do.
1851
+ const recentConsolidations = readEvents({ type: "consolidate_completed" });
1852
+ const lastConsolidation = recentConsolidations.events
1853
+ .filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
1854
+ .sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
1855
+ const lastConsolidateTs = lastConsolidation?.ts;
1856
+ // Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do.
1857
+ // Using file mtime keeps this query DB-free and matches what the indexer
1858
+ // already uses as the canonical `memory.updated_at` proxy.
1859
+ //
1860
+ // Bootstrap: when no successful consolidate_completed event has ever been
1861
+ // recorded, we cannot evaluate the pool-delta — treat as eligible so a
1862
+ // fresh stash runs consolidate once before the steady-state gate kicks in.
1863
+ const memoryUpdatedAfterLastConsolidate = (() => {
1864
+ if (volumeTriggered)
1865
+ return true; // volume override forces the run regardless.
1866
+ if (!lastConsolidateTs)
1867
+ return true; // bootstrap path: never consolidated.
1868
+ if (!primaryStashDir)
1869
+ return false;
1870
+ const memoriesDir = path.join(primaryStashDir, "memories");
1871
+ if (!fs.existsSync(memoriesDir))
1872
+ return false;
1873
+ try {
1874
+ return fs.readdirSync(memoriesDir).some((f) => {
1875
+ if (!f.endsWith(".md"))
1876
+ return false;
1877
+ try {
1878
+ return fs.statSync(path.join(memoriesDir, f)).mtime.toISOString() > lastConsolidateTs;
1879
+ }
1880
+ catch {
1881
+ return false;
1882
+ }
1883
+ });
1884
+ }
1885
+ catch {
1886
+ return false;
1887
+ }
1888
+ })();
1889
+ const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
1890
+ // Profile gate: if profile explicitly disables consolidate, skip the entire pass.
1891
+ const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
1892
+ let consolidation = {
1893
+ schemaVersion: 1,
1894
+ ok: true,
1895
+ shape: "consolidate-result",
1896
+ dryRun: false,
1897
+ previewOnly: false,
1898
+ target: "",
1899
+ processed: 0,
1900
+ merged: 0,
1901
+ deleted: 0,
1902
+ promoted: [],
1903
+ contradicted: 0,
1904
+ warnings: [],
1905
+ durationMs: 0,
1906
+ };
1907
+ let gateAutoAcceptedCount = 0;
1908
+ const consolidateGateCfg = makeGateConfig("consolidate", {
1909
+ globalThreshold: options.autoAccept,
1910
+ dryRun: options.dryRun ?? false,
1911
+ stashDir: primaryStashDir,
1912
+ config: consolidationConfig,
1913
+ eventsCtx,
1914
+ }, { minimumThreshold: 95 });
1915
+ if (consolidateDisabledByProfile) {
1916
+ info("[improve] consolidation skipped (disabled by improve profile)");
1917
+ }
1918
+ else if (!consolidationOnCooldown) {
1919
+ consolidation = await akmConsolidate({
1920
+ ...options.consolidateOptions,
1921
+ config: consolidationConfig,
1922
+ stashDir: options.stashDir,
1923
+ autoTriggered: volumeTriggered,
1924
+ // Incremental consolidation: in steady state (not bootstrap, not volume-
1925
+ // triggered) pass the last-consolidation timestamp so akmConsolidate skips
1926
+ // chunks with no memory changed since then. Converts consolidation cost
1927
+ // from O(pool) to O(changed clusters) — the fix for the rising p95 tail
1928
+ // where full-pool re-judging produced 5–10 min runs that promoted ~0.
1929
+ // undefined → full pass (bootstrap, or volume-triggered large-pool sweep).
1930
+ incrementalSince: volumeTriggered ? undefined : lastConsolidateTs,
1931
+ maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
1932
+ // Honor profile.autoAccept (already merged into options.autoAccept at the
1933
+ // top of akmImprove). The CLI parser always supplies 90 when --auto-accept
1934
+ // is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
1935
+ // (which maps to undefined) from disabling consolidation auto-accept.
1936
+ // options.consolidateOptions.autoAccept (if explicitly provided by caller)
1937
+ // still wins because the spread above runs first.
1938
+ autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
1939
+ });
1940
+ gateAutoAcceptedCount += (await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
1941
+ try {
1942
+ if (!primaryStashDir)
1943
+ return { proposalId, confidence: undefined };
1944
+ const proposal = getProposal(primaryStashDir, proposalId);
1945
+ return { proposalId, confidence: proposal.confidence };
1946
+ }
1947
+ catch {
1948
+ return { proposalId, confidence: undefined };
1949
+ }
1950
+ }), consolidateGateCfg)).promoted.length;
1951
+ if (consolidation.processed > 0) {
1952
+ appendEvent({
1953
+ eventType: "consolidate_completed",
1954
+ ref: "memory:_consolidation",
1955
+ metadata: { processed: consolidation.processed, merged: consolidation.merged },
1956
+ }, eventsCtx);
1957
+ }
1958
+ }
1959
+ else {
1960
+ appendEvent({
1961
+ eventType: "improve_skipped",
1962
+ ref: "memory:_consolidation",
1963
+ metadata: {
1964
+ reason: "consolidation_no_memory_updates",
1965
+ lastEventTs: lastConsolidation?.ts ?? null,
1966
+ },
1967
+ }, eventsCtx);
1968
+ info("[improve] consolidation skipped (no memory updates since last run)");
1969
+ }
1970
+ // D9: track whether consolidation wrote any data so graph extraction can reindex if needed
1971
+ const consolidationRan = !consolidateDisabledByProfile && !consolidationOnCooldown && consolidation.processed > 0;
1972
+ info("[improve] post-loop maintenance starting");
1973
+ const maintenanceResult = await runImproveMaintenancePasses({
1974
+ options,
1975
+ primaryStashDir,
1976
+ actionableRefs,
1977
+ memoryRefsForInference,
1978
+ allWarnings,
1979
+ reindexFn,
1980
+ consolidationRan,
1981
+ // O-1 (#364): forward the budget signal to memory inference + graph extraction.
1982
+ budgetSignal,
1983
+ eventsCtx,
1984
+ improveProfile,
1985
+ });
1986
+ let deadUrls;
1987
+ if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
1988
+ try {
1989
+ const knowledgeEntries = actionableRefs
1990
+ .filter((r) => {
1991
+ try {
1992
+ return parseAssetRef(r.ref).type === "knowledge";
1993
+ }
1994
+ catch {
1995
+ return false;
1996
+ }
1997
+ })
1998
+ .slice(0, 10)
1999
+ .map((r) => ({ ref: r.ref, body: "" }));
2000
+ if (knowledgeEntries.length > 0) {
2001
+ info(`[improve] checking URLs in ${knowledgeEntries.length} knowledge refs`);
2002
+ deadUrls = await checkDeadUrls(primaryStashDir, knowledgeEntries);
2003
+ info(`[improve] URL check complete (${deadUrls.length} dead/timeout URLs)`);
2004
+ }
2005
+ }
2006
+ catch {
2007
+ // best-effort
2008
+ }
2009
+ }
2010
+ return {
2011
+ allWarnings,
2012
+ consolidation,
2013
+ deadUrls,
2014
+ ...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
2015
+ ...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
2016
+ ...(maintenanceResult.stalenessDetection ? { stalenessDetection: maintenanceResult.stalenessDetection } : {}),
2017
+ ...(maintenanceResult.actions && maintenanceResult.actions.length > 0
2018
+ ? { maintenanceActions: maintenanceResult.actions }
2019
+ : {}),
2020
+ memoryInferenceDurationMs: maintenanceResult.memoryInferenceDurationMs,
2021
+ graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
2022
+ orphansPurged: maintenanceResult.orphansPurged,
2023
+ proposalsExpired: maintenanceResult.proposalsExpired,
2024
+ gateAutoAcceptedCount,
2025
+ };
2026
+ }
2027
+ // TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
2028
+ async function runImproveMaintenancePasses(args) {
2029
+ const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
2030
+ if (!primaryStashDir)
2031
+ return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
2032
+ const config = options.config ?? loadConfig();
2033
+ const sources = resolveSourceEntries(options.stashDir, config);
2034
+ const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
2035
+ const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
2036
+ const stalenessDetectionFn = options.stalenessDetectionFn ?? runStalenessDetectionPass;
2037
+ let db;
2038
+ let memoryInference;
2039
+ let graphExtraction;
2040
+ let stalenessDetection;
2041
+ let reindexedAfterInference = false;
2042
+ const actions = [];
2043
+ let memoryInferenceDurationMs = 0;
2044
+ let graphExtractionDurationMs = 0;
2045
+ let orphansPurged = 0;
2046
+ let proposalsExpired = 0;
2047
+ try {
2048
+ db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
2049
+ // Memory inference candidate-discovery (post-Item 9 fix from
2050
+ // memory:akm-improve-critical-review-2026-05-20). Previously this pass
2051
+ // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
2052
+ // candidateRefs filter. But memoryRefsForInference is populated from refs
2053
+ // distilled THIS RUN — by the time that happens, those parents are
2054
+ // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
2055
+ // them. The genuinely-pending parents in the stash never entered the
2056
+ // filter. Result: 0/0/0 for 25 consecutive runs.
2057
+ //
2058
+ // Fix: always run the pass when the feature is enabled; let the pass's
2059
+ // own `collectPendingMemories` + `isPendingMemory` predicate find
2060
+ // candidates from the filesystem-of-truth. The this-run set is still
2061
+ // logged as a hint but no longer used as a filter.
2062
+ const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
2063
+ if (memoryInferenceDisabledByProfile) {
2064
+ info("[improve] memory inference skipped (disabled by improve profile)");
2065
+ }
2066
+ else {
2067
+ const hintRefs = memoryRefsForInference.size;
2068
+ info(hintRefs > 0
2069
+ ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
2070
+ : "[improve] memory inference starting (discovering pending parents)");
2071
+ const inferenceStart = Date.now();
2072
+ try {
2073
+ // O-1 (#364): pass budget signal so a hung inference call is cancelled.
2074
+ memoryInference = await memoryInferenceFn(config, sources, budgetSignal, db, false, (event) => {
2075
+ const current = event.currentRef ? ` ${event.currentRef}` : "";
2076
+ info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
2077
+ });
2078
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
2079
+ actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
2080
+ info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2081
+ }
2082
+ catch (err) {
2083
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
2084
+ allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2085
+ }
2086
+ }
2087
+ if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
2088
+ info("[improve] reindexing after memory inference writes");
2089
+ try {
2090
+ await reindexFn({ stashDir: primaryStashDir });
2091
+ reindexedAfterInference = true;
2092
+ info("[improve] reindex after memory inference complete");
2093
+ }
2094
+ catch (err) {
2095
+ allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2096
+ }
2097
+ }
2098
+ const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
2099
+ const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
2100
+ // Build the set of refs actually touched this run.
2101
+ const touchedRefs = new Set();
2102
+ for (const r of args.actionableRefs)
2103
+ touchedRefs.add(r.ref);
2104
+ for (const r of memoryRefsForInference)
2105
+ touchedRefs.add(r);
2106
+ // INVARIANT: graph extraction must never run on the full corpus from the
2107
+ // improve post-loop. Full-corpus scans belong in `akm index`. We enforce
2108
+ // this by ALWAYS passing `candidatePaths` (possibly an empty Set) to the
2109
+ // extractor — never `undefined`. With an empty Set, the extractor's
2110
+ // filter (graph-extraction.ts ~L452) rejects every file and returns the
2111
+ // empty result without scanning. The pass is still invoked so that the
2112
+ // action is recorded, the D9 post-consolidation reindex still fires, and
2113
+ // mock injection (graphExtractionFn) used by tests stays exercised.
2114
+ if (graphExtractionDisabledByProfile) {
2115
+ info("[improve] graph extraction skipped (disabled by improve profile)");
2116
+ }
2117
+ else if (sources.length > 0 && graphEnabled) {
2118
+ info("[improve] graph extraction starting");
2119
+ const extractionStart = Date.now();
2120
+ try {
2121
+ // D9: if consolidation ran but memory inference did not reindex, force a reindex
2122
+ // so graph extraction sees current DB state after consolidation writes.
2123
+ if (consolidationRan && !reindexedAfterInference) {
2124
+ info("[improve] reindexing after consolidation (graph extraction needs current state)");
2125
+ try {
2126
+ await reindexFn({ stashDir: primaryStashDir });
2127
+ reindexedAfterInference = true;
2128
+ info("[improve] reindex after consolidation complete");
2129
+ }
2130
+ catch (err) {
2131
+ allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
2132
+ }
2133
+ }
2134
+ if (db && reindexedAfterInference) {
2135
+ closeDatabase(db);
2136
+ db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
2137
+ }
2138
+ // Resolve touched refs to absolute file paths. Empty Set is intentional
2139
+ // when no refs were touched — see INVARIANT above.
2140
+ const candidatePaths = new Set();
2141
+ if (primaryStashDir && touchedRefs.size > 0) {
2142
+ const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
2143
+ const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
2144
+ for (const p of resolved) {
2145
+ if (typeof p === "string" && p.length > 0)
2146
+ candidatePaths.add(p);
2147
+ }
2148
+ }
2149
+ const progressHandler = (event) => {
2150
+ const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
2151
+ info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
2152
+ };
2153
+ // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
2154
+ graphExtraction = await graphExtractionFn(config, sources, budgetSignal, db, false, progressHandler, {
2155
+ candidatePaths,
2156
+ });
2157
+ graphExtractionDurationMs = Date.now() - extractionStart;
2158
+ actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
2159
+ info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
2160
+ }
2161
+ catch (err) {
2162
+ graphExtractionDurationMs = Date.now() - extractionStart;
2163
+ allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
2164
+ }
2165
+ }
2166
+ else if (sources.length > 0 && !graphEnabled) {
2167
+ info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
2168
+ }
2169
+ // Orphan proposal purge — reject pending reflect proposals whose target
2170
+ // asset no longer exists on disk. Runs after graph extraction so newly
2171
+ // promoted assets from accept flows during this run are already present.
2172
+ if (primaryStashDir) {
2173
+ try {
2174
+ const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
2175
+ orphansPurged = purgeResult.rejected;
2176
+ if (purgeResult.rejected > 0) {
2177
+ info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
2178
+ }
2179
+ appendEvent({
2180
+ eventType: "proposal_orphan_purge",
2181
+ ref: "proposals:_orphan-purge",
2182
+ metadata: {
2183
+ checked: purgeResult.checked,
2184
+ rejected: purgeResult.rejected,
2185
+ durationMs: purgeResult.durationMs,
2186
+ byType: purgeResult.byType,
2187
+ orphans: purgeResult.orphans.map((o) => o.ref),
2188
+ },
2189
+ }, eventsCtx);
2190
+ }
2191
+ catch (err) {
2192
+ allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
2193
+ }
2194
+ // Phase 6B (Advantage D6b): expire pending proposals that have aged past
2195
+ // the retention window. Runs AFTER orphan purge so we never double-archive
2196
+ // a proposal that orphan-purge already moved. `expireStaleProposals` emits
2197
+ // its own per-proposal `proposal_expired` events; we additionally emit a
2198
+ // single roll-up event here for parity with the orphan-purge surface.
2199
+ try {
2200
+ const expireResult = expireStaleProposals(primaryStashDir, config);
2201
+ proposalsExpired = expireResult.expired;
2202
+ if (expireResult.expired > 0) {
2203
+ info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
2204
+ `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
2205
+ }
2206
+ appendEvent({
2207
+ eventType: "proposal_expiration_pass",
2208
+ ref: "proposals:_expiration",
2209
+ metadata: {
2210
+ checked: expireResult.checked,
2211
+ expired: expireResult.expired,
2212
+ durationMs: expireResult.durationMs,
2213
+ retentionDays: expireResult.retentionDays,
2214
+ expiredProposals: expireResult.expiredProposals,
2215
+ },
2216
+ }, eventsCtx);
2217
+ }
2218
+ catch (err) {
2219
+ allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
2220
+ }
2221
+ }
2222
+ // Fix #2 (observability 0.8.0): trim the events table in state.db so it
2223
+ // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
2224
+ // invocation, and every command surface emits at least one event besides —
2225
+ // without this trim, state.db is a permanent append-only log. Config key
2226
+ // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
2227
+ // window. `purgeOldEvents()` opens its own state.db handle separate from
2228
+ // the index `db` above (different SQLite file).
2229
+ {
2230
+ const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
2231
+ if (retentionDays > 0) {
2232
+ let stateDb;
2233
+ try {
2234
+ stateDb = openStateDatabase();
2235
+ const purgedCount = purgeOldEvents(stateDb, retentionDays);
2236
+ if (purgedCount > 0) {
2237
+ info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
2238
+ }
2239
+ appendEvent({
2240
+ eventType: "events_purged",
2241
+ ref: "events:_purge",
2242
+ metadata: { purgedCount, retentionDays },
2243
+ }, eventsCtx);
2244
+ // improve_runs uses the same retention window as events — both are
2245
+ // observability/audit data, both grow append-only, both have a
2246
+ // dedicated purge helper. Mirroring the events purge here means a
2247
+ // single retention knob (improve.eventRetentionDays) governs both.
2248
+ const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
2249
+ if (improveRunsPurged > 0) {
2250
+ info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
2251
+ }
2252
+ appendEvent({
2253
+ eventType: "improve_runs_purged",
2254
+ ref: "improve_runs:_purge",
2255
+ metadata: { purgedCount: improveRunsPurged, retentionDays },
2256
+ }, eventsCtx);
2257
+ }
2258
+ catch (err) {
2259
+ allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
2260
+ }
2261
+ finally {
2262
+ if (stateDb) {
2263
+ try {
2264
+ stateDb.close();
2265
+ }
2266
+ catch {
2267
+ // best-effort
2268
+ }
2269
+ }
2270
+ }
2271
+ }
2272
+ }
2273
+ // Phase 4A (staleness detection). Activates the `deprecated` belief-state
2274
+ // machinery shipped in Phase 1A. Default OFF — gated by
2275
+ // `features.index.staleness_detection.enabled`. Runs after orphan purge
2276
+ // and before the URL check (which lives in the outer caller).
2277
+ if (sources.length > 0) {
2278
+ try {
2279
+ stalenessDetection = await stalenessDetectionFn(config, sources, budgetSignal, db);
2280
+ if (stalenessDetection.considered > 0) {
2281
+ info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
2282
+ `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
2283
+ `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
2284
+ }
2285
+ for (const w of stalenessDetection.warnings)
2286
+ allWarnings.push(`[improve] staleness detection: ${w}`);
2287
+ }
2288
+ catch (err) {
2289
+ allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
2290
+ }
2291
+ }
2292
+ }
2293
+ finally {
2294
+ if (db)
2295
+ closeDatabase(db);
2296
+ }
2297
+ return {
2298
+ ...(memoryInference ? { memoryInference } : {}),
2299
+ ...(graphExtraction ? { graphExtraction } : {}),
2300
+ ...(stalenessDetection ? { stalenessDetection } : {}),
2301
+ ...(actions.length > 0 ? { actions } : {}),
2302
+ memoryInferenceDurationMs,
2303
+ graphExtractionDurationMs,
2304
+ orphansPurged,
2305
+ proposalsExpired,
2306
+ };
2307
+ }
2308
+ function shouldAnalyzeMemoryCleanup(scope, eligibleMemories, primaryStashDir) {
2309
+ if (!primaryStashDir || eligibleMemories === 0)
2310
+ return false;
2311
+ if (scope.mode === "all")
2312
+ return true;
2313
+ if (scope.mode === "type")
2314
+ return scope.value === "memory";
2315
+ if (!scope.value)
2316
+ return false;
2317
+ return parseAssetRef(scope.value).type === "memory";
2318
+ }
2319
+ function shapeMemoryCleanup(plan) {
2320
+ return {
2321
+ analyzedDerived: plan.analyzedDerived,
2322
+ pruneCandidates: plan.pruneCandidates,
2323
+ contradictionCandidates: plan.contradictionCandidates,
2324
+ beliefStateTransitions: plan.beliefStateTransitions,
2325
+ consolidationCandidates: plan.consolidationCandidates,
2326
+ ...(plan.relativeDateCandidates.length > 0 ? { relativeDateCandidates: plan.relativeDateCandidates } : {}),
2327
+ };
2328
+ }
2329
+ function buildUtilityMap(refs) {
2330
+ const map = new Map();
2331
+ if (refs.length === 0)
2332
+ return map;
2333
+ const refSet = new Set(refs.map((r) => r.ref));
2334
+ let db;
2335
+ try {
2336
+ db = openExistingDatabase();
2337
+ const allDbEntries = getAllEntries(db);
2338
+ const idToRef = new Map();
2339
+ for (const indexed of allDbEntries) {
2340
+ const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
2341
+ if (refSet.has(ref))
2342
+ idToRef.set(indexed.id, ref);
2343
+ }
2344
+ const ids = [...idToRef.keys()];
2345
+ if (ids.length > 0) {
2346
+ const { global: scores } = getUtilityScoresByIds(db, ids);
2347
+ for (const [id, score] of scores) {
2348
+ const ref = idToRef.get(id);
2349
+ if (ref)
2350
+ map.set(ref, score.utility);
2351
+ }
2352
+ }
2353
+ }
2354
+ catch (err) {
2355
+ rethrowIfTestIsolationError(err);
2356
+ // best-effort: if DB unavailable, all utilities default to 0
2357
+ }
2358
+ finally {
2359
+ if (db)
2360
+ closeDatabase(db);
2361
+ }
2362
+ return map;
2363
+ }
2364
+ async function findAssetFilePath(ref, stashDir, writableDirSet) {
2365
+ return resolveAssetPath(ref, {
2366
+ stashDir,
2367
+ mode: "disk-only",
2368
+ writableDirSet,
2369
+ directoryIndexNames: ["SKILL.md"],
2370
+ preserveDirectNameFallback: true,
2371
+ honorOrigin: false,
2372
+ });
2373
+ }