akm-cli 0.8.0-rc2 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (313) hide show
  1. package/{.github/CHANGELOG.md → CHANGELOG.md} +238 -3
  2. package/README.md +22 -6
  3. package/SECURITY.md +93 -0
  4. package/dist/assets/help/help-accept.md +12 -0
  5. package/dist/assets/help/help-improve.md +81 -0
  6. package/dist/{commands → assets}/help/help-proposals.md +7 -4
  7. package/dist/assets/help/help-reject.md +11 -0
  8. package/dist/{output → assets/hints}/cli-hints-full.md +60 -32
  9. package/dist/{output → assets/hints}/cli-hints-short.md +10 -7
  10. package/dist/assets/profiles/default.json +15 -0
  11. package/dist/assets/profiles/graph-refresh.json +13 -0
  12. package/dist/assets/profiles/memory-focus.json +12 -0
  13. package/dist/assets/profiles/quick.json +15 -0
  14. package/dist/assets/profiles/thorough.json +15 -0
  15. package/dist/assets/prompts/extract-session.md +80 -0
  16. package/dist/assets/prompts/graph-extract-user-prompt.md +35 -0
  17. package/dist/assets/tasks/graph-refresh-weekly.yml +10 -0
  18. package/dist/cli/config-migrate.js +144 -0
  19. package/dist/cli/config-validate.js +39 -0
  20. package/dist/cli/confirm.js +73 -0
  21. package/dist/cli/parse-args.js +93 -3
  22. package/dist/cli/shared.js +129 -0
  23. package/dist/cli.js +2141 -1268
  24. package/dist/commands/add-cli.js +279 -0
  25. package/dist/commands/agent-dispatch.js +20 -12
  26. package/dist/commands/agent-support.js +11 -5
  27. package/dist/commands/completions.js +3 -0
  28. package/dist/commands/config-cli.js +129 -517
  29. package/dist/commands/consolidate.js +1557 -147
  30. package/dist/commands/curate.js +44 -3
  31. package/dist/commands/db-cli.js +23 -0
  32. package/dist/commands/distill-promotion-policy.js +5 -3
  33. package/dist/commands/distill.js +906 -100
  34. package/dist/commands/env.js +213 -0
  35. package/dist/commands/eval-cases.js +3 -0
  36. package/dist/commands/events.js +3 -0
  37. package/dist/commands/extract-cli.js +127 -0
  38. package/dist/commands/extract-prompt.js +217 -0
  39. package/dist/commands/extract.js +477 -0
  40. package/dist/commands/feedback-cli.js +331 -0
  41. package/dist/commands/graph.js +260 -5
  42. package/dist/commands/health.js +1042 -55
  43. package/dist/commands/history.js +51 -16
  44. package/dist/commands/improve-auto-accept.js +97 -0
  45. package/dist/commands/improve-cli.js +236 -0
  46. package/dist/commands/improve-profiles.js +138 -0
  47. package/dist/commands/improve-result-file.js +167 -0
  48. package/dist/commands/improve.js +1736 -346
  49. package/dist/commands/info.js +26 -28
  50. package/dist/commands/init.js +49 -1
  51. package/dist/commands/installed-stashes.js +6 -23
  52. package/dist/commands/knowledge.js +3 -0
  53. package/dist/commands/lint/agent-linter.js +3 -0
  54. package/dist/commands/lint/base-linter.js +199 -5
  55. package/dist/commands/lint/command-linter.js +3 -0
  56. package/dist/commands/lint/default-linter.js +3 -0
  57. package/dist/commands/lint/env-key-rules.js +154 -0
  58. package/dist/commands/lint/index.js +92 -3
  59. package/dist/commands/lint/knowledge-linter.js +3 -0
  60. package/dist/commands/lint/markdown-insertion.js +343 -0
  61. package/dist/commands/lint/memory-linter.js +3 -0
  62. package/dist/commands/lint/registry.js +3 -0
  63. package/dist/commands/lint/skill-linter.js +3 -0
  64. package/dist/commands/lint/task-linter.js +15 -12
  65. package/dist/commands/lint/types.js +3 -0
  66. package/dist/commands/lint/workflow-linter.js +3 -0
  67. package/dist/commands/lint.js +3 -0
  68. package/dist/commands/migration-help.js +5 -2
  69. package/dist/commands/proposal-drain-policies.js +128 -0
  70. package/dist/commands/proposal-drain.js +477 -0
  71. package/dist/commands/proposal.js +60 -6
  72. package/dist/commands/propose.js +24 -19
  73. package/dist/commands/reflect.js +1004 -94
  74. package/dist/commands/registry-cli.js +150 -0
  75. package/dist/commands/registry-search.js +3 -0
  76. package/dist/commands/remember-cli.js +257 -0
  77. package/dist/commands/remember.js +15 -6
  78. package/dist/commands/schema-repair.js +88 -15
  79. package/dist/commands/search.js +99 -14
  80. package/dist/commands/secret.js +173 -0
  81. package/dist/commands/self-update.js +3 -0
  82. package/dist/commands/show.js +32 -13
  83. package/dist/commands/source-add.js +7 -35
  84. package/dist/commands/source-clone.js +3 -0
  85. package/dist/commands/source-manage.js +3 -0
  86. package/dist/commands/tasks.js +161 -95
  87. package/dist/commands/url-checker.js +3 -0
  88. package/dist/core/action-contributors.js +3 -0
  89. package/dist/core/asset-ref.js +13 -2
  90. package/dist/core/asset-registry.js +9 -2
  91. package/dist/core/asset-serialize.js +88 -0
  92. package/dist/core/asset-spec.js +61 -5
  93. package/dist/core/common.js +93 -5
  94. package/dist/core/concurrent.js +3 -0
  95. package/dist/core/config-io.js +347 -0
  96. package/dist/core/config-migration.js +622 -0
  97. package/dist/core/config-schema.js +558 -0
  98. package/dist/core/config-sources.js +108 -0
  99. package/dist/core/config-types.js +4 -0
  100. package/dist/core/config-walker.js +337 -0
  101. package/dist/core/config.js +366 -1077
  102. package/dist/core/errors.js +42 -20
  103. package/dist/core/events.js +31 -25
  104. package/dist/core/file-lock.js +104 -0
  105. package/dist/core/frontmatter.js +75 -10
  106. package/dist/core/lesson-lint.js +3 -0
  107. package/dist/core/markdown.js +3 -0
  108. package/dist/core/memory-belief.js +62 -0
  109. package/dist/core/memory-contradiction-detect.js +274 -0
  110. package/dist/core/memory-improve.js +142 -14
  111. package/dist/core/parse.js +3 -0
  112. package/dist/core/paths.js +218 -50
  113. package/dist/core/proposal-quality-validators.js +380 -0
  114. package/dist/core/proposal-validators.js +11 -3
  115. package/dist/core/proposals.js +464 -5
  116. package/dist/core/state-db.js +349 -56
  117. package/dist/core/text-truncation.js +107 -0
  118. package/dist/core/time.js +3 -0
  119. package/dist/core/tty.js +59 -0
  120. package/dist/core/warn.js +7 -2
  121. package/dist/core/write-source.js +12 -0
  122. package/dist/indexer/db-backup.js +391 -0
  123. package/dist/indexer/db-search.js +136 -28
  124. package/dist/indexer/db.js +661 -166
  125. package/dist/indexer/ensure-index.js +3 -0
  126. package/dist/indexer/file-context.js +3 -0
  127. package/dist/indexer/graph-boost.js +162 -40
  128. package/dist/indexer/graph-db.js +241 -51
  129. package/dist/indexer/graph-dedup.js +3 -7
  130. package/dist/indexer/graph-extraction.js +242 -149
  131. package/dist/indexer/index-context.js +3 -9
  132. package/dist/indexer/indexer.js +86 -16
  133. package/dist/indexer/llm-cache.js +24 -19
  134. package/dist/indexer/manifest.js +3 -0
  135. package/dist/indexer/matchers.js +184 -11
  136. package/dist/indexer/memory-inference.js +94 -50
  137. package/dist/indexer/metadata-contributors.js +3 -0
  138. package/dist/indexer/metadata.js +110 -50
  139. package/dist/indexer/path-resolver.js +3 -0
  140. package/dist/indexer/project-context.js +192 -0
  141. package/dist/indexer/ranking-contributors.js +134 -7
  142. package/dist/indexer/ranking.js +8 -1
  143. package/dist/indexer/search-fields.js +5 -9
  144. package/dist/indexer/search-hit-enrichers.js +91 -2
  145. package/dist/indexer/search-source.js +20 -1
  146. package/dist/indexer/semantic-status.js +4 -1
  147. package/dist/indexer/staleness-detect.js +447 -0
  148. package/dist/indexer/usage-events.js +12 -9
  149. package/dist/indexer/walker.js +3 -0
  150. package/dist/integrations/agent/builders.js +135 -0
  151. package/dist/integrations/agent/config.js +121 -401
  152. package/dist/integrations/agent/detect.js +3 -0
  153. package/dist/integrations/agent/index.js +6 -14
  154. package/dist/integrations/agent/model-aliases.js +55 -0
  155. package/dist/integrations/agent/profiles.js +3 -0
  156. package/dist/integrations/agent/prompts.js +137 -8
  157. package/dist/integrations/agent/runner.js +208 -0
  158. package/dist/integrations/agent/sdk-runner.js +8 -2
  159. package/dist/integrations/agent/spawn.js +54 -14
  160. package/dist/integrations/github.js +3 -0
  161. package/dist/integrations/lockfile.js +22 -51
  162. package/dist/integrations/session-logs/index.js +4 -0
  163. package/dist/integrations/session-logs/inline-refs.js +35 -0
  164. package/dist/integrations/session-logs/pre-filter.js +152 -0
  165. package/dist/integrations/session-logs/providers/claude-code.js +226 -0
  166. package/dist/integrations/session-logs/providers/opencode.js +231 -25
  167. package/dist/integrations/session-logs/types.js +3 -0
  168. package/dist/llm/call-ai.js +14 -26
  169. package/dist/llm/client.js +16 -2
  170. package/dist/llm/embedder.js +20 -29
  171. package/dist/llm/embedders/cache.js +3 -7
  172. package/dist/llm/embedders/local.js +42 -1
  173. package/dist/llm/embedders/remote.js +20 -8
  174. package/dist/llm/embedders/types.js +3 -7
  175. package/dist/llm/feature-gate.js +92 -56
  176. package/dist/llm/graph-extract.js +402 -31
  177. package/dist/llm/index-passes.js +44 -29
  178. package/dist/llm/memory-infer.js +30 -2
  179. package/dist/llm/metadata-enhance.js +3 -7
  180. package/dist/output/cli-hints.js +7 -4
  181. package/dist/output/context.js +60 -8
  182. package/dist/output/renderers.js +170 -194
  183. package/dist/output/shapes/curate.js +56 -0
  184. package/dist/output/shapes/distill.js +10 -0
  185. package/dist/output/shapes/env-list.js +19 -0
  186. package/dist/output/shapes/events.js +11 -0
  187. package/dist/output/shapes/helpers.js +424 -0
  188. package/dist/output/shapes/history.js +7 -0
  189. package/dist/output/shapes/passthrough.js +105 -0
  190. package/dist/output/shapes/proposal-accept.js +7 -0
  191. package/dist/output/shapes/proposal-diff.js +7 -0
  192. package/dist/output/shapes/proposal-list.js +7 -0
  193. package/dist/output/shapes/proposal-producer.js +11 -0
  194. package/dist/output/shapes/proposal-reject.js +7 -0
  195. package/dist/output/shapes/proposal-show.js +7 -0
  196. package/dist/output/shapes/registry-search.js +6 -0
  197. package/dist/output/shapes/registry.js +30 -0
  198. package/dist/output/shapes/search.js +6 -0
  199. package/dist/output/shapes/secret-list.js +19 -0
  200. package/dist/output/shapes/show.js +6 -0
  201. package/dist/output/shapes/vault-list.js +19 -0
  202. package/dist/output/shapes.js +51 -549
  203. package/dist/output/text/add.js +6 -0
  204. package/dist/output/text/clone.js +6 -0
  205. package/dist/output/text/config.js +6 -0
  206. package/dist/output/text/curate.js +6 -0
  207. package/dist/output/text/distill.js +7 -0
  208. package/dist/output/text/enable-disable.js +7 -0
  209. package/dist/output/text/events.js +10 -0
  210. package/dist/output/text/feedback.js +6 -0
  211. package/dist/output/text/helpers.js +1059 -0
  212. package/dist/output/text/history.js +7 -0
  213. package/dist/output/text/import.js +6 -0
  214. package/dist/output/text/index.js +6 -0
  215. package/dist/output/text/info.js +6 -0
  216. package/dist/output/text/init.js +6 -0
  217. package/dist/output/text/list.js +6 -0
  218. package/dist/output/text/proposal-producer.js +8 -0
  219. package/dist/output/text/proposal.js +12 -0
  220. package/dist/output/text/registry-commands.js +11 -0
  221. package/dist/output/text/registry.js +30 -0
  222. package/dist/output/text/remember.js +6 -0
  223. package/dist/output/text/remove.js +6 -0
  224. package/dist/output/text/save.js +6 -0
  225. package/dist/output/text/search.js +6 -0
  226. package/dist/output/text/show.js +6 -0
  227. package/dist/output/text/update.js +6 -0
  228. package/dist/output/text/upgrade.js +6 -0
  229. package/dist/output/text/vault.js +16 -0
  230. package/dist/output/text/wiki.js +15 -0
  231. package/dist/output/text/workflow.js +14 -0
  232. package/dist/output/text.js +44 -1329
  233. package/dist/registry/build-index.js +3 -0
  234. package/dist/registry/create-provider-registry.js +3 -0
  235. package/dist/registry/factory.js +4 -1
  236. package/dist/registry/origin-resolve.js +3 -0
  237. package/dist/registry/providers/index.js +3 -0
  238. package/dist/registry/providers/skills-sh.js +11 -2
  239. package/dist/registry/providers/static-index.js +10 -1
  240. package/dist/registry/providers/types.js +3 -24
  241. package/dist/registry/resolve.js +11 -16
  242. package/dist/registry/types.js +3 -0
  243. package/dist/scripts/migrate-storage.js +17767 -0
  244. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +9031 -0
  245. package/dist/scripts/migrations/v16-to-v17.js +141 -0
  246. package/dist/setup/detect.js +3 -0
  247. package/dist/setup/ripgrep-install.js +3 -0
  248. package/dist/setup/ripgrep-resolve.js +3 -0
  249. package/dist/setup/setup.js +306 -67
  250. package/dist/setup/steps.js +3 -15
  251. package/dist/sources/include.js +3 -0
  252. package/dist/sources/provider-factory.js +3 -11
  253. package/dist/sources/provider.js +3 -20
  254. package/dist/sources/providers/filesystem.js +19 -23
  255. package/dist/sources/providers/git.js +171 -21
  256. package/dist/sources/providers/index.js +3 -0
  257. package/dist/sources/providers/install-types.js +3 -13
  258. package/dist/sources/providers/npm.js +3 -4
  259. package/dist/sources/providers/provider-utils.js +3 -0
  260. package/dist/sources/providers/sync-from-ref.js +3 -11
  261. package/dist/sources/providers/tar-utils.js +3 -0
  262. package/dist/sources/providers/website.js +18 -22
  263. package/dist/sources/resolve.js +3 -0
  264. package/dist/sources/types.js +3 -0
  265. package/dist/sources/website-ingest.js +3 -0
  266. package/dist/tasks/backends/cron.js +3 -0
  267. package/dist/tasks/backends/exec-utils.js +3 -0
  268. package/dist/tasks/backends/index.js +3 -11
  269. package/dist/tasks/backends/launchd.js +4 -1
  270. package/dist/tasks/backends/schtasks.js +4 -1
  271. package/dist/tasks/parser.js +51 -38
  272. package/dist/tasks/resolveAkmBin.js +3 -0
  273. package/dist/tasks/runner.js +35 -9
  274. package/dist/tasks/schedule.js +20 -1
  275. package/dist/tasks/schema.js +5 -3
  276. package/dist/tasks/validator.js +6 -3
  277. package/dist/version.js +3 -0
  278. package/dist/wiki/wiki-templates.js +6 -3
  279. package/dist/wiki/wiki.js +4 -1
  280. package/dist/workflows/authoring.js +4 -1
  281. package/dist/workflows/cli.js +3 -0
  282. package/dist/workflows/db.js +140 -10
  283. package/dist/workflows/document-cache.js +3 -10
  284. package/dist/workflows/parser.js +3 -0
  285. package/dist/workflows/renderer.js +3 -0
  286. package/dist/workflows/runs.js +18 -1
  287. package/dist/workflows/schema.js +3 -0
  288. package/dist/workflows/scope-key.js +3 -0
  289. package/dist/workflows/validator.js +5 -9
  290. package/docs/README.md +7 -2
  291. package/docs/data-and-telemetry.md +225 -0
  292. package/docs/migration/release-notes/0.7.5.md +2 -2
  293. package/docs/migration/release-notes/0.8.0.md +57 -5
  294. package/docs/migration/v0.7-to-v0.8.md +1378 -0
  295. package/package.json +28 -11
  296. package/.github/LICENSE +0 -374
  297. package/dist/commands/help/help-accept.md +0 -9
  298. package/dist/commands/help/help-improve.md +0 -53
  299. package/dist/commands/help/help-reject.md +0 -8
  300. package/dist/commands/install-audit.js +0 -385
  301. package/dist/commands/vault.js +0 -310
  302. package/dist/indexer/match-contributors.js +0 -141
  303. package/dist/integrations/agent/pipeline.js +0 -39
  304. package/dist/integrations/agent/runners.js +0 -31
  305. package/dist/llm/prompts/graph-extract-user-prompt.md +0 -12
  306. /package/dist/{tasks → assets}/backends/launchd-template.xml +0 -0
  307. /package/dist/{tasks → assets}/backends/schtasks-template.xml +0 -0
  308. /package/dist/{commands → assets}/help/help-propose.md +0 -0
  309. /package/dist/{wiki → assets/wiki}/index-template.md +0 -0
  310. /package/dist/{wiki → assets/wiki}/ingest-workflow-template.md +0 -0
  311. /package/dist/{wiki → assets/wiki}/log-template.md +0 -0
  312. /package/dist/{wiki → assets/wiki}/schema-template.md +0 -0
  313. /package/dist/{workflows → assets/workflows}/workflow-template.md +0 -0
@@ -1,82 +1,196 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { createHash } from "node:crypto";
1
5
  import fs from "node:fs";
2
6
  import path from "node:path";
3
7
  import readline from "node:readline";
4
- import { stringify as yamlStringify } from "yaml";
8
+ import { parse as yamlParse, stringify as yamlStringify } from "yaml";
5
9
  import { parseAssetRef } from "../core/asset-ref";
10
+ import { assembleAssetFromString } from "../core/asset-serialize";
6
11
  import { resolveStashDir, timestampForFilename } from "../core/common";
7
- import { loadConfig } from "../core/config";
12
+ import { getDefaultLlmConfig, loadConfig } from "../core/config";
8
13
  import { ConfigError } from "../core/errors";
14
+ import { appendEvent } from "../core/events";
9
15
  import { parseFrontmatter } from "../core/frontmatter";
16
+ import { writeContradictEdge } from "../core/memory-belief";
10
17
  import { parseEmbeddedJsonResponse } from "../core/parse";
11
- import { createProposal, listProposals } from "../core/proposals";
18
+ import { hasHotCaptureMode, hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../core/proposal-quality-validators";
19
+ import { createProposal, isProposalSkipped, listProposals } from "../core/proposals";
20
+ import { detectTruncatedDescription } from "../core/text-truncation";
21
+ // Re-export the moved helpers so existing test imports continue to resolve.
22
+ export { hasSupersededStatus, validateProposalFrontmatter };
12
23
  import { warn } from "../core/warn";
13
24
  import { deleteAssetFromSource, resolveWriteTarget, writeAssetToSource } from "../core/write-source";
14
- import { closeDatabase, getAllEntries, openExistingDatabase } from "../indexer/db";
25
+ import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../indexer/db";
26
+ import { resolveImproveProcessRunnerFromProfile } from "../integrations/agent/runner";
15
27
  import { chatCompletion } from "../llm/client";
28
+ import { cosineSimilarity, embedBatch } from "../llm/embedder";
16
29
  import { isLlmFeatureEnabled, tryLlmFeature } from "../llm/feature-gate";
17
30
  // ── Prompts ─────────────────────────────────────────────────────────────────
18
31
  const CONSOLIDATE_SYSTEM_PROMPT = `You are the akm consolidate assistant analyzing memory assets.
19
32
 
20
33
  Rules:
21
34
  1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
22
- 2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion.
23
- 3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory.
24
- 4. KEEP: Memory is unique and currentomit from output.
35
+ 2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
36
+ 3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
37
+ 4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
38
+ 5. KEEP: Memory is unique and current → omit from output.
25
39
 
26
40
  Return ONLY JSON (no prose, no code fences):
27
41
  {
28
42
  "operations": [
29
- { "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize" },
30
- { "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>" },
31
- { "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>" }
43
+ { "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
44
+ { "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
45
+ { "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
46
+ { "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
32
47
  ],
33
48
  "warnings": ["<optional concerns>"]
34
- }`;
49
+ }
50
+
51
+ For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
52
+
53
+ When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
54
+ /**
55
+ * JSON Schema for structured consolidate plans (PR 1 of the asset-writers
56
+ * decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
57
+ * Mirrors the {ops[], warnings?[]} shape currently described in
58
+ * CONSOLIDATE_SYSTEM_PROMPT. Providers with `supportsJsonSchema: true` enforce
59
+ * the shape upstream so the chunk-level "invalid plan from AI — skipping"
60
+ * branch in `runConsolidate` becomes unreachable on schema-honouring providers.
61
+ *
62
+ * The four operation variants (merge / delete / promote / contradict) are
63
+ * modeled as a oneOf so a structured-output provider can still tell them apart
64
+ * by the required `op` discriminator. `parseEmbeddedJsonResponse` keeps
65
+ * working as a fallback parser for providers that ignore the schema.
66
+ */
67
+ export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
68
+ type: "object",
69
+ required: ["operations"],
70
+ additionalProperties: false,
71
+ properties: {
72
+ operations: {
73
+ type: "array",
74
+ description: "Ordered list of consolidate operations the planner proposes.",
75
+ items: {
76
+ oneOf: [
77
+ {
78
+ type: "object",
79
+ required: ["op", "primary", "secondaries", "mergeStrategy"],
80
+ additionalProperties: false,
81
+ properties: {
82
+ op: { type: "string", enum: ["merge"] },
83
+ primary: { type: "string", minLength: 1 },
84
+ secondaries: {
85
+ type: "array",
86
+ minItems: 1,
87
+ items: { type: "string", minLength: 1 },
88
+ },
89
+ mergeStrategy: { type: "string", minLength: 1 },
90
+ confidence: { type: "number", minimum: 0, maximum: 1 },
91
+ },
92
+ },
93
+ {
94
+ type: "object",
95
+ required: ["op", "ref", "reason"],
96
+ additionalProperties: false,
97
+ properties: {
98
+ op: { type: "string", enum: ["delete"] },
99
+ ref: { type: "string", minLength: 1 },
100
+ reason: { type: "string", minLength: 1 },
101
+ confidence: { type: "number", minimum: 0, maximum: 1 },
102
+ },
103
+ },
104
+ {
105
+ type: "object",
106
+ required: ["op", "ref", "knowledgeRef", "reason"],
107
+ additionalProperties: false,
108
+ properties: {
109
+ op: { type: "string", enum: ["promote"] },
110
+ ref: { type: "string", minLength: 1 },
111
+ knowledgeRef: { type: "string", minLength: 1 },
112
+ reason: { type: "string", minLength: 1 },
113
+ description: { type: "string" },
114
+ confidence: { type: "number", minimum: 0, maximum: 1 },
115
+ },
116
+ },
117
+ {
118
+ type: "object",
119
+ required: ["op", "ref", "contradictedByRef", "reason"],
120
+ additionalProperties: false,
121
+ properties: {
122
+ op: { type: "string", enum: ["contradict"] },
123
+ ref: { type: "string", minLength: 1 },
124
+ contradictedByRef: { type: "string", minLength: 1 },
125
+ reason: { type: "string", minLength: 1 },
126
+ confidence: { type: "number", minimum: 0, maximum: 1 },
127
+ },
128
+ },
129
+ ],
130
+ },
131
+ },
132
+ warnings: {
133
+ type: "array",
134
+ description: "Optional list of human-readable concerns the planner wants to surface.",
135
+ items: { type: "string" },
136
+ },
137
+ },
138
+ };
35
139
  export function isConsolidationEligibleMemoryName(name) {
36
140
  return !name.endsWith(".derived");
37
141
  }
38
- function loadMemoriesFromDb(sourceFilterPath) {
39
- let db;
142
+ /**
143
+ * Returns true when the memory file has `captureMode: hot` in its frontmatter.
144
+ *
145
+ * Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
146
+ * The consolidate LLM is forbidden from deleting or auto-merging them — the
147
+ * user wrote them on purpose and only the user can decide to retire them.
148
+ *
149
+ * Reads the file once per check; consolidate runs against ~10 memories per
150
+ * chunk so the IO cost is trivial. Returns false on any read/parse error
151
+ * (fail-safe: an unparseable file is treated as not-hot, but the broader
152
+ * consolidate flow already guards against unparseable memories elsewhere).
153
+ *
154
+ * Defends against four observed defect classes (see
155
+ * `memory:akm-improve-critical-review-2026-05-20`):
156
+ * - LLM marks a memory contradicted then deletes (dangling contradictedBy)
157
+ * - LLM merges two unrelated memories sharing a topic keyword
158
+ * - LLM judges a recent durable design memo as "redundant"
159
+ * - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
160
+ */
161
+ export function isHotCapturedMemory(filePath) {
40
162
  try {
41
- db = openExistingDatabase();
42
- const entries = getAllEntries(db, "memory");
43
- return entries
44
- .filter((e) => {
45
- if (!sourceFilterPath)
46
- return true;
47
- return path.resolve(e.stashDir) === path.resolve(sourceFilterPath);
48
- })
49
- .filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
50
- .map((e) => ({
51
- name: e.entry.name,
52
- filePath: e.filePath,
53
- description: e.entry.description ?? "",
54
- tags: e.entry.tags ?? [],
55
- stashDir: e.stashDir,
56
- }));
163
+ if (!fs.existsSync(filePath))
164
+ return false;
165
+ const content = fs.readFileSync(filePath, "utf8");
166
+ const parsed = parseFrontmatter(content);
167
+ return hasHotCaptureMode(parsed.data);
57
168
  }
58
169
  catch {
59
- return [];
60
- }
61
- finally {
62
- if (db)
63
- closeDatabase(db);
170
+ return false;
64
171
  }
65
172
  }
66
- function loadMemoriesFromFs(memoriesDir, stashDir) {
67
- if (!fs.existsSync(memoriesDir))
68
- return [];
69
- const entries = [];
70
- for (const fname of fs.readdirSync(memoriesDir)) {
71
- if (!fname.endsWith(".md"))
72
- continue;
73
- const filePath = path.join(memoriesDir, fname);
74
- const name = fname.replace(/\.md$/, "");
75
- if (!isConsolidationEligibleMemoryName(name))
76
- continue;
77
- entries.push({ name, filePath, description: "", tags: [], stashDir });
173
+ export function consolidateGuardStatus(filePath) {
174
+ if (!fs.existsSync(filePath))
175
+ return "missing";
176
+ let content;
177
+ try {
178
+ content = fs.readFileSync(filePath, "utf8");
179
+ }
180
+ catch {
181
+ return "unparseable";
182
+ }
183
+ let parsed;
184
+ try {
185
+ parsed = parseFrontmatter(content);
186
+ }
187
+ catch {
188
+ return "unparseable";
78
189
  }
79
- return entries;
190
+ const data = parsed.data;
191
+ if (!data || Object.keys(data).length === 0)
192
+ return "unparseable";
193
+ return hasHotCaptureMode(data) ? "hot" : "safe";
80
194
  }
81
195
  // ── Chunk sizing ─────────────────────────────────────────────────────────────
82
196
  /**
@@ -94,21 +208,21 @@ const CHARS_PER_TOKEN = 3;
94
208
  */
95
209
  const PROMPT_OVERHEAD_TOKENS = 2_000;
96
210
  /**
97
- * Default effective token budget used when `config.llm.contextLength` is not
98
- * set. This is intentionally conservative (4 096) rather than being set to
99
- * the model's actual context window, because:
211
+ * Default effective token budget used when the default LLM profile's
212
+ * `contextLength` is not set. This is intentionally conservative (4 096)
213
+ * rather than being set to the model's actual context window, because:
100
214
  *
101
- * - When the agent path is used (config.agent), the agent CLI (e.g. opencode)
215
+ * - When the agent path is used, the agent CLI (e.g. opencode)
102
216
  * prepends its own large system prompt + conversation history before
103
217
  * forwarding to the model. That overhead easily consumes 30K+ tokens on
104
218
  * a model with a 16K context window, leaving very little room for
105
219
  * chunk content.
106
- * - When the HTTP path is used (config.llm), only the akm system prompt and
107
- * user prompt are sent, so the budget can be set to the model's actual
108
- * context length via config.llm.contextLength.
220
+ * - When the HTTP path is used (an LLM profile is selected), only the akm
221
+ * system prompt and user prompt are sent, so the budget can be set to the
222
+ * model's actual context length via profiles.llm[defaults.llm].contextLength.
109
223
  *
110
- * Set config.llm.contextLength in your config file to the model's actual
111
- * context window to allow larger chunks on the HTTP path.
224
+ * Set profiles.llm[defaults.llm].contextLength in your config file to the
225
+ * model's actual context window to allow larger chunks on the HTTP path.
112
226
  */
113
227
  export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
114
228
  /**
@@ -125,40 +239,181 @@ export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
125
239
  *
126
240
  * @param contextLength - Model context window in tokens.
127
241
  * @param bodyTruncation - Max chars per memory body included in the prompt.
242
+ * @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
128
243
  */
129
- export function computeSafeChunkSize(contextLength, bodyTruncation) {
244
+ export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
130
245
  const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
131
246
  const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
132
247
  const raw = Math.floor(usableTokens / tokensPerMemory);
133
- return Math.max(1, Math.min(50, raw));
248
+ return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
249
+ }
250
+ // ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
251
+ /**
252
+ * Re-order memories so that similar ones are placed adjacent to each other
253
+ * before the memories are sliced into chunks. This ensures high-similarity
254
+ * memories land in the same LLM context window, allowing the consolidate
255
+ * model to detect and merge duplicates that would otherwise be split across
256
+ * chunks and survive indefinitely.
257
+ *
258
+ * Algorithm: greedy nearest-neighbour chain starting from the first memory.
259
+ * Each step selects the unused memory with the highest cosine similarity to
260
+ * the last-placed memory. O(n²) — acceptable for the expected N < 200.
261
+ *
262
+ * mem0 arXiv:2504.19413 — every candidate compared against whole store.
263
+ * A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
264
+ *
265
+ * Returns the original order unchanged when:
266
+ * - The embedding config is not present.
267
+ * - Embedding requests fail (fail-open).
268
+ * - There are fewer than 3 memories (no benefit to reordering).
269
+ */
270
+ async function clusterMemoriesBySimilarity(memories, config) {
271
+ if (memories.length < 3 || !config.embedding)
272
+ return memories;
273
+ const texts = memories.map((m) => {
274
+ const parts = [];
275
+ if (m.description)
276
+ parts.push(m.description);
277
+ if (m.tags.length > 0)
278
+ parts.push(m.tags.join(" "));
279
+ return parts.join(". ") || m.name;
280
+ });
281
+ let embeddings = null;
282
+ try {
283
+ embeddings = await embedBatch(texts, config.embedding);
284
+ }
285
+ catch {
286
+ // Fail open: embedding failures degrade gracefully to original order.
287
+ return memories;
288
+ }
289
+ if (!embeddings || embeddings.length !== memories.length)
290
+ return memories;
291
+ // Greedy nearest-neighbour chain.
292
+ const used = new Array(memories.length).fill(false);
293
+ const ordered = [];
294
+ let current = 0; // start from the first memory
295
+ ordered.push(memories[current]);
296
+ used[current] = true;
297
+ for (let step = 1; step < memories.length; step++) {
298
+ const currentEmb = embeddings[current];
299
+ let bestIdx = -1;
300
+ let bestSim = -Infinity;
301
+ for (let j = 0; j < memories.length; j++) {
302
+ if (used[j])
303
+ continue;
304
+ const sim = cosineSimilarity(currentEmb, embeddings[j]);
305
+ if (sim > bestSim) {
306
+ bestSim = sim;
307
+ bestIdx = j;
308
+ }
309
+ }
310
+ if (bestIdx === -1)
311
+ break;
312
+ ordered.push(memories[bestIdx]);
313
+ used[bestIdx] = true;
314
+ current = bestIdx;
315
+ }
316
+ return ordered;
134
317
  }
135
318
  // ── Chunk helpers ────────────────────────────────────────────────────────────
136
- export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation) {
319
+ /**
320
+ * Build the per-chunk user prompt fed to the consolidate LLM.
321
+ *
322
+ * Each memory is annotated with two flags that drive the system-prompt
323
+ * rules at lines 181-186:
324
+ * - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
325
+ * forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
326
+ * stack before this annotation.
327
+ * - `(already queued)` — the memory's body hash matches a pending
328
+ * consolidate proposal; system prompt rule 3 forbids proposing
329
+ * promote/merge/contradict. ~107/4h before this annotation.
330
+ *
331
+ * Both annotations are visible to the LLM. `pendingProposalBodyHashes`
332
+ * is precomputed once per run by `loadPendingConsolidateProposalHashes`
333
+ * so the cost stays O(memories) inside the chunk loop.
334
+ */
335
+ export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set()) {
137
336
  const start = memories[0] ? `memory:${memories[0].name}` : "";
138
337
  const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
338
+ const annotationsByIndex = [];
339
+ const hotRefs = [];
340
+ for (const m of memories) {
341
+ let body = "";
342
+ try {
343
+ body = fs.readFileSync(m.filePath, "utf8");
344
+ }
345
+ catch {
346
+ body = "(unreadable)";
347
+ }
348
+ const parsed = parseFrontmatter(body);
349
+ const isHot = parsed.data.captureMode === "hot";
350
+ const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
351
+ const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
352
+ annotationsByIndex.push({ isHot, isAlreadyQueued, body });
353
+ if (isHot)
354
+ hotRefs.push(`memory:${m.name}`);
355
+ }
139
356
  const lines = [
140
357
  `Source: ${sourceName}`,
141
358
  `Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
142
359
  "",
143
360
  ];
361
+ // Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
362
+ // op-words like "promote", "merge", "contradict" so the model doesn't
363
+ // accidentally treat the warning as a hint to use that op elsewhere
364
+ // (variant B leaked the word "contradict" into the control sample
365
+ // during the diagnostic).
366
+ if (hotRefs.length > 0) {
367
+ lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
368
+ for (const ref of hotRefs)
369
+ lines.push(` - ${ref}`);
370
+ lines.push("");
371
+ }
144
372
  for (let i = 0; i < memories.length; i++) {
145
373
  const m = memories[i];
146
- lines.push(`[${i + 1}] memory:${m.name}`);
374
+ const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
375
+ const annotations = [];
376
+ if (isHot)
377
+ annotations.push("captureMode: hot");
378
+ if (isAlreadyQueued)
379
+ annotations.push("already queued");
380
+ const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
381
+ lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
147
382
  lines.push(`Description: ${m.description || "(none)"}`);
148
383
  lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
149
384
  lines.push("---");
150
- let body = "";
151
- try {
152
- body = fs.readFileSync(m.filePath, "utf8");
153
- }
154
- catch {
155
- body = "(unreadable)";
156
- }
157
385
  lines.push(body.slice(0, bodyTruncation));
158
386
  lines.push("");
159
387
  }
160
388
  return lines.join("\n");
161
389
  }
390
+ /**
391
+ * Precompute body-hashes of all currently-pending consolidate proposals so
392
+ * the per-chunk prompt can annotate memories whose body would just produce
393
+ * a deterministic `dedup_pending_proposal` skip. Hash domain matches the
394
+ * dedup site at ~line 1510 (sha256 over the post-frontmatter content,
395
+ * trimmed). Empty set on any read/parse error — fail-safe to "annotate
396
+ * nothing" so the LLM still proposes, just slightly more wastefully.
397
+ */
398
+ export function loadPendingConsolidateProposalHashes(stashDir) {
399
+ const hashes = new Set();
400
+ try {
401
+ const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
402
+ for (const p of pending) {
403
+ try {
404
+ const body = parseFrontmatter(p.payload.content).content.trim();
405
+ hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
406
+ }
407
+ catch {
408
+ // skip malformed payloads — they can't dedup anyway
409
+ }
410
+ }
411
+ }
412
+ catch {
413
+ // listProposals throws on missing stash dir during tests — empty set is safe
414
+ }
415
+ return hashes;
416
+ }
162
417
  function isValidOp(op) {
163
418
  if (typeof op !== "object" || op === null)
164
419
  return false;
@@ -172,43 +427,133 @@ function isValidOp(op) {
172
427
  if (o.op === "promote") {
173
428
  return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
174
429
  }
430
+ if (o.op === "contradict") {
431
+ return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
432
+ }
175
433
  return false;
176
434
  }
177
- function mergePlans(chunks) {
435
+ export function mergePlans(chunks, knownRefs) {
178
436
  const mergeOps = new Map();
179
437
  const deleteOps = new Map();
180
438
  const promoteOps = new Map();
439
+ // C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
440
+ const contradictOps = new Map();
181
441
  const warnings = [];
182
442
  for (const chunk of chunks) {
183
443
  for (const op of chunk) {
184
444
  if (op.op === "merge") {
445
+ // Drop ops whose primary the LLM hallucinated (not in the loaded memory
446
+ // pool). Without this guard, a hallucinated primary flows all the way to
447
+ // Phase B where !memoryByRef.has(primary) fires and charges every real
448
+ // secondary with merge_primary_missing — masking LLM hallucinations as
449
+ // filter regressions in health metrics.
450
+ if (knownRefs && !knownRefs.has(op.primary)) {
451
+ warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
452
+ // Use a dedicated skip reason so dashboards can distinguish
453
+ // hallucinated primaries from stale-DB regressions.
454
+ // Secondaries are real refs; they are NOT charged here — they remain
455
+ // available for other ops to claim.
456
+ continue;
457
+ }
458
+ // Filter hallucinated secondaries while preserving real ones.
459
+ let mergeOp = op;
460
+ if (knownRefs) {
461
+ const filteredSecondaries = op.secondaries.filter((sec) => {
462
+ if (!knownRefs.has(sec)) {
463
+ warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
464
+ return false;
465
+ }
466
+ return true;
467
+ });
468
+ if (filteredSecondaries.length !== op.secondaries.length) {
469
+ mergeOp = { ...op, secondaries: filteredSecondaries };
470
+ }
471
+ }
185
472
  // merge wins over delete
186
- if (deleteOps.has(op.primary)) {
187
- deleteOps.delete(op.primary);
473
+ if (deleteOps.has(mergeOp.primary)) {
474
+ deleteOps.delete(mergeOp.primary);
188
475
  }
189
- for (const sec of op.secondaries) {
476
+ for (const sec of mergeOp.secondaries) {
190
477
  if (deleteOps.has(sec))
191
478
  deleteOps.delete(sec);
192
479
  }
193
- mergeOps.set(op.primary, op);
480
+ mergeOps.set(mergeOp.primary, mergeOp);
194
481
  }
195
482
  else if (op.op === "delete") {
196
- if (!mergeOps.has(op.ref)) {
483
+ // merge and promote both win over delete. A promote is non-destructive
484
+ // (creates a proposal) but the source memory is counted in `promoted`;
485
+ // if a delete also fires, the ref lands in both `promoted` and
486
+ // `skipReasons`, breaking the invariant by +1.
487
+ if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
197
488
  deleteOps.set(op.ref, op);
198
489
  }
199
490
  }
200
491
  else if (op.op === "promote") {
201
- const existingMerge = mergeOps.get(op.ref);
202
- if (existingMerge) {
203
- warnings.push(`Conflict: promote and merge both target ${op.ref}; preferring merge.`);
204
- }
205
- else {
206
- promoteOps.set(op.ref, op);
492
+ // C-2 / #381: when both a promote and a merge target the same ref,
493
+ // queue the promote FIRST rather than discarding it. The promote op
494
+ // routes through createProposal (the human-gated proposal queue), so
495
+ // it is non-destructive. The merge follows after the proposal is
496
+ // created. This preserves the human reviewer's ability to inspect the
497
+ // promotion before the source memory is merged/deleted.
498
+ // AGM K*8 — retain the maximally informative consistent subset.
499
+ promoteOps.set(op.ref, op);
500
+ }
501
+ else if (op.op === "contradict") {
502
+ // Deduplicate by ref+contradictedByRef pair.
503
+ const key = `${op.ref}|${op.contradictedByRef}`;
504
+ if (!contradictOps.has(key)) {
505
+ contradictOps.set(key, op);
207
506
  }
208
507
  }
209
508
  }
210
509
  }
211
- const ops = [...mergeOps.values(), ...deleteOps.values(), ...promoteOps.values()];
510
+ // Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
511
+ //
512
+ // 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
513
+ // for secondaries that were already in deleteOps, but misses the case
514
+ // where the delete chunk came first. A full sweep here fixes both orders.
515
+ //
516
+ // 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
517
+ // only the first (insertion-order) retains it. Without this, a successful
518
+ // merge credits A to mergedSecondaries and a later merge's emitMerge-
519
+ // FailureSkips also charges A to skipReasons — double-counting A while
520
+ // processed has it only once.
521
+ //
522
+ // 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
523
+ // a secondary in another, remove A from the secondary list. Both merges
524
+ // would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
525
+ // breaking the invariant the same way.
526
+ // Also remove delete ops for any ref claimed by a promote op (handles the
527
+ // case where the delete chunk appeared before the promote chunk).
528
+ for (const ref of promoteOps.keys()) {
529
+ deleteOps.delete(ref);
530
+ }
531
+ const claimedSecondaries = new Set();
532
+ for (const mergeOp of mergeOps.values()) {
533
+ deleteOps.delete(mergeOp.primary);
534
+ mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
535
+ if (mergeOps.has(sec)) {
536
+ warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
537
+ return false;
538
+ }
539
+ if (claimedSecondaries.has(sec)) {
540
+ warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
541
+ return false;
542
+ }
543
+ claimedSecondaries.add(sec);
544
+ deleteOps.delete(sec);
545
+ return true;
546
+ });
547
+ }
548
+ // C-2 / #381: promote ops are ordered BEFORE merge ops so that the
549
+ // human-gated proposal queue entry is created before any destructive merge.
550
+ // Phase B processes ops in array order, so promote executes first.
551
+ const ops = [
552
+ ...promoteOps.values(),
553
+ ...mergeOps.values(),
554
+ ...deleteOps.values(),
555
+ ...contradictOps.values(),
556
+ ];
212
557
  return { ops, warnings };
213
558
  }
214
559
  function getJournalPath(stashDir) {
@@ -359,8 +704,7 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
359
704
  ...(supersededBy ? { superseded_by: supersededBy } : {}),
360
705
  superseded_reason: reason,
361
706
  };
362
- const fmStr = yamlStringify(newFm).trimEnd();
363
- content = `---\n${fmStr}\n---\n${parsed.content}`;
707
+ content = assembleAssetFromString(yamlStringify(newFm).trimEnd(), parsed.content);
364
708
  }
365
709
  catch {
366
710
  if (warnings)
@@ -377,9 +721,44 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
377
721
  warnings.push(`archiveMemory: write failed for ${ref}: ${String(e)}`);
378
722
  }
379
723
  }
724
+ // ── LLM resolution ──────────────────────────────────────────────────────────
725
+ /**
726
+ * Resolve the LLM connection for the consolidate pass.
727
+ *
728
+ * Priority order (mirrors extract / reflect / distill — see
729
+ * `src/commands/extract.ts:421-438` and the canonical
730
+ * `resolveImproveProcessRunnerFromProfile` pattern):
731
+ *
732
+ * 1. `profiles.improve.default.processes.consolidate.profile` (or `mode`)
733
+ * via {@link resolveImproveProcessRunnerFromProfile}. Lets the user pin
734
+ * a dedicated model (e.g. `ministral-3b`) for consolidation instead of
735
+ * whatever `defaults.llm` happens to be.
736
+ * 2. `getDefaultLlmConfig(config)` — the baseline default LLM profile.
737
+ *
738
+ * Regression guard (2026-05-26): before this resolver, `akmConsolidate`
739
+ * called `getDefaultLlmConfig` directly and silently ignored a configured
740
+ * `processes.consolidate.profile`, sending every chunk to the default LLM
741
+ * (often a long-context model loaded with a smaller runtime `n_ctx`, causing
742
+ * silent 400s from LM Studio). The investigation lives at
743
+ * `/tmp/akm-health-investigations/consolidation-no-op.md`.
744
+ */
745
+ function resolveConsolidateLlmConfig(config) {
746
+ const consolidateProcess = config.profiles?.improve?.default?.processes?.consolidate;
747
+ const runnerSpec = resolveImproveProcessRunnerFromProfile(consolidateProcess, config);
748
+ if (runnerSpec && runnerSpec.kind === "llm") {
749
+ return runnerSpec.connection;
750
+ }
751
+ // Non-LLM runner modes (agent/sdk) don't apply to consolidate's HTTP path;
752
+ // fall back to the default LLM profile rather than disabling the pass.
753
+ return getDefaultLlmConfig(config);
754
+ }
380
755
  // ── Main entry point ─────────────────────────────────────────────────────────
381
756
  export async function akmConsolidate(opts = {}) {
382
757
  const startMs = Date.now();
758
+ // Derive a stable PROV-DM token for this run. Callers (e.g. akmImprove)
759
+ // should pass opts.sourceRun to tie proposals back to the parent run;
760
+ // standalone `akm consolidate` gets a self-contained token.
761
+ const sourceRun = opts.sourceRun ?? `consolidate-${startMs}`;
383
762
  const config = opts.config ?? loadConfig();
384
763
  const stashDir = opts.stashDir ?? resolveStashDir();
385
764
  if (!isLlmFeatureEnabled(config, "memory_consolidation")) {
@@ -394,13 +773,24 @@ export async function akmConsolidate(opts = {}) {
394
773
  merged: 0,
395
774
  deleted: 0,
396
775
  promoted: [],
776
+ contradicted: 0,
397
777
  warnings: [],
398
778
  durationMs: Date.now() - startMs,
399
779
  };
400
780
  }
401
781
  const warnings = [];
402
782
  checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
403
- const memories = loadMemoriesForSource(opts.target, stashDir, warnings);
783
+ let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
784
+ // Pre-flight: filter out stale DB entries whose files no longer exist on
785
+ // disk. Without this, memories deleted by a prior run (but not yet
786
+ // reindexed) appear in chunk prompts, causing the LLM to generate plans
787
+ // against ghost refs and wasting tokens. Filtering here ensures the chunk
788
+ // pool and memoryByRef are authoritative against the actual filesystem state.
789
+ const staleCount = memories.filter((m) => !fs.existsSync(m.filePath)).length;
790
+ if (staleCount > 0) {
791
+ warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
792
+ }
793
+ memories = memories.filter((m) => fs.existsSync(m.filePath));
404
794
  if (memories.length === 0) {
405
795
  return {
406
796
  schemaVersion: 1,
@@ -413,55 +803,181 @@ export async function akmConsolidate(opts = {}) {
413
803
  merged: 0,
414
804
  deleted: 0,
415
805
  promoted: [],
806
+ contradicted: 0,
416
807
  warnings,
417
808
  durationMs: Date.now() - startMs,
418
809
  };
419
810
  }
811
+ if (opts.incrementalSince) {
812
+ memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings);
813
+ if (memories.length === 0) {
814
+ return {
815
+ schemaVersion: 1,
816
+ ok: true,
817
+ shape: "consolidate-result",
818
+ dryRun: opts.dryRun ?? false,
819
+ previewOnly: false,
820
+ target: opts.target ?? stashDir,
821
+ processed: 0,
822
+ merged: 0,
823
+ deleted: 0,
824
+ promoted: [],
825
+ contradicted: 0,
826
+ warnings,
827
+ durationMs: Date.now() - startMs,
828
+ };
829
+ }
830
+ }
420
831
  // Consolidation always uses the HTTP LLM client directly — never the agent
421
832
  // CLI. The agent CLI is for interactive agent sessions (reflect, propose);
422
833
  // structured JSON generation works better and faster via HTTP.
423
- const isHttpPath = !!config.llm;
834
+ //
835
+ // Honor `profiles.improve.default.processes.consolidate.profile` first; fall
836
+ // back to the default LLM. See {@link resolveConsolidateLlmConfig}.
837
+ const llmConfig = resolveConsolidateLlmConfig(config);
838
+ const isHttpPath = !!llmConfig;
424
839
  // Chunk sizing: derive a safe chunk size from the configured model context
425
- // window (config.llm.contextLength) so that the full prompt (system prompt +
426
- // chunk user prompt) never exceeds the model's n_ctx limit. When no context
427
- // length is configured we fall back to DEFAULT_CONTEXT_LENGTH_TOKENS (8 000)
428
- // which is conservative enough for most 8K–16K local models.
840
+ // window so that the full prompt (system prompt + chunk user prompt) never
841
+ // exceeds the model's n_ctx limit. When no context length is configured we
842
+ // fall back to DEFAULT_CONTEXT_LENGTH_TOKENS (8 000) which is conservative
843
+ // enough for most 8K–16K local models.
429
844
  //
430
845
  // bodyTruncation caps the body excerpt included per memory in the prompt.
431
846
  // Reducing it further than 500 chars degrades consolidation quality, so we
432
847
  // keep it fixed and let computeSafeChunkSize vary the number of memories
433
848
  // per chunk instead.
434
849
  const bodyTruncation = 500;
435
- const modelContextLength = config.llm?.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS;
436
- const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation);
850
+ const modelContextLength = llmConfig?.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS;
851
+ const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
437
852
  // -- Phase A: plan generation -----------------------------------------------
438
853
  const sourceName = opts.target ?? stashDir;
854
+ // C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
855
+ // This ensures that semantically similar memories land in the same LLM
856
+ // context window, allowing the model to detect and merge duplicates that
857
+ // would otherwise be split across chunks and survive indefinitely.
858
+ // mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
859
+ // Fails open: if embeddings are unavailable or fail, original order is used.
860
+ const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
439
861
  const chunks = [];
440
- for (let i = 0; i < memories.length; i += chunkSize) {
441
- chunks.push(memories.slice(i, i + chunkSize));
862
+ for (let i = 0; i < clusteredMemories.length; i += chunkSize) {
863
+ chunks.push(clusteredMemories.slice(i, i + chunkSize));
442
864
  }
443
- warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}`);
865
+ // 2026-05-27 prompt-context fix: precompute body-hashes of pending
866
+ // consolidate proposals once, so the per-chunk prompt can annotate
867
+ // memories whose body would just produce a deterministic
868
+ // `dedup_pending_proposal` skip. Cuts ~110 wasted LLM proposals per
869
+ // 4h on this user's stack. See
870
+ // /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
871
+ const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
872
+ warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
873
+ ` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
444
874
  const chunkOpsArrays = [];
445
- let consecutiveFailures = 0;
875
+ // Structured skip-reason histogram (2026-05-26): every deterministic
876
+ // post-LLM op rejection site below also calls `pushSkipReason` so the
877
+ // health rollup can aggregate without regex-parsing English warning
878
+ // strings. See `/tmp/akm-health-investigations/tuning-reasons-investigation.md` §Q2.
879
+ const skipReasons = [];
880
+ // Tracks refs already emitted to skipReasons. A ref can only occupy one
881
+ // accounting bucket; subsequent skip ops for the same ref are recorded as
882
+ // warnings but must not push a second skipReasons entry (that would inflate
883
+ // Σ(skipReasons) and break the invariant by +1 per duplicate).
884
+ const skipReasonEmittedRefs = new Set();
885
+ const pushSkipReason = (op, ref, reason) => {
886
+ // 2026-05-27 cross-chunk double-count fix: if `ref` already contributed
887
+ // to judgedNoAction in its own chunk (a different chunk proposed an op
888
+ // for it that is now being rejected here), promote it from the
889
+ // judgedNoAction bucket into the more specific skipReason bucket.
890
+ // Preserves the invariant: processed == actioned + judgedNoAction +
891
+ // Σ(skipReasons) + failedChunkMemories.
892
+ if (judgedNoActionRefs.delete(ref))
893
+ judgedNoAction--;
894
+ if (skipReasonEmittedRefs.has(ref)) {
895
+ // Already counted once. Record the extra skip for observability but
896
+ // don't push to skipReasons — that would break the accounting invariant.
897
+ warnings.push(`Skip: ${ref} already in skipReasons (${reason} via ${op}); not re-counted.`);
898
+ return;
899
+ }
900
+ skipReasonEmittedRefs.add(ref);
901
+ skipReasons.push({ op, ref, reason });
902
+ };
903
+ // judgedNoAction tracks memories the LLM saw inside a chunk but proposed
904
+ // no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
905
+ let judgedNoAction = 0;
906
+ // 2026-05-27 cross-chunk double-count fix: refs that contributed to
907
+ // judgedNoAction in their own chunk. When a different chunk's op references
908
+ // one of these as a secondary and that op later fails, the ref would land
909
+ // in BOTH judgedNoAction and skipReasons (delta +1 per occurrence). Track
910
+ // the set so the merge-failure path can decrement and re-bucket.
911
+ const judgedNoActionRefs = new Set();
912
+ // 2026-05-26 accounting-leak fix: memories that belong to a chunk whose
913
+ // LLM call failed before any per-chunk noAction calculation runs. They
914
+ // would otherwise vanish from the envelope's accounting (no judgedNoAction
915
+ // bump, no skipReasons entry, no actioned counter).
916
+ let failedChunkMemories = 0;
917
+ // 2026-05-26 accounting-leak fix: per-secondary tally so successful merges
918
+ // account for `1 + secondaries.length` memories instead of 1.
919
+ let mergedSecondaries = 0;
920
+ // C-6 / #392: Replace two-consecutive-failures abort with failure-rate threshold.
921
+ // Consecutive-count policies are brittle against transient LM Studio reloads:
922
+ // two transient failures abort the run even though the next chunk would succeed.
923
+ // Rate-based abort (≥50% failure over ≥4 chunks) is more robust.
924
+ // Tanenbaum, Distributed Systems §8 — rate-based policies with minimum sample sizes.
925
+ let totalChunksProcessed = 0;
926
+ let totalChunksFailed = 0;
927
+ const ABORT_MIN_CHUNKS = 4;
928
+ const ABORT_FAILURE_RATE = 0.5;
446
929
  for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
447
- // Abort early if the first chunk failed the LLM/agent is likely unavailable
448
- // and continuing would waste minutes processing chunks that will all fail the same way.
449
- if (chunkIdx > 0 && consecutiveFailures >= 2) {
450
- const skipped = chunks.length - chunkIdx;
451
- warnings.push(`Consolidation aborted after ${consecutiveFailures} consecutive chunk failures — LLM may be unavailable. ${skipped} chunk(s) skipped.`);
452
- break;
930
+ // Abort if failure rate >= 50% over at least 4 processed chunks.
931
+ if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
932
+ const failureRate = totalChunksFailed / totalChunksProcessed;
933
+ if (failureRate >= ABORT_FAILURE_RATE) {
934
+ const skipped = chunks.length - chunkIdx;
935
+ const abortMsg = `Consolidation aborted — failure rate ${(failureRate * 100).toFixed(0)}% over ${totalChunksProcessed} chunks (>= ${ABORT_FAILURE_RATE * 100}% threshold). LLM may be unavailable. ${skipped} chunk(s) skipped.`;
936
+ warn(abortMsg);
937
+ warnings.push(abortMsg);
938
+ // Account for memories in chunks we never attempted: they are
939
+ // neither judgedNoAction (no plan parsed) nor skipReason (no op
940
+ // rejected). Without this, the accounting invariant fails by
941
+ // `Σ(unattempted_chunk.length)` whenever the abort fires.
942
+ for (let i = chunkIdx; i < chunks.length; i++) {
943
+ failedChunkMemories += chunks[i].length;
944
+ }
945
+ break;
946
+ }
453
947
  }
454
948
  const chunk = chunks[chunkIdx];
949
+ // All-hot chunk early-exit. The per-prompt hot-list block (see
950
+ // buildChunkPrompt) only *discourages* delete proposals on a mixed chunk;
951
+ // when EVERY memory in the chunk is captureMode: hot, the only ops the LLM
952
+ // could ever propose are deletes — all of which the downstream guard
953
+ // refuses unconditionally. Calling the model is therefore pure token waste.
954
+ // Skip the request entirely and bucket every memory as judgedNoAction (we
955
+ // judged "no action" without spending an LLM call), preserving the
956
+ // accounting invariant `processed == actioned + judgedNoAction +
957
+ // Σ(skipReasons) + failedChunkMemories`. Not counted toward the
958
+ // LLM-failure-rate abort policy — no request was attempted.
959
+ if (chunk.length > 0 && chunk.every((m) => isHotCapturedMemory(m.filePath))) {
960
+ for (const m of chunk)
961
+ judgedNoActionRefs.add(`memory:${m.name}`);
962
+ judgedNoAction += chunk.length;
963
+ warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length}: all ${chunk.length} memories are captureMode: hot — skipping LLM (judged no-action).`);
964
+ continue;
965
+ }
455
966
  warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
456
- const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation);
457
- const raw = await tryLlmFeature("memory_consolidation", config, async () => {
458
- if (!config.llm)
967
+ const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes);
968
+ let raw = await tryLlmFeature("memory_consolidation", config, async () => {
969
+ if (!llmConfig)
459
970
  return { ok: false, error: "No LLM configured for consolidation" };
460
971
  try {
461
- const content = await chatCompletion(config.llm, [
972
+ // responseSchema lift (PR 1, asset-writers-investigation §5): pass
973
+ // the consolidate plan schema so providers with
974
+ // `supportsJsonSchema: true` enforce shape upstream. Providers that
975
+ // ignore the option fall through to the existing
976
+ // `parseEmbeddedJsonResponse` path on the response side.
977
+ const content = await chatCompletion(llmConfig, [
462
978
  { role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
463
979
  { role: "user", content: userPrompt },
464
- ]);
980
+ ], { responseSchema: CONSOLIDATE_PLAN_JSON_SCHEMA, enableThinking: false });
465
981
  return { ok: true, content };
466
982
  }
467
983
  catch (e) {
@@ -469,9 +985,37 @@ export async function akmConsolidate(opts = {}) {
469
985
  }
470
986
  }, { ok: false, error: `chunk ${chunkIdx + 1} failed` });
471
987
  if (!raw.ok) {
472
- warnings.push(raw.error ?? `chunk ${chunkIdx + 1} failed`);
473
- consecutiveFailures++;
474
- continue;
988
+ // Single retry with 2s backoff before recording chunk as lost.
989
+ // Recovers transient Shredder LM Studio timeouts without significantly
990
+ // extending run time. Only marks failed if both attempts fail.
991
+ await new Promise((r) => setTimeout(r, 2_000));
992
+ const retry = await tryLlmFeature("memory_consolidation", config, async () => {
993
+ if (!llmConfig)
994
+ return { ok: false, error: "No LLM configured for consolidation" };
995
+ try {
996
+ const content = await chatCompletion(llmConfig, [
997
+ { role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
998
+ { role: "user", content: userPrompt },
999
+ ], { responseSchema: CONSOLIDATE_PLAN_JSON_SCHEMA, enableThinking: false });
1000
+ return { ok: true, content };
1001
+ }
1002
+ catch (e) {
1003
+ return { ok: false, error: String(e) };
1004
+ }
1005
+ }, { ok: false, error: `chunk ${chunkIdx + 1} retry failed` });
1006
+ if (!retry.ok) {
1007
+ warn(retry.error ?? `chunk ${chunkIdx + 1} failed after retry`);
1008
+ warnings.push(retry.error ?? `chunk ${chunkIdx + 1} failed after retry`);
1009
+ totalChunksProcessed++;
1010
+ totalChunksFailed++;
1011
+ // Account for the chunk's memories under the failed-chunk bucket.
1012
+ // judgedNoAction does NOT run on this path (it's after the success
1013
+ // guards) so without this the accounting invariant breaks on every
1014
+ // chunk-level transport/parse failure.
1015
+ failedChunkMemories += chunk.length;
1016
+ continue;
1017
+ }
1018
+ raw = retry;
475
1019
  }
476
1020
  if (process.env.AKM_DEBUG_LLM) {
477
1021
  const preview = (raw.content ?? "").slice(0, 500);
@@ -482,11 +1026,14 @@ export async function akmConsolidate(opts = {}) {
482
1026
  const hint = raw.content !== undefined && raw.content.trim() === ""
483
1027
  ? " (empty response — if using a thinking model, disable thinking mode)"
484
1028
  : "";
1029
+ warn(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
485
1030
  warnings.push(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
486
- consecutiveFailures++;
1031
+ totalChunksProcessed++;
1032
+ totalChunksFailed++;
1033
+ failedChunkMemories += chunk.length;
487
1034
  continue;
488
1035
  }
489
- consecutiveFailures = 0; // reset on success
1036
+ totalChunksProcessed++; // success
490
1037
  const ops = [];
491
1038
  for (const op of parsed.operations) {
492
1039
  if (isValidOp(op)) {
@@ -502,9 +1049,37 @@ export async function akmConsolidate(opts = {}) {
502
1049
  warnings.push(w);
503
1050
  }
504
1051
  }
1052
+ // Per-chunk judgedNoAction: count memories the LLM saw but proposed no
1053
+ // op for. Membership is by `memory:<name>` ref against the targets of
1054
+ // each op (primary + secondaries for merge; ref otherwise). 2026-05-26:
1055
+ // pre-fix this was a 78/119 (66%) silent drop in the cron run — no
1056
+ // warning, event, or counter. See tuning investigation §Q2.
1057
+ const targetRefs = new Set();
1058
+ for (const op of ops) {
1059
+ if (op.op === "merge") {
1060
+ targetRefs.add(op.primary);
1061
+ for (const s of op.secondaries)
1062
+ targetRefs.add(s);
1063
+ }
1064
+ else {
1065
+ targetRefs.add(op.ref);
1066
+ }
1067
+ }
1068
+ let chunkNoAction = 0;
1069
+ for (const m of chunk) {
1070
+ const memRef = `memory:${m.name}`;
1071
+ if (!targetRefs.has(memRef)) {
1072
+ chunkNoAction++;
1073
+ judgedNoActionRefs.add(memRef);
1074
+ }
1075
+ }
1076
+ judgedNoAction += chunkNoAction;
505
1077
  chunkOpsArrays.push(ops);
506
1078
  }
507
- const { ops: allOps, warnings: mergeWarnings } = mergePlans(chunkOpsArrays);
1079
+ // Build the known-refs set from the already-filtered memory pool so
1080
+ // mergePlans() can reject LLM-hallucinated primary refs before execution.
1081
+ const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
1082
+ const { ops: allOps, warnings: mergeWarnings } = mergePlans(chunkOpsArrays, knownRefs);
508
1083
  warnings.push(...mergeWarnings);
509
1084
  // -- Dry-run: show AI plan without executing any writes --------------------
510
1085
  if (opts.dryRun) {
@@ -519,6 +1094,13 @@ export async function akmConsolidate(opts = {}) {
519
1094
  merged: 0,
520
1095
  deleted: 0,
521
1096
  promoted: [],
1097
+ contradicted: 0,
1098
+ failedChunks: totalChunksFailed,
1099
+ totalChunks: chunks.length,
1100
+ judgedNoAction,
1101
+ skipReasons,
1102
+ mergedSecondaries,
1103
+ failedChunkMemories,
522
1104
  planned: allOps,
523
1105
  warnings,
524
1106
  durationMs: Date.now() - startMs,
@@ -528,9 +1110,20 @@ export async function akmConsolidate(opts = {}) {
528
1110
  // -- HTTP path: warn about quality and confirm unless auto-accepted --------
529
1111
  if (isHttpPath) {
530
1112
  warnings.push("Running on HTTP path — plan generated from truncated memory excerpts; quality may vary.");
531
- if (!opts.autoAccept) {
1113
+ // Per-proposal confidence gating is handled by the caller (improve.ts)
1114
+ // via runAutoAcceptGate after this function returns. The gate reads
1115
+ // proposal.confidence (forwarded from op.confidence above) and applies
1116
+ // a minimumThreshold floor of 95 for consolidate's destructive ops.
1117
+ // Here we only gate the interactive-confirm path for manual/HTTP invocations.
1118
+ if (opts.autoAccept === undefined && allOps.length > 0) {
532
1119
  const n = allOps.length;
533
- const answer = await promptConfirm(`Apply ${n} operations? [y/N] `);
1120
+ // Non-interactive contexts (CI / test runners / piped stdin) must not
1121
+ // block on an unanswerable prompt. Default to a non-destructive "no"
1122
+ // so callers in those contexts get the same "aborted, preview only"
1123
+ // shape they'd get from explicit user dismissal. AKM_NON_INTERACTIVE
1124
+ // lets callers force this path even when stdin happens to be a TTY.
1125
+ const nonInteractive = process.stdin.isTTY === false || process.env.AKM_NON_INTERACTIVE === "1";
1126
+ const answer = nonInteractive ? false : await promptConfirm(`Apply ${n} operations? [y/N] `);
534
1127
  if (!answer) {
535
1128
  return {
536
1129
  schemaVersion: 1,
@@ -543,8 +1136,15 @@ export async function akmConsolidate(opts = {}) {
543
1136
  merged: 0,
544
1137
  deleted: 0,
545
1138
  promoted: [],
1139
+ contradicted: 0,
1140
+ failedChunks: totalChunksFailed,
1141
+ totalChunks: chunks.length,
1142
+ judgedNoAction,
1143
+ skipReasons,
1144
+ mergedSecondaries,
1145
+ failedChunkMemories,
546
1146
  planned: allOps,
547
- warnings: [...warnings, "Aborted by user."],
1147
+ warnings: [...warnings, nonInteractive ? "Non-interactive context: skipped apply." : "Aborted by user."],
548
1148
  durationMs: Date.now() - startMs,
549
1149
  };
550
1150
  }
@@ -559,6 +1159,12 @@ export async function akmConsolidate(opts = {}) {
559
1159
  let merged = 0;
560
1160
  let deleted = 0;
561
1161
  const promoted = [];
1162
+ let contradicted = 0; // C-3 / #382: count of contradiction edges written
1163
+ // Within-run dedup: track source refs for which a promote proposal was
1164
+ // already created this run. The LLM can return multiple promote ops for
1165
+ // different source memories that happen to have identical content (all are
1166
+ // duplicate memories), so we also need a content-hash guard below.
1167
+ const promotedSourceRefs = new Set();
562
1168
  // Build a lookup map: ref → MemoryEntry
563
1169
  const memoryByRef = new Map();
564
1170
  for (const m of memories) {
@@ -566,11 +1172,40 @@ export async function akmConsolidate(opts = {}) {
566
1172
  }
567
1173
  for (let opIndex = 0; opIndex < allOps.length; opIndex++) {
568
1174
  const op = allOps[opIndex];
569
- warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${op.op === "merge" ? op.primary : op.ref}`);
1175
+ const opDisplayRef = op.op === "merge" ? op.primary : op.op === "contradict" ? `${op.ref} ↔ ${op.contradictedByRef}` : op.ref;
1176
+ warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${opDisplayRef}`);
570
1177
  if (op.op === "merge") {
1178
+ // Accounting helper: emit a per-participant skipReason for failed
1179
+ // merges so primary + every loaded-memory secondary land in the
1180
+ // structured skip histogram. Pre-2026-05-26 only the primary was
1181
+ // counted (1 skipReason per failed merge), leaving N secondaries
1182
+ // unaccounted for in the `processed == actioned + noAction + Σskips`
1183
+ // invariant — the source of the 4–11 silent leaks per run.
1184
+ const emitMergeFailureSkips = (reason) => {
1185
+ if (memoryByRef.has(op.primary))
1186
+ pushSkipReason("merge", op.primary, reason);
1187
+ for (const secRef of op.secondaries) {
1188
+ if (memoryByRef.has(secRef))
1189
+ pushSkipReason("merge", secRef, reason);
1190
+ }
1191
+ };
571
1192
  const primaryEntry = memoryByRef.get(op.primary);
572
1193
  if (!primaryEntry) {
573
- warnings.push(`Merge: primary ${op.primary} not found in loaded memories skipping.`);
1194
+ // This fires when a prior op in the same run consumed this ref as a
1195
+ // secondary and Fix-A pruned it from memoryByRef. It should NOT fire
1196
+ // for hallucinated primaries (those are dropped by mergePlans() before
1197
+ // reaching here). If this counter is non-zero, suspect an intra-run
1198
+ // cross-chunk race, not a filter regression.
1199
+ warnings.push(`Merge: primary ${op.primary} not found in loaded memories (pruned by prior op this run) — skipping.`);
1200
+ emitMergeFailureSkips("merge_primary_missing");
1201
+ continue;
1202
+ }
1203
+ // Defense-in-depth: even if the entry is in memoryByRef (pre-flight ran
1204
+ // before this run's own ops), the file may have been deleted by a
1205
+ // concurrent process or an edge case the pre-flight filter missed.
1206
+ if (!fs.existsSync(primaryEntry.filePath)) {
1207
+ warnings.push(`Merge: primary ${op.primary} file gone at execution time (stale entry) — skipping.`);
1208
+ emitMergeFailureSkips("merge_primary_file_gone");
574
1209
  continue;
575
1210
  }
576
1211
  // Phase B: generate merged content
@@ -579,29 +1214,106 @@ export async function akmConsolidate(opts = {}) {
579
1214
  const secEntry = memoryByRef.get(secRef);
580
1215
  if (!secEntry) {
581
1216
  warnings.push(`Merge: secondary ${secRef} not found — skipping merge op.`);
1217
+ // No accounting impact: a missing secondary is a phantom ref and
1218
+ // never contributed to any chunk's targetRefs reduction. We still
1219
+ // continue the loop to gather the remaining valid secondaries.
582
1220
  continue;
583
1221
  }
584
1222
  secondaryBodies.push(secRef);
585
1223
  }
586
- if (secondaryBodies.length === 0)
1224
+ if (secondaryBodies.length === 0) {
1225
+ warnings.push(`Merge: ${op.primary} has no valid secondaries — skipping.`);
1226
+ emitMergeFailureSkips("merge_no_valid_secondaries");
1227
+ continue;
1228
+ }
1229
+ // Pre-flight hot guard — skip the LLM call entirely if any participant
1230
+ // is hot or unparseable. Without this, mixed chunks still send hot merges
1231
+ // to the planner which proposes them; generateMergedContent() is then
1232
+ // called, produces output without `description`, and the skip is
1233
+ // misattributed to merge_missing_description instead of the real cause.
1234
+ const preflightParticipants = [op.primary, ...op.secondaries];
1235
+ const preflightBlocked = preflightParticipants.flatMap((ref) => {
1236
+ const e = memoryByRef.get(ref);
1237
+ if (!e)
1238
+ return [];
1239
+ const verdict = consolidateGuardStatus(e.filePath);
1240
+ if (verdict === "hot" || verdict === "unparseable")
1241
+ return [{ ref, verdict }];
1242
+ return [];
1243
+ });
1244
+ if (preflightBlocked.length > 0) {
1245
+ const detail = preflightBlocked.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1246
+ warnings.push(`Merge: refused for ${op.primary} — ${preflightBlocked.length} participant(s) blocked by hot/unparseable frontmatter guard (pre-flight): ${detail}`);
1247
+ emitMergeFailureSkips("merge_participant_blocked");
587
1248
  continue;
1249
+ }
588
1250
  let primaryBody = "";
589
1251
  try {
590
1252
  primaryBody = fs.readFileSync(primaryEntry.filePath, "utf8");
591
1253
  }
592
1254
  catch {
593
1255
  warnings.push(`Merge: could not read primary ${op.primary} — skipping.`);
1256
+ emitMergeFailureSkips("merge_read_failed");
594
1257
  continue;
595
1258
  }
596
- const mergedContent = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef, warnings);
597
- if (mergedContent === null)
1259
+ const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef);
1260
+ if ("error" in mergeResult) {
1261
+ warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
1262
+ emitMergeFailureSkips(mergeResult.error);
598
1263
  continue;
599
- // Validate frontmatter of merged content
1264
+ }
1265
+ const mergedContent = mergeResult.content;
1266
+ // Validate frontmatter of merged content — must have a `---` block
1267
+ // with at minimum a `description` field. We parse via the hand-rolled
1268
+ // parser (cheap) AND require non-empty description. This guards against
1269
+ // the historical defect where merged memories were written back with
1270
+ // empty `description` and later polluted the promote path.
1271
+ let parsedMerged;
600
1272
  try {
601
- parseFrontmatter(mergedContent);
1273
+ parsedMerged = parseFrontmatter(mergedContent);
602
1274
  }
603
1275
  catch {
604
1276
  warnings.push(`Merge: merged content for ${op.primary} has invalid frontmatter — skipping.`);
1277
+ emitMergeFailureSkips("merge_invalid_frontmatter");
1278
+ continue;
1279
+ }
1280
+ if (parsedMerged.frontmatter === null) {
1281
+ warnings.push(`Merge: merged content for ${op.primary} has no frontmatter block — skipping.`);
1282
+ emitMergeFailureSkips("merge_invalid_frontmatter");
1283
+ continue;
1284
+ }
1285
+ const mergedDesc = parsedMerged.data.description;
1286
+ if (typeof mergedDesc !== "string" || mergedDesc.trim().length === 0) {
1287
+ warnings.push(`Merge: merged content for ${op.primary} missing description — skipping.`);
1288
+ emitMergeFailureSkips("merge_missing_description");
1289
+ continue;
1290
+ }
1291
+ const truncReason = detectTruncatedDescription(mergedDesc);
1292
+ if (truncReason) {
1293
+ warnings.push(`Merge: merged content for ${op.primary} has truncated description (${truncReason}) — skipping.`);
1294
+ emitMergeFailureSkips("merge_truncated_description");
1295
+ continue;
1296
+ }
1297
+ // captureMode:hot guard — refuse the merge if ANY participating memory
1298
+ // (primary or secondary) was user-captured or has unparseable frontmatter
1299
+ // (could have hidden a hot flag). Hot memories are user-explicit and
1300
+ // must not be deleted/overwritten by the consolidate LLM. 14 user
1301
+ // memories were silent-deleted by consolidate before this guard landed;
1302
+ // recovery required copying from .akm/archive/ by hand.
1303
+ const mergeParticipants = [op.primary, ...op.secondaries];
1304
+ const blockedParticipants = mergeParticipants.flatMap((ref) => {
1305
+ const e = memoryByRef.get(ref);
1306
+ if (!e)
1307
+ return [];
1308
+ const verdict = consolidateGuardStatus(e.filePath);
1309
+ if (verdict === "hot" || verdict === "unparseable")
1310
+ return [{ ref, verdict }];
1311
+ return [];
1312
+ });
1313
+ if (blockedParticipants.length > 0) {
1314
+ const detail = blockedParticipants.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1315
+ warnings.push(`Merge: refused for ${op.primary} — ${blockedParticipants.length} participant(s) blocked by hot/unparseable frontmatter guard: ${detail}`);
1316
+ emitMergeFailureSkips("merge_participant_blocked");
605
1317
  continue;
606
1318
  }
607
1319
  // Backup secondaries before deleting
@@ -618,6 +1330,7 @@ export async function akmConsolidate(opts = {}) {
618
1330
  }
619
1331
  catch (e) {
620
1332
  warnings.push(`Merge: write failed for ${op.primary}: ${String(e)}`);
1333
+ emitMergeFailureSkips("merge_write_failed");
621
1334
  continue;
622
1335
  }
623
1336
  // Archive and delete secondaries (P1-B: soft-invalidation)
@@ -639,11 +1352,45 @@ export async function akmConsolidate(opts = {}) {
639
1352
  }
640
1353
  markJournalCompleted(stashDir, op.primary);
641
1354
  merged++;
1355
+ // 2026-05-26 accounting-leak fix: `merged` is op-level, but each
1356
+ // successful merge actions `1 + secondaries.length` memories. Without
1357
+ // this counter the accounting invariant breaks by `secondaries.length`
1358
+ // per successful merge (chunk loop excluded all secondaries from
1359
+ // judgedNoAction via targetRefs, but only the primary is credited to
1360
+ // `merged`). Count only loaded-memory secondaries; phantom secondary
1361
+ // refs never affected any chunk's targetRefs in the first place.
1362
+ for (const secRef of op.secondaries) {
1363
+ if (memoryByRef.has(secRef))
1364
+ mergedSecondaries++;
1365
+ }
1366
+ // Prune consumed refs from memoryByRef so later ops in this run cannot
1367
+ // reference an absorbed secondary as a merge primary and proceed with a
1368
+ // stale entry. Primary is rewritten (not deleted), so we only remove
1369
+ // secondaries; the primary ref remains valid under its new content.
1370
+ for (const secRef of op.secondaries) {
1371
+ memoryByRef.delete(secRef);
1372
+ }
642
1373
  }
643
1374
  else if (op.op === "delete") {
644
1375
  const entry = memoryByRef.get(op.ref);
645
1376
  if (!entry) {
646
1377
  warnings.push(`Delete: ${op.ref} not found in loaded memories — skipping.`);
1378
+ // Phantom ref: not in the batch so not in processed. Pushing to
1379
+ // skipReasons would inflate Σ(skipReasons) without a matching processed
1380
+ // entry, breaking the accounting invariant. Visibility is preserved via
1381
+ // the warnings array above.
1382
+ continue;
1383
+ }
1384
+ // captureMode:hot guard — refuse to delete user-captured memories OR
1385
+ // memories whose frontmatter is unparseable (could have hidden the hot
1386
+ // flag). The consolidate LLM was deleting hot-captured user memos as
1387
+ // "redundant" — 14 such deletes were silently archived between
1388
+ // 2026-05-19 and 2026-05-20 before this guard. Hot memories are
1389
+ // user-explicit and may only be deleted by the user.
1390
+ const guard = consolidateGuardStatus(entry.filePath);
1391
+ if (guard === "hot" || guard === "unparseable") {
1392
+ warnings.push(`Delete: refused for ${op.ref} — ${guard === "hot" ? "captureMode:hot (user-explicit; never auto-delete)" : "frontmatter unparseable (cannot verify hot flag absent)"}. Reason from LLM: "${op.reason ?? "n/a"}"`);
1393
+ pushSkipReason("delete", op.ref, "captureMode_hot_refused");
647
1394
  continue;
648
1395
  }
649
1396
  if (fs.existsSync(entry.filePath)) {
@@ -656,15 +1403,40 @@ export async function akmConsolidate(opts = {}) {
656
1403
  await deleteAssetFromSource(target.source, target.config, parsedRef);
657
1404
  markJournalCompleted(stashDir, op.ref);
658
1405
  deleted++;
1406
+ // Prune from memoryByRef so later ops in this run cannot reference a
1407
+ // deleted memory as a merge primary or secondary.
1408
+ memoryByRef.delete(op.ref);
659
1409
  }
660
1410
  catch (e) {
661
- warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
1411
+ // Distinguish "file already absent" from genuine failures. A prior run
1412
+ // may have deleted the file but the DB was not yet re-indexed, so the
1413
+ // ref still appeared in memoryByRef. The delete goal is already met.
1414
+ const msg = e instanceof Error ? e.message : String(e);
1415
+ if (msg.includes("not found in source")) {
1416
+ warnings.push(`Delete: ${op.ref} — file already absent (stale DB entry); skipping.`);
1417
+ pushSkipReason("delete", op.ref, "delete_already_gone");
1418
+ }
1419
+ else {
1420
+ warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
1421
+ pushSkipReason("delete", op.ref, "delete_failed");
1422
+ }
662
1423
  }
663
1424
  }
664
1425
  else if (op.op === "promote") {
665
1426
  const entry = memoryByRef.get(op.ref);
666
1427
  if (!entry) {
667
1428
  warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
1429
+ // Phantom ref: not in processed, so no skipReason (same rationale as
1430
+ // delete_ref_missing above).
1431
+ continue;
1432
+ }
1433
+ // Within-run source-ref dedup: skip if this source memory was already
1434
+ // promoted earlier in this run (safety belt — mergePlans already
1435
+ // deduplicates promote ops by source ref via Map, but this guard also
1436
+ // catches any future code paths that bypass mergePlans).
1437
+ if (promotedSourceRefs.has(op.ref)) {
1438
+ warnings.push(`Skipping promote: ${op.ref} already promoted in this run`);
1439
+ pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
668
1440
  continue;
669
1441
  }
670
1442
  let knowledgeRef = op.knowledgeRef;
@@ -679,10 +1451,11 @@ export async function akmConsolidate(opts = {}) {
679
1451
  knowledgeRef = `knowledge:${slug}`;
680
1452
  warnings.push(`Normalized invalid ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
681
1453
  }
682
- // Idempotency: check pending proposals
1454
+ // Idempotency: check pending proposals by target ref
683
1455
  const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
684
1456
  if (existingProposals.some((p) => p.status === "pending")) {
685
1457
  warnings.push(`Skipping promote: pending proposal already exists for ${knowledgeRef}`);
1458
+ pushSkipReason("promote", op.ref, "promote_pending_proposal_exists");
686
1459
  continue;
687
1460
  }
688
1461
  // Idempotency: check if knowledge asset already exists
@@ -690,6 +1463,7 @@ export async function akmConsolidate(opts = {}) {
690
1463
  const destPath = path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`);
691
1464
  if (fs.existsSync(destPath)) {
692
1465
  warnings.push(`Skipping promote: ${knowledgeRef} already exists in source`);
1466
+ pushSkipReason("promote", op.ref, "promote_already_exists");
693
1467
  continue;
694
1468
  }
695
1469
  let memoryContent = "";
@@ -698,24 +1472,173 @@ export async function akmConsolidate(opts = {}) {
698
1472
  }
699
1473
  catch (e) {
700
1474
  warnings.push(`Promote: could not read ${op.ref}: ${String(e)}`);
1475
+ pushSkipReason("promote", op.ref, "promote_read_failed");
1476
+ continue;
1477
+ }
1478
+ // Defensive sanitization: legacy memory files written by older
1479
+ // consolidate runs may still carry outer code fences or broken YAML.
1480
+ // Strip them here so we never propose a polluted asset.
1481
+ const promoteSanitized = sanitizeMergedContent(memoryContent);
1482
+ if (!promoteSanitized.ok) {
1483
+ warnings.push(`Promote: rejected ${op.ref} — source memory failed sanitization (${promoteSanitized.reason}).`);
1484
+ pushSkipReason("promote", op.ref, "promote_sanitization_failed");
1485
+ continue;
1486
+ }
1487
+ memoryContent = promoteSanitized.result.content;
1488
+ // SOURCE_SUPERSEDED guard: refuse to promote a memory whose source
1489
+ // frontmatter carries `status: superseded`. Predicate at module top
1490
+ // (`hasSupersededStatus`) so tests can exercise it directly.
1491
+ if (hasSupersededStatus(promoteSanitized.result.frontmatter)) {
1492
+ warnings.push(`Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
1493
+ pushSkipReason("promote", op.ref, "promote_superseded");
1494
+ continue;
1495
+ }
1496
+ // Parse the source memory up-front so the body/frontmatter checks below
1497
+ // share the same parsed view.
1498
+ const parsedMemory = parseFrontmatter(memoryContent);
1499
+ // Reject sources whose body is too small to make useful knowledge.
1500
+ // Observed failure: memory files whose body is literally a tags string
1501
+ // ("discord,notification,send-notification") get promoted to knowledge
1502
+ // proposals that no reviewer would accept. Threshold is conservative —
1503
+ // 100 chars catches single-line tag dumps without rejecting genuinely
1504
+ // terse but valid notes.
1505
+ const PROMOTE_BODY_MIN_CHARS = 100;
1506
+ const sourceBody = parsedMemory.content.trim();
1507
+ if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
1508
+ warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
1509
+ pushSkipReason("promote", op.ref, "promote_source_too_small");
1510
+ continue;
1511
+ }
1512
+ // Cross-run + within-run content dedup: if an identical body already
1513
+ // exists in ANY pending consolidate proposal (regardless of target ref),
1514
+ // skip. This prevents duplicate proposals when:
1515
+ // (a) Multiple source memories have identical bodies but differ only
1516
+ // in noise frontmatter (`inferenceProcessed: true` twin alongside
1517
+ // the original; differing `updated:` timestamps; etc.) — the body
1518
+ // is the load-bearing content, so dedup must hash on body only.
1519
+ // (b) A prior run created a proposal for the same body under a
1520
+ // different knowledgeRef slug.
1521
+ const bodyHash = createHash("sha256").update(sourceBody, "utf8").digest("hex");
1522
+ const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
1523
+ const contentDupProposal = allPendingConsolidateProposals.find((p) => {
1524
+ const otherBody = parseFrontmatter(p.payload.content).content.trim();
1525
+ return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
1526
+ });
1527
+ if (contentDupProposal) {
1528
+ warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
1529
+ pushSkipReason("promote", op.ref, "dedup_pending_proposal");
701
1530
  continue;
702
1531
  }
703
1532
  try {
704
- const proposal = createProposal(stashDir, {
1533
+ // Use LLM-provided description; fall back to memory's own description
1534
+ // (post-sanitization frontmatter is authoritative).
1535
+ const description = (typeof op.description === "string" && op.description.trim()
1536
+ ? op.description.trim()
1537
+ : parsedMemory.data?.description?.trim()) ?? "";
1538
+ // Validate the resolved frontmatter before emitting a proposal.
1539
+ // Required field: non-empty description. Reject obvious truncation
1540
+ // markers (description ends with `,`/`;`/`:`/`...`/hanging connector)
1541
+ // so the queue never sees half-formed metadata that the reviewer
1542
+ // would only reject.
1543
+ const fmCheck = validateProposalFrontmatter({ description });
1544
+ if (!fmCheck.ok) {
1545
+ warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
1546
+ pushSkipReason("promote", op.ref, "promote_invalid_frontmatter");
1547
+ continue;
1548
+ }
1549
+ // Merge `description` INTO the body's YAML frontmatter so it lands in
1550
+ // the on-disk asset when the proposal is accepted. The descriptionQuality
1551
+ // validator parses `payload.content` body (not the envelope
1552
+ // `payload.frontmatter`), and a memory's native frontmatter has
1553
+ // `captureMode`/`beliefState`/etc. but never `description` — without
1554
+ // this merge, 60+ pending proposals were blocked at accept-time with
1555
+ // MISSING_FRONTMATTER_DESCRIPTION even though the envelope had it.
1556
+ // (The body-frontmatter assumption baked into the 2026-05-20 comment
1557
+ // below was wrong: body fm and envelope fm only converge when the
1558
+ // writer explicitly merges them, which it now does.)
1559
+ const mergedBodyFm = {
1560
+ ...(parsedMemory.data ?? {}),
1561
+ description,
1562
+ };
1563
+ const serializedMergedFm = yamlStringify(mergedBodyFm).trimEnd();
1564
+ const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
1565
+ // Pre-emit dedup against pending consolidate proposals from the
1566
+ // same improve run (slug-variant match). The cross-run content-hash
1567
+ // dedup inside `mergePlans` handles duplicates against existing
1568
+ // stash assets — see commit history for the deletion of the
1569
+ // unbounded embedding + cross-type slug branches.
1570
+ const dedup = await checkPreEmitDedup({
1571
+ candidateRef: knowledgeRef,
1572
+ candidateText: `${description}. ${memoryContent}`,
1573
+ stashDir,
1574
+ config,
1575
+ });
1576
+ if (dedup.duplicate) {
1577
+ warnings.push(`Promote: skipped ${op.ref} → ${knowledgeRef} — ${dedup.reason}.`);
1578
+ pushSkipReason("promote", op.ref, "promote_dedup_window");
1579
+ continue;
1580
+ }
1581
+ const proposalResult = createProposal(stashDir, {
705
1582
  ref: knowledgeRef,
706
1583
  source: "consolidate",
707
- payload: { content: memoryContent },
1584
+ sourceRun,
1585
+ payload: {
1586
+ content: proposalContent,
1587
+ frontmatter: { description },
1588
+ },
1589
+ ...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
708
1590
  });
709
- promoted.push(proposal.id);
710
- markJournalCompleted(stashDir, op.ref);
1591
+ if (isProposalSkipped(proposalResult)) {
1592
+ warnings.push(`Promote: skipped proposal for ${op.ref} (${proposalResult.reason}): ${proposalResult.message}`);
1593
+ pushSkipReason("promote", op.ref, `promote_proposal_${proposalResult.reason}`);
1594
+ }
1595
+ else {
1596
+ promoted.push(proposalResult.id);
1597
+ promotedSourceRefs.add(op.ref);
1598
+ markJournalCompleted(stashDir, op.ref);
1599
+ }
711
1600
  }
712
1601
  catch (e) {
713
1602
  warnings.push(`Promote: createProposal failed for ${op.ref}: ${String(e)}`);
1603
+ pushSkipReason("promote", op.ref, "promote_create_failed");
1604
+ }
1605
+ }
1606
+ else if (op.op === "contradict") {
1607
+ // C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
1608
+ // (the SCC resolver in memory-improve.ts) has edges to work on.
1609
+ // Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
1610
+ const entry = memoryByRef.get(op.ref);
1611
+ const contradictorEntry = memoryByRef.get(op.contradictedByRef);
1612
+ if (!entry) {
1613
+ warnings.push(`Contradict: ${op.ref} not found in loaded memories — skipping.`);
1614
+ // Phantom ref: not in processed, so no skipReason (same rationale as
1615
+ // delete_ref_missing).
1616
+ continue;
1617
+ }
1618
+ if (!contradictorEntry) {
1619
+ warnings.push(`Contradict: ${op.contradictedByRef} not found — skipping.`);
1620
+ // op.ref IS in the batch (entry found above) so the skipReason is
1621
+ // correctly charged against a real processed memory.
1622
+ pushSkipReason("contradict", op.ref, "contradict_target_missing");
1623
+ continue;
1624
+ }
1625
+ try {
1626
+ // Write the contradiction edge: op.ref is contradicted by op.contradictedByRef
1627
+ writeContradictEdge(entry.filePath, op.contradictedByRef);
1628
+ contradicted++;
1629
+ markJournalCompleted(stashDir, op.ref);
1630
+ }
1631
+ catch (e) {
1632
+ warnings.push(`Contradict: failed to write edge for ${op.ref}: ${String(e)}`);
1633
+ pushSkipReason("contradict", op.ref, "contradict_write_failed");
714
1634
  }
715
1635
  }
716
1636
  }
717
1637
  cleanupJournal(stashDir, timestamp);
718
- // TTL cleanup: remove archive entries older than archiveRetentionDays (default 90)
1638
+ // TTL cleanup: remove archive entries older than archiveRetentionDays (default 90).
1639
+ // C-5 / #391: emit an `archive_cleanup` event before each deletion so the
1640
+ // audit trail records what was lost. Outbox pattern (EIP, Hohpe-Woolf) —
1641
+ // any event that is recorded must be queryable; silent deletes are an anti-pattern.
719
1642
  const archiveDir = path.join(stashDir, ".akm", "archive");
720
1643
  if (fs.existsSync(archiveDir)) {
721
1644
  const retentionMs = (config.archiveRetentionDays ?? 90) * 86_400_000;
@@ -723,8 +1646,20 @@ export async function akmConsolidate(opts = {}) {
723
1646
  for (const fname of fs.readdirSync(archiveDir)) {
724
1647
  const fp = path.join(archiveDir, fname);
725
1648
  try {
726
- if (fs.statSync(fp).mtimeMs < cutoff)
1649
+ const stat = fs.statSync(fp);
1650
+ if (stat.mtimeMs < cutoff) {
1651
+ // Emit event before deletion so the record survives the purge.
1652
+ appendEvent({
1653
+ eventType: "archive_cleanup",
1654
+ metadata: {
1655
+ file: fname,
1656
+ filePath: fp,
1657
+ ageMs: Date.now() - stat.mtimeMs,
1658
+ retentionMs,
1659
+ },
1660
+ });
727
1661
  fs.unlinkSync(fp);
1662
+ }
728
1663
  }
729
1664
  catch {
730
1665
  /* ignore race conditions */
@@ -742,57 +1677,465 @@ export async function akmConsolidate(opts = {}) {
742
1677
  merged,
743
1678
  deleted,
744
1679
  promoted,
1680
+ contradicted,
1681
+ failedChunks: totalChunksFailed,
1682
+ totalChunks: chunks.length,
1683
+ judgedNoAction,
1684
+ skipReasons,
1685
+ mergedSecondaries,
1686
+ failedChunkMemories,
745
1687
  warnings,
746
1688
  durationMs: Date.now() - startMs,
747
1689
  };
748
1690
  }
749
1691
  // ── Helpers ─────────────────────────────────────────────────────────────────
1692
+ // ── LLM-output sanitization ─────────────────────────────────────────────────
1693
+ //
1694
+ // Three classes of LLM defect have been observed across hundreds of
1695
+ // consolidate proposals (see audit notes in this branch):
1696
+ //
1697
+ // 1. Code-fence leakage: the entire merged asset is wrapped in
1698
+ // ```markdown … ``` (or ```yaml … ```) despite the prompt forbidding
1699
+ // fences. The post-processor used to pass this through verbatim, so the
1700
+ // first character of the asset content became a backtick rather than
1701
+ // `---`, defeating the frontmatter parser.
1702
+ // 2. YAML quote-escaping bugs: descriptions like `'"Specialty intro...:`
1703
+ // with unbalanced quotes that break the YAML reader. The post-processor
1704
+ // historically passed the LLM's raw scalar straight into a manually
1705
+ // assembled `description: <raw>` line.
1706
+ // 3. Truncated descriptions hitting token cutoffs — the model's max_tokens
1707
+ // runs out mid-sentence, leaving things like
1708
+ // `description: "Tables in narrow column containers need max-width:100% +"`
1709
+ // with no closing context.
1710
+ //
1711
+ // `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
1712
+ // all three at the point where LLM output is consumed.
1713
+ /**
1714
+ * Attempt to recover a frontmatter block that is missing its closing `---`.
1715
+ *
1716
+ * Scans lines after the opening `---` for the first blank line or the first
1717
+ * line that cannot be a YAML scalar (i.e. not a key-value, indented
1718
+ * continuation, comment, or list item). Injects `---` before that line so
1719
+ * the normal parser can proceed.
1720
+ *
1721
+ * Returns the patched string on success, or `null` if the structure is too
1722
+ * ambiguous to recover safely (e.g. no opening `---`, or no body content
1723
+ * found after the frontmatter key-value lines).
1724
+ */
1725
+ function recoverMalformedFrontmatter(raw) {
1726
+ if (!raw.startsWith("---"))
1727
+ return null;
1728
+ const lines = raw.split(/\r?\n/);
1729
+ // Skip the opening `---` line (index 0).
1730
+ let insertAt = -1;
1731
+ for (let i = 1; i < lines.length; i++) {
1732
+ const line = lines[i];
1733
+ // A blank line marks the end of the frontmatter block in many YAML variants.
1734
+ if (line.trim() === "") {
1735
+ insertAt = i;
1736
+ break;
1737
+ }
1738
+ // A line that is clearly body content: doesn't look like a YAML key, an
1739
+ // indented continuation, a comment, or a sequence item.
1740
+ const isYaml = /^\w[\w-]*\s*:/.test(line) || // key: value
1741
+ /^\s+\S/.test(line) || // indented continuation / nested
1742
+ /^\s*#/.test(line) || // YAML comment
1743
+ /^\s*-\s/.test(line); // sequence item
1744
+ if (!isYaml) {
1745
+ insertAt = i;
1746
+ break;
1747
+ }
1748
+ }
1749
+ if (insertAt < 0)
1750
+ return null;
1751
+ const result = [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
1752
+ return result;
1753
+ }
1754
+ /**
1755
+ * Outer-fence stripper specific to consolidate. Unlike the shared
1756
+ * `stripMarkdownFences` helper (which only handles markdown fences), this
1757
+ * variant additionally recognises `yaml` and bare-language fences and refuses
1758
+ * to strip an unbalanced fence — i.e. a leading ``` with no trailing ``` is
1759
+ * treated as a malformed response, not partially sanitized.
1760
+ *
1761
+ * Returns `null` when only one half of a fence pair is present (caller
1762
+ * should reject the response entirely).
1763
+ */
1764
+ export function stripOuterCodeFence(raw) {
1765
+ const trimmed = raw.trim();
1766
+ const leading = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n/i);
1767
+ const trailing = trimmed.match(/\r?\n```\s*$/);
1768
+ if (!leading && !trailing)
1769
+ return { content: trimmed, stripped: false };
1770
+ if (!leading || !trailing)
1771
+ return null; // unbalanced — refuse
1772
+ const inner = trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim();
1773
+ return { content: inner, stripped: true };
1774
+ }
1775
+ export function sanitizeMergedContent(raw) {
1776
+ // Step 1: Strip outer code fence.
1777
+ // Recovery path: if only the leading fence is present, strip it and continue
1778
+ // provided the inner content starts with `---`. Trailing-only fences are NOT
1779
+ // recovered — a trailing ``` is more likely a body code block than a forgotten
1780
+ // wrapper, so recovering would silently corrupt the body.
1781
+ let body;
1782
+ {
1783
+ const fenceResult = stripOuterCodeFence(raw);
1784
+ if (fenceResult) {
1785
+ body = fenceResult.content;
1786
+ }
1787
+ else {
1788
+ const trimmed = raw.trim();
1789
+ const leadingMatch = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i);
1790
+ const inner = leadingMatch ? leadingMatch[1].trim() : null;
1791
+ if (!inner?.startsWith("---")) {
1792
+ return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
1793
+ }
1794
+ body = inner;
1795
+ }
1796
+ }
1797
+ // Strip <think> blocks (some local models still emit them despite system prompts).
1798
+ body = body.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
1799
+ // Step 2: Verify frontmatter sentinel.
1800
+ // Recovery path: LLM sometimes emits 1-2 lines of preamble (e.g. "Here is the
1801
+ // merged content:") before the `---`. Accept if `---` appears within 300 chars.
1802
+ // Beyond that it's more likely a body section divider, not a frontmatter start.
1803
+ if (!body.startsWith("---")) {
1804
+ const nlIdx = body.indexOf("\n---");
1805
+ if (nlIdx >= 0 && nlIdx < 300) {
1806
+ body = body.slice(nlIdx + 1);
1807
+ }
1808
+ else {
1809
+ return { ok: false, reason: "MISSING_FRONTMATTER_SENTINEL" };
1810
+ }
1811
+ }
1812
+ // Extract frontmatter block.
1813
+ // Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
1814
+ // by scanning lines after the opening `---` for the first blank line or the
1815
+ // first line that isn't a YAML key-value pair, then inject `---` there.
1816
+ let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1817
+ if (!match) {
1818
+ const recovered = recoverMalformedFrontmatter(body);
1819
+ if (recovered) {
1820
+ match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1821
+ }
1822
+ if (!match) {
1823
+ return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
1824
+ }
1825
+ }
1826
+ // Re-parse via the yaml library so any quote-escaping mistakes either get
1827
+ // normalised or surface as a parse error we can reject.
1828
+ // Recovery: if the strict yaml library fails, fall back to the lenient
1829
+ // hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
1830
+ // quirks (unescaped special chars, bare scalars, etc.). If it recovers
1831
+ // at least one key, proceed — yamlStringify below will re-serialize
1832
+ // cleanly. Only reject if both parsers fail to extract any data.
1833
+ let parsedFm;
1834
+ try {
1835
+ parsedFm = yamlParse(match[1]);
1836
+ }
1837
+ catch (e) {
1838
+ const fallback = parseFrontmatter(`---\n${match[1]}\n---\n${match[2]}`);
1839
+ if (fallback.frontmatter !== null && Object.keys(fallback.data).length > 0) {
1840
+ parsedFm = fallback.data;
1841
+ }
1842
+ else {
1843
+ return { ok: false, reason: `INVALID_YAML: ${e instanceof Error ? e.message : String(e)}` };
1844
+ }
1845
+ }
1846
+ if (parsedFm === null || typeof parsedFm !== "object" || Array.isArray(parsedFm)) {
1847
+ return { ok: false, reason: "FRONTMATTER_NOT_OBJECT" };
1848
+ }
1849
+ const fm = parsedFm;
1850
+ // Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
1851
+ // `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
1852
+ // these, but small models still do. Replace any such leak with today's ISO
1853
+ // date OR drop the field if we can't safely normalise it.
1854
+ normalizeUpdatedField(fm);
1855
+ // Re-serialise via yaml.stringify to fix any quoting quirks.
1856
+ let serialized;
1857
+ try {
1858
+ serialized = yamlStringify(fm).trimEnd();
1859
+ }
1860
+ catch (e) {
1861
+ return { ok: false, reason: `YAML_STRINGIFY_FAILED: ${e instanceof Error ? e.message : String(e)}` };
1862
+ }
1863
+ const cleaned = assembleAssetFromString(serialized, match[2]);
1864
+ return { ok: true, result: { content: cleaned, frontmatter: fm } };
1865
+ }
1866
+ /**
1867
+ * Mutate `fm.updated` in place to normalise placeholder leaks emitted by the
1868
+ * LLM. The consolidate prompt forbids these, but small models still produce
1869
+ * literal `today` / `{today: null}` / `now` values.
1870
+ *
1871
+ * Rules:
1872
+ * - A real ISO-style date string (YYYY-MM-DD, optionally with time) stays as-is.
1873
+ * - A Date object (some YAML parsers materialise dates) is converted to its
1874
+ * ISO yyyy-mm-dd form.
1875
+ * - A placeholder string ("today", "now", "{today}", "${today}", template
1876
+ * variables) is replaced with today's ISO date.
1877
+ * - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
1878
+ * - `null`, empty string, missing → left alone (no field added; reviewers
1879
+ * should not silently gain metadata they didn't write).
1880
+ *
1881
+ * Exported for unit testing.
1882
+ */
1883
+ export function normalizeUpdatedField(fm) {
1884
+ if (!("updated" in fm))
1885
+ return;
1886
+ const v = fm.updated;
1887
+ if (v === null || v === undefined || v === "")
1888
+ return;
1889
+ const todayIso = new Date().toISOString().slice(0, 10);
1890
+ if (v instanceof Date) {
1891
+ fm.updated = v.toISOString().slice(0, 10);
1892
+ return;
1893
+ }
1894
+ if (typeof v === "string") {
1895
+ const trimmed = v.trim().toLowerCase();
1896
+ if (/^\d{4}-\d{2}-\d{2}/.test(v.trim()))
1897
+ return; // already a real date
1898
+ if (trimmed === "today" ||
1899
+ trimmed === "now" ||
1900
+ trimmed === "{today}" ||
1901
+ // biome-ignore lint/suspicious/noTemplateCurlyInString: matches the literal user-typed placeholder text "${today}" so we can normalize it to today's ISO date
1902
+ trimmed === "${today}" ||
1903
+ trimmed === "{{today}}" ||
1904
+ /^\{?\s*today\s*\}?$/.test(trimmed)) {
1905
+ fm.updated = todayIso;
1906
+ return;
1907
+ }
1908
+ // Unknown string format — leave alone so it's visible in the diff.
1909
+ return;
1910
+ }
1911
+ if (typeof v === "object") {
1912
+ // Maps like `{today: null}`, `{now: null}` — clearly a template leak.
1913
+ fm.updated = todayIso;
1914
+ return;
1915
+ }
1916
+ }
1917
+ /**
1918
+ * Normalise a knowledge slug for variant-aware deduplication. Collapses:
1919
+ * - date suffixes (`-may-2026`, `-2026-05-03`, `-2026`)
1920
+ * - numeric counter suffixes (`-2`, `-3`)
1921
+ * - trailing -patterns / -2026-05-03 styles
1922
+ * - word reorderings via alphabetical sort of the remaining tokens.
1923
+ *
1924
+ * Two slugs that normalise to the same string are considered the same asset
1925
+ * for dedup purposes even if they don't share an exact ref.
1926
+ */
1927
+ export function normalizeSlugForDedup(ref) {
1928
+ const slug = ref.replace(/^[^:]+:/, "");
1929
+ const monthRe = /(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i;
1930
+ const tokens = slug
1931
+ .toLowerCase()
1932
+ .split("-")
1933
+ .filter((tok) => tok.length > 0)
1934
+ // Strip purely-numeric tokens (years, dates, counter suffixes like -2 / -3).
1935
+ // Numbers carry no semantic information for our dedup purposes — every
1936
+ // observed defective slug variant differs only in dates or counters.
1937
+ .filter((tok) => !/^\d+$/.test(tok))
1938
+ .filter((tok) => !monthRe.test(tok));
1939
+ // Sort to absorb word reorderings.
1940
+ tokens.sort();
1941
+ return tokens.join("-");
1942
+ }
1943
+ /**
1944
+ * Pre-emit dedup check: compare the candidate ref against pending consolidate
1945
+ * proposals only. Returns a reason string if a slug-variant match is found,
1946
+ * else null.
1947
+ *
1948
+ * Historical context (REMOVED 2026-05-20): this function previously also ran
1949
+ * (a) a normalised-slug match against existing knowledge AND memory entries
1950
+ * in the DB, and
1951
+ * (b) an embedding cosine-similarity check (>= 0.85) against ALL knowledge
1952
+ * and non-derived memory entries.
1953
+ * Both branches had ZERO observed fires across 30 sampled runs in the
1954
+ * post-fix window. The 29 actual dedup catches all came from the SEPARATE
1955
+ * content-hash dedup inside `mergePlans` (the older SHA-256 helper). The
1956
+ * embedding branch in particular had unbounded cost per promote (embedded
1957
+ * every knowledge + non-derived memory entry, every time) with no observed
1958
+ * benefit. Empirical signal → deleted.
1959
+ *
1960
+ * What remains: a check against pending consolidate proposals in the SAME
1961
+ * improve run. This catches duplicates queued back-to-back within a single
1962
+ * improve invocation — a different concern from the cross-run content-hash
1963
+ * dedup, and cheap (no embeddings, no DB query).
1964
+ */
1965
+ export async function checkPreEmitDedup(opts) {
1966
+ const normCandidate = normalizeSlugForDedup(opts.candidateRef);
1967
+ // Pending consolidate proposals (slug match) — within the same improve run.
1968
+ const pendingConsolidate = listProposals(opts.stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
1969
+ for (const p of pendingConsolidate) {
1970
+ if (normalizeSlugForDedup(p.ref) === normCandidate) {
1971
+ return { duplicate: true, reason: `slug-variant of pending proposal ${p.id} (${p.ref})` };
1972
+ }
1973
+ }
1974
+ return { duplicate: false };
1975
+ }
1976
+ /**
1977
+ * Incremental candidate set: {changed} ∪ {top-k persisted-vector neighbours of
1978
+ * each changed memory}, intersected with the loaded pool. Returns [] when
1979
+ * nothing changed (caller emits a no-op envelope), the full pool when
1980
+ * everything changed or the index can't answer (fail-open to preserve merge
1981
+ * correctness). `since` is an ISO timestamp.
1982
+ */
1983
+ export function narrowToIncrementalCandidates(memories, since, warnings) {
1984
+ const isChanged = (m) => {
1985
+ try {
1986
+ return fs.statSync(m.filePath).mtime.toISOString() > since;
1987
+ }
1988
+ catch {
1989
+ return true; // never silently drop a memory we cannot stat
1990
+ }
1991
+ };
1992
+ const changed = memories.filter(isChanged);
1993
+ if (changed.length === 0)
1994
+ return [];
1995
+ if (changed.length === memories.length)
1996
+ return memories;
1997
+ const NEIGHBORS_PER_CHANGED = 5;
1998
+ const byName = new Map(memories.map((m) => [m.name, m]));
1999
+ const keep = new Set(changed.map((m) => m.name));
2000
+ let db;
2001
+ try {
2002
+ db = openExistingDatabase();
2003
+ for (const m of changed) {
2004
+ const id = findEntryIdByRef(db, `memory:${m.name}`);
2005
+ if (id === undefined)
2006
+ continue;
2007
+ for (const hit of getNeighborsByEntryId(db, id, NEIGHBORS_PER_CHANGED + 1)) {
2008
+ if (hit.id === id)
2009
+ continue;
2010
+ const entry = getEntryById(db, hit.id);
2011
+ if (!entry)
2012
+ continue;
2013
+ const name = entry.entry.name;
2014
+ if (byName.has(name))
2015
+ keep.add(name); // only neighbours present in the loaded pool
2016
+ }
2017
+ }
2018
+ }
2019
+ catch {
2020
+ warnings.push("Incremental consolidation: index unavailable — processing full pool.");
2021
+ return memories;
2022
+ }
2023
+ finally {
2024
+ if (db)
2025
+ closeDatabase(db);
2026
+ }
2027
+ const candidates = memories.filter((m) => keep.has(m.name));
2028
+ warnings.push(`Incremental consolidation: ${changed.length} changed + neighbours → ${candidates.length}/${memories.length} memories considered (since ${since}).`);
2029
+ return candidates;
2030
+ }
750
2031
  function loadMemoriesForSource(source, stashDir, warnings) {
751
- let memories = loadMemoriesFromDb(source ? resolveSourcePath(source) : undefined);
2032
+ // Load from DB first
2033
+ let memories = [];
2034
+ let db;
2035
+ try {
2036
+ db = openExistingDatabase();
2037
+ const entries = getAllEntries(db, "memory");
2038
+ memories = entries
2039
+ .filter((e) => {
2040
+ if (!source)
2041
+ return true;
2042
+ return path.resolve(e.stashDir) === path.resolve(source);
2043
+ })
2044
+ .filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
2045
+ // Skip stale DB entries whose file was deleted by a prior run but not yet
2046
+ // re-indexed. Without this guard the deleted file's ref appears in chunks
2047
+ // sent to the LLM, which then proposes a second delete → delete_failed
2048
+ // because the file is already gone. Re-indexing runs on a cron cadence so
2049
+ // several successful deletes can accumulate before the DB catches up.
2050
+ .filter((e) => fs.existsSync(e.filePath))
2051
+ .map((e) => ({
2052
+ name: e.entry.name,
2053
+ filePath: e.filePath,
2054
+ description: e.entry.description ?? "",
2055
+ tags: e.entry.tags ?? [],
2056
+ stashDir: e.stashDir,
2057
+ }));
2058
+ }
2059
+ catch {
2060
+ memories = [];
2061
+ }
2062
+ finally {
2063
+ if (db)
2064
+ closeDatabase(db);
2065
+ }
752
2066
  if (memories.length === 0) {
753
2067
  // DB fallback: walk filesystem
754
2068
  const memoriesDir = path.join(source ?? stashDir, "memories");
755
- memories = loadMemoriesFromFs(memoriesDir, source ?? stashDir);
2069
+ const fsStashDir = source ?? stashDir;
2070
+ if (fs.existsSync(memoriesDir)) {
2071
+ for (const fname of fs.readdirSync(memoriesDir)) {
2072
+ if (!fname.endsWith(".md"))
2073
+ continue;
2074
+ const filePath = path.join(memoriesDir, fname);
2075
+ const name = fname.replace(/\.md$/, "");
2076
+ if (!isConsolidationEligibleMemoryName(name))
2077
+ continue;
2078
+ memories.push({ name, filePath, description: "", tags: [], stashDir: fsStashDir });
2079
+ }
2080
+ }
756
2081
  if (memories.length > 0) {
757
2082
  warnings.push("DB not found or empty — loaded memories directly from filesystem.");
758
2083
  }
759
2084
  }
760
2085
  return memories;
761
2086
  }
762
- function resolveSourcePath(sourceName) {
763
- // If it looks like an absolute path, use directly
764
- if (path.isAbsolute(sourceName))
765
- return sourceName;
766
- return sourceName;
767
- }
768
- async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef, warnings) {
2087
+ async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef) {
769
2088
  // Only handle single-secondary merges per design (one call per merge op)
770
2089
  const secRef = secondaryRefs[0];
771
2090
  const secEntry = memoryByRef.get(secRef);
772
2091
  if (!secEntry)
773
- return null;
2092
+ return { error: "merge_read_failed", detail: `secondary ${secRef} not in memoryByRef` };
774
2093
  let secBody = "";
775
2094
  try {
776
2095
  secBody = fs.readFileSync(secEntry.filePath, "utf8");
777
2096
  }
778
2097
  catch {
779
- warnings.push(`Merge: could not read secondary ${secRef} — skipping.`);
780
- return null;
2098
+ return { error: "merge_read_failed", detail: `could not read secondary ${secRef}` };
781
2099
  }
2100
+ const primaryFmKeys = Object.keys(parseFrontmatter(primaryBody).data);
2101
+ const secFmKeys = Object.keys(parseFrontmatter(secBody).data);
2102
+ const requiredFmKeys = [...new Set([...primaryFmKeys, ...secFmKeys])];
782
2103
  const prompt = [
783
2104
  "Merge these two memory assets into one. Output ONLY the merged markdown (with YAML frontmatter). Do not explain, do not use code fences.",
784
2105
  "",
2106
+ "## OUTPUT FORMAT (MANDATORY)",
2107
+ "Return raw markdown content beginning DIRECTLY with the `---` frontmatter delimiter.",
2108
+ "DO NOT wrap your entire response in a code fence.",
2109
+ "",
2110
+ 'GOOD: "---\\ndescription: ...\\n---\\nBody content."',
2111
+ 'BAD: "```markdown\\n---\\ndescription: ...\\n---\\nBody content.\\n```"',
2112
+ 'BAD: "```yaml\\n---\\ndescription: ...\\n---\\nBody content.\\n```"',
2113
+ "",
2114
+ "## FRONTMATTER RULES (MANDATORY)",
2115
+ "- The `updated:` field, if present, MUST be a real ISO date (e.g. `updated: 2026-05-20`). NEVER emit `updated: today`, `updated: now`, or `updated: {today: null}`. If you don't have a real date, OMIT the field — the post-processor will not invent one.",
2116
+ "- REQUIRED: The merged frontmatter MUST include a `description` field with a concise one-sentence summary of the merged asset's content. If neither source has a `description` field, synthesize one from the content.",
2117
+ requiredFmKeys.length > 0
2118
+ ? `- CRITICAL: The merged frontmatter MUST include ALL of these keys from both source memories: ${requiredFmKeys.join(", ")}. Do NOT drop any of them.`
2119
+ : null,
2120
+ "",
785
2121
  `=== Primary memory (${primaryRef}) ===`,
786
2122
  primaryBody,
787
2123
  "",
788
2124
  `=== Secondary memory (${secRef}) ===`,
789
2125
  secBody,
790
- ].join("\n");
2126
+ ]
2127
+ .filter((line) => line !== null)
2128
+ .join("\n");
2129
+ // Use the same per-process profile resolution as the chunk-plan call above
2130
+ // so the merge generation step doesn't silently revert to the default LLM.
2131
+ const llmConfig = resolveConsolidateLlmConfig(config);
791
2132
  const result = await tryLlmFeature("memory_consolidation", config, async () => {
792
- if (!config.llm)
2133
+ if (!llmConfig)
793
2134
  return { ok: false, error: "No LLM configured for consolidation" };
794
2135
  try {
795
- const content = await chatCompletion(config.llm, [{ role: "user", content: prompt }]);
2136
+ const content = await chatCompletion(llmConfig, [{ role: "user", content: prompt }], {
2137
+ enableThinking: false,
2138
+ });
796
2139
  return { ok: true, content };
797
2140
  }
798
2141
  catch (e) {
@@ -800,10 +2143,77 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
800
2143
  }
801
2144
  }, { ok: false, error: `merge content generation failed for ${primaryRef}` });
802
2145
  if (!result.ok) {
803
- warnings.push(result.error ?? `merge content generation failed for ${primaryRef}`);
804
- return null;
2146
+ return {
2147
+ error: "merge_transport_failed",
2148
+ detail: result.error ?? `merge content generation failed for ${primaryRef}`,
2149
+ };
2150
+ }
2151
+ // Sanitize LLM output: strip outer code fences (defends against the
2152
+ // ```markdown … ``` leak observed in production), re-serialise frontmatter
2153
+ // through the yaml lib (fixes quote-escaping mistakes), and reject empty
2154
+ // or fence-only responses.
2155
+ const sanitized = sanitizeMergedContent(result.content ?? "");
2156
+ if (!sanitized.ok) {
2157
+ const reason = sanitized.reason;
2158
+ const isFenceError = reason === "UNBALANCED_CODE_FENCE" ||
2159
+ reason === "MISSING_FRONTMATTER_SENTINEL" ||
2160
+ reason === "MALFORMED_FRONTMATTER_BLOCK" ||
2161
+ reason === "FRONTMATTER_NOT_OBJECT";
2162
+ const mergeReason = isFenceError ? "merge_fence_rejected" : "merge_yaml_invalid";
2163
+ return { error: mergeReason, detail: `${primaryRef} — ${reason}` };
2164
+ }
2165
+ const mergedRaw = sanitized.result.content;
2166
+ // C-4 / #383: Content-preservation lint (mem0 §3.2, arXiv:2504.19413).
2167
+ // Guards against LLM-generated merged content that silently drops information
2168
+ // from the source assets. Two checks:
2169
+ // 1. Body size: merged body must be >= 50% of the larger source body.
2170
+ // 2. Frontmatter superset: merged frontmatter must contain all keys present
2171
+ // in both source frontmatters.
2172
+ // Failures return a discriminated error so the call site can emit a specific
2173
+ // skip-reason key in the histogram.
2174
+ try {
2175
+ const primaryFm = parseFrontmatter(primaryBody);
2176
+ const secFm = parseFrontmatter(secBody);
2177
+ const mergedFm = parseFrontmatter(mergedRaw);
2178
+ // Check body size — blended floor: max(ratio × largerLen, absoluteFloor).
2179
+ // Deduplication is expected, so the ratio is lower than the reflect gate
2180
+ // (0.3 vs 0.5). The absolute floor protects very short memory pairs where
2181
+ // the ratio alone would produce a near-zero threshold.
2182
+ const primaryBodyLen = (primaryFm.content ?? "").trim().length;
2183
+ const secBodyLen = (secFm.content ?? "").trim().length;
2184
+ const mergedBodyLen = (mergedFm.content ?? "").trim().length;
2185
+ const largerBodyLen = Math.max(primaryBodyLen, secBodyLen);
2186
+ const mergeFloor = Math.max(MERGE_SHRINK_RATIO_MIN * largerBodyLen, MERGE_ABSOLUTE_FLOOR_CHARS);
2187
+ if (largerBodyLen > 0 && mergedBodyLen < mergeFloor) {
2188
+ return {
2189
+ error: "merge_content_too_short",
2190
+ detail: `${primaryRef} — merged body (${mergedBodyLen} chars) is less than floor (${Math.round(mergeFloor)} chars; max(${MERGE_SHRINK_RATIO_MIN}×${largerBodyLen}, ${MERGE_ABSOLUTE_FLOOR_CHARS}))`,
2191
+ };
2192
+ }
2193
+ // Check frontmatter superset — attempt repair before rejecting.
2194
+ const primaryKeys = Object.keys(primaryFm.data ?? {});
2195
+ const secKeys = Object.keys(secFm.data ?? {});
2196
+ const mergedKeys = new Set(Object.keys(mergedFm.data ?? {}));
2197
+ const missingKeys = [...new Set([...primaryKeys, ...secKeys])].filter((k) => !mergedKeys.has(k));
2198
+ if (missingKeys.length > 0) {
2199
+ // Inject missing keys from source FMs. Primary value wins on conflict.
2200
+ const repairedFmData = { ...mergedFm.data };
2201
+ for (const key of missingKeys) {
2202
+ repairedFmData[key] =
2203
+ key in primaryFm.data
2204
+ ? primaryFm.data[key]
2205
+ : secFm.data[key];
2206
+ }
2207
+ normalizeUpdatedField(repairedFmData);
2208
+ const repairedYaml = yamlStringify(repairedFmData).trimEnd();
2209
+ const bodyPart = mergedFm.content ?? "";
2210
+ return { content: `---\n${repairedYaml}\n---\n${bodyPart}` };
2211
+ }
2212
+ }
2213
+ catch {
2214
+ // parseFrontmatter failures are non-fatal — allow the merge to proceed.
805
2215
  }
806
- return result.content;
2216
+ return { content: mergedRaw };
807
2217
  }
808
2218
  async function promptConfirm(message) {
809
2219
  process.stdout.write(message);