akm-cli 0.8.0-rc1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (295) hide show
  1. package/{.github/CHANGELOG.md → CHANGELOG.md} +191 -3
  2. package/README.md +22 -6
  3. package/SECURITY.md +93 -0
  4. package/dist/cli/config-migrate.js +144 -0
  5. package/dist/cli/config-validate.js +39 -0
  6. package/dist/cli/confirm.js +73 -0
  7. package/dist/cli/parse-args.js +93 -3
  8. package/dist/cli/shared.js +129 -0
  9. package/dist/cli.js +2162 -1258
  10. package/dist/commands/add-cli.js +279 -0
  11. package/dist/commands/agent-dispatch.js +20 -12
  12. package/dist/commands/agent-support.js +11 -5
  13. package/dist/commands/completions.js +3 -0
  14. package/dist/commands/config-cli.js +129 -517
  15. package/dist/commands/consolidate.js +1533 -144
  16. package/dist/commands/curate.js +44 -3
  17. package/dist/commands/db-cli.js +23 -0
  18. package/dist/commands/distill-promotion-policy.js +5 -3
  19. package/dist/commands/distill.js +906 -100
  20. package/dist/commands/env.js +213 -0
  21. package/dist/commands/eval-cases.js +3 -0
  22. package/dist/commands/events.js +3 -0
  23. package/dist/commands/extract-cli.js +127 -0
  24. package/dist/commands/extract-prompt.js +204 -0
  25. package/dist/commands/extract.js +477 -0
  26. package/dist/commands/feedback-cli.js +331 -0
  27. package/dist/commands/graph.js +260 -5
  28. package/dist/commands/health.js +977 -51
  29. package/dist/commands/help/help-accept.md +6 -3
  30. package/dist/commands/help/help-improve.md +36 -8
  31. package/dist/commands/help/help-proposals.md +7 -4
  32. package/dist/commands/help/help-reject.md +5 -2
  33. package/dist/commands/history.js +51 -16
  34. package/dist/commands/improve-auto-accept.js +97 -0
  35. package/dist/commands/improve-cli.js +236 -0
  36. package/dist/commands/improve-profiles.js +184 -0
  37. package/dist/commands/improve-result-file.js +167 -0
  38. package/dist/commands/improve.js +1725 -332
  39. package/dist/commands/info.js +3 -0
  40. package/dist/commands/init.js +49 -1
  41. package/dist/commands/installed-stashes.js +6 -23
  42. package/dist/commands/knowledge.js +3 -0
  43. package/dist/commands/lint/agent-linter.js +3 -0
  44. package/dist/commands/lint/base-linter.js +233 -5
  45. package/dist/commands/lint/command-linter.js +3 -0
  46. package/dist/commands/lint/default-linter.js +3 -0
  47. package/dist/commands/lint/env-key-rules.js +154 -0
  48. package/dist/commands/lint/index.js +92 -3
  49. package/dist/commands/lint/knowledge-linter.js +3 -0
  50. package/dist/commands/lint/markdown-insertion.js +343 -0
  51. package/dist/commands/lint/memory-linter.js +3 -0
  52. package/dist/commands/lint/registry.js +3 -0
  53. package/dist/commands/lint/skill-linter.js +3 -0
  54. package/dist/commands/lint/task-linter.js +15 -12
  55. package/dist/commands/lint/types.js +3 -0
  56. package/dist/commands/lint/workflow-linter.js +3 -0
  57. package/dist/commands/lint.js +3 -0
  58. package/dist/commands/migration-help.js +5 -2
  59. package/dist/commands/proposal-drain-policies.js +128 -0
  60. package/dist/commands/proposal-drain.js +477 -0
  61. package/dist/commands/proposal.js +60 -6
  62. package/dist/commands/propose.js +24 -19
  63. package/dist/commands/reflect.js +1004 -94
  64. package/dist/commands/registry-cli.js +150 -0
  65. package/dist/commands/registry-search.js +3 -0
  66. package/dist/commands/remember-cli.js +257 -0
  67. package/dist/commands/remember.js +15 -6
  68. package/dist/commands/schema-repair.js +88 -15
  69. package/dist/commands/search.js +99 -14
  70. package/dist/commands/secret.js +173 -0
  71. package/dist/commands/self-update.js +3 -0
  72. package/dist/commands/show.js +32 -13
  73. package/dist/commands/source-add.js +7 -35
  74. package/dist/commands/source-clone.js +3 -0
  75. package/dist/commands/source-manage.js +3 -0
  76. package/dist/commands/tasks.js +161 -95
  77. package/dist/commands/url-checker.js +3 -0
  78. package/dist/core/action-contributors.js +3 -0
  79. package/dist/core/asset-ref.js +17 -2
  80. package/dist/core/asset-registry.js +9 -2
  81. package/dist/core/asset-serialize.js +88 -0
  82. package/dist/core/asset-spec.js +61 -5
  83. package/dist/core/common.js +93 -5
  84. package/dist/core/concurrent.js +3 -0
  85. package/dist/core/config-io.js +347 -0
  86. package/dist/core/config-migration.js +622 -0
  87. package/dist/core/config-schema.js +558 -0
  88. package/dist/core/config-sources.js +108 -0
  89. package/dist/core/config-types.js +4 -0
  90. package/dist/core/config-walker.js +337 -0
  91. package/dist/core/config.js +366 -1077
  92. package/dist/core/errors.js +42 -20
  93. package/dist/core/events.js +31 -25
  94. package/dist/core/file-lock.js +104 -0
  95. package/dist/core/frontmatter.js +75 -10
  96. package/dist/core/lesson-lint.js +3 -0
  97. package/dist/core/markdown.js +3 -0
  98. package/dist/core/memory-belief.js +62 -0
  99. package/dist/core/memory-contradiction-detect.js +274 -0
  100. package/dist/core/memory-improve.js +142 -14
  101. package/dist/core/parse.js +3 -0
  102. package/dist/core/paths.js +218 -50
  103. package/dist/core/proposal-quality-validators.js +380 -0
  104. package/dist/core/proposal-validators.js +11 -3
  105. package/dist/core/proposals.js +464 -5
  106. package/dist/core/state-db.js +349 -56
  107. package/dist/core/text-truncation.js +107 -0
  108. package/dist/core/time.js +3 -0
  109. package/dist/core/tty.js +59 -0
  110. package/dist/core/warn.js +7 -2
  111. package/dist/core/write-source.js +12 -0
  112. package/dist/indexer/db-backup.js +391 -0
  113. package/dist/indexer/db-search.js +136 -28
  114. package/dist/indexer/db.js +662 -166
  115. package/dist/indexer/ensure-index.js +3 -0
  116. package/dist/indexer/file-context.js +3 -0
  117. package/dist/indexer/graph-boost.js +162 -40
  118. package/dist/indexer/graph-db.js +241 -51
  119. package/dist/indexer/graph-dedup.js +3 -7
  120. package/dist/indexer/graph-extraction.js +242 -149
  121. package/dist/indexer/index-context.js +3 -9
  122. package/dist/indexer/indexer.js +84 -14
  123. package/dist/indexer/llm-cache.js +24 -19
  124. package/dist/indexer/manifest.js +3 -0
  125. package/dist/indexer/matchers.js +184 -11
  126. package/dist/indexer/memory-inference.js +94 -50
  127. package/dist/indexer/metadata-contributors.js +3 -0
  128. package/dist/indexer/metadata.js +114 -48
  129. package/dist/indexer/path-resolver.js +3 -0
  130. package/dist/indexer/project-context.js +192 -0
  131. package/dist/indexer/ranking-contributors.js +134 -7
  132. package/dist/indexer/ranking.js +8 -1
  133. package/dist/indexer/search-fields.js +5 -9
  134. package/dist/indexer/search-hit-enrichers.js +91 -2
  135. package/dist/indexer/search-source.js +20 -1
  136. package/dist/indexer/semantic-status.js +4 -1
  137. package/dist/indexer/staleness-detect.js +447 -0
  138. package/dist/indexer/usage-events.js +12 -9
  139. package/dist/indexer/walker.js +3 -0
  140. package/dist/integrations/agent/builders.js +135 -0
  141. package/dist/integrations/agent/config.js +121 -401
  142. package/dist/integrations/agent/detect.js +3 -0
  143. package/dist/integrations/agent/index.js +6 -14
  144. package/dist/integrations/agent/model-aliases.js +55 -0
  145. package/dist/integrations/agent/profiles.js +3 -0
  146. package/dist/integrations/agent/prompts.js +137 -8
  147. package/dist/integrations/agent/runner.js +208 -0
  148. package/dist/integrations/agent/sdk-runner.js +8 -2
  149. package/dist/integrations/agent/spawn.js +54 -14
  150. package/dist/integrations/github.js +3 -0
  151. package/dist/integrations/lockfile.js +22 -51
  152. package/dist/integrations/session-logs/index.js +4 -0
  153. package/dist/integrations/session-logs/inline-refs.js +35 -0
  154. package/dist/integrations/session-logs/pre-filter.js +152 -0
  155. package/dist/integrations/session-logs/providers/claude-code.js +226 -0
  156. package/dist/integrations/session-logs/providers/opencode.js +231 -25
  157. package/dist/integrations/session-logs/types.js +3 -0
  158. package/dist/llm/call-ai.js +14 -26
  159. package/dist/llm/client.js +16 -2
  160. package/dist/llm/embedder.js +20 -29
  161. package/dist/llm/embedders/cache.js +3 -7
  162. package/dist/llm/embedders/local.js +42 -1
  163. package/dist/llm/embedders/remote.js +20 -8
  164. package/dist/llm/embedders/types.js +3 -7
  165. package/dist/llm/feature-gate.js +92 -56
  166. package/dist/llm/graph-extract.js +401 -30
  167. package/dist/llm/index-passes.js +44 -29
  168. package/dist/llm/memory-infer.js +30 -2
  169. package/dist/llm/metadata-enhance.js +3 -7
  170. package/dist/llm/prompts/extract-session.md +80 -0
  171. package/dist/llm/prompts/graph-extract-user-prompt.md +24 -1
  172. package/dist/output/cli-hints-full.md +60 -32
  173. package/dist/output/cli-hints-short.md +10 -7
  174. package/dist/output/cli-hints.js +5 -2
  175. package/dist/output/context.js +60 -8
  176. package/dist/output/renderers.js +170 -194
  177. package/dist/output/shapes/curate.js +56 -0
  178. package/dist/output/shapes/distill.js +10 -0
  179. package/dist/output/shapes/env-list.js +19 -0
  180. package/dist/output/shapes/events.js +11 -0
  181. package/dist/output/shapes/helpers.js +424 -0
  182. package/dist/output/shapes/history.js +7 -0
  183. package/dist/output/shapes/passthrough.js +105 -0
  184. package/dist/output/shapes/proposal-accept.js +7 -0
  185. package/dist/output/shapes/proposal-diff.js +7 -0
  186. package/dist/output/shapes/proposal-list.js +7 -0
  187. package/dist/output/shapes/proposal-producer.js +11 -0
  188. package/dist/output/shapes/proposal-reject.js +7 -0
  189. package/dist/output/shapes/proposal-show.js +7 -0
  190. package/dist/output/shapes/registry-search.js +6 -0
  191. package/dist/output/shapes/registry.js +30 -0
  192. package/dist/output/shapes/search.js +6 -0
  193. package/dist/output/shapes/secret-list.js +19 -0
  194. package/dist/output/shapes/show.js +6 -0
  195. package/dist/output/shapes/vault-list.js +19 -0
  196. package/dist/output/shapes.js +51 -549
  197. package/dist/output/text/add.js +6 -0
  198. package/dist/output/text/clone.js +6 -0
  199. package/dist/output/text/config.js +6 -0
  200. package/dist/output/text/curate.js +6 -0
  201. package/dist/output/text/distill.js +7 -0
  202. package/dist/output/text/enable-disable.js +7 -0
  203. package/dist/output/text/events.js +10 -0
  204. package/dist/output/text/feedback.js +6 -0
  205. package/dist/output/text/helpers.js +1059 -0
  206. package/dist/output/text/history.js +7 -0
  207. package/dist/output/text/import.js +6 -0
  208. package/dist/output/text/index.js +6 -0
  209. package/dist/output/text/info.js +6 -0
  210. package/dist/output/text/init.js +6 -0
  211. package/dist/output/text/list.js +6 -0
  212. package/dist/output/text/proposal-producer.js +8 -0
  213. package/dist/output/text/proposal.js +12 -0
  214. package/dist/output/text/registry-commands.js +11 -0
  215. package/dist/output/text/registry.js +30 -0
  216. package/dist/output/text/remember.js +6 -0
  217. package/dist/output/text/remove.js +6 -0
  218. package/dist/output/text/save.js +6 -0
  219. package/dist/output/text/search.js +6 -0
  220. package/dist/output/text/show.js +6 -0
  221. package/dist/output/text/update.js +6 -0
  222. package/dist/output/text/upgrade.js +6 -0
  223. package/dist/output/text/vault.js +16 -0
  224. package/dist/output/text/wiki.js +15 -0
  225. package/dist/output/text/workflow.js +14 -0
  226. package/dist/output/text.js +44 -1329
  227. package/dist/registry/build-index.js +3 -0
  228. package/dist/registry/create-provider-registry.js +3 -0
  229. package/dist/registry/factory.js +4 -1
  230. package/dist/registry/origin-resolve.js +3 -0
  231. package/dist/registry/providers/index.js +3 -0
  232. package/dist/registry/providers/skills-sh.js +11 -2
  233. package/dist/registry/providers/static-index.js +10 -1
  234. package/dist/registry/providers/types.js +3 -24
  235. package/dist/registry/resolve.js +11 -16
  236. package/dist/registry/types.js +3 -0
  237. package/dist/scripts/migrate-storage.js +17767 -0
  238. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +9031 -0
  239. package/dist/scripts/migrations/v16-to-v17.js +141 -0
  240. package/dist/setup/detect.js +3 -0
  241. package/dist/setup/ripgrep-install.js +3 -0
  242. package/dist/setup/ripgrep-resolve.js +3 -0
  243. package/dist/setup/setup.js +306 -67
  244. package/dist/setup/steps.js +3 -15
  245. package/dist/sources/include.js +3 -0
  246. package/dist/sources/provider-factory.js +3 -11
  247. package/dist/sources/provider.js +3 -20
  248. package/dist/sources/providers/filesystem.js +19 -23
  249. package/dist/sources/providers/git.js +171 -21
  250. package/dist/sources/providers/index.js +3 -0
  251. package/dist/sources/providers/install-types.js +3 -13
  252. package/dist/sources/providers/npm.js +3 -4
  253. package/dist/sources/providers/provider-utils.js +3 -0
  254. package/dist/sources/providers/sync-from-ref.js +3 -11
  255. package/dist/sources/providers/tar-utils.js +3 -0
  256. package/dist/sources/providers/website.js +18 -22
  257. package/dist/sources/resolve.js +3 -0
  258. package/dist/sources/types.js +3 -0
  259. package/dist/sources/website-ingest.js +3 -0
  260. package/dist/tasks/backends/cron.js +3 -0
  261. package/dist/tasks/backends/exec-utils.js +3 -0
  262. package/dist/tasks/backends/index.js +3 -11
  263. package/dist/tasks/backends/launchd.js +3 -0
  264. package/dist/tasks/backends/schtasks.js +3 -0
  265. package/dist/tasks/parser.js +51 -38
  266. package/dist/tasks/resolveAkmBin.js +3 -0
  267. package/dist/tasks/runner.js +35 -9
  268. package/dist/tasks/schedule.js +20 -1
  269. package/dist/tasks/schema.js +5 -3
  270. package/dist/tasks/validator.js +6 -3
  271. package/dist/version.js +3 -0
  272. package/dist/wiki/wiki-templates.js +3 -0
  273. package/dist/wiki/wiki.js +3 -0
  274. package/dist/workflows/authoring.js +3 -0
  275. package/dist/workflows/cli.js +3 -0
  276. package/dist/workflows/db.js +140 -10
  277. package/dist/workflows/document-cache.js +3 -10
  278. package/dist/workflows/parser.js +3 -0
  279. package/dist/workflows/renderer.js +3 -0
  280. package/dist/workflows/runs.js +18 -1
  281. package/dist/workflows/schema.js +3 -0
  282. package/dist/workflows/scope-key.js +3 -0
  283. package/dist/workflows/validator.js +5 -9
  284. package/docs/README.md +7 -2
  285. package/docs/data-and-telemetry.md +225 -0
  286. package/docs/migration/release-notes/0.7.5.md +2 -2
  287. package/docs/migration/release-notes/0.8.0.md +57 -5
  288. package/docs/migration/v0.7-to-v0.8.md +1378 -0
  289. package/package.json +28 -11
  290. package/.github/LICENSE +0 -374
  291. package/dist/commands/install-audit.js +0 -385
  292. package/dist/commands/vault.js +0 -307
  293. package/dist/indexer/match-contributors.js +0 -141
  294. package/dist/integrations/agent/pipeline.js +0 -39
  295. package/dist/integrations/agent/runners.js +0 -31
@@ -1,82 +1,196 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { createHash } from "node:crypto";
1
5
  import fs from "node:fs";
2
6
  import path from "node:path";
3
7
  import readline from "node:readline";
4
- import { stringify as yamlStringify } from "yaml";
8
+ import { parse as yamlParse, stringify as yamlStringify } from "yaml";
5
9
  import { parseAssetRef } from "../core/asset-ref";
10
+ import { assembleAssetFromString } from "../core/asset-serialize";
6
11
  import { resolveStashDir, timestampForFilename } from "../core/common";
7
- import { loadConfig } from "../core/config";
12
+ import { getDefaultLlmConfig, loadConfig } from "../core/config";
8
13
  import { ConfigError } from "../core/errors";
14
+ import { appendEvent } from "../core/events";
9
15
  import { parseFrontmatter } from "../core/frontmatter";
16
+ import { writeContradictEdge } from "../core/memory-belief";
10
17
  import { parseEmbeddedJsonResponse } from "../core/parse";
11
- import { createProposal, listProposals } from "../core/proposals";
18
+ import { hasHotCaptureMode, hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../core/proposal-quality-validators";
19
+ import { createProposal, isProposalSkipped, listProposals } from "../core/proposals";
20
+ import { detectTruncatedDescription } from "../core/text-truncation";
21
+ // Re-export the moved helpers so existing test imports continue to resolve.
22
+ export { hasSupersededStatus, validateProposalFrontmatter };
12
23
  import { warn } from "../core/warn";
13
24
  import { deleteAssetFromSource, resolveWriteTarget, writeAssetToSource } from "../core/write-source";
14
- import { closeDatabase, getAllEntries, openExistingDatabase } from "../indexer/db";
25
+ import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../indexer/db";
26
+ import { resolveImproveProcessRunnerFromProfile } from "../integrations/agent/runner";
15
27
  import { chatCompletion } from "../llm/client";
28
+ import { cosineSimilarity, embedBatch } from "../llm/embedder";
16
29
  import { isLlmFeatureEnabled, tryLlmFeature } from "../llm/feature-gate";
17
30
  // ── Prompts ─────────────────────────────────────────────────────────────────
18
31
  const CONSOLIDATE_SYSTEM_PROMPT = `You are the akm consolidate assistant analyzing memory assets.
19
32
 
20
33
  Rules:
21
34
  1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
22
- 2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion.
23
- 3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory.
24
- 4. KEEP: Memory is unique and currentomit from output.
35
+ 2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
36
+ 3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
37
+ 4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
38
+ 5. KEEP: Memory is unique and current → omit from output.
25
39
 
26
40
  Return ONLY JSON (no prose, no code fences):
27
41
  {
28
42
  "operations": [
29
- { "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize" },
30
- { "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>" },
31
- { "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>" }
43
+ { "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
44
+ { "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
45
+ { "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
46
+ { "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
32
47
  ],
33
48
  "warnings": ["<optional concerns>"]
34
- }`;
49
+ }
50
+
51
+ For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
52
+
53
+ When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
54
+ /**
55
+ * JSON Schema for structured consolidate plans (PR 1 of the asset-writers
56
+ * decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
57
+ * Mirrors the {ops[], warnings?[]} shape currently described in
58
+ * CONSOLIDATE_SYSTEM_PROMPT. Providers with `supportsJsonSchema: true` enforce
59
+ * the shape upstream so the chunk-level "invalid plan from AI — skipping"
60
+ * branch in `runConsolidate` becomes unreachable on schema-honouring providers.
61
+ *
62
+ * The four operation variants (merge / delete / promote / contradict) are
63
+ * modeled as a oneOf so a structured-output provider can still tell them apart
64
+ * by the required `op` discriminator. `parseEmbeddedJsonResponse` keeps
65
+ * working as a fallback parser for providers that ignore the schema.
66
+ */
67
+ export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
68
+ type: "object",
69
+ required: ["operations"],
70
+ additionalProperties: false,
71
+ properties: {
72
+ operations: {
73
+ type: "array",
74
+ description: "Ordered list of consolidate operations the planner proposes.",
75
+ items: {
76
+ oneOf: [
77
+ {
78
+ type: "object",
79
+ required: ["op", "primary", "secondaries", "mergeStrategy"],
80
+ additionalProperties: false,
81
+ properties: {
82
+ op: { type: "string", enum: ["merge"] },
83
+ primary: { type: "string", minLength: 1 },
84
+ secondaries: {
85
+ type: "array",
86
+ minItems: 1,
87
+ items: { type: "string", minLength: 1 },
88
+ },
89
+ mergeStrategy: { type: "string", minLength: 1 },
90
+ confidence: { type: "number", minimum: 0, maximum: 1 },
91
+ },
92
+ },
93
+ {
94
+ type: "object",
95
+ required: ["op", "ref", "reason"],
96
+ additionalProperties: false,
97
+ properties: {
98
+ op: { type: "string", enum: ["delete"] },
99
+ ref: { type: "string", minLength: 1 },
100
+ reason: { type: "string", minLength: 1 },
101
+ confidence: { type: "number", minimum: 0, maximum: 1 },
102
+ },
103
+ },
104
+ {
105
+ type: "object",
106
+ required: ["op", "ref", "knowledgeRef", "reason"],
107
+ additionalProperties: false,
108
+ properties: {
109
+ op: { type: "string", enum: ["promote"] },
110
+ ref: { type: "string", minLength: 1 },
111
+ knowledgeRef: { type: "string", minLength: 1 },
112
+ reason: { type: "string", minLength: 1 },
113
+ description: { type: "string" },
114
+ confidence: { type: "number", minimum: 0, maximum: 1 },
115
+ },
116
+ },
117
+ {
118
+ type: "object",
119
+ required: ["op", "ref", "contradictedByRef", "reason"],
120
+ additionalProperties: false,
121
+ properties: {
122
+ op: { type: "string", enum: ["contradict"] },
123
+ ref: { type: "string", minLength: 1 },
124
+ contradictedByRef: { type: "string", minLength: 1 },
125
+ reason: { type: "string", minLength: 1 },
126
+ confidence: { type: "number", minimum: 0, maximum: 1 },
127
+ },
128
+ },
129
+ ],
130
+ },
131
+ },
132
+ warnings: {
133
+ type: "array",
134
+ description: "Optional list of human-readable concerns the planner wants to surface.",
135
+ items: { type: "string" },
136
+ },
137
+ },
138
+ };
35
139
  export function isConsolidationEligibleMemoryName(name) {
36
140
  return !name.endsWith(".derived");
37
141
  }
38
- function loadMemoriesFromDb(sourceFilterPath) {
39
- let db;
142
+ /**
143
+ * Returns true when the memory file has `captureMode: hot` in its frontmatter.
144
+ *
145
+ * Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
146
+ * The consolidate LLM is forbidden from deleting or auto-merging them — the
147
+ * user wrote them on purpose and only the user can decide to retire them.
148
+ *
149
+ * Reads the file once per check; consolidate runs against ~10 memories per
150
+ * chunk so the IO cost is trivial. Returns false on any read/parse error
151
+ * (fail-safe: an unparseable file is treated as not-hot, but the broader
152
+ * consolidate flow already guards against unparseable memories elsewhere).
153
+ *
154
+ * Defends against four observed defect classes (see
155
+ * `memory:akm-improve-critical-review-2026-05-20`):
156
+ * - LLM marks a memory contradicted then deletes (dangling contradictedBy)
157
+ * - LLM merges two unrelated memories sharing a topic keyword
158
+ * - LLM judges a recent durable design memo as "redundant"
159
+ * - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
160
+ */
161
+ export function isHotCapturedMemory(filePath) {
40
162
  try {
41
- db = openExistingDatabase();
42
- const entries = getAllEntries(db, "memory");
43
- return entries
44
- .filter((e) => {
45
- if (!sourceFilterPath)
46
- return true;
47
- return path.resolve(e.stashDir) === path.resolve(sourceFilterPath);
48
- })
49
- .filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
50
- .map((e) => ({
51
- name: e.entry.name,
52
- filePath: e.filePath,
53
- description: e.entry.description ?? "",
54
- tags: e.entry.tags ?? [],
55
- stashDir: e.stashDir,
56
- }));
163
+ if (!fs.existsSync(filePath))
164
+ return false;
165
+ const content = fs.readFileSync(filePath, "utf8");
166
+ const parsed = parseFrontmatter(content);
167
+ return hasHotCaptureMode(parsed.data);
57
168
  }
58
169
  catch {
59
- return [];
60
- }
61
- finally {
62
- if (db)
63
- closeDatabase(db);
170
+ return false;
64
171
  }
65
172
  }
66
- function loadMemoriesFromFs(memoriesDir, stashDir) {
67
- if (!fs.existsSync(memoriesDir))
68
- return [];
69
- const entries = [];
70
- for (const fname of fs.readdirSync(memoriesDir)) {
71
- if (!fname.endsWith(".md"))
72
- continue;
73
- const filePath = path.join(memoriesDir, fname);
74
- const name = fname.replace(/\.md$/, "");
75
- if (!isConsolidationEligibleMemoryName(name))
76
- continue;
77
- entries.push({ name, filePath, description: "", tags: [], stashDir });
173
+ export function consolidateGuardStatus(filePath) {
174
+ if (!fs.existsSync(filePath))
175
+ return "missing";
176
+ let content;
177
+ try {
178
+ content = fs.readFileSync(filePath, "utf8");
179
+ }
180
+ catch {
181
+ return "unparseable";
182
+ }
183
+ let parsed;
184
+ try {
185
+ parsed = parseFrontmatter(content);
186
+ }
187
+ catch {
188
+ return "unparseable";
78
189
  }
79
- return entries;
190
+ const data = parsed.data;
191
+ if (!data || Object.keys(data).length === 0)
192
+ return "unparseable";
193
+ return hasHotCaptureMode(data) ? "hot" : "safe";
80
194
  }
81
195
  // ── Chunk sizing ─────────────────────────────────────────────────────────────
82
196
  /**
@@ -94,21 +208,21 @@ const CHARS_PER_TOKEN = 3;
94
208
  */
95
209
  const PROMPT_OVERHEAD_TOKENS = 2_000;
96
210
  /**
97
- * Default effective token budget used when `config.llm.contextLength` is not
98
- * set. This is intentionally conservative (4 096) rather than being set to
99
- * the model's actual context window, because:
211
+ * Default effective token budget used when the default LLM profile's
212
+ * `contextLength` is not set. This is intentionally conservative (4 096)
213
+ * rather than being set to the model's actual context window, because:
100
214
  *
101
- * - When the agent path is used (config.agent), the agent CLI (e.g. opencode)
215
+ * - When the agent path is used, the agent CLI (e.g. opencode)
102
216
  * prepends its own large system prompt + conversation history before
103
217
  * forwarding to the model. That overhead easily consumes 30K+ tokens on
104
218
  * a model with a 16K context window, leaving very little room for
105
219
  * chunk content.
106
- * - When the HTTP path is used (config.llm), only the akm system prompt and
107
- * user prompt are sent, so the budget can be set to the model's actual
108
- * context length via config.llm.contextLength.
220
+ * - When the HTTP path is used (an LLM profile is selected), only the akm
221
+ * system prompt and user prompt are sent, so the budget can be set to the
222
+ * model's actual context length via profiles.llm[defaults.llm].contextLength.
109
223
  *
110
- * Set config.llm.contextLength in your config file to the model's actual
111
- * context window to allow larger chunks on the HTTP path.
224
+ * Set profiles.llm[defaults.llm].contextLength in your config file to the
225
+ * model's actual context window to allow larger chunks on the HTTP path.
112
226
  */
113
227
  export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
114
228
  /**
@@ -125,40 +239,181 @@ export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
125
239
  *
126
240
  * @param contextLength - Model context window in tokens.
127
241
  * @param bodyTruncation - Max chars per memory body included in the prompt.
242
+ * @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
128
243
  */
129
- export function computeSafeChunkSize(contextLength, bodyTruncation) {
244
+ export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
130
245
  const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
131
246
  const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
132
247
  const raw = Math.floor(usableTokens / tokensPerMemory);
133
- return Math.max(1, Math.min(50, raw));
248
+ return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
249
+ }
250
+ // ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
251
+ /**
252
+ * Re-order memories so that similar ones are placed adjacent to each other
253
+ * before the memories are sliced into chunks. This ensures high-similarity
254
+ * memories land in the same LLM context window, allowing the consolidate
255
+ * model to detect and merge duplicates that would otherwise be split across
256
+ * chunks and survive indefinitely.
257
+ *
258
+ * Algorithm: greedy nearest-neighbour chain starting from the first memory.
259
+ * Each step selects the unused memory with the highest cosine similarity to
260
+ * the last-placed memory. O(n²) — acceptable for the expected N < 200.
261
+ *
262
+ * mem0 arXiv:2504.19413 — every candidate compared against whole store.
263
+ * A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
264
+ *
265
+ * Returns the original order unchanged when:
266
+ * - The embedding config is not present.
267
+ * - Embedding requests fail (fail-open).
268
+ * - There are fewer than 3 memories (no benefit to reordering).
269
+ */
270
+ async function clusterMemoriesBySimilarity(memories, config) {
271
+ if (memories.length < 3 || !config.embedding)
272
+ return memories;
273
+ const texts = memories.map((m) => {
274
+ const parts = [];
275
+ if (m.description)
276
+ parts.push(m.description);
277
+ if (m.tags.length > 0)
278
+ parts.push(m.tags.join(" "));
279
+ return parts.join(". ") || m.name;
280
+ });
281
+ let embeddings = null;
282
+ try {
283
+ embeddings = await embedBatch(texts, config.embedding);
284
+ }
285
+ catch {
286
+ // Fail open: embedding failures degrade gracefully to original order.
287
+ return memories;
288
+ }
289
+ if (!embeddings || embeddings.length !== memories.length)
290
+ return memories;
291
+ // Greedy nearest-neighbour chain.
292
+ const used = new Array(memories.length).fill(false);
293
+ const ordered = [];
294
+ let current = 0; // start from the first memory
295
+ ordered.push(memories[current]);
296
+ used[current] = true;
297
+ for (let step = 1; step < memories.length; step++) {
298
+ const currentEmb = embeddings[current];
299
+ let bestIdx = -1;
300
+ let bestSim = -Infinity;
301
+ for (let j = 0; j < memories.length; j++) {
302
+ if (used[j])
303
+ continue;
304
+ const sim = cosineSimilarity(currentEmb, embeddings[j]);
305
+ if (sim > bestSim) {
306
+ bestSim = sim;
307
+ bestIdx = j;
308
+ }
309
+ }
310
+ if (bestIdx === -1)
311
+ break;
312
+ ordered.push(memories[bestIdx]);
313
+ used[bestIdx] = true;
314
+ current = bestIdx;
315
+ }
316
+ return ordered;
134
317
  }
135
318
  // ── Chunk helpers ────────────────────────────────────────────────────────────
136
- export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation) {
319
+ /**
320
+ * Build the per-chunk user prompt fed to the consolidate LLM.
321
+ *
322
+ * Each memory is annotated with two flags that drive the system-prompt
323
+ * rules at lines 181-186:
324
+ * - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
325
+ * forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
326
+ * stack before this annotation.
327
+ * - `(already queued)` — the memory's body hash matches a pending
328
+ * consolidate proposal; system prompt rule 3 forbids proposing
329
+ * promote/merge/contradict. ~107/4h before this annotation.
330
+ *
331
+ * Both annotations are visible to the LLM. `pendingProposalBodyHashes`
332
+ * is precomputed once per run by `loadPendingConsolidateProposalHashes`
333
+ * so the cost stays O(memories) inside the chunk loop.
334
+ */
335
+ export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set()) {
137
336
  const start = memories[0] ? `memory:${memories[0].name}` : "";
138
337
  const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
338
+ const annotationsByIndex = [];
339
+ const hotRefs = [];
340
+ for (const m of memories) {
341
+ let body = "";
342
+ try {
343
+ body = fs.readFileSync(m.filePath, "utf8");
344
+ }
345
+ catch {
346
+ body = "(unreadable)";
347
+ }
348
+ const parsed = parseFrontmatter(body);
349
+ const isHot = parsed.data.captureMode === "hot";
350
+ const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
351
+ const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
352
+ annotationsByIndex.push({ isHot, isAlreadyQueued, body });
353
+ if (isHot)
354
+ hotRefs.push(`memory:${m.name}`);
355
+ }
139
356
  const lines = [
140
357
  `Source: ${sourceName}`,
141
358
  `Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
142
359
  "",
143
360
  ];
361
+ // Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
362
+ // op-words like "promote", "merge", "contradict" so the model doesn't
363
+ // accidentally treat the warning as a hint to use that op elsewhere
364
+ // (variant B leaked the word "contradict" into the control sample
365
+ // during the diagnostic).
366
+ if (hotRefs.length > 0) {
367
+ lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
368
+ for (const ref of hotRefs)
369
+ lines.push(` - ${ref}`);
370
+ lines.push("");
371
+ }
144
372
  for (let i = 0; i < memories.length; i++) {
145
373
  const m = memories[i];
146
- lines.push(`[${i + 1}] memory:${m.name}`);
374
+ const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
375
+ const annotations = [];
376
+ if (isHot)
377
+ annotations.push("captureMode: hot");
378
+ if (isAlreadyQueued)
379
+ annotations.push("already queued");
380
+ const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
381
+ lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
147
382
  lines.push(`Description: ${m.description || "(none)"}`);
148
383
  lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
149
384
  lines.push("---");
150
- let body = "";
151
- try {
152
- body = fs.readFileSync(m.filePath, "utf8");
153
- }
154
- catch {
155
- body = "(unreadable)";
156
- }
157
385
  lines.push(body.slice(0, bodyTruncation));
158
386
  lines.push("");
159
387
  }
160
388
  return lines.join("\n");
161
389
  }
390
+ /**
391
+ * Precompute body-hashes of all currently-pending consolidate proposals so
392
+ * the per-chunk prompt can annotate memories whose body would just produce
393
+ * a deterministic `dedup_pending_proposal` skip. Hash domain matches the
394
+ * dedup site at ~line 1510 (sha256 over the post-frontmatter content,
395
+ * trimmed). Empty set on any read/parse error — fail-safe to "annotate
396
+ * nothing" so the LLM still proposes, just slightly more wastefully.
397
+ */
398
+ export function loadPendingConsolidateProposalHashes(stashDir) {
399
+ const hashes = new Set();
400
+ try {
401
+ const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
402
+ for (const p of pending) {
403
+ try {
404
+ const body = parseFrontmatter(p.payload.content).content.trim();
405
+ hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
406
+ }
407
+ catch {
408
+ // skip malformed payloads — they can't dedup anyway
409
+ }
410
+ }
411
+ }
412
+ catch {
413
+ // listProposals throws on missing stash dir during tests — empty set is safe
414
+ }
415
+ return hashes;
416
+ }
162
417
  function isValidOp(op) {
163
418
  if (typeof op !== "object" || op === null)
164
419
  return false;
@@ -172,43 +427,133 @@ function isValidOp(op) {
172
427
  if (o.op === "promote") {
173
428
  return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
174
429
  }
430
+ if (o.op === "contradict") {
431
+ return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
432
+ }
175
433
  return false;
176
434
  }
177
- function mergePlans(chunks) {
435
+ export function mergePlans(chunks, knownRefs) {
178
436
  const mergeOps = new Map();
179
437
  const deleteOps = new Map();
180
438
  const promoteOps = new Map();
439
+ // C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
440
+ const contradictOps = new Map();
181
441
  const warnings = [];
182
442
  for (const chunk of chunks) {
183
443
  for (const op of chunk) {
184
444
  if (op.op === "merge") {
445
+ // Drop ops whose primary the LLM hallucinated (not in the loaded memory
446
+ // pool). Without this guard, a hallucinated primary flows all the way to
447
+ // Phase B where !memoryByRef.has(primary) fires and charges every real
448
+ // secondary with merge_primary_missing — masking LLM hallucinations as
449
+ // filter regressions in health metrics.
450
+ if (knownRefs && !knownRefs.has(op.primary)) {
451
+ warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
452
+ // Use a dedicated skip reason so dashboards can distinguish
453
+ // hallucinated primaries from stale-DB regressions.
454
+ // Secondaries are real refs; they are NOT charged here — they remain
455
+ // available for other ops to claim.
456
+ continue;
457
+ }
458
+ // Filter hallucinated secondaries while preserving real ones.
459
+ let mergeOp = op;
460
+ if (knownRefs) {
461
+ const filteredSecondaries = op.secondaries.filter((sec) => {
462
+ if (!knownRefs.has(sec)) {
463
+ warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
464
+ return false;
465
+ }
466
+ return true;
467
+ });
468
+ if (filteredSecondaries.length !== op.secondaries.length) {
469
+ mergeOp = { ...op, secondaries: filteredSecondaries };
470
+ }
471
+ }
185
472
  // merge wins over delete
186
- if (deleteOps.has(op.primary)) {
187
- deleteOps.delete(op.primary);
473
+ if (deleteOps.has(mergeOp.primary)) {
474
+ deleteOps.delete(mergeOp.primary);
188
475
  }
189
- for (const sec of op.secondaries) {
476
+ for (const sec of mergeOp.secondaries) {
190
477
  if (deleteOps.has(sec))
191
478
  deleteOps.delete(sec);
192
479
  }
193
- mergeOps.set(op.primary, op);
480
+ mergeOps.set(mergeOp.primary, mergeOp);
194
481
  }
195
482
  else if (op.op === "delete") {
196
- if (!mergeOps.has(op.ref)) {
483
+ // merge and promote both win over delete. A promote is non-destructive
484
+ // (creates a proposal) but the source memory is counted in `promoted`;
485
+ // if a delete also fires, the ref lands in both `promoted` and
486
+ // `skipReasons`, breaking the invariant by +1.
487
+ if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
197
488
  deleteOps.set(op.ref, op);
198
489
  }
199
490
  }
200
491
  else if (op.op === "promote") {
201
- const existingMerge = mergeOps.get(op.ref);
202
- if (existingMerge) {
203
- warnings.push(`Conflict: promote and merge both target ${op.ref}; preferring merge.`);
204
- }
205
- else {
206
- promoteOps.set(op.ref, op);
492
+ // C-2 / #381: when both a promote and a merge target the same ref,
493
+ // queue the promote FIRST rather than discarding it. The promote op
494
+ // routes through createProposal (the human-gated proposal queue), so
495
+ // it is non-destructive. The merge follows after the proposal is
496
+ // created. This preserves the human reviewer's ability to inspect the
497
+ // promotion before the source memory is merged/deleted.
498
+ // AGM K*8 — retain the maximally informative consistent subset.
499
+ promoteOps.set(op.ref, op);
500
+ }
501
+ else if (op.op === "contradict") {
502
+ // Deduplicate by ref+contradictedByRef pair.
503
+ const key = `${op.ref}|${op.contradictedByRef}`;
504
+ if (!contradictOps.has(key)) {
505
+ contradictOps.set(key, op);
207
506
  }
208
507
  }
209
508
  }
210
509
  }
211
- const ops = [...mergeOps.values(), ...deleteOps.values(), ...promoteOps.values()];
510
+ // Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
511
+ //
512
+ // 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
513
+ // for secondaries that were already in deleteOps, but misses the case
514
+ // where the delete chunk came first. A full sweep here fixes both orders.
515
+ //
516
+ // 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
517
+ // only the first (insertion-order) retains it. Without this, a successful
518
+ // merge credits A to mergedSecondaries and a later merge's emitMerge-
519
+ // FailureSkips also charges A to skipReasons — double-counting A while
520
+ // processed has it only once.
521
+ //
522
+ // 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
523
+ // a secondary in another, remove A from the secondary list. Both merges
524
+ // would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
525
+ // breaking the invariant the same way.
526
+ // Also remove delete ops for any ref claimed by a promote op (handles the
527
+ // case where the delete chunk appeared before the promote chunk).
528
+ for (const ref of promoteOps.keys()) {
529
+ deleteOps.delete(ref);
530
+ }
531
+ const claimedSecondaries = new Set();
532
+ for (const mergeOp of mergeOps.values()) {
533
+ deleteOps.delete(mergeOp.primary);
534
+ mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
535
+ if (mergeOps.has(sec)) {
536
+ warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
537
+ return false;
538
+ }
539
+ if (claimedSecondaries.has(sec)) {
540
+ warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
541
+ return false;
542
+ }
543
+ claimedSecondaries.add(sec);
544
+ deleteOps.delete(sec);
545
+ return true;
546
+ });
547
+ }
548
+ // C-2 / #381: promote ops are ordered BEFORE merge ops so that the
549
+ // human-gated proposal queue entry is created before any destructive merge.
550
+ // Phase B processes ops in array order, so promote executes first.
551
+ const ops = [
552
+ ...promoteOps.values(),
553
+ ...mergeOps.values(),
554
+ ...deleteOps.values(),
555
+ ...contradictOps.values(),
556
+ ];
212
557
  return { ops, warnings };
213
558
  }
214
559
  function getJournalPath(stashDir) {
@@ -359,8 +704,7 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
359
704
  ...(supersededBy ? { superseded_by: supersededBy } : {}),
360
705
  superseded_reason: reason,
361
706
  };
362
- const fmStr = yamlStringify(newFm).trimEnd();
363
- content = `---\n${fmStr}\n---\n${parsed.content}`;
707
+ content = assembleAssetFromString(yamlStringify(newFm).trimEnd(), parsed.content);
364
708
  }
365
709
  catch {
366
710
  if (warnings)
@@ -377,9 +721,44 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
377
721
  warnings.push(`archiveMemory: write failed for ${ref}: ${String(e)}`);
378
722
  }
379
723
  }
724
+ // ── LLM resolution ──────────────────────────────────────────────────────────
725
+ /**
726
+ * Resolve the LLM connection for the consolidate pass.
727
+ *
728
+ * Priority order (mirrors extract / reflect / distill — see
729
+ * `src/commands/extract.ts:421-438` and the canonical
730
+ * `resolveImproveProcessRunnerFromProfile` pattern):
731
+ *
732
+ * 1. `profiles.improve.default.processes.consolidate.profile` (or `mode`)
733
+ * via {@link resolveImproveProcessRunnerFromProfile}. Lets the user pin
734
+ * a dedicated model (e.g. `ministral-3b`) for consolidation instead of
735
+ * whatever `defaults.llm` happens to be.
736
+ * 2. `getDefaultLlmConfig(config)` — the baseline default LLM profile.
737
+ *
738
+ * Regression guard (2026-05-26): before this resolver, `akmConsolidate`
739
+ * called `getDefaultLlmConfig` directly and silently ignored a configured
740
+ * `processes.consolidate.profile`, sending every chunk to the default LLM
741
+ * (often a long-context model loaded with a smaller runtime `n_ctx`, causing
742
+ * silent 400s from LM Studio). The investigation lives at
743
+ * `/tmp/akm-health-investigations/consolidation-no-op.md`.
744
+ */
745
+ function resolveConsolidateLlmConfig(config) {
746
+ const consolidateProcess = config.profiles?.improve?.default?.processes?.consolidate;
747
+ const runnerSpec = resolveImproveProcessRunnerFromProfile(consolidateProcess, config);
748
+ if (runnerSpec && runnerSpec.kind === "llm") {
749
+ return runnerSpec.connection;
750
+ }
751
+ // Non-LLM runner modes (agent/sdk) don't apply to consolidate's HTTP path;
752
+ // fall back to the default LLM profile rather than disabling the pass.
753
+ return getDefaultLlmConfig(config);
754
+ }
380
755
  // ── Main entry point ─────────────────────────────────────────────────────────
381
756
  export async function akmConsolidate(opts = {}) {
382
757
  const startMs = Date.now();
758
+ // Derive a stable PROV-DM token for this run. Callers (e.g. akmImprove)
759
+ // should pass opts.sourceRun to tie proposals back to the parent run;
760
+ // standalone `akm consolidate` gets a self-contained token.
761
+ const sourceRun = opts.sourceRun ?? `consolidate-${startMs}`;
383
762
  const config = opts.config ?? loadConfig();
384
763
  const stashDir = opts.stashDir ?? resolveStashDir();
385
764
  if (!isLlmFeatureEnabled(config, "memory_consolidation")) {
@@ -394,13 +773,24 @@ export async function akmConsolidate(opts = {}) {
394
773
  merged: 0,
395
774
  deleted: 0,
396
775
  promoted: [],
776
+ contradicted: 0,
397
777
  warnings: [],
398
778
  durationMs: Date.now() - startMs,
399
779
  };
400
780
  }
401
781
  const warnings = [];
402
782
  checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
403
- const memories = loadMemoriesForSource(opts.target, stashDir, warnings);
783
+ let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
784
+ // Pre-flight: filter out stale DB entries whose files no longer exist on
785
+ // disk. Without this, memories deleted by a prior run (but not yet
786
+ // reindexed) appear in chunk prompts, causing the LLM to generate plans
787
+ // against ghost refs and wasting tokens. Filtering here ensures the chunk
788
+ // pool and memoryByRef are authoritative against the actual filesystem state.
789
+ const staleCount = memories.filter((m) => !fs.existsSync(m.filePath)).length;
790
+ if (staleCount > 0) {
791
+ warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
792
+ }
793
+ memories = memories.filter((m) => fs.existsSync(m.filePath));
404
794
  if (memories.length === 0) {
405
795
  return {
406
796
  schemaVersion: 1,
@@ -413,55 +803,181 @@ export async function akmConsolidate(opts = {}) {
413
803
  merged: 0,
414
804
  deleted: 0,
415
805
  promoted: [],
806
+ contradicted: 0,
416
807
  warnings,
417
808
  durationMs: Date.now() - startMs,
418
809
  };
419
810
  }
811
+ if (opts.incrementalSince) {
812
+ memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings);
813
+ if (memories.length === 0) {
814
+ return {
815
+ schemaVersion: 1,
816
+ ok: true,
817
+ shape: "consolidate-result",
818
+ dryRun: opts.dryRun ?? false,
819
+ previewOnly: false,
820
+ target: opts.target ?? stashDir,
821
+ processed: 0,
822
+ merged: 0,
823
+ deleted: 0,
824
+ promoted: [],
825
+ contradicted: 0,
826
+ warnings,
827
+ durationMs: Date.now() - startMs,
828
+ };
829
+ }
830
+ }
420
831
  // Consolidation always uses the HTTP LLM client directly — never the agent
421
832
  // CLI. The agent CLI is for interactive agent sessions (reflect, propose);
422
833
  // structured JSON generation works better and faster via HTTP.
423
- const isHttpPath = !!config.llm;
834
+ //
835
+ // Honor `profiles.improve.default.processes.consolidate.profile` first; fall
836
+ // back to the default LLM. See {@link resolveConsolidateLlmConfig}.
837
+ const llmConfig = resolveConsolidateLlmConfig(config);
838
+ const isHttpPath = !!llmConfig;
424
839
  // Chunk sizing: derive a safe chunk size from the configured model context
425
- // window (config.llm.contextLength) so that the full prompt (system prompt +
426
- // chunk user prompt) never exceeds the model's n_ctx limit. When no context
427
- // length is configured we fall back to DEFAULT_CONTEXT_LENGTH_TOKENS (8 000)
428
- // which is conservative enough for most 8K–16K local models.
840
+ // window so that the full prompt (system prompt + chunk user prompt) never
841
+ // exceeds the model's n_ctx limit. When no context length is configured we
842
+ // fall back to DEFAULT_CONTEXT_LENGTH_TOKENS (8 000) which is conservative
843
+ // enough for most 8K–16K local models.
429
844
  //
430
845
  // bodyTruncation caps the body excerpt included per memory in the prompt.
431
846
  // Reducing it further than 500 chars degrades consolidation quality, so we
432
847
  // keep it fixed and let computeSafeChunkSize vary the number of memories
433
848
  // per chunk instead.
434
849
  const bodyTruncation = 500;
435
- const modelContextLength = config.llm?.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS;
436
- const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation);
850
+ const modelContextLength = llmConfig?.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS;
851
+ const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
437
852
  // -- Phase A: plan generation -----------------------------------------------
438
853
  const sourceName = opts.target ?? stashDir;
854
+ // C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
855
+ // This ensures that semantically similar memories land in the same LLM
856
+ // context window, allowing the model to detect and merge duplicates that
857
+ // would otherwise be split across chunks and survive indefinitely.
858
+ // mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
859
+ // Fails open: if embeddings are unavailable or fail, original order is used.
860
+ const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
439
861
  const chunks = [];
440
- for (let i = 0; i < memories.length; i += chunkSize) {
441
- chunks.push(memories.slice(i, i + chunkSize));
862
+ for (let i = 0; i < clusteredMemories.length; i += chunkSize) {
863
+ chunks.push(clusteredMemories.slice(i, i + chunkSize));
442
864
  }
443
- warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}`);
865
+ // 2026-05-27 prompt-context fix: precompute body-hashes of pending
866
+ // consolidate proposals once, so the per-chunk prompt can annotate
867
+ // memories whose body would just produce a deterministic
868
+ // `dedup_pending_proposal` skip. Cuts ~110 wasted LLM proposals per
869
+ // 4h on this user's stack. See
870
+ // /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
871
+ const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
872
+ warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
873
+ ` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
444
874
  const chunkOpsArrays = [];
445
- let consecutiveFailures = 0;
875
+ // Structured skip-reason histogram (2026-05-26): every deterministic
876
+ // post-LLM op rejection site below also calls `pushSkipReason` so the
877
+ // health rollup can aggregate without regex-parsing English warning
878
+ // strings. See `/tmp/akm-health-investigations/tuning-reasons-investigation.md` §Q2.
879
+ const skipReasons = [];
880
+ // Tracks refs already emitted to skipReasons. A ref can only occupy one
881
+ // accounting bucket; subsequent skip ops for the same ref are recorded as
882
+ // warnings but must not push a second skipReasons entry (that would inflate
883
+ // Σ(skipReasons) and break the invariant by +1 per duplicate).
884
+ const skipReasonEmittedRefs = new Set();
885
+ const pushSkipReason = (op, ref, reason) => {
886
+ // 2026-05-27 cross-chunk double-count fix: if `ref` already contributed
887
+ // to judgedNoAction in its own chunk (a different chunk proposed an op
888
+ // for it that is now being rejected here), promote it from the
889
+ // judgedNoAction bucket into the more specific skipReason bucket.
890
+ // Preserves the invariant: processed == actioned + judgedNoAction +
891
+ // Σ(skipReasons) + failedChunkMemories.
892
+ if (judgedNoActionRefs.delete(ref))
893
+ judgedNoAction--;
894
+ if (skipReasonEmittedRefs.has(ref)) {
895
+ // Already counted once. Record the extra skip for observability but
896
+ // don't push to skipReasons — that would break the accounting invariant.
897
+ warnings.push(`Skip: ${ref} already in skipReasons (${reason} via ${op}); not re-counted.`);
898
+ return;
899
+ }
900
+ skipReasonEmittedRefs.add(ref);
901
+ skipReasons.push({ op, ref, reason });
902
+ };
903
+ // judgedNoAction tracks memories the LLM saw inside a chunk but proposed
904
+ // no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
905
+ let judgedNoAction = 0;
906
+ // 2026-05-27 cross-chunk double-count fix: refs that contributed to
907
+ // judgedNoAction in their own chunk. When a different chunk's op references
908
+ // one of these as a secondary and that op later fails, the ref would land
909
+ // in BOTH judgedNoAction and skipReasons (delta +1 per occurrence). Track
910
+ // the set so the merge-failure path can decrement and re-bucket.
911
+ const judgedNoActionRefs = new Set();
912
+ // 2026-05-26 accounting-leak fix: memories that belong to a chunk whose
913
+ // LLM call failed before any per-chunk noAction calculation runs. They
914
+ // would otherwise vanish from the envelope's accounting (no judgedNoAction
915
+ // bump, no skipReasons entry, no actioned counter).
916
+ let failedChunkMemories = 0;
917
+ // 2026-05-26 accounting-leak fix: per-secondary tally so successful merges
918
+ // account for `1 + secondaries.length` memories instead of 1.
919
+ let mergedSecondaries = 0;
920
+ // C-6 / #392: Replace two-consecutive-failures abort with failure-rate threshold.
921
+ // Consecutive-count policies are brittle against transient LM Studio reloads:
922
+ // two transient failures abort the run even though the next chunk would succeed.
923
+ // Rate-based abort (≥50% failure over ≥4 chunks) is more robust.
924
+ // Tanenbaum, Distributed Systems §8 — rate-based policies with minimum sample sizes.
925
+ let totalChunksProcessed = 0;
926
+ let totalChunksFailed = 0;
927
+ const ABORT_MIN_CHUNKS = 4;
928
+ const ABORT_FAILURE_RATE = 0.5;
446
929
  for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
447
- // Abort early if the first chunk failed the LLM/agent is likely unavailable
448
- // and continuing would waste minutes processing chunks that will all fail the same way.
449
- if (chunkIdx > 0 && consecutiveFailures >= 2) {
450
- const skipped = chunks.length - chunkIdx;
451
- warnings.push(`Consolidation aborted after ${consecutiveFailures} consecutive chunk failures — LLM may be unavailable. ${skipped} chunk(s) skipped.`);
452
- break;
930
+ // Abort if failure rate >= 50% over at least 4 processed chunks.
931
+ if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
932
+ const failureRate = totalChunksFailed / totalChunksProcessed;
933
+ if (failureRate >= ABORT_FAILURE_RATE) {
934
+ const skipped = chunks.length - chunkIdx;
935
+ const abortMsg = `Consolidation aborted — failure rate ${(failureRate * 100).toFixed(0)}% over ${totalChunksProcessed} chunks (>= ${ABORT_FAILURE_RATE * 100}% threshold). LLM may be unavailable. ${skipped} chunk(s) skipped.`;
936
+ warn(abortMsg);
937
+ warnings.push(abortMsg);
938
+ // Account for memories in chunks we never attempted: they are
939
+ // neither judgedNoAction (no plan parsed) nor skipReason (no op
940
+ // rejected). Without this, the accounting invariant fails by
941
+ // `Σ(unattempted_chunk.length)` whenever the abort fires.
942
+ for (let i = chunkIdx; i < chunks.length; i++) {
943
+ failedChunkMemories += chunks[i].length;
944
+ }
945
+ break;
946
+ }
453
947
  }
454
948
  const chunk = chunks[chunkIdx];
949
+ // All-hot chunk early-exit. The per-prompt hot-list block (see
950
+ // buildChunkPrompt) only *discourages* delete proposals on a mixed chunk;
951
+ // when EVERY memory in the chunk is captureMode: hot, the only ops the LLM
952
+ // could ever propose are deletes — all of which the downstream guard
953
+ // refuses unconditionally. Calling the model is therefore pure token waste.
954
+ // Skip the request entirely and bucket every memory as judgedNoAction (we
955
+ // judged "no action" without spending an LLM call), preserving the
956
+ // accounting invariant `processed == actioned + judgedNoAction +
957
+ // Σ(skipReasons) + failedChunkMemories`. Not counted toward the
958
+ // LLM-failure-rate abort policy — no request was attempted.
959
+ if (chunk.length > 0 && chunk.every((m) => isHotCapturedMemory(m.filePath))) {
960
+ for (const m of chunk)
961
+ judgedNoActionRefs.add(`memory:${m.name}`);
962
+ judgedNoAction += chunk.length;
963
+ warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length}: all ${chunk.length} memories are captureMode: hot — skipping LLM (judged no-action).`);
964
+ continue;
965
+ }
455
966
  warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
456
- const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation);
967
+ const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes);
457
968
  const raw = await tryLlmFeature("memory_consolidation", config, async () => {
458
- if (!config.llm)
969
+ if (!llmConfig)
459
970
  return { ok: false, error: "No LLM configured for consolidation" };
460
971
  try {
461
- const content = await chatCompletion(config.llm, [
972
+ // responseSchema lift (PR 1, asset-writers-investigation §5): pass
973
+ // the consolidate plan schema so providers with
974
+ // `supportsJsonSchema: true` enforce shape upstream. Providers that
975
+ // ignore the option fall through to the existing
976
+ // `parseEmbeddedJsonResponse` path on the response side.
977
+ const content = await chatCompletion(llmConfig, [
462
978
  { role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
463
979
  { role: "user", content: userPrompt },
464
- ]);
980
+ ], { responseSchema: CONSOLIDATE_PLAN_JSON_SCHEMA, enableThinking: false });
465
981
  return { ok: true, content };
466
982
  }
467
983
  catch (e) {
@@ -469,8 +985,15 @@ export async function akmConsolidate(opts = {}) {
469
985
  }
470
986
  }, { ok: false, error: `chunk ${chunkIdx + 1} failed` });
471
987
  if (!raw.ok) {
988
+ warn(raw.error ?? `chunk ${chunkIdx + 1} failed`);
472
989
  warnings.push(raw.error ?? `chunk ${chunkIdx + 1} failed`);
473
- consecutiveFailures++;
990
+ totalChunksProcessed++;
991
+ totalChunksFailed++;
992
+ // Account for the chunk's memories under the failed-chunk bucket.
993
+ // judgedNoAction does NOT run on this path (it's after the success
994
+ // guards) so without this the accounting invariant breaks on every
995
+ // chunk-level transport/parse failure.
996
+ failedChunkMemories += chunk.length;
474
997
  continue;
475
998
  }
476
999
  if (process.env.AKM_DEBUG_LLM) {
@@ -482,11 +1005,14 @@ export async function akmConsolidate(opts = {}) {
482
1005
  const hint = raw.content !== undefined && raw.content.trim() === ""
483
1006
  ? " (empty response — if using a thinking model, disable thinking mode)"
484
1007
  : "";
1008
+ warn(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
485
1009
  warnings.push(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
486
- consecutiveFailures++;
1010
+ totalChunksProcessed++;
1011
+ totalChunksFailed++;
1012
+ failedChunkMemories += chunk.length;
487
1013
  continue;
488
1014
  }
489
- consecutiveFailures = 0; // reset on success
1015
+ totalChunksProcessed++; // success
490
1016
  const ops = [];
491
1017
  for (const op of parsed.operations) {
492
1018
  if (isValidOp(op)) {
@@ -502,9 +1028,37 @@ export async function akmConsolidate(opts = {}) {
502
1028
  warnings.push(w);
503
1029
  }
504
1030
  }
1031
+ // Per-chunk judgedNoAction: count memories the LLM saw but proposed no
1032
+ // op for. Membership is by `memory:<name>` ref against the targets of
1033
+ // each op (primary + secondaries for merge; ref otherwise). 2026-05-26:
1034
+ // pre-fix this was a 78/119 (66%) silent drop in the cron run — no
1035
+ // warning, event, or counter. See tuning investigation §Q2.
1036
+ const targetRefs = new Set();
1037
+ for (const op of ops) {
1038
+ if (op.op === "merge") {
1039
+ targetRefs.add(op.primary);
1040
+ for (const s of op.secondaries)
1041
+ targetRefs.add(s);
1042
+ }
1043
+ else {
1044
+ targetRefs.add(op.ref);
1045
+ }
1046
+ }
1047
+ let chunkNoAction = 0;
1048
+ for (const m of chunk) {
1049
+ const memRef = `memory:${m.name}`;
1050
+ if (!targetRefs.has(memRef)) {
1051
+ chunkNoAction++;
1052
+ judgedNoActionRefs.add(memRef);
1053
+ }
1054
+ }
1055
+ judgedNoAction += chunkNoAction;
505
1056
  chunkOpsArrays.push(ops);
506
1057
  }
507
- const { ops: allOps, warnings: mergeWarnings } = mergePlans(chunkOpsArrays);
1058
+ // Build the known-refs set from the already-filtered memory pool so
1059
+ // mergePlans() can reject LLM-hallucinated primary refs before execution.
1060
+ const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
1061
+ const { ops: allOps, warnings: mergeWarnings } = mergePlans(chunkOpsArrays, knownRefs);
508
1062
  warnings.push(...mergeWarnings);
509
1063
  // -- Dry-run: show AI plan without executing any writes --------------------
510
1064
  if (opts.dryRun) {
@@ -519,6 +1073,13 @@ export async function akmConsolidate(opts = {}) {
519
1073
  merged: 0,
520
1074
  deleted: 0,
521
1075
  promoted: [],
1076
+ contradicted: 0,
1077
+ failedChunks: totalChunksFailed,
1078
+ totalChunks: chunks.length,
1079
+ judgedNoAction,
1080
+ skipReasons,
1081
+ mergedSecondaries,
1082
+ failedChunkMemories,
522
1083
  planned: allOps,
523
1084
  warnings,
524
1085
  durationMs: Date.now() - startMs,
@@ -528,9 +1089,20 @@ export async function akmConsolidate(opts = {}) {
528
1089
  // -- HTTP path: warn about quality and confirm unless auto-accepted --------
529
1090
  if (isHttpPath) {
530
1091
  warnings.push("Running on HTTP path — plan generated from truncated memory excerpts; quality may vary.");
531
- if (!opts.autoAccept) {
1092
+ // Per-proposal confidence gating is handled by the caller (improve.ts)
1093
+ // via runAutoAcceptGate after this function returns. The gate reads
1094
+ // proposal.confidence (forwarded from op.confidence above) and applies
1095
+ // a minimumThreshold floor of 95 for consolidate's destructive ops.
1096
+ // Here we only gate the interactive-confirm path for manual/HTTP invocations.
1097
+ if (opts.autoAccept === undefined && allOps.length > 0) {
532
1098
  const n = allOps.length;
533
- const answer = await promptConfirm(`Apply ${n} operations? [y/N] `);
1099
+ // Non-interactive contexts (CI / test runners / piped stdin) must not
1100
+ // block on an unanswerable prompt. Default to a non-destructive "no"
1101
+ // so callers in those contexts get the same "aborted, preview only"
1102
+ // shape they'd get from explicit user dismissal. AKM_NON_INTERACTIVE
1103
+ // lets callers force this path even when stdin happens to be a TTY.
1104
+ const nonInteractive = process.stdin.isTTY === false || process.env.AKM_NON_INTERACTIVE === "1";
1105
+ const answer = nonInteractive ? false : await promptConfirm(`Apply ${n} operations? [y/N] `);
534
1106
  if (!answer) {
535
1107
  return {
536
1108
  schemaVersion: 1,
@@ -543,8 +1115,15 @@ export async function akmConsolidate(opts = {}) {
543
1115
  merged: 0,
544
1116
  deleted: 0,
545
1117
  promoted: [],
1118
+ contradicted: 0,
1119
+ failedChunks: totalChunksFailed,
1120
+ totalChunks: chunks.length,
1121
+ judgedNoAction,
1122
+ skipReasons,
1123
+ mergedSecondaries,
1124
+ failedChunkMemories,
546
1125
  planned: allOps,
547
- warnings: [...warnings, "Aborted by user."],
1126
+ warnings: [...warnings, nonInteractive ? "Non-interactive context: skipped apply." : "Aborted by user."],
548
1127
  durationMs: Date.now() - startMs,
549
1128
  };
550
1129
  }
@@ -559,6 +1138,12 @@ export async function akmConsolidate(opts = {}) {
559
1138
  let merged = 0;
560
1139
  let deleted = 0;
561
1140
  const promoted = [];
1141
+ let contradicted = 0; // C-3 / #382: count of contradiction edges written
1142
+ // Within-run dedup: track source refs for which a promote proposal was
1143
+ // already created this run. The LLM can return multiple promote ops for
1144
+ // different source memories that happen to have identical content (all are
1145
+ // duplicate memories), so we also need a content-hash guard below.
1146
+ const promotedSourceRefs = new Set();
562
1147
  // Build a lookup map: ref → MemoryEntry
563
1148
  const memoryByRef = new Map();
564
1149
  for (const m of memories) {
@@ -566,11 +1151,40 @@ export async function akmConsolidate(opts = {}) {
566
1151
  }
567
1152
  for (let opIndex = 0; opIndex < allOps.length; opIndex++) {
568
1153
  const op = allOps[opIndex];
569
- warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${op.op === "merge" ? op.primary : op.ref}`);
1154
+ const opDisplayRef = op.op === "merge" ? op.primary : op.op === "contradict" ? `${op.ref} ↔ ${op.contradictedByRef}` : op.ref;
1155
+ warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${opDisplayRef}`);
570
1156
  if (op.op === "merge") {
1157
+ // Accounting helper: emit a per-participant skipReason for failed
1158
+ // merges so primary + every loaded-memory secondary land in the
1159
+ // structured skip histogram. Pre-2026-05-26 only the primary was
1160
+ // counted (1 skipReason per failed merge), leaving N secondaries
1161
+ // unaccounted for in the `processed == actioned + noAction + Σskips`
1162
+ // invariant — the source of the 4–11 silent leaks per run.
1163
+ const emitMergeFailureSkips = (reason) => {
1164
+ if (memoryByRef.has(op.primary))
1165
+ pushSkipReason("merge", op.primary, reason);
1166
+ for (const secRef of op.secondaries) {
1167
+ if (memoryByRef.has(secRef))
1168
+ pushSkipReason("merge", secRef, reason);
1169
+ }
1170
+ };
571
1171
  const primaryEntry = memoryByRef.get(op.primary);
572
1172
  if (!primaryEntry) {
573
- warnings.push(`Merge: primary ${op.primary} not found in loaded memories skipping.`);
1173
+ // This fires when a prior op in the same run consumed this ref as a
1174
+ // secondary and Fix-A pruned it from memoryByRef. It should NOT fire
1175
+ // for hallucinated primaries (those are dropped by mergePlans() before
1176
+ // reaching here). If this counter is non-zero, suspect an intra-run
1177
+ // cross-chunk race, not a filter regression.
1178
+ warnings.push(`Merge: primary ${op.primary} not found in loaded memories (pruned by prior op this run) — skipping.`);
1179
+ emitMergeFailureSkips("merge_primary_missing");
1180
+ continue;
1181
+ }
1182
+ // Defense-in-depth: even if the entry is in memoryByRef (pre-flight ran
1183
+ // before this run's own ops), the file may have been deleted by a
1184
+ // concurrent process or an edge case the pre-flight filter missed.
1185
+ if (!fs.existsSync(primaryEntry.filePath)) {
1186
+ warnings.push(`Merge: primary ${op.primary} file gone at execution time (stale entry) — skipping.`);
1187
+ emitMergeFailureSkips("merge_primary_file_gone");
574
1188
  continue;
575
1189
  }
576
1190
  // Phase B: generate merged content
@@ -579,29 +1193,106 @@ export async function akmConsolidate(opts = {}) {
579
1193
  const secEntry = memoryByRef.get(secRef);
580
1194
  if (!secEntry) {
581
1195
  warnings.push(`Merge: secondary ${secRef} not found — skipping merge op.`);
1196
+ // No accounting impact: a missing secondary is a phantom ref and
1197
+ // never contributed to any chunk's targetRefs reduction. We still
1198
+ // continue the loop to gather the remaining valid secondaries.
582
1199
  continue;
583
1200
  }
584
1201
  secondaryBodies.push(secRef);
585
1202
  }
586
- if (secondaryBodies.length === 0)
1203
+ if (secondaryBodies.length === 0) {
1204
+ warnings.push(`Merge: ${op.primary} has no valid secondaries — skipping.`);
1205
+ emitMergeFailureSkips("merge_no_valid_secondaries");
587
1206
  continue;
1207
+ }
1208
+ // Pre-flight hot guard — skip the LLM call entirely if any participant
1209
+ // is hot or unparseable. Without this, mixed chunks still send hot merges
1210
+ // to the planner which proposes them; generateMergedContent() is then
1211
+ // called, produces output without `description`, and the skip is
1212
+ // misattributed to merge_missing_description instead of the real cause.
1213
+ const preflightParticipants = [op.primary, ...op.secondaries];
1214
+ const preflightBlocked = preflightParticipants.flatMap((ref) => {
1215
+ const e = memoryByRef.get(ref);
1216
+ if (!e)
1217
+ return [];
1218
+ const verdict = consolidateGuardStatus(e.filePath);
1219
+ if (verdict === "hot" || verdict === "unparseable")
1220
+ return [{ ref, verdict }];
1221
+ return [];
1222
+ });
1223
+ if (preflightBlocked.length > 0) {
1224
+ const detail = preflightBlocked.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1225
+ warnings.push(`Merge: refused for ${op.primary} — ${preflightBlocked.length} participant(s) blocked by hot/unparseable frontmatter guard (pre-flight): ${detail}`);
1226
+ emitMergeFailureSkips("merge_participant_blocked");
1227
+ continue;
1228
+ }
588
1229
  let primaryBody = "";
589
1230
  try {
590
1231
  primaryBody = fs.readFileSync(primaryEntry.filePath, "utf8");
591
1232
  }
592
1233
  catch {
593
1234
  warnings.push(`Merge: could not read primary ${op.primary} — skipping.`);
1235
+ emitMergeFailureSkips("merge_read_failed");
594
1236
  continue;
595
1237
  }
596
- const mergedContent = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef, warnings);
597
- if (mergedContent === null)
1238
+ const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef);
1239
+ if ("error" in mergeResult) {
1240
+ warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
1241
+ emitMergeFailureSkips(mergeResult.error);
598
1242
  continue;
599
- // Validate frontmatter of merged content
1243
+ }
1244
+ const mergedContent = mergeResult.content;
1245
+ // Validate frontmatter of merged content — must have a `---` block
1246
+ // with at minimum a `description` field. We parse via the hand-rolled
1247
+ // parser (cheap) AND require non-empty description. This guards against
1248
+ // the historical defect where merged memories were written back with
1249
+ // empty `description` and later polluted the promote path.
1250
+ let parsedMerged;
600
1251
  try {
601
- parseFrontmatter(mergedContent);
1252
+ parsedMerged = parseFrontmatter(mergedContent);
602
1253
  }
603
1254
  catch {
604
1255
  warnings.push(`Merge: merged content for ${op.primary} has invalid frontmatter — skipping.`);
1256
+ emitMergeFailureSkips("merge_invalid_frontmatter");
1257
+ continue;
1258
+ }
1259
+ if (parsedMerged.frontmatter === null) {
1260
+ warnings.push(`Merge: merged content for ${op.primary} has no frontmatter block — skipping.`);
1261
+ emitMergeFailureSkips("merge_invalid_frontmatter");
1262
+ continue;
1263
+ }
1264
+ const mergedDesc = parsedMerged.data.description;
1265
+ if (typeof mergedDesc !== "string" || mergedDesc.trim().length === 0) {
1266
+ warnings.push(`Merge: merged content for ${op.primary} missing description — skipping.`);
1267
+ emitMergeFailureSkips("merge_missing_description");
1268
+ continue;
1269
+ }
1270
+ const truncReason = detectTruncatedDescription(mergedDesc);
1271
+ if (truncReason) {
1272
+ warnings.push(`Merge: merged content for ${op.primary} has truncated description (${truncReason}) — skipping.`);
1273
+ emitMergeFailureSkips("merge_truncated_description");
1274
+ continue;
1275
+ }
1276
+ // captureMode:hot guard — refuse the merge if ANY participating memory
1277
+ // (primary or secondary) was user-captured or has unparseable frontmatter
1278
+ // (could have hidden a hot flag). Hot memories are user-explicit and
1279
+ // must not be deleted/overwritten by the consolidate LLM. 14 user
1280
+ // memories were silent-deleted by consolidate before this guard landed;
1281
+ // recovery required copying from .akm/archive/ by hand.
1282
+ const mergeParticipants = [op.primary, ...op.secondaries];
1283
+ const blockedParticipants = mergeParticipants.flatMap((ref) => {
1284
+ const e = memoryByRef.get(ref);
1285
+ if (!e)
1286
+ return [];
1287
+ const verdict = consolidateGuardStatus(e.filePath);
1288
+ if (verdict === "hot" || verdict === "unparseable")
1289
+ return [{ ref, verdict }];
1290
+ return [];
1291
+ });
1292
+ if (blockedParticipants.length > 0) {
1293
+ const detail = blockedParticipants.map((p) => `${p.ref} (${p.verdict})`).join(", ");
1294
+ warnings.push(`Merge: refused for ${op.primary} — ${blockedParticipants.length} participant(s) blocked by hot/unparseable frontmatter guard: ${detail}`);
1295
+ emitMergeFailureSkips("merge_participant_blocked");
605
1296
  continue;
606
1297
  }
607
1298
  // Backup secondaries before deleting
@@ -618,6 +1309,7 @@ export async function akmConsolidate(opts = {}) {
618
1309
  }
619
1310
  catch (e) {
620
1311
  warnings.push(`Merge: write failed for ${op.primary}: ${String(e)}`);
1312
+ emitMergeFailureSkips("merge_write_failed");
621
1313
  continue;
622
1314
  }
623
1315
  // Archive and delete secondaries (P1-B: soft-invalidation)
@@ -639,11 +1331,45 @@ export async function akmConsolidate(opts = {}) {
639
1331
  }
640
1332
  markJournalCompleted(stashDir, op.primary);
641
1333
  merged++;
1334
+ // 2026-05-26 accounting-leak fix: `merged` is op-level, but each
1335
+ // successful merge actions `1 + secondaries.length` memories. Without
1336
+ // this counter the accounting invariant breaks by `secondaries.length`
1337
+ // per successful merge (chunk loop excluded all secondaries from
1338
+ // judgedNoAction via targetRefs, but only the primary is credited to
1339
+ // `merged`). Count only loaded-memory secondaries; phantom secondary
1340
+ // refs never affected any chunk's targetRefs in the first place.
1341
+ for (const secRef of op.secondaries) {
1342
+ if (memoryByRef.has(secRef))
1343
+ mergedSecondaries++;
1344
+ }
1345
+ // Prune consumed refs from memoryByRef so later ops in this run cannot
1346
+ // reference an absorbed secondary as a merge primary and proceed with a
1347
+ // stale entry. Primary is rewritten (not deleted), so we only remove
1348
+ // secondaries; the primary ref remains valid under its new content.
1349
+ for (const secRef of op.secondaries) {
1350
+ memoryByRef.delete(secRef);
1351
+ }
642
1352
  }
643
1353
  else if (op.op === "delete") {
644
1354
  const entry = memoryByRef.get(op.ref);
645
1355
  if (!entry) {
646
1356
  warnings.push(`Delete: ${op.ref} not found in loaded memories — skipping.`);
1357
+ // Phantom ref: not in the batch so not in processed. Pushing to
1358
+ // skipReasons would inflate Σ(skipReasons) without a matching processed
1359
+ // entry, breaking the accounting invariant. Visibility is preserved via
1360
+ // the warnings array above.
1361
+ continue;
1362
+ }
1363
+ // captureMode:hot guard — refuse to delete user-captured memories OR
1364
+ // memories whose frontmatter is unparseable (could have hidden the hot
1365
+ // flag). The consolidate LLM was deleting hot-captured user memos as
1366
+ // "redundant" — 14 such deletes were silently archived between
1367
+ // 2026-05-19 and 2026-05-20 before this guard. Hot memories are
1368
+ // user-explicit and may only be deleted by the user.
1369
+ const guard = consolidateGuardStatus(entry.filePath);
1370
+ if (guard === "hot" || guard === "unparseable") {
1371
+ warnings.push(`Delete: refused for ${op.ref} — ${guard === "hot" ? "captureMode:hot (user-explicit; never auto-delete)" : "frontmatter unparseable (cannot verify hot flag absent)"}. Reason from LLM: "${op.reason ?? "n/a"}"`);
1372
+ pushSkipReason("delete", op.ref, "captureMode_hot_refused");
647
1373
  continue;
648
1374
  }
649
1375
  if (fs.existsSync(entry.filePath)) {
@@ -656,15 +1382,40 @@ export async function akmConsolidate(opts = {}) {
656
1382
  await deleteAssetFromSource(target.source, target.config, parsedRef);
657
1383
  markJournalCompleted(stashDir, op.ref);
658
1384
  deleted++;
1385
+ // Prune from memoryByRef so later ops in this run cannot reference a
1386
+ // deleted memory as a merge primary or secondary.
1387
+ memoryByRef.delete(op.ref);
659
1388
  }
660
1389
  catch (e) {
661
- warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
1390
+ // Distinguish "file already absent" from genuine failures. A prior run
1391
+ // may have deleted the file but the DB was not yet re-indexed, so the
1392
+ // ref still appeared in memoryByRef. The delete goal is already met.
1393
+ const msg = e instanceof Error ? e.message : String(e);
1394
+ if (msg.includes("not found in source")) {
1395
+ warnings.push(`Delete: ${op.ref} — file already absent (stale DB entry); skipping.`);
1396
+ pushSkipReason("delete", op.ref, "delete_already_gone");
1397
+ }
1398
+ else {
1399
+ warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
1400
+ pushSkipReason("delete", op.ref, "delete_failed");
1401
+ }
662
1402
  }
663
1403
  }
664
1404
  else if (op.op === "promote") {
665
1405
  const entry = memoryByRef.get(op.ref);
666
1406
  if (!entry) {
667
1407
  warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
1408
+ // Phantom ref: not in processed, so no skipReason (same rationale as
1409
+ // delete_ref_missing above).
1410
+ continue;
1411
+ }
1412
+ // Within-run source-ref dedup: skip if this source memory was already
1413
+ // promoted earlier in this run (safety belt — mergePlans already
1414
+ // deduplicates promote ops by source ref via Map, but this guard also
1415
+ // catches any future code paths that bypass mergePlans).
1416
+ if (promotedSourceRefs.has(op.ref)) {
1417
+ warnings.push(`Skipping promote: ${op.ref} already promoted in this run`);
1418
+ pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
668
1419
  continue;
669
1420
  }
670
1421
  let knowledgeRef = op.knowledgeRef;
@@ -679,10 +1430,11 @@ export async function akmConsolidate(opts = {}) {
679
1430
  knowledgeRef = `knowledge:${slug}`;
680
1431
  warnings.push(`Normalized invalid ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
681
1432
  }
682
- // Idempotency: check pending proposals
1433
+ // Idempotency: check pending proposals by target ref
683
1434
  const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
684
1435
  if (existingProposals.some((p) => p.status === "pending")) {
685
1436
  warnings.push(`Skipping promote: pending proposal already exists for ${knowledgeRef}`);
1437
+ pushSkipReason("promote", op.ref, "promote_pending_proposal_exists");
686
1438
  continue;
687
1439
  }
688
1440
  // Idempotency: check if knowledge asset already exists
@@ -690,6 +1442,7 @@ export async function akmConsolidate(opts = {}) {
690
1442
  const destPath = path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`);
691
1443
  if (fs.existsSync(destPath)) {
692
1444
  warnings.push(`Skipping promote: ${knowledgeRef} already exists in source`);
1445
+ pushSkipReason("promote", op.ref, "promote_already_exists");
693
1446
  continue;
694
1447
  }
695
1448
  let memoryContent = "";
@@ -698,24 +1451,173 @@ export async function akmConsolidate(opts = {}) {
698
1451
  }
699
1452
  catch (e) {
700
1453
  warnings.push(`Promote: could not read ${op.ref}: ${String(e)}`);
1454
+ pushSkipReason("promote", op.ref, "promote_read_failed");
1455
+ continue;
1456
+ }
1457
+ // Defensive sanitization: legacy memory files written by older
1458
+ // consolidate runs may still carry outer code fences or broken YAML.
1459
+ // Strip them here so we never propose a polluted asset.
1460
+ const promoteSanitized = sanitizeMergedContent(memoryContent);
1461
+ if (!promoteSanitized.ok) {
1462
+ warnings.push(`Promote: rejected ${op.ref} — source memory failed sanitization (${promoteSanitized.reason}).`);
1463
+ pushSkipReason("promote", op.ref, "promote_sanitization_failed");
1464
+ continue;
1465
+ }
1466
+ memoryContent = promoteSanitized.result.content;
1467
+ // SOURCE_SUPERSEDED guard: refuse to promote a memory whose source
1468
+ // frontmatter carries `status: superseded`. Predicate at module top
1469
+ // (`hasSupersededStatus`) so tests can exercise it directly.
1470
+ if (hasSupersededStatus(promoteSanitized.result.frontmatter)) {
1471
+ warnings.push(`Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
1472
+ pushSkipReason("promote", op.ref, "promote_superseded");
1473
+ continue;
1474
+ }
1475
+ // Parse the source memory up-front so the body/frontmatter checks below
1476
+ // share the same parsed view.
1477
+ const parsedMemory = parseFrontmatter(memoryContent);
1478
+ // Reject sources whose body is too small to make useful knowledge.
1479
+ // Observed failure: memory files whose body is literally a tags string
1480
+ // ("discord,notification,send-notification") get promoted to knowledge
1481
+ // proposals that no reviewer would accept. Threshold is conservative —
1482
+ // 100 chars catches single-line tag dumps without rejecting genuinely
1483
+ // terse but valid notes.
1484
+ const PROMOTE_BODY_MIN_CHARS = 100;
1485
+ const sourceBody = parsedMemory.content.trim();
1486
+ if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
1487
+ warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
1488
+ pushSkipReason("promote", op.ref, "promote_source_too_small");
1489
+ continue;
1490
+ }
1491
+ // Cross-run + within-run content dedup: if an identical body already
1492
+ // exists in ANY pending consolidate proposal (regardless of target ref),
1493
+ // skip. This prevents duplicate proposals when:
1494
+ // (a) Multiple source memories have identical bodies but differ only
1495
+ // in noise frontmatter (`inferenceProcessed: true` twin alongside
1496
+ // the original; differing `updated:` timestamps; etc.) — the body
1497
+ // is the load-bearing content, so dedup must hash on body only.
1498
+ // (b) A prior run created a proposal for the same body under a
1499
+ // different knowledgeRef slug.
1500
+ const bodyHash = createHash("sha256").update(sourceBody, "utf8").digest("hex");
1501
+ const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
1502
+ const contentDupProposal = allPendingConsolidateProposals.find((p) => {
1503
+ const otherBody = parseFrontmatter(p.payload.content).content.trim();
1504
+ return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
1505
+ });
1506
+ if (contentDupProposal) {
1507
+ warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
1508
+ pushSkipReason("promote", op.ref, "dedup_pending_proposal");
701
1509
  continue;
702
1510
  }
703
1511
  try {
704
- const proposal = createProposal(stashDir, {
1512
+ // Use LLM-provided description; fall back to memory's own description
1513
+ // (post-sanitization frontmatter is authoritative).
1514
+ const description = (typeof op.description === "string" && op.description.trim()
1515
+ ? op.description.trim()
1516
+ : parsedMemory.data?.description?.trim()) ?? "";
1517
+ // Validate the resolved frontmatter before emitting a proposal.
1518
+ // Required field: non-empty description. Reject obvious truncation
1519
+ // markers (description ends with `,`/`;`/`:`/`...`/hanging connector)
1520
+ // so the queue never sees half-formed metadata that the reviewer
1521
+ // would only reject.
1522
+ const fmCheck = validateProposalFrontmatter({ description });
1523
+ if (!fmCheck.ok) {
1524
+ warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
1525
+ pushSkipReason("promote", op.ref, "promote_invalid_frontmatter");
1526
+ continue;
1527
+ }
1528
+ // Merge `description` INTO the body's YAML frontmatter so it lands in
1529
+ // the on-disk asset when the proposal is accepted. The descriptionQuality
1530
+ // validator parses `payload.content` body (not the envelope
1531
+ // `payload.frontmatter`), and a memory's native frontmatter has
1532
+ // `captureMode`/`beliefState`/etc. but never `description` — without
1533
+ // this merge, 60+ pending proposals were blocked at accept-time with
1534
+ // MISSING_FRONTMATTER_DESCRIPTION even though the envelope had it.
1535
+ // (The body-frontmatter assumption baked into the 2026-05-20 comment
1536
+ // below was wrong: body fm and envelope fm only converge when the
1537
+ // writer explicitly merges them, which it now does.)
1538
+ const mergedBodyFm = {
1539
+ ...(parsedMemory.data ?? {}),
1540
+ description,
1541
+ };
1542
+ const serializedMergedFm = yamlStringify(mergedBodyFm).trimEnd();
1543
+ const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
1544
+ // Pre-emit dedup against pending consolidate proposals from the
1545
+ // same improve run (slug-variant match). The cross-run content-hash
1546
+ // dedup inside `mergePlans` handles duplicates against existing
1547
+ // stash assets — see commit history for the deletion of the
1548
+ // unbounded embedding + cross-type slug branches.
1549
+ const dedup = await checkPreEmitDedup({
1550
+ candidateRef: knowledgeRef,
1551
+ candidateText: `${description}. ${memoryContent}`,
1552
+ stashDir,
1553
+ config,
1554
+ });
1555
+ if (dedup.duplicate) {
1556
+ warnings.push(`Promote: skipped ${op.ref} → ${knowledgeRef} — ${dedup.reason}.`);
1557
+ pushSkipReason("promote", op.ref, "promote_dedup_window");
1558
+ continue;
1559
+ }
1560
+ const proposalResult = createProposal(stashDir, {
705
1561
  ref: knowledgeRef,
706
1562
  source: "consolidate",
707
- payload: { content: memoryContent },
1563
+ sourceRun,
1564
+ payload: {
1565
+ content: proposalContent,
1566
+ frontmatter: { description },
1567
+ },
1568
+ ...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
708
1569
  });
709
- promoted.push(proposal.id);
710
- markJournalCompleted(stashDir, op.ref);
1570
+ if (isProposalSkipped(proposalResult)) {
1571
+ warnings.push(`Promote: skipped proposal for ${op.ref} (${proposalResult.reason}): ${proposalResult.message}`);
1572
+ pushSkipReason("promote", op.ref, `promote_proposal_${proposalResult.reason}`);
1573
+ }
1574
+ else {
1575
+ promoted.push(proposalResult.id);
1576
+ promotedSourceRefs.add(op.ref);
1577
+ markJournalCompleted(stashDir, op.ref);
1578
+ }
711
1579
  }
712
1580
  catch (e) {
713
1581
  warnings.push(`Promote: createProposal failed for ${op.ref}: ${String(e)}`);
1582
+ pushSkipReason("promote", op.ref, "promote_create_failed");
1583
+ }
1584
+ }
1585
+ else if (op.op === "contradict") {
1586
+ // C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
1587
+ // (the SCC resolver in memory-improve.ts) has edges to work on.
1588
+ // Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
1589
+ const entry = memoryByRef.get(op.ref);
1590
+ const contradictorEntry = memoryByRef.get(op.contradictedByRef);
1591
+ if (!entry) {
1592
+ warnings.push(`Contradict: ${op.ref} not found in loaded memories — skipping.`);
1593
+ // Phantom ref: not in processed, so no skipReason (same rationale as
1594
+ // delete_ref_missing).
1595
+ continue;
1596
+ }
1597
+ if (!contradictorEntry) {
1598
+ warnings.push(`Contradict: ${op.contradictedByRef} not found — skipping.`);
1599
+ // op.ref IS in the batch (entry found above) so the skipReason is
1600
+ // correctly charged against a real processed memory.
1601
+ pushSkipReason("contradict", op.ref, "contradict_target_missing");
1602
+ continue;
1603
+ }
1604
+ try {
1605
+ // Write the contradiction edge: op.ref is contradicted by op.contradictedByRef
1606
+ writeContradictEdge(entry.filePath, op.contradictedByRef);
1607
+ contradicted++;
1608
+ markJournalCompleted(stashDir, op.ref);
1609
+ }
1610
+ catch (e) {
1611
+ warnings.push(`Contradict: failed to write edge for ${op.ref}: ${String(e)}`);
1612
+ pushSkipReason("contradict", op.ref, "contradict_write_failed");
714
1613
  }
715
1614
  }
716
1615
  }
717
1616
  cleanupJournal(stashDir, timestamp);
718
- // TTL cleanup: remove archive entries older than archiveRetentionDays (default 90)
1617
+ // TTL cleanup: remove archive entries older than archiveRetentionDays (default 90).
1618
+ // C-5 / #391: emit an `archive_cleanup` event before each deletion so the
1619
+ // audit trail records what was lost. Outbox pattern (EIP, Hohpe-Woolf) —
1620
+ // any event that is recorded must be queryable; silent deletes are an anti-pattern.
719
1621
  const archiveDir = path.join(stashDir, ".akm", "archive");
720
1622
  if (fs.existsSync(archiveDir)) {
721
1623
  const retentionMs = (config.archiveRetentionDays ?? 90) * 86_400_000;
@@ -723,8 +1625,20 @@ export async function akmConsolidate(opts = {}) {
723
1625
  for (const fname of fs.readdirSync(archiveDir)) {
724
1626
  const fp = path.join(archiveDir, fname);
725
1627
  try {
726
- if (fs.statSync(fp).mtimeMs < cutoff)
1628
+ const stat = fs.statSync(fp);
1629
+ if (stat.mtimeMs < cutoff) {
1630
+ // Emit event before deletion so the record survives the purge.
1631
+ appendEvent({
1632
+ eventType: "archive_cleanup",
1633
+ metadata: {
1634
+ file: fname,
1635
+ filePath: fp,
1636
+ ageMs: Date.now() - stat.mtimeMs,
1637
+ retentionMs,
1638
+ },
1639
+ });
727
1640
  fs.unlinkSync(fp);
1641
+ }
728
1642
  }
729
1643
  catch {
730
1644
  /* ignore race conditions */
@@ -742,57 +1656,465 @@ export async function akmConsolidate(opts = {}) {
742
1656
  merged,
743
1657
  deleted,
744
1658
  promoted,
1659
+ contradicted,
1660
+ failedChunks: totalChunksFailed,
1661
+ totalChunks: chunks.length,
1662
+ judgedNoAction,
1663
+ skipReasons,
1664
+ mergedSecondaries,
1665
+ failedChunkMemories,
745
1666
  warnings,
746
1667
  durationMs: Date.now() - startMs,
747
1668
  };
748
1669
  }
749
1670
  // ── Helpers ─────────────────────────────────────────────────────────────────
1671
+ // ── LLM-output sanitization ─────────────────────────────────────────────────
1672
+ //
1673
+ // Three classes of LLM defect have been observed across hundreds of
1674
+ // consolidate proposals (see audit notes in this branch):
1675
+ //
1676
+ // 1. Code-fence leakage: the entire merged asset is wrapped in
1677
+ // ```markdown … ``` (or ```yaml … ```) despite the prompt forbidding
1678
+ // fences. The post-processor used to pass this through verbatim, so the
1679
+ // first character of the asset content became a backtick rather than
1680
+ // `---`, defeating the frontmatter parser.
1681
+ // 2. YAML quote-escaping bugs: descriptions like `'"Specialty intro...:`
1682
+ // with unbalanced quotes that break the YAML reader. The post-processor
1683
+ // historically passed the LLM's raw scalar straight into a manually
1684
+ // assembled `description: <raw>` line.
1685
+ // 3. Truncated descriptions hitting token cutoffs — the model's max_tokens
1686
+ // runs out mid-sentence, leaving things like
1687
+ // `description: "Tables in narrow column containers need max-width:100% +"`
1688
+ // with no closing context.
1689
+ //
1690
+ // `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
1691
+ // all three at the point where LLM output is consumed.
1692
+ /**
1693
+ * Attempt to recover a frontmatter block that is missing its closing `---`.
1694
+ *
1695
+ * Scans lines after the opening `---` for the first blank line or the first
1696
+ * line that cannot be a YAML scalar (i.e. not a key-value, indented
1697
+ * continuation, comment, or list item). Injects `---` before that line so
1698
+ * the normal parser can proceed.
1699
+ *
1700
+ * Returns the patched string on success, or `null` if the structure is too
1701
+ * ambiguous to recover safely (e.g. no opening `---`, or no body content
1702
+ * found after the frontmatter key-value lines).
1703
+ */
1704
+ function recoverMalformedFrontmatter(raw) {
1705
+ if (!raw.startsWith("---"))
1706
+ return null;
1707
+ const lines = raw.split(/\r?\n/);
1708
+ // Skip the opening `---` line (index 0).
1709
+ let insertAt = -1;
1710
+ for (let i = 1; i < lines.length; i++) {
1711
+ const line = lines[i];
1712
+ // A blank line marks the end of the frontmatter block in many YAML variants.
1713
+ if (line.trim() === "") {
1714
+ insertAt = i;
1715
+ break;
1716
+ }
1717
+ // A line that is clearly body content: doesn't look like a YAML key, an
1718
+ // indented continuation, a comment, or a sequence item.
1719
+ const isYaml = /^\w[\w-]*\s*:/.test(line) || // key: value
1720
+ /^\s+\S/.test(line) || // indented continuation / nested
1721
+ /^\s*#/.test(line) || // YAML comment
1722
+ /^\s*-\s/.test(line); // sequence item
1723
+ if (!isYaml) {
1724
+ insertAt = i;
1725
+ break;
1726
+ }
1727
+ }
1728
+ if (insertAt < 0)
1729
+ return null;
1730
+ const result = [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
1731
+ return result;
1732
+ }
1733
+ /**
1734
+ * Outer-fence stripper specific to consolidate. Unlike the shared
1735
+ * `stripMarkdownFences` helper (which only handles markdown fences), this
1736
+ * variant additionally recognises `yaml` and bare-language fences and refuses
1737
+ * to strip an unbalanced fence — i.e. a leading ``` with no trailing ``` is
1738
+ * treated as a malformed response, not partially sanitized.
1739
+ *
1740
+ * Returns `null` when only one half of a fence pair is present (caller
1741
+ * should reject the response entirely).
1742
+ */
1743
+ export function stripOuterCodeFence(raw) {
1744
+ const trimmed = raw.trim();
1745
+ const leading = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n/i);
1746
+ const trailing = trimmed.match(/\r?\n```\s*$/);
1747
+ if (!leading && !trailing)
1748
+ return { content: trimmed, stripped: false };
1749
+ if (!leading || !trailing)
1750
+ return null; // unbalanced — refuse
1751
+ const inner = trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim();
1752
+ return { content: inner, stripped: true };
1753
+ }
1754
+ export function sanitizeMergedContent(raw) {
1755
+ // Step 1: Strip outer code fence.
1756
+ // Recovery path: if only the leading fence is present, strip it and continue
1757
+ // provided the inner content starts with `---`. Trailing-only fences are NOT
1758
+ // recovered — a trailing ``` is more likely a body code block than a forgotten
1759
+ // wrapper, so recovering would silently corrupt the body.
1760
+ let body;
1761
+ {
1762
+ const fenceResult = stripOuterCodeFence(raw);
1763
+ if (fenceResult) {
1764
+ body = fenceResult.content;
1765
+ }
1766
+ else {
1767
+ const trimmed = raw.trim();
1768
+ const leadingMatch = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i);
1769
+ const inner = leadingMatch ? leadingMatch[1].trim() : null;
1770
+ if (!inner?.startsWith("---")) {
1771
+ return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
1772
+ }
1773
+ body = inner;
1774
+ }
1775
+ }
1776
+ // Strip <think> blocks (some local models still emit them despite system prompts).
1777
+ body = body.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
1778
+ // Step 2: Verify frontmatter sentinel.
1779
+ // Recovery path: LLM sometimes emits 1-2 lines of preamble (e.g. "Here is the
1780
+ // merged content:") before the `---`. Accept if `---` appears within 300 chars.
1781
+ // Beyond that it's more likely a body section divider, not a frontmatter start.
1782
+ if (!body.startsWith("---")) {
1783
+ const nlIdx = body.indexOf("\n---");
1784
+ if (nlIdx >= 0 && nlIdx < 300) {
1785
+ body = body.slice(nlIdx + 1);
1786
+ }
1787
+ else {
1788
+ return { ok: false, reason: "MISSING_FRONTMATTER_SENTINEL" };
1789
+ }
1790
+ }
1791
+ // Extract frontmatter block.
1792
+ // Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
1793
+ // by scanning lines after the opening `---` for the first blank line or the
1794
+ // first line that isn't a YAML key-value pair, then inject `---` there.
1795
+ let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1796
+ if (!match) {
1797
+ const recovered = recoverMalformedFrontmatter(body);
1798
+ if (recovered) {
1799
+ match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
1800
+ }
1801
+ if (!match) {
1802
+ return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
1803
+ }
1804
+ }
1805
+ // Re-parse via the yaml library so any quote-escaping mistakes either get
1806
+ // normalised or surface as a parse error we can reject.
1807
+ // Recovery: if the strict yaml library fails, fall back to the lenient
1808
+ // hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
1809
+ // quirks (unescaped special chars, bare scalars, etc.). If it recovers
1810
+ // at least one key, proceed — yamlStringify below will re-serialize
1811
+ // cleanly. Only reject if both parsers fail to extract any data.
1812
+ let parsedFm;
1813
+ try {
1814
+ parsedFm = yamlParse(match[1]);
1815
+ }
1816
+ catch (e) {
1817
+ const fallback = parseFrontmatter(`---\n${match[1]}\n---\n${match[2]}`);
1818
+ if (fallback.frontmatter !== null && Object.keys(fallback.data).length > 0) {
1819
+ parsedFm = fallback.data;
1820
+ }
1821
+ else {
1822
+ return { ok: false, reason: `INVALID_YAML: ${e instanceof Error ? e.message : String(e)}` };
1823
+ }
1824
+ }
1825
+ if (parsedFm === null || typeof parsedFm !== "object" || Array.isArray(parsedFm)) {
1826
+ return { ok: false, reason: "FRONTMATTER_NOT_OBJECT" };
1827
+ }
1828
+ const fm = parsedFm;
1829
+ // Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
1830
+ // `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
1831
+ // these, but small models still do. Replace any such leak with today's ISO
1832
+ // date OR drop the field if we can't safely normalise it.
1833
+ normalizeUpdatedField(fm);
1834
+ // Re-serialise via yaml.stringify to fix any quoting quirks.
1835
+ let serialized;
1836
+ try {
1837
+ serialized = yamlStringify(fm).trimEnd();
1838
+ }
1839
+ catch (e) {
1840
+ return { ok: false, reason: `YAML_STRINGIFY_FAILED: ${e instanceof Error ? e.message : String(e)}` };
1841
+ }
1842
+ const cleaned = assembleAssetFromString(serialized, match[2]);
1843
+ return { ok: true, result: { content: cleaned, frontmatter: fm } };
1844
+ }
1845
+ /**
1846
+ * Mutate `fm.updated` in place to normalise placeholder leaks emitted by the
1847
+ * LLM. The consolidate prompt forbids these, but small models still produce
1848
+ * literal `today` / `{today: null}` / `now` values.
1849
+ *
1850
+ * Rules:
1851
+ * - A real ISO-style date string (YYYY-MM-DD, optionally with time) stays as-is.
1852
+ * - A Date object (some YAML parsers materialise dates) is converted to its
1853
+ * ISO yyyy-mm-dd form.
1854
+ * - A placeholder string ("today", "now", "{today}", "${today}", template
1855
+ * variables) is replaced with today's ISO date.
1856
+ * - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
1857
+ * - `null`, empty string, missing → left alone (no field added; reviewers
1858
+ * should not silently gain metadata they didn't write).
1859
+ *
1860
+ * Exported for unit testing.
1861
+ */
1862
+ export function normalizeUpdatedField(fm) {
1863
+ if (!("updated" in fm))
1864
+ return;
1865
+ const v = fm.updated;
1866
+ if (v === null || v === undefined || v === "")
1867
+ return;
1868
+ const todayIso = new Date().toISOString().slice(0, 10);
1869
+ if (v instanceof Date) {
1870
+ fm.updated = v.toISOString().slice(0, 10);
1871
+ return;
1872
+ }
1873
+ if (typeof v === "string") {
1874
+ const trimmed = v.trim().toLowerCase();
1875
+ if (/^\d{4}-\d{2}-\d{2}/.test(v.trim()))
1876
+ return; // already a real date
1877
+ if (trimmed === "today" ||
1878
+ trimmed === "now" ||
1879
+ trimmed === "{today}" ||
1880
+ // biome-ignore lint/suspicious/noTemplateCurlyInString: matches the literal user-typed placeholder text "${today}" so we can normalize it to today's ISO date
1881
+ trimmed === "${today}" ||
1882
+ trimmed === "{{today}}" ||
1883
+ /^\{?\s*today\s*\}?$/.test(trimmed)) {
1884
+ fm.updated = todayIso;
1885
+ return;
1886
+ }
1887
+ // Unknown string format — leave alone so it's visible in the diff.
1888
+ return;
1889
+ }
1890
+ if (typeof v === "object") {
1891
+ // Maps like `{today: null}`, `{now: null}` — clearly a template leak.
1892
+ fm.updated = todayIso;
1893
+ return;
1894
+ }
1895
+ }
1896
+ /**
1897
+ * Normalise a knowledge slug for variant-aware deduplication. Collapses:
1898
+ * - date suffixes (`-may-2026`, `-2026-05-03`, `-2026`)
1899
+ * - numeric counter suffixes (`-2`, `-3`)
1900
+ * - trailing -patterns / -2026-05-03 styles
1901
+ * - word reorderings via alphabetical sort of the remaining tokens.
1902
+ *
1903
+ * Two slugs that normalise to the same string are considered the same asset
1904
+ * for dedup purposes even if they don't share an exact ref.
1905
+ */
1906
+ export function normalizeSlugForDedup(ref) {
1907
+ const slug = ref.replace(/^[^:]+:/, "");
1908
+ const monthRe = /(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i;
1909
+ const tokens = slug
1910
+ .toLowerCase()
1911
+ .split("-")
1912
+ .filter((tok) => tok.length > 0)
1913
+ // Strip purely-numeric tokens (years, dates, counter suffixes like -2 / -3).
1914
+ // Numbers carry no semantic information for our dedup purposes — every
1915
+ // observed defective slug variant differs only in dates or counters.
1916
+ .filter((tok) => !/^\d+$/.test(tok))
1917
+ .filter((tok) => !monthRe.test(tok));
1918
+ // Sort to absorb word reorderings.
1919
+ tokens.sort();
1920
+ return tokens.join("-");
1921
+ }
1922
+ /**
1923
+ * Pre-emit dedup check: compare the candidate ref against pending consolidate
1924
+ * proposals only. Returns a reason string if a slug-variant match is found,
1925
+ * else null.
1926
+ *
1927
+ * Historical context (REMOVED 2026-05-20): this function previously also ran
1928
+ * (a) a normalised-slug match against existing knowledge AND memory entries
1929
+ * in the DB, and
1930
+ * (b) an embedding cosine-similarity check (>= 0.85) against ALL knowledge
1931
+ * and non-derived memory entries.
1932
+ * Both branches had ZERO observed fires across 30 sampled runs in the
1933
+ * post-fix window. The 29 actual dedup catches all came from the SEPARATE
1934
+ * content-hash dedup inside `mergePlans` (the older SHA-256 helper). The
1935
+ * embedding branch in particular had unbounded cost per promote (embedded
1936
+ * every knowledge + non-derived memory entry, every time) with no observed
1937
+ * benefit. Empirical signal → deleted.
1938
+ *
1939
+ * What remains: a check against pending consolidate proposals in the SAME
1940
+ * improve run. This catches duplicates queued back-to-back within a single
1941
+ * improve invocation — a different concern from the cross-run content-hash
1942
+ * dedup, and cheap (no embeddings, no DB query).
1943
+ */
1944
+ export async function checkPreEmitDedup(opts) {
1945
+ const normCandidate = normalizeSlugForDedup(opts.candidateRef);
1946
+ // Pending consolidate proposals (slug match) — within the same improve run.
1947
+ const pendingConsolidate = listProposals(opts.stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
1948
+ for (const p of pendingConsolidate) {
1949
+ if (normalizeSlugForDedup(p.ref) === normCandidate) {
1950
+ return { duplicate: true, reason: `slug-variant of pending proposal ${p.id} (${p.ref})` };
1951
+ }
1952
+ }
1953
+ return { duplicate: false };
1954
+ }
1955
+ /**
1956
+ * Incremental candidate set: {changed} ∪ {top-k persisted-vector neighbours of
1957
+ * each changed memory}, intersected with the loaded pool. Returns [] when
1958
+ * nothing changed (caller emits a no-op envelope), the full pool when
1959
+ * everything changed or the index can't answer (fail-open to preserve merge
1960
+ * correctness). `since` is an ISO timestamp.
1961
+ */
1962
+ export function narrowToIncrementalCandidates(memories, since, warnings) {
1963
+ const isChanged = (m) => {
1964
+ try {
1965
+ return fs.statSync(m.filePath).mtime.toISOString() > since;
1966
+ }
1967
+ catch {
1968
+ return true; // never silently drop a memory we cannot stat
1969
+ }
1970
+ };
1971
+ const changed = memories.filter(isChanged);
1972
+ if (changed.length === 0)
1973
+ return [];
1974
+ if (changed.length === memories.length)
1975
+ return memories;
1976
+ const NEIGHBORS_PER_CHANGED = 5;
1977
+ const byName = new Map(memories.map((m) => [m.name, m]));
1978
+ const keep = new Set(changed.map((m) => m.name));
1979
+ let db;
1980
+ try {
1981
+ db = openExistingDatabase();
1982
+ for (const m of changed) {
1983
+ const id = findEntryIdByRef(db, `memory:${m.name}`);
1984
+ if (id === undefined)
1985
+ continue;
1986
+ for (const hit of getNeighborsByEntryId(db, id, NEIGHBORS_PER_CHANGED + 1)) {
1987
+ if (hit.id === id)
1988
+ continue;
1989
+ const entry = getEntryById(db, hit.id);
1990
+ if (!entry)
1991
+ continue;
1992
+ const name = entry.entry.name;
1993
+ if (byName.has(name))
1994
+ keep.add(name); // only neighbours present in the loaded pool
1995
+ }
1996
+ }
1997
+ }
1998
+ catch {
1999
+ warnings.push("Incremental consolidation: index unavailable — processing full pool.");
2000
+ return memories;
2001
+ }
2002
+ finally {
2003
+ if (db)
2004
+ closeDatabase(db);
2005
+ }
2006
+ const candidates = memories.filter((m) => keep.has(m.name));
2007
+ warnings.push(`Incremental consolidation: ${changed.length} changed + neighbours → ${candidates.length}/${memories.length} memories considered (since ${since}).`);
2008
+ return candidates;
2009
+ }
750
2010
  function loadMemoriesForSource(source, stashDir, warnings) {
751
- let memories = loadMemoriesFromDb(source ? resolveSourcePath(source) : undefined);
2011
+ // Load from DB first
2012
+ let memories = [];
2013
+ let db;
2014
+ try {
2015
+ db = openExistingDatabase();
2016
+ const entries = getAllEntries(db, "memory");
2017
+ memories = entries
2018
+ .filter((e) => {
2019
+ if (!source)
2020
+ return true;
2021
+ return path.resolve(e.stashDir) === path.resolve(source);
2022
+ })
2023
+ .filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
2024
+ // Skip stale DB entries whose file was deleted by a prior run but not yet
2025
+ // re-indexed. Without this guard the deleted file's ref appears in chunks
2026
+ // sent to the LLM, which then proposes a second delete → delete_failed
2027
+ // because the file is already gone. Re-indexing runs on a cron cadence so
2028
+ // several successful deletes can accumulate before the DB catches up.
2029
+ .filter((e) => fs.existsSync(e.filePath))
2030
+ .map((e) => ({
2031
+ name: e.entry.name,
2032
+ filePath: e.filePath,
2033
+ description: e.entry.description ?? "",
2034
+ tags: e.entry.tags ?? [],
2035
+ stashDir: e.stashDir,
2036
+ }));
2037
+ }
2038
+ catch {
2039
+ memories = [];
2040
+ }
2041
+ finally {
2042
+ if (db)
2043
+ closeDatabase(db);
2044
+ }
752
2045
  if (memories.length === 0) {
753
2046
  // DB fallback: walk filesystem
754
2047
  const memoriesDir = path.join(source ?? stashDir, "memories");
755
- memories = loadMemoriesFromFs(memoriesDir, source ?? stashDir);
2048
+ const fsStashDir = source ?? stashDir;
2049
+ if (fs.existsSync(memoriesDir)) {
2050
+ for (const fname of fs.readdirSync(memoriesDir)) {
2051
+ if (!fname.endsWith(".md"))
2052
+ continue;
2053
+ const filePath = path.join(memoriesDir, fname);
2054
+ const name = fname.replace(/\.md$/, "");
2055
+ if (!isConsolidationEligibleMemoryName(name))
2056
+ continue;
2057
+ memories.push({ name, filePath, description: "", tags: [], stashDir: fsStashDir });
2058
+ }
2059
+ }
756
2060
  if (memories.length > 0) {
757
2061
  warnings.push("DB not found or empty — loaded memories directly from filesystem.");
758
2062
  }
759
2063
  }
760
2064
  return memories;
761
2065
  }
762
- function resolveSourcePath(sourceName) {
763
- // If it looks like an absolute path, use directly
764
- if (path.isAbsolute(sourceName))
765
- return sourceName;
766
- return sourceName;
767
- }
768
- async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef, warnings) {
2066
+ async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef) {
769
2067
  // Only handle single-secondary merges per design (one call per merge op)
770
2068
  const secRef = secondaryRefs[0];
771
2069
  const secEntry = memoryByRef.get(secRef);
772
2070
  if (!secEntry)
773
- return null;
2071
+ return { error: "merge_read_failed", detail: `secondary ${secRef} not in memoryByRef` };
774
2072
  let secBody = "";
775
2073
  try {
776
2074
  secBody = fs.readFileSync(secEntry.filePath, "utf8");
777
2075
  }
778
2076
  catch {
779
- warnings.push(`Merge: could not read secondary ${secRef} — skipping.`);
780
- return null;
2077
+ return { error: "merge_read_failed", detail: `could not read secondary ${secRef}` };
781
2078
  }
2079
+ const primaryFmKeys = Object.keys(parseFrontmatter(primaryBody).data);
2080
+ const secFmKeys = Object.keys(parseFrontmatter(secBody).data);
2081
+ const requiredFmKeys = [...new Set([...primaryFmKeys, ...secFmKeys])];
782
2082
  const prompt = [
783
2083
  "Merge these two memory assets into one. Output ONLY the merged markdown (with YAML frontmatter). Do not explain, do not use code fences.",
784
2084
  "",
2085
+ "## OUTPUT FORMAT (MANDATORY)",
2086
+ "Return raw markdown content beginning DIRECTLY with the `---` frontmatter delimiter.",
2087
+ "DO NOT wrap your entire response in a code fence.",
2088
+ "",
2089
+ 'GOOD: "---\\ndescription: ...\\n---\\nBody content."',
2090
+ 'BAD: "```markdown\\n---\\ndescription: ...\\n---\\nBody content.\\n```"',
2091
+ 'BAD: "```yaml\\n---\\ndescription: ...\\n---\\nBody content.\\n```"',
2092
+ "",
2093
+ "## FRONTMATTER RULES (MANDATORY)",
2094
+ "- The `updated:` field, if present, MUST be a real ISO date (e.g. `updated: 2026-05-20`). NEVER emit `updated: today`, `updated: now`, or `updated: {today: null}`. If you don't have a real date, OMIT the field — the post-processor will not invent one.",
2095
+ "- REQUIRED: The merged frontmatter MUST include a `description` field with a concise one-sentence summary of the merged asset's content. If neither source has a `description` field, synthesize one from the content.",
2096
+ requiredFmKeys.length > 0
2097
+ ? `- CRITICAL: The merged frontmatter MUST include ALL of these keys from both source memories: ${requiredFmKeys.join(", ")}. Do NOT drop any of them.`
2098
+ : null,
2099
+ "",
785
2100
  `=== Primary memory (${primaryRef}) ===`,
786
2101
  primaryBody,
787
2102
  "",
788
2103
  `=== Secondary memory (${secRef}) ===`,
789
2104
  secBody,
790
- ].join("\n");
2105
+ ]
2106
+ .filter((line) => line !== null)
2107
+ .join("\n");
2108
+ // Use the same per-process profile resolution as the chunk-plan call above
2109
+ // so the merge generation step doesn't silently revert to the default LLM.
2110
+ const llmConfig = resolveConsolidateLlmConfig(config);
791
2111
  const result = await tryLlmFeature("memory_consolidation", config, async () => {
792
- if (!config.llm)
2112
+ if (!llmConfig)
793
2113
  return { ok: false, error: "No LLM configured for consolidation" };
794
2114
  try {
795
- const content = await chatCompletion(config.llm, [{ role: "user", content: prompt }]);
2115
+ const content = await chatCompletion(llmConfig, [{ role: "user", content: prompt }], {
2116
+ enableThinking: false,
2117
+ });
796
2118
  return { ok: true, content };
797
2119
  }
798
2120
  catch (e) {
@@ -800,10 +2122,77 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
800
2122
  }
801
2123
  }, { ok: false, error: `merge content generation failed for ${primaryRef}` });
802
2124
  if (!result.ok) {
803
- warnings.push(result.error ?? `merge content generation failed for ${primaryRef}`);
804
- return null;
2125
+ return {
2126
+ error: "merge_transport_failed",
2127
+ detail: result.error ?? `merge content generation failed for ${primaryRef}`,
2128
+ };
2129
+ }
2130
+ // Sanitize LLM output: strip outer code fences (defends against the
2131
+ // ```markdown … ``` leak observed in production), re-serialise frontmatter
2132
+ // through the yaml lib (fixes quote-escaping mistakes), and reject empty
2133
+ // or fence-only responses.
2134
+ const sanitized = sanitizeMergedContent(result.content ?? "");
2135
+ if (!sanitized.ok) {
2136
+ const reason = sanitized.reason;
2137
+ const isFenceError = reason === "UNBALANCED_CODE_FENCE" ||
2138
+ reason === "MISSING_FRONTMATTER_SENTINEL" ||
2139
+ reason === "MALFORMED_FRONTMATTER_BLOCK" ||
2140
+ reason === "FRONTMATTER_NOT_OBJECT";
2141
+ const mergeReason = isFenceError ? "merge_fence_rejected" : "merge_yaml_invalid";
2142
+ return { error: mergeReason, detail: `${primaryRef} — ${reason}` };
2143
+ }
2144
+ const mergedRaw = sanitized.result.content;
2145
+ // C-4 / #383: Content-preservation lint (mem0 §3.2, arXiv:2504.19413).
2146
+ // Guards against LLM-generated merged content that silently drops information
2147
+ // from the source assets. Two checks:
2148
+ // 1. Body size: merged body must be >= 50% of the larger source body.
2149
+ // 2. Frontmatter superset: merged frontmatter must contain all keys present
2150
+ // in both source frontmatters.
2151
+ // Failures return a discriminated error so the call site can emit a specific
2152
+ // skip-reason key in the histogram.
2153
+ try {
2154
+ const primaryFm = parseFrontmatter(primaryBody);
2155
+ const secFm = parseFrontmatter(secBody);
2156
+ const mergedFm = parseFrontmatter(mergedRaw);
2157
+ // Check body size — blended floor: max(ratio × largerLen, absoluteFloor).
2158
+ // Deduplication is expected, so the ratio is lower than the reflect gate
2159
+ // (0.3 vs 0.5). The absolute floor protects very short memory pairs where
2160
+ // the ratio alone would produce a near-zero threshold.
2161
+ const primaryBodyLen = (primaryFm.content ?? "").trim().length;
2162
+ const secBodyLen = (secFm.content ?? "").trim().length;
2163
+ const mergedBodyLen = (mergedFm.content ?? "").trim().length;
2164
+ const largerBodyLen = Math.max(primaryBodyLen, secBodyLen);
2165
+ const mergeFloor = Math.max(MERGE_SHRINK_RATIO_MIN * largerBodyLen, MERGE_ABSOLUTE_FLOOR_CHARS);
2166
+ if (largerBodyLen > 0 && mergedBodyLen < mergeFloor) {
2167
+ return {
2168
+ error: "merge_content_too_short",
2169
+ detail: `${primaryRef} — merged body (${mergedBodyLen} chars) is less than floor (${Math.round(mergeFloor)} chars; max(${MERGE_SHRINK_RATIO_MIN}×${largerBodyLen}, ${MERGE_ABSOLUTE_FLOOR_CHARS}))`,
2170
+ };
2171
+ }
2172
+ // Check frontmatter superset — attempt repair before rejecting.
2173
+ const primaryKeys = Object.keys(primaryFm.data ?? {});
2174
+ const secKeys = Object.keys(secFm.data ?? {});
2175
+ const mergedKeys = new Set(Object.keys(mergedFm.data ?? {}));
2176
+ const missingKeys = [...new Set([...primaryKeys, ...secKeys])].filter((k) => !mergedKeys.has(k));
2177
+ if (missingKeys.length > 0) {
2178
+ // Inject missing keys from source FMs. Primary value wins on conflict.
2179
+ const repairedFmData = { ...mergedFm.data };
2180
+ for (const key of missingKeys) {
2181
+ repairedFmData[key] =
2182
+ key in primaryFm.data
2183
+ ? primaryFm.data[key]
2184
+ : secFm.data[key];
2185
+ }
2186
+ normalizeUpdatedField(repairedFmData);
2187
+ const repairedYaml = yamlStringify(repairedFmData).trimEnd();
2188
+ const bodyPart = mergedFm.content ?? "";
2189
+ return { content: `---\n${repairedYaml}\n---\n${bodyPart}` };
2190
+ }
2191
+ }
2192
+ catch {
2193
+ // parseFrontmatter failures are non-fatal — allow the merge to proceed.
805
2194
  }
806
- return result.content;
2195
+ return { content: mergedRaw };
807
2196
  }
808
2197
  async function promptConfirm(message) {
809
2198
  process.stdout.write(message);