akm-cli 0.8.0-rc2 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (313) hide show
  1. package/{.github/CHANGELOG.md → CHANGELOG.md} +238 -3
  2. package/README.md +22 -6
  3. package/SECURITY.md +93 -0
  4. package/dist/assets/help/help-accept.md +12 -0
  5. package/dist/assets/help/help-improve.md +81 -0
  6. package/dist/{commands → assets}/help/help-proposals.md +7 -4
  7. package/dist/assets/help/help-reject.md +11 -0
  8. package/dist/{output → assets/hints}/cli-hints-full.md +60 -32
  9. package/dist/{output → assets/hints}/cli-hints-short.md +10 -7
  10. package/dist/assets/profiles/default.json +15 -0
  11. package/dist/assets/profiles/graph-refresh.json +13 -0
  12. package/dist/assets/profiles/memory-focus.json +12 -0
  13. package/dist/assets/profiles/quick.json +15 -0
  14. package/dist/assets/profiles/thorough.json +15 -0
  15. package/dist/assets/prompts/extract-session.md +80 -0
  16. package/dist/assets/prompts/graph-extract-user-prompt.md +35 -0
  17. package/dist/assets/tasks/graph-refresh-weekly.yml +10 -0
  18. package/dist/cli/config-migrate.js +144 -0
  19. package/dist/cli/config-validate.js +39 -0
  20. package/dist/cli/confirm.js +73 -0
  21. package/dist/cli/parse-args.js +93 -3
  22. package/dist/cli/shared.js +129 -0
  23. package/dist/cli.js +2141 -1268
  24. package/dist/commands/add-cli.js +279 -0
  25. package/dist/commands/agent-dispatch.js +20 -12
  26. package/dist/commands/agent-support.js +11 -5
  27. package/dist/commands/completions.js +3 -0
  28. package/dist/commands/config-cli.js +129 -517
  29. package/dist/commands/consolidate.js +1557 -147
  30. package/dist/commands/curate.js +44 -3
  31. package/dist/commands/db-cli.js +23 -0
  32. package/dist/commands/distill-promotion-policy.js +5 -3
  33. package/dist/commands/distill.js +906 -100
  34. package/dist/commands/env.js +213 -0
  35. package/dist/commands/eval-cases.js +3 -0
  36. package/dist/commands/events.js +3 -0
  37. package/dist/commands/extract-cli.js +127 -0
  38. package/dist/commands/extract-prompt.js +217 -0
  39. package/dist/commands/extract.js +477 -0
  40. package/dist/commands/feedback-cli.js +331 -0
  41. package/dist/commands/graph.js +260 -5
  42. package/dist/commands/health.js +1042 -55
  43. package/dist/commands/history.js +51 -16
  44. package/dist/commands/improve-auto-accept.js +97 -0
  45. package/dist/commands/improve-cli.js +236 -0
  46. package/dist/commands/improve-profiles.js +138 -0
  47. package/dist/commands/improve-result-file.js +167 -0
  48. package/dist/commands/improve.js +1736 -346
  49. package/dist/commands/info.js +26 -28
  50. package/dist/commands/init.js +49 -1
  51. package/dist/commands/installed-stashes.js +6 -23
  52. package/dist/commands/knowledge.js +3 -0
  53. package/dist/commands/lint/agent-linter.js +3 -0
  54. package/dist/commands/lint/base-linter.js +199 -5
  55. package/dist/commands/lint/command-linter.js +3 -0
  56. package/dist/commands/lint/default-linter.js +3 -0
  57. package/dist/commands/lint/env-key-rules.js +154 -0
  58. package/dist/commands/lint/index.js +92 -3
  59. package/dist/commands/lint/knowledge-linter.js +3 -0
  60. package/dist/commands/lint/markdown-insertion.js +343 -0
  61. package/dist/commands/lint/memory-linter.js +3 -0
  62. package/dist/commands/lint/registry.js +3 -0
  63. package/dist/commands/lint/skill-linter.js +3 -0
  64. package/dist/commands/lint/task-linter.js +15 -12
  65. package/dist/commands/lint/types.js +3 -0
  66. package/dist/commands/lint/workflow-linter.js +3 -0
  67. package/dist/commands/lint.js +3 -0
  68. package/dist/commands/migration-help.js +5 -2
  69. package/dist/commands/proposal-drain-policies.js +128 -0
  70. package/dist/commands/proposal-drain.js +477 -0
  71. package/dist/commands/proposal.js +60 -6
  72. package/dist/commands/propose.js +24 -19
  73. package/dist/commands/reflect.js +1004 -94
  74. package/dist/commands/registry-cli.js +150 -0
  75. package/dist/commands/registry-search.js +3 -0
  76. package/dist/commands/remember-cli.js +257 -0
  77. package/dist/commands/remember.js +15 -6
  78. package/dist/commands/schema-repair.js +88 -15
  79. package/dist/commands/search.js +99 -14
  80. package/dist/commands/secret.js +173 -0
  81. package/dist/commands/self-update.js +3 -0
  82. package/dist/commands/show.js +32 -13
  83. package/dist/commands/source-add.js +7 -35
  84. package/dist/commands/source-clone.js +3 -0
  85. package/dist/commands/source-manage.js +3 -0
  86. package/dist/commands/tasks.js +161 -95
  87. package/dist/commands/url-checker.js +3 -0
  88. package/dist/core/action-contributors.js +3 -0
  89. package/dist/core/asset-ref.js +13 -2
  90. package/dist/core/asset-registry.js +9 -2
  91. package/dist/core/asset-serialize.js +88 -0
  92. package/dist/core/asset-spec.js +61 -5
  93. package/dist/core/common.js +93 -5
  94. package/dist/core/concurrent.js +3 -0
  95. package/dist/core/config-io.js +347 -0
  96. package/dist/core/config-migration.js +622 -0
  97. package/dist/core/config-schema.js +558 -0
  98. package/dist/core/config-sources.js +108 -0
  99. package/dist/core/config-types.js +4 -0
  100. package/dist/core/config-walker.js +337 -0
  101. package/dist/core/config.js +366 -1077
  102. package/dist/core/errors.js +42 -20
  103. package/dist/core/events.js +31 -25
  104. package/dist/core/file-lock.js +104 -0
  105. package/dist/core/frontmatter.js +75 -10
  106. package/dist/core/lesson-lint.js +3 -0
  107. package/dist/core/markdown.js +3 -0
  108. package/dist/core/memory-belief.js +62 -0
  109. package/dist/core/memory-contradiction-detect.js +274 -0
  110. package/dist/core/memory-improve.js +142 -14
  111. package/dist/core/parse.js +3 -0
  112. package/dist/core/paths.js +218 -50
  113. package/dist/core/proposal-quality-validators.js +380 -0
  114. package/dist/core/proposal-validators.js +11 -3
  115. package/dist/core/proposals.js +464 -5
  116. package/dist/core/state-db.js +349 -56
  117. package/dist/core/text-truncation.js +107 -0
  118. package/dist/core/time.js +3 -0
  119. package/dist/core/tty.js +59 -0
  120. package/dist/core/warn.js +7 -2
  121. package/dist/core/write-source.js +12 -0
  122. package/dist/indexer/db-backup.js +391 -0
  123. package/dist/indexer/db-search.js +136 -28
  124. package/dist/indexer/db.js +661 -166
  125. package/dist/indexer/ensure-index.js +3 -0
  126. package/dist/indexer/file-context.js +3 -0
  127. package/dist/indexer/graph-boost.js +162 -40
  128. package/dist/indexer/graph-db.js +241 -51
  129. package/dist/indexer/graph-dedup.js +3 -7
  130. package/dist/indexer/graph-extraction.js +242 -149
  131. package/dist/indexer/index-context.js +3 -9
  132. package/dist/indexer/indexer.js +86 -16
  133. package/dist/indexer/llm-cache.js +24 -19
  134. package/dist/indexer/manifest.js +3 -0
  135. package/dist/indexer/matchers.js +184 -11
  136. package/dist/indexer/memory-inference.js +94 -50
  137. package/dist/indexer/metadata-contributors.js +3 -0
  138. package/dist/indexer/metadata.js +110 -50
  139. package/dist/indexer/path-resolver.js +3 -0
  140. package/dist/indexer/project-context.js +192 -0
  141. package/dist/indexer/ranking-contributors.js +134 -7
  142. package/dist/indexer/ranking.js +8 -1
  143. package/dist/indexer/search-fields.js +5 -9
  144. package/dist/indexer/search-hit-enrichers.js +91 -2
  145. package/dist/indexer/search-source.js +20 -1
  146. package/dist/indexer/semantic-status.js +4 -1
  147. package/dist/indexer/staleness-detect.js +447 -0
  148. package/dist/indexer/usage-events.js +12 -9
  149. package/dist/indexer/walker.js +3 -0
  150. package/dist/integrations/agent/builders.js +135 -0
  151. package/dist/integrations/agent/config.js +121 -401
  152. package/dist/integrations/agent/detect.js +3 -0
  153. package/dist/integrations/agent/index.js +6 -14
  154. package/dist/integrations/agent/model-aliases.js +55 -0
  155. package/dist/integrations/agent/profiles.js +3 -0
  156. package/dist/integrations/agent/prompts.js +137 -8
  157. package/dist/integrations/agent/runner.js +208 -0
  158. package/dist/integrations/agent/sdk-runner.js +8 -2
  159. package/dist/integrations/agent/spawn.js +54 -14
  160. package/dist/integrations/github.js +3 -0
  161. package/dist/integrations/lockfile.js +22 -51
  162. package/dist/integrations/session-logs/index.js +4 -0
  163. package/dist/integrations/session-logs/inline-refs.js +35 -0
  164. package/dist/integrations/session-logs/pre-filter.js +152 -0
  165. package/dist/integrations/session-logs/providers/claude-code.js +226 -0
  166. package/dist/integrations/session-logs/providers/opencode.js +231 -25
  167. package/dist/integrations/session-logs/types.js +3 -0
  168. package/dist/llm/call-ai.js +14 -26
  169. package/dist/llm/client.js +16 -2
  170. package/dist/llm/embedder.js +20 -29
  171. package/dist/llm/embedders/cache.js +3 -7
  172. package/dist/llm/embedders/local.js +42 -1
  173. package/dist/llm/embedders/remote.js +20 -8
  174. package/dist/llm/embedders/types.js +3 -7
  175. package/dist/llm/feature-gate.js +92 -56
  176. package/dist/llm/graph-extract.js +402 -31
  177. package/dist/llm/index-passes.js +44 -29
  178. package/dist/llm/memory-infer.js +30 -2
  179. package/dist/llm/metadata-enhance.js +3 -7
  180. package/dist/output/cli-hints.js +7 -4
  181. package/dist/output/context.js +60 -8
  182. package/dist/output/renderers.js +170 -194
  183. package/dist/output/shapes/curate.js +56 -0
  184. package/dist/output/shapes/distill.js +10 -0
  185. package/dist/output/shapes/env-list.js +19 -0
  186. package/dist/output/shapes/events.js +11 -0
  187. package/dist/output/shapes/helpers.js +424 -0
  188. package/dist/output/shapes/history.js +7 -0
  189. package/dist/output/shapes/passthrough.js +105 -0
  190. package/dist/output/shapes/proposal-accept.js +7 -0
  191. package/dist/output/shapes/proposal-diff.js +7 -0
  192. package/dist/output/shapes/proposal-list.js +7 -0
  193. package/dist/output/shapes/proposal-producer.js +11 -0
  194. package/dist/output/shapes/proposal-reject.js +7 -0
  195. package/dist/output/shapes/proposal-show.js +7 -0
  196. package/dist/output/shapes/registry-search.js +6 -0
  197. package/dist/output/shapes/registry.js +30 -0
  198. package/dist/output/shapes/search.js +6 -0
  199. package/dist/output/shapes/secret-list.js +19 -0
  200. package/dist/output/shapes/show.js +6 -0
  201. package/dist/output/shapes/vault-list.js +19 -0
  202. package/dist/output/shapes.js +51 -549
  203. package/dist/output/text/add.js +6 -0
  204. package/dist/output/text/clone.js +6 -0
  205. package/dist/output/text/config.js +6 -0
  206. package/dist/output/text/curate.js +6 -0
  207. package/dist/output/text/distill.js +7 -0
  208. package/dist/output/text/enable-disable.js +7 -0
  209. package/dist/output/text/events.js +10 -0
  210. package/dist/output/text/feedback.js +6 -0
  211. package/dist/output/text/helpers.js +1059 -0
  212. package/dist/output/text/history.js +7 -0
  213. package/dist/output/text/import.js +6 -0
  214. package/dist/output/text/index.js +6 -0
  215. package/dist/output/text/info.js +6 -0
  216. package/dist/output/text/init.js +6 -0
  217. package/dist/output/text/list.js +6 -0
  218. package/dist/output/text/proposal-producer.js +8 -0
  219. package/dist/output/text/proposal.js +12 -0
  220. package/dist/output/text/registry-commands.js +11 -0
  221. package/dist/output/text/registry.js +30 -0
  222. package/dist/output/text/remember.js +6 -0
  223. package/dist/output/text/remove.js +6 -0
  224. package/dist/output/text/save.js +6 -0
  225. package/dist/output/text/search.js +6 -0
  226. package/dist/output/text/show.js +6 -0
  227. package/dist/output/text/update.js +6 -0
  228. package/dist/output/text/upgrade.js +6 -0
  229. package/dist/output/text/vault.js +16 -0
  230. package/dist/output/text/wiki.js +15 -0
  231. package/dist/output/text/workflow.js +14 -0
  232. package/dist/output/text.js +44 -1329
  233. package/dist/registry/build-index.js +3 -0
  234. package/dist/registry/create-provider-registry.js +3 -0
  235. package/dist/registry/factory.js +4 -1
  236. package/dist/registry/origin-resolve.js +3 -0
  237. package/dist/registry/providers/index.js +3 -0
  238. package/dist/registry/providers/skills-sh.js +11 -2
  239. package/dist/registry/providers/static-index.js +10 -1
  240. package/dist/registry/providers/types.js +3 -24
  241. package/dist/registry/resolve.js +11 -16
  242. package/dist/registry/types.js +3 -0
  243. package/dist/scripts/migrate-storage.js +17767 -0
  244. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +9031 -0
  245. package/dist/scripts/migrations/v16-to-v17.js +141 -0
  246. package/dist/setup/detect.js +3 -0
  247. package/dist/setup/ripgrep-install.js +3 -0
  248. package/dist/setup/ripgrep-resolve.js +3 -0
  249. package/dist/setup/setup.js +306 -67
  250. package/dist/setup/steps.js +3 -15
  251. package/dist/sources/include.js +3 -0
  252. package/dist/sources/provider-factory.js +3 -11
  253. package/dist/sources/provider.js +3 -20
  254. package/dist/sources/providers/filesystem.js +19 -23
  255. package/dist/sources/providers/git.js +171 -21
  256. package/dist/sources/providers/index.js +3 -0
  257. package/dist/sources/providers/install-types.js +3 -13
  258. package/dist/sources/providers/npm.js +3 -4
  259. package/dist/sources/providers/provider-utils.js +3 -0
  260. package/dist/sources/providers/sync-from-ref.js +3 -11
  261. package/dist/sources/providers/tar-utils.js +3 -0
  262. package/dist/sources/providers/website.js +18 -22
  263. package/dist/sources/resolve.js +3 -0
  264. package/dist/sources/types.js +3 -0
  265. package/dist/sources/website-ingest.js +3 -0
  266. package/dist/tasks/backends/cron.js +3 -0
  267. package/dist/tasks/backends/exec-utils.js +3 -0
  268. package/dist/tasks/backends/index.js +3 -11
  269. package/dist/tasks/backends/launchd.js +4 -1
  270. package/dist/tasks/backends/schtasks.js +4 -1
  271. package/dist/tasks/parser.js +51 -38
  272. package/dist/tasks/resolveAkmBin.js +3 -0
  273. package/dist/tasks/runner.js +35 -9
  274. package/dist/tasks/schedule.js +20 -1
  275. package/dist/tasks/schema.js +5 -3
  276. package/dist/tasks/validator.js +6 -3
  277. package/dist/version.js +3 -0
  278. package/dist/wiki/wiki-templates.js +6 -3
  279. package/dist/wiki/wiki.js +4 -1
  280. package/dist/workflows/authoring.js +4 -1
  281. package/dist/workflows/cli.js +3 -0
  282. package/dist/workflows/db.js +140 -10
  283. package/dist/workflows/document-cache.js +3 -10
  284. package/dist/workflows/parser.js +3 -0
  285. package/dist/workflows/renderer.js +3 -0
  286. package/dist/workflows/runs.js +18 -1
  287. package/dist/workflows/schema.js +3 -0
  288. package/dist/workflows/scope-key.js +3 -0
  289. package/dist/workflows/validator.js +5 -9
  290. package/docs/README.md +7 -2
  291. package/docs/data-and-telemetry.md +225 -0
  292. package/docs/migration/release-notes/0.7.5.md +2 -2
  293. package/docs/migration/release-notes/0.8.0.md +57 -5
  294. package/docs/migration/v0.7-to-v0.8.md +1378 -0
  295. package/package.json +28 -11
  296. package/.github/LICENSE +0 -374
  297. package/dist/commands/help/help-accept.md +0 -9
  298. package/dist/commands/help/help-improve.md +0 -53
  299. package/dist/commands/help/help-reject.md +0 -8
  300. package/dist/commands/install-audit.js +0 -385
  301. package/dist/commands/vault.js +0 -310
  302. package/dist/indexer/match-contributors.js +0 -141
  303. package/dist/integrations/agent/pipeline.js +0 -39
  304. package/dist/integrations/agent/runners.js +0 -31
  305. package/dist/llm/prompts/graph-extract-user-prompt.md +0 -12
  306. /package/dist/{tasks → assets}/backends/launchd-template.xml +0 -0
  307. /package/dist/{tasks → assets}/backends/schtasks-template.xml +0 -0
  308. /package/dist/{commands → assets}/help/help-propose.md +0 -0
  309. /package/dist/{wiki → assets/wiki}/index-template.md +0 -0
  310. /package/dist/{wiki → assets/wiki}/ingest-workflow-template.md +0 -0
  311. /package/dist/{wiki → assets/wiki}/log-template.md +0 -0
  312. /package/dist/{wiki → assets/wiki}/schema-template.md +0 -0
  313. /package/dist/{workflows → assets/workflows}/workflow-template.md +0 -0
@@ -1,42 +1,15 @@
1
- /**
2
- * Memory inference pass for `akm index` (#201).
3
- *
4
- * Detects memories pending inference, asks the configured LLM to compress each
5
- * into one higher-signal derived memory, and writes the result back as a new
6
- * memory file with frontmatter `inferred: true` + a `source:` backref to the
7
- * parent memory.
8
- *
9
- * Pending predicate (see {@link isPendingMemory}):
10
- * - File lives under `<stashRoot>/memories/` and ends in `.md`.
11
- * - Frontmatter does NOT have `inferenceProcessed: true` (parent already split).
12
- * - Frontmatter does NOT have `inferred: true` (this is itself a child fact).
13
- *
14
- * Idempotency: after a successful split the parent's frontmatter is rewritten
15
- * with `inferenceProcessed: true`. A subsequent `akm index` therefore skips
16
- * the parent without re-running the LLM.
17
- *
18
- * Disabling — two orthogonal gates per v1 spec §14:
19
- * 1. `llm.features.memory_inference = false` blocks the pass at the
20
- * locked feature-flag layer (no network call may ever issue).
21
- * 2. `index.memory.llm = false` (or no `akm.llm` block at all) opts the
22
- * pass out at the per-pass layer (#208).
23
- * A pass runs iff both layers allow it. Existing inferred children are
24
- * NEVER deleted — the user keeps what was already produced.
25
- *
26
- * Locked v1 contract:
27
- * - LLM access is exclusively via `resolveIndexPassLLM("memory", config)`.
28
- * - All child memory writes go through `writeAssetToSource` in
29
- * `src/core/write-source.ts`. The parent's frontmatter rewrite is an
30
- * explicit narrow exception — see {@link markParentProcessed}.
31
- */
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
32
4
  import fs from "node:fs";
33
5
  import path from "node:path";
34
- import { stringify as yamlStringify } from "yaml";
35
6
  import { parseAssetRef } from "../core/asset-ref";
7
+ import { assembleAsset } from "../core/asset-serialize";
36
8
  import { concurrentMap } from "../core/concurrent";
37
9
  import { parseFrontmatter, parseFrontmatterBlock } from "../core/frontmatter";
38
10
  import { warn } from "../core/warn";
39
11
  import { writeAssetToSource } from "../core/write-source";
12
+ import { isProcessEnabled } from "../llm/feature-gate";
40
13
  import { resolveIndexPassLLM } from "../llm/index-passes";
41
14
  import * as memoryInfer from "../llm/memory-infer";
42
15
  import { withLlmCache } from "./llm-cache";
@@ -48,14 +21,15 @@ import { walkMarkdownFiles } from "./walker";
48
21
  const FM_INFERRED = "inferred";
49
22
  const FM_INFERENCE_PROCESSED = "inferenceProcessed";
50
23
  const FM_SOURCE = "source";
24
+ const FM_CAPTURE_MODE = "captureMode";
51
25
  /**
52
26
  * Top-level entry point. Returns a no-op result when the pass is disabled.
53
27
  *
54
- * Two orthogonal gates per v1 spec §14:
28
+ * Two orthogonal gates:
55
29
  *
56
- * 1. **Feature gate** — `llm.features.memory_inference` (defaults to
57
- * `true`). When `false`, no network call may issue regardless of
58
- * per-pass settings. This is the locked spec-§14 gate.
30
+ * 1. **Feature gate** — `profiles.improve.default.processes.memoryInference.enabled`
31
+ * (defaults to `true`). When `false`, no network call may issue regardless
32
+ * of per-pass settings.
59
33
  * 2. **Per-pass gate** — `resolveIndexPassLLM("memory", config)` (which
60
34
  * reads `index.memory.llm`). When `false`, the indexer simply skips
61
35
  * this pass for the current run.
@@ -66,13 +40,18 @@ const FM_SOURCE = "source";
66
40
  export async function runMemoryInferencePass(config, sources, signal, db, reEnrich, onProgress, options = {}) {
67
41
  const result = {
68
42
  considered: 0,
43
+ cacheHits: 0,
69
44
  splitParents: 0,
70
45
  writtenFacts: 0,
71
46
  skippedNoFacts: 0,
47
+ skippedChildExists: 0,
48
+ skippedAborted: 0,
49
+ unaccounted: 0,
72
50
  };
73
- // Gate 1 — locked feature flag (§14). Defaults to enabled; only an
74
- // explicit `false` disables the pass entirely.
75
- if (config.llm?.features?.memory_inference === false)
51
+ // Gate 1 — feature gate via isProcessEnabled, which reads the 0.8.0 path
52
+ // (profiles.improve.default.processes.memoryInference.enabled). Defaults to
53
+ // enabled when the key is absent.
54
+ if (!isProcessEnabled("index", "memory_inference", config))
76
55
  return result;
77
56
  // Gate 2 — per-pass opt-out (#208). Returns the resolved llm config or
78
57
  // `undefined` when the pass should not run.
@@ -93,8 +72,13 @@ export async function runMemoryInferencePass(config, sources, signal, db, reEnri
93
72
  const total = pending.length;
94
73
  onProgress?.({ processed, total, writtenFacts: 0, skippedNoFacts: 0 });
95
74
  const perRecordResults = await concurrentMap(pending, async (record) => {
75
+ // Aborted BEFORE a fresh LLM call. Returned as a typed outcome so the
76
+ // for-loop below increments `skippedAborted` instead of silently
77
+ // dropping the record (which historically inflated freshAttempts and
78
+ // dragged the health-reported yield rate down — see investigation
79
+ // 2026-05-26).
96
80
  if (signal?.aborted)
97
- return undefined;
81
+ return { aborted: true };
98
82
  // Incremental cache: skip LLM call when body hash is unchanged and
99
83
  // --re-enrich was not requested. The cache ref is the absolute file path.
100
84
  const validate = (raw) => {
@@ -113,22 +97,37 @@ export async function runMemoryInferencePass(config, sources, signal, db, reEnri
113
97
  }
114
98
  return undefined;
115
99
  };
100
+ // Track whether THIS candidate's result came from the body-hash
101
+ // cache vs. a fresh LLM call. The cache short-circuits when the
102
+ // parent body has not changed since a prior derived write — surfacing
103
+ // the hit count separately so the operational yield rate
104
+ // (writtenFacts / freshAttempts) is interpretable as the cache warms.
105
+ let fromCache = false;
116
106
  const derived = db
117
107
  ? await withLlmCache(db, record.filePath, record.body, reEnrich ?? false, () => memoryInfer.compressMemoryToDerivedMemory(llmConfig, record.body, signal, config, (evt) => {
118
108
  warn(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
119
- }), validate)
109
+ }), validate, undefined, "", {
110
+ onCacheHit: () => {
111
+ fromCache = true;
112
+ },
113
+ })
120
114
  : await memoryInfer.compressMemoryToDerivedMemory(llmConfig, record.body, signal, config, (evt) => {
121
115
  warn(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
122
116
  });
123
117
  if (!derived) {
124
- return { skipped: true };
118
+ return { skipped: true, fromCache };
125
119
  }
126
120
  const written = await writeDerivedMemory(record, derived);
127
121
  if (written > 0) {
128
122
  markParentProcessed(record);
129
- return { skipped: false, splitParent: true, written };
123
+ return { skipped: false, splitParent: true, written, fromCache };
130
124
  }
131
- return { skipped: false, splitParent: false, written: 0 };
125
+ // LLM produced a valid derived draft but no file was written either
126
+ // because `<parent>.derived.md` already exists on disk or
127
+ // `writeAssetToSource` threw. Categorise as `childExists` so the
128
+ // attempt is accounted for in health metrics rather than vanishing
129
+ // into the freshAttempts denominator.
130
+ return { skipped: false, splitParent: false, written: 0, fromCache, childExists: true };
132
131
  },
133
132
  // Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
134
133
  // in config.json for local model servers (LM Studio, Ollama).
@@ -137,6 +136,21 @@ export async function runMemoryInferencePass(config, sources, signal, db, reEnri
137
136
  const res = perRecordResults[i];
138
137
  if (!res)
139
138
  continue;
139
+ if ("aborted" in res && res.aborted) {
140
+ result.skippedAborted += 1;
141
+ processed++;
142
+ onProgress?.({
143
+ processed,
144
+ total,
145
+ writtenFacts: result.writtenFacts,
146
+ skippedNoFacts: result.skippedNoFacts,
147
+ currentRef: pending[i]?.ref,
148
+ });
149
+ continue;
150
+ }
151
+ if (res.fromCache) {
152
+ result.cacheHits += 1;
153
+ }
140
154
  if (res.skipped) {
141
155
  result.skippedNoFacts += 1;
142
156
  // Intentionally NOT marked processed — a transient LLM failure should
@@ -146,6 +160,20 @@ export async function runMemoryInferencePass(config, sources, signal, db, reEnri
146
160
  result.splitParents += 1;
147
161
  result.writtenFacts += res.written;
148
162
  }
163
+ else if ("childExists" in res && res.childExists) {
164
+ // LLM call was consumed but the derived file already existed (or the
165
+ // write threw). Track separately so this category is observable in
166
+ // health output and stops bleeding into the freshAttempts denominator.
167
+ result.skippedChildExists += 1;
168
+ warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
169
+ }
170
+ else {
171
+ // The per-record state machine should cover every outcome. A hit here
172
+ // means a new code path slipped past the categorisation — surface it
173
+ // loudly so health metrics stay honest and we get a signal to fix.
174
+ result.unaccounted += 1;
175
+ warn(`memory inference: unaccounted per-record outcome for ${pending[i]?.ref ?? "<unknown>"}`);
176
+ }
149
177
  processed++;
150
178
  onProgress?.({
151
179
  processed,
@@ -177,7 +205,7 @@ export function collectPendingMemories(stashRoot) {
177
205
  continue;
178
206
  }
179
207
  const parsed = parseFrontmatter(raw);
180
- if (!isPendingMemory(parsed.data))
208
+ if (!isPendingMemory(parsed.data, filePath))
181
209
  continue;
182
210
  const relName = toMemoryName(memoriesDir, filePath);
183
211
  if (!relName)
@@ -197,10 +225,27 @@ export function collectPendingMemories(stashRoot) {
197
225
  * Predicate: true when the parsed frontmatter indicates the memory has not
198
226
  * yet been split AND is not itself an inferred child.
199
227
  *
228
+ * Also guards against `.derived` files whose `inferred:` frontmatter key has
229
+ * been dropped by a manual edit or schema-repair rewrite. The file name suffix
230
+ * is structural and immutable; frontmatter flags are mutable. A file whose
231
+ * path contains `.derived` is always treated as a derived child regardless of
232
+ * its frontmatter state — this prevents `<name>.derived.derived.md` chains.
233
+ *
234
+ * @param frontmatter - Parsed YAML frontmatter from the memory file.
235
+ * @param filePath - Optional absolute path to the memory file. When
236
+ * supplied, the name-based guard is applied.
237
+ *
200
238
  * Exported for direct unit testing — keeping the predicate in one place
201
239
  * avoids drift between the walker, tests, and any future consumers.
202
240
  */
203
- export function isPendingMemory(frontmatter) {
241
+ export function isPendingMemory(frontmatter, filePath) {
242
+ // Name-based guard: a `.derived` suffix in the path means this file is a
243
+ // derived child regardless of what its frontmatter currently says.
244
+ if (filePath !== undefined) {
245
+ const base = path.basename(filePath, ".md");
246
+ if (base.endsWith(".derived"))
247
+ return false;
248
+ }
204
249
  if (frontmatter[FM_INFERRED] === true)
205
250
  return false;
206
251
  if (frontmatter[FM_INFERENCE_PROCESSED] === true)
@@ -248,6 +293,7 @@ async function writeDerivedMemory(parent, derived) {
248
293
  function renderDerivedMemory(parent, derived) {
249
294
  const fm = {
250
295
  [FM_INFERRED]: true,
296
+ [FM_CAPTURE_MODE]: "background",
251
297
  [FM_SOURCE]: parent.ref,
252
298
  description: derived.description,
253
299
  tags: derived.tags,
@@ -255,8 +301,7 @@ function renderDerivedMemory(parent, derived) {
255
301
  title: derived.title,
256
302
  derivedFrom: parent.name,
257
303
  };
258
- const yaml = yamlStringify(fm).trimEnd();
259
- return `---\n${yaml}\n---\n\n# ${derived.title.trim()}\n\n${derived.content.trim()}\n`;
304
+ return assembleAsset(fm, `# ${derived.title.trim()}\n\n${derived.content.trim()}\n`);
260
305
  }
261
306
  function markParentProcessed(parent) {
262
307
  // Frontmatter-only rewrite of an existing asset: not a new asset write,
@@ -273,10 +318,9 @@ function markParentProcessed(parent) {
273
318
  return;
274
319
  }
275
320
  const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
276
- const yaml = yamlStringify(updatedFm).trimEnd();
277
321
  const block = parseFrontmatterBlock(raw);
278
322
  const body = block?.content ?? raw;
279
- const next = `---\n${yaml}\n---\n${body.startsWith("\n") ? "" : "\n"}${body}`;
323
+ const next = assembleAsset(updatedFm, body);
280
324
  try {
281
325
  fs.writeFileSync(parent.filePath, next, "utf8");
282
326
  }
@@ -1,3 +1,6 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
1
4
  const contributors = [];
2
5
  let builtinsPromise;
3
6
  async function ensureBuiltinMetadataContributorsRegistered() {
@@ -1,8 +1,11 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
1
4
  import fs from "node:fs";
2
5
  import path from "node:path";
3
6
  import { deriveCanonicalAssetName, deriveCanonicalAssetNameFromStashRoot, isRelevantAssetFile, } from "../core/asset-spec";
4
- import { isAssetType, writeFileAtomic } from "../core/common";
5
- import { parseFrontmatter, toStringOrUndefined } from "../core/frontmatter";
7
+ import { asNonEmptyString, isAssetType, writeFileAtomic } from "../core/common";
8
+ import { parseFrontmatter } from "../core/frontmatter";
6
9
  import { isVerbose, warn } from "../core/warn";
7
10
  import { buildFileContext, buildRenderContext, getRenderer, runMatchers } from "./file-context";
8
11
  import { applyMetadataContributors } from "./metadata-contributors";
@@ -173,43 +176,38 @@ export function validateStashEntry(entry) {
173
176
  if (typeof e.pageKind === "string" && e.pageKind.trim().length > 0) {
174
177
  result.pageKind = e.pageKind.trim();
175
178
  }
176
- if (Array.isArray(e.xrefs)) {
177
- const filtered = e.xrefs
178
- .filter((x) => typeof x === "string" && x.trim().length > 0)
179
- .map((x) => x.trim());
180
- if (filtered.length > 0)
181
- result.xrefs = filtered;
182
- }
183
- if (Array.isArray(e.sources)) {
184
- const filtered = e.sources
185
- .filter((s) => typeof s === "string" && s.trim().length > 0)
186
- .map((s) => s.trim());
187
- if (filtered.length > 0)
188
- result.sources = filtered;
189
- }
179
+ const xrefs = normalizeNonEmptyStringList(e.xrefs);
180
+ if (xrefs)
181
+ result.xrefs = xrefs;
182
+ const sources = normalizeNonEmptyStringList(e.sources);
183
+ if (sources)
184
+ result.sources = sources;
190
185
  if (typeof e.beliefState === "string" && e.beliefState.trim().length > 0) {
191
186
  result.beliefState = e.beliefState.trim();
192
187
  }
193
- if (Array.isArray(e.supersededBy)) {
194
- const filtered = e.supersededBy
195
- .filter((s) => typeof s === "string" && s.trim().length > 0)
196
- .map((s) => s.trim());
197
- if (filtered.length > 0)
198
- result.supersededBy = filtered;
188
+ const supersededBy = normalizeNonEmptyStringList(e.supersededBy);
189
+ if (supersededBy)
190
+ result.supersededBy = supersededBy;
191
+ const contradictedBy = normalizeNonEmptyStringList(e.contradictedBy);
192
+ if (contradictedBy)
193
+ result.contradictedBy = contradictedBy;
194
+ const currentBeliefRefs = normalizeNonEmptyStringList(e.currentBeliefRefs);
195
+ if (currentBeliefRefs)
196
+ result.currentBeliefRefs = currentBeliefRefs;
197
+ if (e.captureMode === "hot" || e.captureMode === "background") {
198
+ result.captureMode = e.captureMode;
199
199
  }
200
- if (Array.isArray(e.contradictedBy)) {
201
- const filtered = e.contradictedBy
202
- .filter((s) => typeof s === "string" && s.trim().length > 0)
203
- .map((s) => s.trim());
204
- if (filtered.length > 0)
205
- result.contradictedBy = filtered;
200
+ if (typeof e.whenToUse === "string" && e.whenToUse.trim().length > 0) {
201
+ result.whenToUse = e.whenToUse.trim();
206
202
  }
207
- if (Array.isArray(e.currentBeliefRefs)) {
208
- const filtered = e.currentBeliefRefs
209
- .filter((s) => typeof s === "string" && s.trim().length > 0)
210
- .map((s) => s.trim());
211
- if (filtered.length > 0)
212
- result.currentBeliefRefs = filtered;
203
+ if (typeof e.lessonStrength === "number" && Number.isFinite(e.lessonStrength) && e.lessonStrength >= 0) {
204
+ result.lessonStrength = Math.floor(e.lessonStrength);
205
+ }
206
+ const evidenceSources = normalizeNonEmptyStringList(e.evidenceSources);
207
+ if (evidenceSources)
208
+ result.evidenceSources = evidenceSources;
209
+ if (typeof e.derivedFrom === "string" && e.derivedFrom.trim().length > 0) {
210
+ result.derivedFrom = e.derivedFrom.trim();
213
211
  }
214
212
  if (typeof e.scope === "object" && e.scope !== null && !Array.isArray(e.scope)) {
215
213
  const scope = normalizeScopeObject(e.scope);
@@ -287,9 +285,9 @@ function normalizeIntent(value) {
287
285
  return undefined;
288
286
  const raw = value;
289
287
  const intent = {};
290
- const when = toStringOrUndefined(raw.when);
291
- const input = toStringOrUndefined(raw.input);
292
- const output = toStringOrUndefined(raw.output);
288
+ const when = asNonEmptyString(raw.when);
289
+ const input = asNonEmptyString(raw.input);
290
+ const output = asNonEmptyString(raw.output);
293
291
  if (when)
294
292
  intent.when = when;
295
293
  if (input)
@@ -302,7 +300,7 @@ function normalizeStringListOrUndefined(value) {
302
300
  return normalizeNonEmptyStringList(value);
303
301
  }
304
302
  export function applyCuratedFrontmatter(entry, fmData) {
305
- const description = toStringOrUndefined(fmData.description);
303
+ const description = asNonEmptyString(fmData.description);
306
304
  if (description) {
307
305
  entry.description = description;
308
306
  entry.source = "frontmatter";
@@ -323,19 +321,19 @@ export function applyCuratedFrontmatter(entry, fmData) {
323
321
  const examples = normalizeStringListOrUndefined(fmData.examples);
324
322
  if (examples)
325
323
  entry.examples = examples;
326
- const run = toStringOrUndefined(fmData.run);
324
+ const run = asNonEmptyString(fmData.run);
327
325
  if (run)
328
326
  entry.run = run;
329
- const setup = toStringOrUndefined(fmData.setup);
327
+ const setup = asNonEmptyString(fmData.setup);
330
328
  if (setup)
331
329
  entry.setup = setup;
332
- const cwd = toStringOrUndefined(fmData.cwd);
330
+ const cwd = asNonEmptyString(fmData.cwd);
333
331
  if (cwd)
334
332
  entry.cwd = cwd;
335
- const quality = toStringOrUndefined(fmData.quality);
333
+ const quality = asNonEmptyString(fmData.quality);
336
334
  if (quality)
337
335
  entry.quality = normalizeQuality(quality);
338
- const beliefState = toStringOrUndefined(fmData.beliefState);
336
+ const beliefState = asNonEmptyString(fmData.beliefState);
339
337
  if (beliefState)
340
338
  entry.beliefState = beliefState;
341
339
  const supersededBy = normalizeStringListOrUndefined(fmData.supersededBy);
@@ -347,6 +345,48 @@ export function applyCuratedFrontmatter(entry, fmData) {
347
345
  const currentBeliefRefs = normalizeStringListOrUndefined(fmData.currentBeliefRefs);
348
346
  if (currentBeliefRefs)
349
347
  entry.currentBeliefRefs = currentBeliefRefs;
348
+ // captureMode: "hot" | "background" — strict whitelist; unknown values are ignored.
349
+ if (fmData.captureMode === "hot" || fmData.captureMode === "background") {
350
+ entry.captureMode = fmData.captureMode;
351
+ }
352
+ // when_to_use → whenToUse — free-form guidance for retrieval/intent matching.
353
+ const whenToUse = asNonEmptyString(fmData.when_to_use);
354
+ if (whenToUse)
355
+ entry.whenToUse = whenToUse;
356
+ // lessonStrength: array → length, number → direct. Negative numbers clamp to 0.
357
+ if (Array.isArray(fmData.lessonStrength)) {
358
+ entry.lessonStrength = fmData.lessonStrength.length;
359
+ }
360
+ else if (typeof fmData.lessonStrength === "number" && Number.isFinite(fmData.lessonStrength)) {
361
+ entry.lessonStrength = Math.max(0, Math.floor(fmData.lessonStrength));
362
+ }
363
+ const evidenceSources = normalizeStringListOrUndefined(fmData.evidenceSources);
364
+ if (evidenceSources)
365
+ entry.evidenceSources = evidenceSources;
366
+ // Phase 5A / Advantage D5: capture parent ref for derived memories.
367
+ // Memory-inference writes `source: "memory:<parent>"` and `inferred: true`
368
+ // (and a derived child name suffix `.derived`). We mirror that source ref
369
+ // into `entry.derivedFrom` so the indexer can populate the dedicated
370
+ // `derived_from` column. Non-derived entries leave this field unset.
371
+ if (entry.type === "memory") {
372
+ const isDerivedByName = entry.name.toLowerCase().endsWith(".derived");
373
+ const isDerivedByFm = fmData.inferred === true;
374
+ if (isDerivedByName || isDerivedByFm) {
375
+ const sourceStr = asNonEmptyString(fmData.source);
376
+ if (sourceStr?.includes(":")) {
377
+ entry.derivedFrom = sourceStr;
378
+ }
379
+ else {
380
+ // Fallback: some legacy renderings store only `derivedFrom: <name>`
381
+ // (a bare parent name). Promote it to a `memory:` ref so the lookup
382
+ // column stays consistent.
383
+ const derivedFromName = asNonEmptyString(fmData.derivedFrom);
384
+ if (derivedFromName) {
385
+ entry.derivedFrom = derivedFromName.includes(":") ? derivedFromName : `memory:${derivedFromName}`;
386
+ }
387
+ }
388
+ }
389
+ }
350
390
  const intent = normalizeIntent(fmData.intent);
351
391
  if (intent)
352
392
  entry.intent = intent;
@@ -440,12 +480,29 @@ export function shouldIndexStashFile(stashRoot, file, options) {
440
480
  const segments = relPath.split(/[\\/]+/).filter(Boolean);
441
481
  if (segments.length === 0)
442
482
  return true;
443
- // Skip vault .env files that have a sibling .sensitive marker file.
444
- if (segments[0] === "vaults" && (file.endsWith(".env") || path.basename(file) === ".env")) {
483
+ // Skip env / vault .env files that have a sibling .sensitive marker file.
484
+ if ((segments[0] === "env" || segments[0] === "vaults") &&
485
+ (file.endsWith(".env") || path.basename(file) === ".env")) {
445
486
  const markerPath = file.replace(/\.env$/, ".sensitive");
446
487
  if (fs.existsSync(markerPath))
447
488
  return false;
448
489
  }
490
+ // Deprecation: once a stash has migrated to the `env/` directory, the legacy
491
+ // `vaults/` copy is frozen. Skip indexing it so the same keys are not
492
+ // double-surfaced under both `vault:` and `env:`. (Pre-migration stashes
493
+ // with no `env/` dir still index `vaults/` normally.)
494
+ if (segments[0] === "vaults" && fs.existsSync(path.join(stashRoot, "env"))) {
495
+ return false;
496
+ }
497
+ // Skip secret files that are themselves a `.sensitive` marker, or that have a
498
+ // sibling `<name>.sensitive` marker. Secrets are otherwise indexed by name
499
+ // only (their bytes are never read — see buildEntryFromFile guards).
500
+ if (segments[0] === "secrets") {
501
+ if (file.endsWith(".sensitive") || file.endsWith(".lock"))
502
+ return false;
503
+ if (fs.existsSync(`${file}.sensitive`))
504
+ return false;
505
+ }
449
506
  if (options?.treatStashRootAsWikiRoot) {
450
507
  return !(segments.length === 1 && WIKI_INFRA_FILES.has(segments[0]));
451
508
  }
@@ -775,7 +832,9 @@ async function buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMe
775
832
  entry.tags = normalizeTerms(pkgMeta.keywords);
776
833
  }
777
834
  // Priority 2: Frontmatter (for .md files -- overrides package.json description)
778
- if (ext === ".md") {
835
+ // Secrets are excluded even when the file happens to be `.md`: the whole file
836
+ // is the secret value and must never be read for frontmatter or any metadata.
837
+ if (ext === ".md" && assetType !== "secret") {
779
838
  const content = ctx.content();
780
839
  const parsed = parseFrontmatter(content);
781
840
  applyCuratedFrontmatter(entry, parsed.data);
@@ -794,10 +853,11 @@ async function buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMe
794
853
  }
795
854
  }
796
855
  // Extract @param from script files.
797
- // Vault files (.env) are deliberately excluded their contents are secrets
798
- // and must never be parsed for @param or any other metadata that could
799
- // embed a value into the entry.
800
- if (ext !== ".md" && assetType !== "vault") {
856
+ // Env / vault files (.env) and secret files (whole-file secrets) are
857
+ // deliberately excluded their contents are secrets and must never be
858
+ // parsed for @param or any other metadata that could embed a value into the
859
+ // entry.
860
+ if (ext !== ".md" && assetType !== "env" && assetType !== "vault" && assetType !== "secret") {
801
861
  const content = ctx.content();
802
862
  const scriptParams = extractScriptParameters(file, content);
803
863
  if (scriptParams)
@@ -1,3 +1,6 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
1
4
  import fs from "node:fs";
2
5
  import path from "node:path";
3
6
  import { parseAssetRef } from "../core/asset-ref";