akm-cli 0.9.0-beta.2 → 0.9.0-beta.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +660 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +5 -1
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/templates/html/default.html +78 -0
  16. package/dist/assets/templates/html/health.html +730 -0
  17. package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
  18. package/dist/cli/shared.js +21 -5
  19. package/dist/cli.js +47 -5
  20. package/dist/commands/agent/contribute-cli.js +16 -3
  21. package/dist/commands/feedback-cli.js +15 -6
  22. package/dist/commands/graph/graph.js +75 -71
  23. package/dist/commands/health/checks.js +48 -0
  24. package/dist/commands/health/html-report.js +790 -0
  25. package/dist/commands/health.js +478 -15
  26. package/dist/commands/improve/calibration.js +161 -0
  27. package/dist/commands/improve/consolidate.js +634 -111
  28. package/dist/commands/improve/dedup.js +482 -0
  29. package/dist/commands/improve/distill.js +145 -69
  30. package/dist/commands/improve/encoding-salience.js +205 -0
  31. package/dist/commands/improve/extract-cli.js +115 -1
  32. package/dist/commands/improve/extract-prompt.js +33 -2
  33. package/dist/commands/improve/extract-watch.js +140 -0
  34. package/dist/commands/improve/extract.js +280 -35
  35. package/dist/commands/improve/feedback-valence.js +54 -0
  36. package/dist/commands/improve/homeostatic.js +467 -0
  37. package/dist/commands/improve/improve-auto-accept.js +139 -6
  38. package/dist/commands/improve/improve-profiles.js +12 -0
  39. package/dist/commands/improve/improve.js +2079 -608
  40. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  41. package/dist/commands/improve/outcome-loop.js +256 -0
  42. package/dist/commands/improve/proactive-maintenance.js +87 -0
  43. package/dist/commands/improve/procedural.js +409 -0
  44. package/dist/commands/improve/recombine.js +488 -0
  45. package/dist/commands/improve/reflect-noise.js +0 -0
  46. package/dist/commands/improve/reflect.js +51 -1
  47. package/dist/commands/improve/related-sessions.js +120 -0
  48. package/dist/commands/improve/salience.js +386 -0
  49. package/dist/commands/improve/triage.js +95 -0
  50. package/dist/commands/lint/agent-linter.js +19 -24
  51. package/dist/commands/lint/base-linter.js +173 -60
  52. package/dist/commands/lint/command-linter.js +19 -24
  53. package/dist/commands/lint/env-key-rules.js +34 -1
  54. package/dist/commands/lint/fact-linter.js +39 -0
  55. package/dist/commands/lint/index.js +31 -13
  56. package/dist/commands/lint/memory-linter.js +1 -1
  57. package/dist/commands/lint/registry.js +7 -2
  58. package/dist/commands/lint/task-linter.js +3 -3
  59. package/dist/commands/lint/workflow-linter.js +26 -1
  60. package/dist/commands/proposal/drain.js +73 -6
  61. package/dist/commands/proposal/proposal-cli.js +22 -10
  62. package/dist/commands/proposal/proposal.js +17 -1
  63. package/dist/commands/proposal/validators/proposals.js +369 -329
  64. package/dist/commands/read/curate.js +344 -80
  65. package/dist/commands/read/search-cli.js +7 -0
  66. package/dist/commands/read/search.js +1 -0
  67. package/dist/commands/read/show.js +67 -2
  68. package/dist/commands/remember.js +6 -2
  69. package/dist/commands/sources/installed-stashes.js +5 -1
  70. package/dist/commands/sources/stash-cli.js +10 -2
  71. package/dist/core/asset/asset-registry.js +2 -0
  72. package/dist/core/asset/asset-spec.js +14 -0
  73. package/dist/core/asset/frontmatter.js +166 -167
  74. package/dist/core/asset/markdown.js +8 -0
  75. package/dist/core/config/config-schema.js +255 -2
  76. package/dist/core/config/config.js +2 -2
  77. package/dist/core/logs-db.js +305 -0
  78. package/dist/core/paths.js +3 -0
  79. package/dist/core/state-db.js +706 -42
  80. package/dist/indexer/db/db.js +364 -38
  81. package/dist/indexer/db/graph-db.js +129 -86
  82. package/dist/indexer/ensure-index.js +152 -17
  83. package/dist/indexer/graph/graph-boost.js +51 -41
  84. package/dist/indexer/graph/graph-extraction.js +203 -3
  85. package/dist/indexer/index-writer-lock.js +99 -0
  86. package/dist/indexer/indexer.js +114 -111
  87. package/dist/indexer/passes/memory-inference.js +71 -25
  88. package/dist/indexer/passes/staleness-detect.js +2 -5
  89. package/dist/indexer/search/db-search.js +15 -4
  90. package/dist/indexer/search/ranking-contributors.js +22 -0
  91. package/dist/indexer/search/ranking.js +4 -0
  92. package/dist/indexer/walk/matchers.js +9 -0
  93. package/dist/integrations/agent/prompts.js +1 -0
  94. package/dist/integrations/harnesses/claude/session-log.js +27 -5
  95. package/dist/integrations/harnesses/opencode/session-log.js +9 -0
  96. package/dist/integrations/session-logs/index.js +16 -0
  97. package/dist/llm/client.js +38 -4
  98. package/dist/llm/embedder.js +27 -3
  99. package/dist/llm/embedders/local.js +66 -2
  100. package/dist/llm/graph-extract.js +2 -1
  101. package/dist/llm/memory-infer.js +4 -8
  102. package/dist/llm/metadata-enhance.js +9 -1
  103. package/dist/llm/usage-persist.js +77 -0
  104. package/dist/llm/usage-telemetry.js +103 -0
  105. package/dist/output/context.js +3 -2
  106. package/dist/output/html-render.js +73 -0
  107. package/dist/output/renderers.js +73 -1
  108. package/dist/output/shapes/curate.js +14 -2
  109. package/dist/output/shapes/helpers.js +17 -1
  110. package/dist/output/text/helpers.js +78 -1
  111. package/dist/runtime.js +25 -1
  112. package/dist/scripts/migrate-storage.js +1262 -591
  113. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +485 -270
  114. package/dist/sources/providers/tar-utils.js +16 -8
  115. package/dist/storage/sqlite-pragmas.js +146 -0
  116. package/dist/tasks/runner.js +99 -16
  117. package/dist/workflows/db.js +5 -2
  118. package/dist/workflows/validate-summary.js +2 -7
  119. package/docs/data-and-telemetry.md +1 -0
  120. package/package.json +9 -6
@@ -12,6 +12,7 @@ import { resolveIndexPassLLM } from "../llm/index-passes.js";
12
12
  import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
13
13
  import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
14
14
  import { deleteStoredGraph } from "./db/graph-db.js";
15
+ import { withIndexWriterLease } from "./index-writer-lock.js";
15
16
  import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
16
17
  import { buildSearchText } from "./search/search-fields.js";
17
18
  import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
@@ -225,119 +226,121 @@ function runCleanPass(db, dryRun) {
225
226
  }
226
227
  // ── Indexer ──────────────────────────────────────────────────────────────────
227
228
  export async function akmIndex(options) {
228
- const stashDir = options?.stashDir || resolveStashDir();
229
- const onProgress = options?.onProgress ?? (() => { });
230
- const signal = options?.signal;
231
- const reEnrich = options?.reEnrich === true;
232
- const full = options?.full === true;
233
- const clean = options?.clean === true;
234
- const dryRun = options?.dryRun === true;
235
- // Load config and resolve all stash sources
236
- const { loadConfig } = await import("../core/config/config.js");
237
- const config = loadConfig();
238
- // One-time, read-only guard: warn if the writable stash still holds an
239
- // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
240
- // entirely, so an unmigrated vault's `.env` data would silently never be
241
- // indexed. Non-destructive only stats, never reads/writes/deletes.
242
- const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
243
- warnOnUnmigratedVaults(stashDir);
244
- // Ensure git stash caches are extracted before resolving stash dirs,
245
- // so their content directories exist on disk for the walker to discover.
246
- const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
247
- await ensureSourceCaches(config, { force: full });
248
- const allSourceEntries = resolveSourceEntries(stashDir, config);
249
- const allSourceDirs = allSourceEntries.map((s) => s.path);
250
- const t0 = Date.now();
251
- // Open database — pass embedding dimension from config if available
252
- const dbPath = getDbPath();
253
- const embeddingDim = config.embedding?.dimension;
254
- const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
255
- try {
256
- // Determine incremental vs full mode
257
- const prevStashDir = getMeta(db, "stashDir");
258
- const prevBuiltAt = getMeta(db, "builtAt");
259
- const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
260
- const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
261
- // Assemble the run context
262
- const ctx = {
263
- db,
264
- config,
265
- sources: allSourceEntries,
266
- sourceDirs: allSourceDirs,
267
- full,
268
- reEnrich,
269
- stashDir,
270
- onProgress,
271
- signal,
272
- timing: {
273
- t0,
274
- tWalkStart: t0,
275
- tWalkEnd: t0,
276
- tLlmEnd: t0,
277
- tFtsEnd: t0,
278
- tEmbedEnd: t0,
279
- },
280
- isIncremental,
281
- builtAtMs,
282
- hadRemovedSources: false,
283
- scannedDirs: 0,
284
- skippedDirs: 0,
285
- generatedCount: 0,
286
- walkWarnings: [],
287
- dirsNeedingLlm: [],
288
- embeddingResult: null,
289
- graphExtractionResult: null,
290
- };
291
- onProgress({
292
- phase: "summary",
293
- message: buildIndexSummaryMessage({
229
+ return withIndexWriterLease({ purpose: "akm-index", signal: options?.signal }, async () => {
230
+ const stashDir = options?.stashDir || resolveStashDir();
231
+ const onProgress = options?.onProgress ?? (() => { });
232
+ const signal = options?.signal;
233
+ const reEnrich = options?.reEnrich === true;
234
+ const full = options?.full === true;
235
+ const clean = options?.clean === true;
236
+ const dryRun = options?.dryRun === true;
237
+ // Load config and resolve all stash sources
238
+ const { loadConfig } = await import("../core/config/config.js");
239
+ const config = loadConfig();
240
+ // One-time, read-only guard: warn if the writable stash still holds an
241
+ // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
242
+ // entirely, so an unmigrated vault's `.env` data would silently never be
243
+ // indexed. Non-destructive only stats, never reads/writes/deletes.
244
+ const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
245
+ warnOnUnmigratedVaults(stashDir);
246
+ // Ensure git stash caches are extracted before resolving stash dirs,
247
+ // so their content directories exist on disk for the walker to discover.
248
+ const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
249
+ await ensureSourceCaches(config, { force: full });
250
+ const allSourceEntries = resolveSourceEntries(stashDir, config);
251
+ const allSourceDirs = allSourceEntries.map((s) => s.path);
252
+ const t0 = Date.now();
253
+ // Open database — pass embedding dimension from config if available
254
+ const dbPath = getDbPath();
255
+ const embeddingDim = config.embedding?.dimension;
256
+ const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
257
+ try {
258
+ // Determine incremental vs full mode
259
+ const prevStashDir = getMeta(db, "stashDir");
260
+ const prevBuiltAt = getMeta(db, "builtAt");
261
+ const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
262
+ const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
263
+ // Assemble the run context
264
+ const ctx = {
265
+ db,
266
+ config,
267
+ sources: allSourceEntries,
268
+ sourceDirs: allSourceDirs,
269
+ full,
270
+ reEnrich,
271
+ stashDir,
272
+ onProgress,
273
+ signal,
274
+ timing: {
275
+ t0,
276
+ tWalkStart: t0,
277
+ tWalkEnd: t0,
278
+ tLlmEnd: t0,
279
+ tFtsEnd: t0,
280
+ tEmbedEnd: t0,
281
+ },
282
+ isIncremental,
283
+ builtAtMs,
284
+ hadRemovedSources: false,
285
+ scannedDirs: 0,
286
+ skippedDirs: 0,
287
+ generatedCount: 0,
288
+ walkWarnings: [],
289
+ dirsNeedingLlm: [],
290
+ embeddingResult: null,
291
+ graphExtractionResult: null,
292
+ };
293
+ onProgress({
294
+ phase: "summary",
295
+ message: buildIndexSummaryMessage({
296
+ mode: isIncremental ? "incremental" : "full",
297
+ sourcesCount: allSourceDirs.length,
298
+ semanticSearchMode: config.semanticSearchMode,
299
+ embeddingProvider: getEmbeddingProvider(config.embedding),
300
+ llmEnabled: !!resolveIndexPassLLM("enrichment", config),
301
+ vecAvailable: isVecAvailable(db),
302
+ }),
303
+ });
304
+ // ── Phase sequence ───────────────────────────────────────────────────────
305
+ await runSourceCachePhase(ctx);
306
+ await runWalkPhase(ctx);
307
+ await runEmbeddingPhase(ctx);
308
+ await runFinalizePhase(ctx);
309
+ // ────────────────────────────────────────────────────────────────────────
310
+ const { _verification: verification, _totalEntries: totalEntries } = ctx;
311
+ const { timing } = ctx;
312
+ // ── Clean pass ───────────────────────────────────────────────────────────
313
+ // After the normal index completes, remove entries whose source files no
314
+ // longer exist on disk. Remote entries (empty file_path) are skipped.
315
+ let cleanResult;
316
+ if (clean) {
317
+ cleanResult = runCleanPass(db, dryRun);
318
+ }
319
+ // ────────────────────────────────────────────────────────────────────────
320
+ return {
321
+ stashDir,
322
+ totalEntries,
323
+ generatedMetadata: ctx.generatedCount,
324
+ indexPath: dbPath,
294
325
  mode: isIncremental ? "incremental" : "full",
295
- sourcesCount: allSourceDirs.length,
296
- semanticSearchMode: config.semanticSearchMode,
297
- embeddingProvider: getEmbeddingProvider(config.embedding),
298
- llmEnabled: !!resolveIndexPassLLM("enrichment", config),
299
- vecAvailable: isVecAvailable(db),
300
- }),
301
- });
302
- // ── Phase sequence ───────────────────────────────────────────────────────
303
- await runSourceCachePhase(ctx);
304
- await runWalkPhase(ctx);
305
- await runEmbeddingPhase(ctx);
306
- await runFinalizePhase(ctx);
307
- // ────────────────────────────────────────────────────────────────────────
308
- const { _verification: verification, _totalEntries: totalEntries } = ctx;
309
- const { timing } = ctx;
310
- // ── Clean pass ───────────────────────────────────────────────────────────
311
- // After the normal index completes, remove entries whose source files no
312
- // longer exist on disk. Remote entries (empty file_path) are skipped.
313
- let cleanResult;
314
- if (clean) {
315
- cleanResult = runCleanPass(db, dryRun);
326
+ directoriesScanned: ctx.scannedDirs,
327
+ directoriesSkipped: ctx.skippedDirs,
328
+ ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
329
+ verification,
330
+ timing: {
331
+ totalMs: Date.now() - timing.t0,
332
+ walkMs: timing.tWalkEnd - timing.tWalkStart,
333
+ llmMs: timing.tLlmEnd - timing.tWalkEnd,
334
+ embedMs: timing.tEmbedEnd - timing.tLlmEnd,
335
+ ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
336
+ },
337
+ ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
338
+ };
316
339
  }
317
- // ────────────────────────────────────────────────────────────────────────
318
- return {
319
- stashDir,
320
- totalEntries,
321
- generatedMetadata: ctx.generatedCount,
322
- indexPath: dbPath,
323
- mode: isIncremental ? "incremental" : "full",
324
- directoriesScanned: ctx.scannedDirs,
325
- directoriesSkipped: ctx.skippedDirs,
326
- ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
327
- verification,
328
- timing: {
329
- totalMs: Date.now() - timing.t0,
330
- walkMs: timing.tWalkEnd - timing.tWalkStart,
331
- llmMs: timing.tLlmEnd - timing.tWalkEnd,
332
- embedMs: timing.tEmbedEnd - timing.tLlmEnd,
333
- ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
334
- },
335
- ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
336
- };
337
- }
338
- finally {
339
- closeDatabase(db);
340
- }
340
+ finally {
341
+ closeDatabase(db);
342
+ }
343
+ });
341
344
  }
342
345
  // ── Extracted helpers for indexing ────────────────────────────────────────────
343
346
  async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
@@ -119,6 +119,26 @@ export async function runMemoryInferencePass(ctx) {
119
119
  // 2026-05-26).
120
120
  if (signal?.aborted)
121
121
  return { aborted: true };
122
+ // Pre-check (#588): when `<parent>.derived.md` is already on disk the
123
+ // inference is by definition complete — the parent only looks pending
124
+ // because `markParentProcessed` never ran (process killed between the
125
+ // child write and the mark) or the child was created externally (e.g.
126
+ // consolidation). Skip the LLM/cache call entirely and mark the parent
127
+ // so it never re-pends. Before this check, production measurements
128
+ // showed ~55% of the pass's LLM budget re-deriving such parents only to
129
+ // discover the existing child after the fact.
130
+ if (fs.existsSync(derivedChildPath(record))) {
131
+ markParentProcessed(record);
132
+ return {
133
+ skipped: false,
134
+ splitParent: false,
135
+ written: 0,
136
+ fromCache: false,
137
+ retryAttempts: 0,
138
+ childExists: true,
139
+ precheck: true,
140
+ };
141
+ }
122
142
  // Incremental cache: skip LLM call when body hash is unchanged and
123
143
  // --re-enrich was not requested. The cache ref is the absolute file path.
124
144
  const validate = (raw) => {
@@ -171,23 +191,30 @@ export async function runMemoryInferencePass(ctx) {
171
191
  return { skipped: false, splitParent: true, written: writeOutcome.written, fromCache, retryAttempts };
172
192
  }
173
193
  // LLM produced a valid derived draft but no file was written — either
174
- // because `<parent>.derived.md` already exists on disk or
175
- // `writeAssetToSource` threw. Categorise as `childExists` so the
176
- // attempt is accounted for in health metrics rather than vanishing
177
- // into the freshAttempts denominator.
194
+ // because `<parent>.derived.md` appeared on disk after the pre-check
195
+ // above (a rare mid-flight race) or `writeAssetToSource` threw.
196
+ // Categorise as `childExists` so the consumed attempt is accounted for
197
+ // in health metrics rather than vanishing into the freshAttempts
198
+ // denominator.
178
199
  //
179
- // When the child already exists on disk the inference is, by definition,
180
- // already complete — so mark the parent processed here too (#550).
181
- // Without this, `isPendingMemory()` re-queues the same parent every run
182
- // (the `written > 0` path was previously the only site that marks it),
183
- // causing permanent re-queueing and wasted LLM calls. A genuine write
184
- // *failure* (`writeAssetToSource` threw) must NOT mark the parent it
185
- // should be retried next run — so we key off the explicit `childExists`
186
- // outcome rather than the conflated `written === 0`.
200
+ // When the child exists the inference is, by definition, complete — so
201
+ // mark the parent processed here too (#550), otherwise
202
+ // `isPendingMemory()` re-queues the same parent every run. A genuine
203
+ // write *failure* (`writeAssetToSource` threw) must NOT mark the parent
204
+ // it should be retried next run so we key off the explicit
205
+ // `childExists` outcome rather than the conflated `written === 0`.
187
206
  if (writeOutcome.childExists) {
188
207
  markParentProcessed(record);
189
208
  }
190
- return { skipped: false, splitParent: false, written: 0, fromCache, retryAttempts, childExists: true };
209
+ return {
210
+ skipped: false,
211
+ splitParent: false,
212
+ written: 0,
213
+ fromCache,
214
+ retryAttempts,
215
+ childExists: true,
216
+ precheck: false,
217
+ };
191
218
  },
192
219
  // Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
193
220
  // in config.json for local model servers (LM Studio, Ollama).
@@ -224,11 +251,16 @@ export async function runMemoryInferencePass(ctx) {
224
251
  result.writtenFacts += res.written;
225
252
  }
226
253
  else if ("childExists" in res && res.childExists) {
227
- // LLM call was consumed but the derived file already existed (or the
228
- // write threw). Track separately so this category is observable in
229
- // health output and stops bleeding into the freshAttempts denominator.
254
+ // Derived child already on disk. Track separately so this category is
255
+ // observable in health output and stops bleeding into the
256
+ // freshAttempts denominator. Pre-check skips (#588) are the routine
257
+ // self-healing path — no LLM attempt was consumed and the parent has
258
+ // been marked processed — so only the rare post-LLM case (mid-flight
259
+ // race or write failure) warrants a per-ref warning.
230
260
  result.skippedChildExists += 1;
231
- warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
261
+ if (!res.precheck) {
262
+ warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
263
+ }
232
264
  }
233
265
  else {
234
266
  // The per-record state machine should cover every outcome. A hit here
@@ -324,6 +356,14 @@ function toMemoryName(memoriesDir, filePath) {
324
356
  // user has organised under memories/.
325
357
  return rel.replace(/\\/g, "/").replace(/\.md$/i, "");
326
358
  }
359
+ /**
360
+ * Absolute path of the derived child for a parent memory. Single source of
361
+ * truth for the `<parent>.derived.md` naming convention — used both by the
362
+ * pre-LLM existence check (#588) and the write path.
363
+ */
364
+ function derivedChildPath(parent) {
365
+ return path.join(parent.stashRoot, "memories", `${parent.name}.derived.md`);
366
+ }
327
367
  async function writeDerivedMemory(parent, derived) {
328
368
  const writeTarget = {
329
369
  kind: "filesystem",
@@ -338,11 +378,10 @@ async function writeDerivedMemory(parent, derived) {
338
378
  };
339
379
  const childName = `${parent.name}.derived`;
340
380
  const childRefStr = `memory:${childName}`;
341
- const childPath = path.join(parent.stashRoot, "memories", `${childName}.md`);
342
- if (fs.existsSync(childPath)) {
343
- // The derived child is already on disk inference for this parent is
344
- // complete. Report `childExists` so the caller marks the parent processed
345
- // (#550) instead of re-queueing it forever.
381
+ if (fs.existsSync(derivedChildPath(parent))) {
382
+ // The derived child appeared on disk after the caller's pre-check (#588)
383
+ // a rare mid-flight race. Report `childExists` so the caller marks the
384
+ // parent processed (#550) instead of re-queueing it forever.
346
385
  return { written: 0, childExists: true };
347
386
  }
348
387
  try {
@@ -386,10 +425,17 @@ function markParentProcessed(parent) {
386
425
  warn(`memory inference: failed to re-read parent ${parent.filePath}: ${err instanceof Error ? err.message : String(err)}`);
387
426
  return;
388
427
  }
389
- const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
390
428
  const block = parseFrontmatterBlock(raw);
391
- const body = block?.content ?? raw;
392
- const next = assembleAsset(updatedFm, body);
429
+ if (!block) {
430
+ // Cannot safely rewrite malformed frontmatter — skip marking so the memory
431
+ // is retried on the next run once the frontmatter is repaired. Writing with
432
+ // `body = raw` would wrap the entire file (including the bad frontmatter)
433
+ // in a new block, producing a duplicate-frontmatter corruption.
434
+ warn(`memory inference: skipping markParentProcessed for ${parent.filePath} — could not parse frontmatter block`);
435
+ return;
436
+ }
437
+ const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
438
+ const next = assembleAsset(updatedFm, block.content);
393
439
  try {
394
440
  fs.writeFileSync(parent.filePath, next, "utf8");
395
441
  }
@@ -44,6 +44,7 @@
44
44
  import { createHash } from "node:crypto";
45
45
  import fs from "node:fs";
46
46
  import path from "node:path";
47
+ import stalenessDetectSystemPrompt from "../../assets/prompts/staleness-detect-system.md" with { type: "text" };
47
48
  import { assembleAsset } from "../../core/asset/asset-serialize.js";
48
49
  import { parseFrontmatter, parseFrontmatterBlock } from "../../core/asset/frontmatter.js";
49
50
  import { concurrentMap } from "../../core/concurrent.js";
@@ -319,11 +320,7 @@ function pickSimilar(candidate, all) {
319
320
  return scored.slice(0, TOP_K_SIMILAR).map((s) => s.snap);
320
321
  }
321
322
  // ── LLM dispatch ────────────────────────────────────────────────────────────
322
- const SYSTEM_PROMPT = "You are a belief-state classifier for a memory store. Given a candidate memory and a list of more-recent similar memories from the same store, decide whether the candidate is still current or has been superseded.\n\n" +
323
- "Respond on the first line with exactly YES or NO.\n" +
324
- "If YES, the second line MUST be of the form `SUPERSEDED_BY: <ref>` where <ref> is the exact ref of the superseding memory from the list provided. Do NOT invent refs.\n" +
325
- "If NO, do not include any additional lines.\n" +
326
- "No prose, no preamble, no markdown.";
323
+ const SYSTEM_PROMPT = stalenessDetectSystemPrompt;
327
324
  async function askValidator(connection, candidate, allMemories, signal, timeoutMs) {
328
325
  const similar = pickSimilar(candidate, allMemories);
329
326
  if (similar.length === 0) {
@@ -65,6 +65,7 @@ export async function searchLocal(input) {
65
65
  const includeProposed = input.includeProposed === true;
66
66
  const beliefFilter = input.beliefFilter ?? "all";
67
67
  const restrictToSources = input.restrictToSources === true;
68
+ const includeExcludedTypes = input.includeExcludedTypes === true;
68
69
  const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
69
70
  const allSourceDirs = sources.map((s) => s.path);
70
71
  const rawStatus = readSemanticStatus();
@@ -114,7 +115,7 @@ export async function searchLocal(input) {
114
115
  mode: "keyword",
115
116
  };
116
117
  }
117
- const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources);
118
+ const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources, includeExcludedTypes);
118
119
  return {
119
120
  hits,
120
121
  tip: hits.length === 0
@@ -131,14 +132,19 @@ export async function searchLocal(input) {
131
132
  }
132
133
  }
133
134
  // ── Database search ─────────────────────────────────────────────────────────
134
- async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false) {
135
+ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false) {
135
136
  const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
137
+ // #627 — resolve the default type-exclusion policy. It applies ONLY on the
138
+ // untyped ('any') path and only when the caller did not opt back in via
139
+ // `includeExcludedTypes`. When the config key is ABSENT a built-in default of
140
+ // ['session'] is applied; an explicit empty list disables exclusion.
141
+ const defaultExcludes = searchType === "any" && !includeExcludedTypes ? (config.search?.defaultExcludeTypes ?? ["session"]) : [];
136
142
  // Empty queries — including ones that sanitize down to no searchable FTS
137
143
  // tokens such as "." — should enumerate matching entries instead of
138
144
  // returning an empty result set from FTS.
139
145
  if (!hasSearchableTokens) {
140
146
  const typeFilter = searchType === "any" ? undefined : searchType;
141
- const allEntries = getAllEntries(db, typeFilter);
147
+ const allEntries = getAllEntries(db, typeFilter, defaultExcludes);
142
148
  // Deduplicate by file path — multiple entries can share the same file
143
149
  const seenFilePaths = new Set();
144
150
  const uniqueEntries = allEntries.filter((ie) => {
@@ -187,7 +193,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
187
193
  const typeFilter = searchType === "any" ? undefined : searchType;
188
194
  const tEmbed0 = Date.now();
189
195
  const embeddingPromise = tryVecScores(db, query, limit * 3, config);
190
- const ftsResults = searchFts(db, query, limit * 3, typeFilter);
196
+ const ftsResults = searchFts(db, query, limit * 3, typeFilter, defaultExcludes);
191
197
  const embeddingScores = await embeddingPromise;
192
198
  const embedMs = Date.now() - tEmbed0;
193
199
  const tRank0 = Date.now();
@@ -208,6 +214,11 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
208
214
  embedScoreMap,
209
215
  getEntryById: (id) => getEntryById(db, id) ?? undefined,
210
216
  typeFilter,
217
+ // #627 — also exclude default-hidden types from the vector-only branch so a
218
+ // session asset that is a top-k vector neighbor (but not an FTS match) does
219
+ // not leak into default ('any') results. defaultExcludes is already []
220
+ // unless this is the untyped path without includeExcludedTypes.
221
+ excludeTypes: defaultExcludes,
211
222
  });
212
223
  // ── Scoring Phase ──────────────────────────────────────────────────────
213
224
  // Apply boosts as multiplicative factors (all boosts in a single phase
@@ -9,6 +9,9 @@ const TYPE_BOOST = {
9
9
  agent: 0.3,
10
10
  script: 0.2,
11
11
  knowledge: 0.22,
12
+ // Facts are authoritative, durable declarations about the stash — rank them
13
+ // alongside knowledge so they surface reliably when relevant.
14
+ fact: 0.22,
12
15
  memory: -0.02,
13
16
  };
14
17
  const MAX_BOOST_SUM = 3.0;
@@ -206,6 +209,24 @@ const lessonStrengthContributor = {
206
209
  return Math.min(0.3, 0.06 * strength);
207
210
  },
208
211
  };
212
+ /**
213
+ * Pinned-fact boost.
214
+ *
215
+ * Facts marked `pinned: true` form the small always-injected "core context"
216
+ * (see docs/design/fact-asset-type.md). The fact metadata contributor records
217
+ * a `pinned` search hint; here we give those facts a modest additive boost so
218
+ * the core outranks ordinary facts on otherwise-equal queries. Capped small so
219
+ * it cannot overpower an exact-name match.
220
+ */
221
+ const pinnedFactRankingContributor = {
222
+ name: "pinned-fact-ranking",
223
+ appliesTo(item) {
224
+ return item.entry.type === "fact" && (item.entry.searchHints?.includes("pinned") ?? false);
225
+ },
226
+ adjust() {
227
+ return 0.15;
228
+ },
229
+ };
209
230
  /**
210
231
  * Blend ratio for scoped vs. global utility signals.
211
232
  *
@@ -310,6 +331,7 @@ export const defaultRankingContributors = [
310
331
  graphRankingContributor,
311
332
  captureModeRankingContributor,
312
333
  lessonStrengthContributor,
334
+ pinnedFactRankingContributor,
313
335
  projectContextRankingContributor,
314
336
  ];
315
337
  export const defaultUtilityRankingContributors = [utilityRankingContributor];
@@ -20,6 +20,7 @@ export function normalizeFtsScores(results) {
20
20
  export function combineSearchScores(options) {
21
21
  const FTS_WEIGHT = 0.7;
22
22
  const VEC_WEIGHT = 0.3;
23
+ const excludeTypeSet = options.excludeTypes && options.excludeTypes.length > 0 ? new Set(options.excludeTypes) : null;
23
24
  const scored = [];
24
25
  const seenIds = new Set();
25
26
  for (const [id, { score: ftsScore, result }] of options.ftsScoreMap) {
@@ -42,6 +43,9 @@ export function combineSearchScores(options) {
42
43
  continue;
43
44
  if (options.typeFilter && found.entry.type !== options.typeFilter)
44
45
  continue;
46
+ // #627 — drop vector-only neighbors whose type is excluded on the default path.
47
+ if (excludeTypeSet?.has(found.entry.type))
48
+ continue;
45
49
  scored.push({
46
50
  id,
47
51
  entry: found.entry,
@@ -78,6 +78,15 @@ const DIR_TYPE_MAP = [
78
78
  type: "session",
79
79
  test: (ext) => ext === ".md",
80
80
  },
81
+ {
82
+ // Durable stash-level facts live under `facts/<category>/<name>.md`.
83
+ // classifyByDirectory walks every ancestor dir, so nested category
84
+ // subdirs still match. Without this entry a fact file would fall through
85
+ // to classifyBySmartMd and be mistyped as `knowledge`.
86
+ dir: "facts",
87
+ type: "fact",
88
+ test: (ext) => ext === ".md",
89
+ },
81
90
  ];
82
91
  const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
83
92
  // Files that should never be treated as the typed asset for the surrounding
@@ -43,6 +43,7 @@ const TYPE_HINTS = {
43
43
  script: "script assets are executable text files. Include a shebang and minimal usage comment.",
44
44
  env: "env assets are `.env` files holding a group of related CONFIGURATION for an app/service (KEY=VALUE pairs, `#` comments) — URLs, flags, and any credentials it needs. Values may or may not be sensitive; all are protected (key names discoverable, values stay on disk). Inject with `akm env run env:<name> -- <cmd>` (the safe path — values never reach stdout/your context); do NOT run `akm env export` and read its output, as that prints values. For a single sensitive value used on its own for authentication (token, key, cert) use a `secret` instead. Never echo values back to the user.",
45
45
  wiki: "wiki assets are markdown reference pages with `# Title` and structured headings.",
46
+ fact: "fact assets are durable stash-level facts (personal/team/project details, coding conventions, stash-meta). Frontmatter SHOULD include `description` and a `category` (personal|team|project|convention|meta); set `pinned: true` only for the small always-injected core. Keep each fact short, high-signal, and self-contained — it is durable context, not an episodic note.",
46
47
  };
47
48
  function hintForType(type) {
48
49
  return TYPE_HINTS[type] ?? `assets of type "${type}" — produce sensible markdown with optional frontmatter.`;
@@ -5,7 +5,19 @@ import fs from "node:fs";
5
5
  import os from "node:os";
6
6
  import path from "node:path";
7
7
  import { extractInlineRefMentions } from "../../session-logs/inline-refs.js";
8
- const CLAUDE_PROJECTS_DIR = path.join(os.homedir(), ".claude", "projects");
8
+ /**
9
+ * Root directory holding Claude Code's per-project JSONL session logs.
10
+ *
11
+ * Resolved per call (not memoized at module load) so the `AKM_CLAUDE_PROJECTS_DIR`
12
+ * override can be set after import. The override exists so tests — and the
13
+ * isolated-storage sandbox — can point the scan at an empty fixture directory
14
+ * instead of the real `~/.claude/projects`, which on an actively-used machine
15
+ * holds many large session files and would make `akm health` (which scans it
16
+ * synchronously) slow and non-hermetic.
17
+ */
18
+ function claudeProjectsDir() {
19
+ return process.env.AKM_CLAUDE_PROJECTS_DIR ?? path.join(os.homedir(), ".claude", "projects");
20
+ }
9
21
  /**
10
22
  * Parse a single Claude Code JSONL event into a normalized {@link SessionEvent}.
11
23
  * Returns `undefined` for events that don't carry textual content (file
@@ -93,11 +105,21 @@ export class ClaudeCodeProvider {
93
105
  // HARNESS_BY_ID.get("claude").runtimeId.
94
106
  name = "claude-code";
95
107
  isAvailable() {
96
- return fs.existsSync(CLAUDE_PROJECTS_DIR);
108
+ return fs.existsSync(claudeProjectsDir());
109
+ }
110
+ /**
111
+ * Directory holding Claude Code's per-project session JSONL files
112
+ * (`~/.claude/projects`, honoring `AKM_CLAUDE_PROJECTS_DIR`). Returns `[]`
113
+ * when the directory does not exist on this machine. See {@link
114
+ * SessionLogHarness.watchRoots}.
115
+ */
116
+ watchRoots() {
117
+ const dir = claudeProjectsDir();
118
+ return fs.existsSync(dir) ? [dir] : [];
97
119
  }
98
120
  *readEvents(input) {
99
121
  try {
100
- for (const jsonlPath of this.#walkJsonl(CLAUDE_PROJECTS_DIR)) {
122
+ for (const jsonlPath of this.#walkJsonl(claudeProjectsDir())) {
101
123
  const stat = fs.statSync(jsonlPath);
102
124
  if (stat.mtimeMs < input.sinceMs)
103
125
  continue;
@@ -128,7 +150,7 @@ export class ClaudeCodeProvider {
128
150
  }
129
151
  }
130
152
  listSessions(input = {}) {
131
- const root = input.location ?? CLAUDE_PROJECTS_DIR;
153
+ const root = input.location ?? claudeProjectsDir();
132
154
  const sinceMs = input.sinceMs ?? 0;
133
155
  const summaries = [];
134
156
  try {
@@ -286,7 +308,7 @@ export class ClaudeCodeProvider {
286
308
  const full = path.join(dir, entry.name);
287
309
  if (entry.isDirectory())
288
310
  yield* this.#walkJsonl(full);
289
- else if (entry.name.endsWith(".jsonl"))
311
+ else if (entry.name.endsWith(".jsonl") && entry.name !== "journal.jsonl")
290
312
  yield full;
291
313
  }
292
314
  }