akm-cli 0.9.0-beta.3 → 0.9.0-beta.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/CHANGELOG.md +613 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +5 -1
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/templates/html/health.html +281 -111
  16. package/dist/cli.js +14 -3
  17. package/dist/commands/agent/contribute-cli.js +16 -3
  18. package/dist/commands/feedback-cli.js +15 -6
  19. package/dist/commands/graph/graph.js +75 -71
  20. package/dist/commands/health/checks.js +48 -0
  21. package/dist/commands/health/html-report.js +422 -80
  22. package/dist/commands/health.js +381 -9
  23. package/dist/commands/improve/calibration.js +161 -0
  24. package/dist/commands/improve/consolidate.js +634 -111
  25. package/dist/commands/improve/dedup.js +482 -0
  26. package/dist/commands/improve/distill.js +145 -69
  27. package/dist/commands/improve/encoding-salience.js +205 -0
  28. package/dist/commands/improve/extract-cli.js +115 -1
  29. package/dist/commands/improve/extract-prompt.js +33 -2
  30. package/dist/commands/improve/extract-watch.js +140 -0
  31. package/dist/commands/improve/extract.js +244 -35
  32. package/dist/commands/improve/feedback-valence.js +54 -0
  33. package/dist/commands/improve/homeostatic.js +467 -0
  34. package/dist/commands/improve/improve-auto-accept.js +113 -6
  35. package/dist/commands/improve/improve-profiles.js +12 -0
  36. package/dist/commands/improve/improve.js +1974 -614
  37. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  38. package/dist/commands/improve/outcome-loop.js +256 -0
  39. package/dist/commands/improve/proactive-maintenance.js +87 -0
  40. package/dist/commands/improve/procedural.js +409 -0
  41. package/dist/commands/improve/recombine.js +593 -0
  42. package/dist/commands/improve/reflect.js +26 -1
  43. package/dist/commands/improve/related-sessions.js +120 -0
  44. package/dist/commands/improve/salience.js +386 -0
  45. package/dist/commands/improve/triage.js +95 -0
  46. package/dist/commands/lint/agent-linter.js +19 -24
  47. package/dist/commands/lint/base-linter.js +173 -60
  48. package/dist/commands/lint/command-linter.js +19 -24
  49. package/dist/commands/lint/env-key-rules.js +34 -1
  50. package/dist/commands/lint/fact-linter.js +39 -0
  51. package/dist/commands/lint/index.js +31 -13
  52. package/dist/commands/lint/memory-linter.js +1 -1
  53. package/dist/commands/lint/registry.js +7 -2
  54. package/dist/commands/lint/task-linter.js +3 -3
  55. package/dist/commands/lint/workflow-linter.js +26 -1
  56. package/dist/commands/proposal/proposal.js +5 -0
  57. package/dist/commands/proposal/validators/proposals.js +71 -54
  58. package/dist/commands/read/curate.js +344 -80
  59. package/dist/commands/read/search-cli.js +7 -0
  60. package/dist/commands/read/search.js +1 -0
  61. package/dist/commands/read/show.js +67 -2
  62. package/dist/commands/sources/installed-stashes.js +5 -1
  63. package/dist/commands/sources/stash-cli.js +10 -2
  64. package/dist/core/asset/asset-registry.js +2 -0
  65. package/dist/core/asset/asset-spec.js +14 -0
  66. package/dist/core/asset/frontmatter.js +166 -167
  67. package/dist/core/asset/markdown.js +8 -0
  68. package/dist/core/config/config-schema.js +259 -2
  69. package/dist/core/config/config.js +2 -2
  70. package/dist/core/logs-db.js +4 -3
  71. package/dist/core/paths.js +3 -0
  72. package/dist/core/state-db.js +649 -30
  73. package/dist/indexer/db/db.js +364 -38
  74. package/dist/indexer/db/graph-db.js +129 -86
  75. package/dist/indexer/ensure-index.js +152 -17
  76. package/dist/indexer/graph/graph-boost.js +51 -41
  77. package/dist/indexer/graph/graph-extraction.js +203 -3
  78. package/dist/indexer/index-writer-lock.js +99 -0
  79. package/dist/indexer/indexer.js +114 -111
  80. package/dist/indexer/passes/memory-inference.js +10 -3
  81. package/dist/indexer/passes/staleness-detect.js +2 -5
  82. package/dist/indexer/search/db-search.js +15 -4
  83. package/dist/indexer/search/ranking-contributors.js +22 -0
  84. package/dist/indexer/search/ranking.js +4 -0
  85. package/dist/indexer/walk/matchers.js +9 -0
  86. package/dist/integrations/agent/prompts.js +1 -0
  87. package/dist/integrations/harnesses/claude/session-log.js +11 -1
  88. package/dist/integrations/harnesses/opencode/session-log.js +9 -0
  89. package/dist/integrations/session-logs/index.js +16 -0
  90. package/dist/llm/client.js +23 -4
  91. package/dist/llm/embedder.js +27 -3
  92. package/dist/llm/embedders/local.js +66 -2
  93. package/dist/llm/graph-extract.js +2 -1
  94. package/dist/llm/memory-infer.js +4 -8
  95. package/dist/llm/metadata-enhance.js +9 -1
  96. package/dist/output/renderers.js +73 -1
  97. package/dist/output/shapes/curate.js +14 -2
  98. package/dist/output/text/helpers.js +9 -0
  99. package/dist/runtime.js +25 -1
  100. package/dist/scripts/migrate-storage.js +1242 -594
  101. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +473 -270
  102. package/dist/sources/providers/tar-utils.js +16 -8
  103. package/dist/storage/sqlite-pragmas.js +146 -0
  104. package/dist/workflows/db.js +3 -4
  105. package/dist/workflows/validate-summary.js +2 -7
  106. package/docs/data-and-telemetry.md +1 -0
  107. package/package.json +9 -6
@@ -12,6 +12,7 @@ import { resolveIndexPassLLM } from "../llm/index-passes.js";
12
12
  import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
13
13
  import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
14
14
  import { deleteStoredGraph } from "./db/graph-db.js";
15
+ import { withIndexWriterLease } from "./index-writer-lock.js";
15
16
  import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
16
17
  import { buildSearchText } from "./search/search-fields.js";
17
18
  import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
@@ -225,119 +226,121 @@ function runCleanPass(db, dryRun) {
225
226
  }
226
227
  // ── Indexer ──────────────────────────────────────────────────────────────────
227
228
  export async function akmIndex(options) {
228
- const stashDir = options?.stashDir || resolveStashDir();
229
- const onProgress = options?.onProgress ?? (() => { });
230
- const signal = options?.signal;
231
- const reEnrich = options?.reEnrich === true;
232
- const full = options?.full === true;
233
- const clean = options?.clean === true;
234
- const dryRun = options?.dryRun === true;
235
- // Load config and resolve all stash sources
236
- const { loadConfig } = await import("../core/config/config.js");
237
- const config = loadConfig();
238
- // One-time, read-only guard: warn if the writable stash still holds an
239
- // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
240
- // entirely, so an unmigrated vault's `.env` data would silently never be
241
- // indexed. Non-destructive only stats, never reads/writes/deletes.
242
- const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
243
- warnOnUnmigratedVaults(stashDir);
244
- // Ensure git stash caches are extracted before resolving stash dirs,
245
- // so their content directories exist on disk for the walker to discover.
246
- const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
247
- await ensureSourceCaches(config, { force: full });
248
- const allSourceEntries = resolveSourceEntries(stashDir, config);
249
- const allSourceDirs = allSourceEntries.map((s) => s.path);
250
- const t0 = Date.now();
251
- // Open database — pass embedding dimension from config if available
252
- const dbPath = getDbPath();
253
- const embeddingDim = config.embedding?.dimension;
254
- const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
255
- try {
256
- // Determine incremental vs full mode
257
- const prevStashDir = getMeta(db, "stashDir");
258
- const prevBuiltAt = getMeta(db, "builtAt");
259
- const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
260
- const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
261
- // Assemble the run context
262
- const ctx = {
263
- db,
264
- config,
265
- sources: allSourceEntries,
266
- sourceDirs: allSourceDirs,
267
- full,
268
- reEnrich,
269
- stashDir,
270
- onProgress,
271
- signal,
272
- timing: {
273
- t0,
274
- tWalkStart: t0,
275
- tWalkEnd: t0,
276
- tLlmEnd: t0,
277
- tFtsEnd: t0,
278
- tEmbedEnd: t0,
279
- },
280
- isIncremental,
281
- builtAtMs,
282
- hadRemovedSources: false,
283
- scannedDirs: 0,
284
- skippedDirs: 0,
285
- generatedCount: 0,
286
- walkWarnings: [],
287
- dirsNeedingLlm: [],
288
- embeddingResult: null,
289
- graphExtractionResult: null,
290
- };
291
- onProgress({
292
- phase: "summary",
293
- message: buildIndexSummaryMessage({
229
+ return withIndexWriterLease({ purpose: "akm-index", signal: options?.signal }, async () => {
230
+ const stashDir = options?.stashDir || resolveStashDir();
231
+ const onProgress = options?.onProgress ?? (() => { });
232
+ const signal = options?.signal;
233
+ const reEnrich = options?.reEnrich === true;
234
+ const full = options?.full === true;
235
+ const clean = options?.clean === true;
236
+ const dryRun = options?.dryRun === true;
237
+ // Load config and resolve all stash sources
238
+ const { loadConfig } = await import("../core/config/config.js");
239
+ const config = loadConfig();
240
+ // One-time, read-only guard: warn if the writable stash still holds an
241
+ // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
242
+ // entirely, so an unmigrated vault's `.env` data would silently never be
243
+ // indexed. Non-destructive only stats, never reads/writes/deletes.
244
+ const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
245
+ warnOnUnmigratedVaults(stashDir);
246
+ // Ensure git stash caches are extracted before resolving stash dirs,
247
+ // so their content directories exist on disk for the walker to discover.
248
+ const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
249
+ await ensureSourceCaches(config, { force: full });
250
+ const allSourceEntries = resolveSourceEntries(stashDir, config);
251
+ const allSourceDirs = allSourceEntries.map((s) => s.path);
252
+ const t0 = Date.now();
253
+ // Open database — pass embedding dimension from config if available
254
+ const dbPath = getDbPath();
255
+ const embeddingDim = config.embedding?.dimension;
256
+ const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
257
+ try {
258
+ // Determine incremental vs full mode
259
+ const prevStashDir = getMeta(db, "stashDir");
260
+ const prevBuiltAt = getMeta(db, "builtAt");
261
+ const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
262
+ const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
263
+ // Assemble the run context
264
+ const ctx = {
265
+ db,
266
+ config,
267
+ sources: allSourceEntries,
268
+ sourceDirs: allSourceDirs,
269
+ full,
270
+ reEnrich,
271
+ stashDir,
272
+ onProgress,
273
+ signal,
274
+ timing: {
275
+ t0,
276
+ tWalkStart: t0,
277
+ tWalkEnd: t0,
278
+ tLlmEnd: t0,
279
+ tFtsEnd: t0,
280
+ tEmbedEnd: t0,
281
+ },
282
+ isIncremental,
283
+ builtAtMs,
284
+ hadRemovedSources: false,
285
+ scannedDirs: 0,
286
+ skippedDirs: 0,
287
+ generatedCount: 0,
288
+ walkWarnings: [],
289
+ dirsNeedingLlm: [],
290
+ embeddingResult: null,
291
+ graphExtractionResult: null,
292
+ };
293
+ onProgress({
294
+ phase: "summary",
295
+ message: buildIndexSummaryMessage({
296
+ mode: isIncremental ? "incremental" : "full",
297
+ sourcesCount: allSourceDirs.length,
298
+ semanticSearchMode: config.semanticSearchMode,
299
+ embeddingProvider: getEmbeddingProvider(config.embedding),
300
+ llmEnabled: !!resolveIndexPassLLM("enrichment", config),
301
+ vecAvailable: isVecAvailable(db),
302
+ }),
303
+ });
304
+ // ── Phase sequence ───────────────────────────────────────────────────────
305
+ await runSourceCachePhase(ctx);
306
+ await runWalkPhase(ctx);
307
+ await runEmbeddingPhase(ctx);
308
+ await runFinalizePhase(ctx);
309
+ // ────────────────────────────────────────────────────────────────────────
310
+ const { _verification: verification, _totalEntries: totalEntries } = ctx;
311
+ const { timing } = ctx;
312
+ // ── Clean pass ───────────────────────────────────────────────────────────
313
+ // After the normal index completes, remove entries whose source files no
314
+ // longer exist on disk. Remote entries (empty file_path) are skipped.
315
+ let cleanResult;
316
+ if (clean) {
317
+ cleanResult = runCleanPass(db, dryRun);
318
+ }
319
+ // ────────────────────────────────────────────────────────────────────────
320
+ return {
321
+ stashDir,
322
+ totalEntries,
323
+ generatedMetadata: ctx.generatedCount,
324
+ indexPath: dbPath,
294
325
  mode: isIncremental ? "incremental" : "full",
295
- sourcesCount: allSourceDirs.length,
296
- semanticSearchMode: config.semanticSearchMode,
297
- embeddingProvider: getEmbeddingProvider(config.embedding),
298
- llmEnabled: !!resolveIndexPassLLM("enrichment", config),
299
- vecAvailable: isVecAvailable(db),
300
- }),
301
- });
302
- // ── Phase sequence ───────────────────────────────────────────────────────
303
- await runSourceCachePhase(ctx);
304
- await runWalkPhase(ctx);
305
- await runEmbeddingPhase(ctx);
306
- await runFinalizePhase(ctx);
307
- // ────────────────────────────────────────────────────────────────────────
308
- const { _verification: verification, _totalEntries: totalEntries } = ctx;
309
- const { timing } = ctx;
310
- // ── Clean pass ───────────────────────────────────────────────────────────
311
- // After the normal index completes, remove entries whose source files no
312
- // longer exist on disk. Remote entries (empty file_path) are skipped.
313
- let cleanResult;
314
- if (clean) {
315
- cleanResult = runCleanPass(db, dryRun);
326
+ directoriesScanned: ctx.scannedDirs,
327
+ directoriesSkipped: ctx.skippedDirs,
328
+ ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
329
+ verification,
330
+ timing: {
331
+ totalMs: Date.now() - timing.t0,
332
+ walkMs: timing.tWalkEnd - timing.tWalkStart,
333
+ llmMs: timing.tLlmEnd - timing.tWalkEnd,
334
+ embedMs: timing.tEmbedEnd - timing.tLlmEnd,
335
+ ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
336
+ },
337
+ ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
338
+ };
316
339
  }
317
- // ────────────────────────────────────────────────────────────────────────
318
- return {
319
- stashDir,
320
- totalEntries,
321
- generatedMetadata: ctx.generatedCount,
322
- indexPath: dbPath,
323
- mode: isIncremental ? "incremental" : "full",
324
- directoriesScanned: ctx.scannedDirs,
325
- directoriesSkipped: ctx.skippedDirs,
326
- ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
327
- verification,
328
- timing: {
329
- totalMs: Date.now() - timing.t0,
330
- walkMs: timing.tWalkEnd - timing.tWalkStart,
331
- llmMs: timing.tLlmEnd - timing.tWalkEnd,
332
- embedMs: timing.tEmbedEnd - timing.tLlmEnd,
333
- ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
334
- },
335
- ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
336
- };
337
- }
338
- finally {
339
- closeDatabase(db);
340
- }
340
+ finally {
341
+ closeDatabase(db);
342
+ }
343
+ });
341
344
  }
342
345
  // ── Extracted helpers for indexing ────────────────────────────────────────────
343
346
  async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
@@ -425,10 +425,17 @@ function markParentProcessed(parent) {
425
425
  warn(`memory inference: failed to re-read parent ${parent.filePath}: ${err instanceof Error ? err.message : String(err)}`);
426
426
  return;
427
427
  }
428
- const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
429
428
  const block = parseFrontmatterBlock(raw);
430
- const body = block?.content ?? raw;
431
- const next = assembleAsset(updatedFm, body);
429
+ if (!block) {
430
+ // Cannot safely rewrite malformed frontmatter — skip marking so the memory
431
+ // is retried on the next run once the frontmatter is repaired. Writing with
432
+ // `body = raw` would wrap the entire file (including the bad frontmatter)
433
+ // in a new block, producing a duplicate-frontmatter corruption.
434
+ warn(`memory inference: skipping markParentProcessed for ${parent.filePath} — could not parse frontmatter block`);
435
+ return;
436
+ }
437
+ const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
438
+ const next = assembleAsset(updatedFm, block.content);
432
439
  try {
433
440
  fs.writeFileSync(parent.filePath, next, "utf8");
434
441
  }
@@ -44,6 +44,7 @@
44
44
  import { createHash } from "node:crypto";
45
45
  import fs from "node:fs";
46
46
  import path from "node:path";
47
+ import stalenessDetectSystemPrompt from "../../assets/prompts/staleness-detect-system.md" with { type: "text" };
47
48
  import { assembleAsset } from "../../core/asset/asset-serialize.js";
48
49
  import { parseFrontmatter, parseFrontmatterBlock } from "../../core/asset/frontmatter.js";
49
50
  import { concurrentMap } from "../../core/concurrent.js";
@@ -319,11 +320,7 @@ function pickSimilar(candidate, all) {
319
320
  return scored.slice(0, TOP_K_SIMILAR).map((s) => s.snap);
320
321
  }
321
322
  // ── LLM dispatch ────────────────────────────────────────────────────────────
322
- const SYSTEM_PROMPT = "You are a belief-state classifier for a memory store. Given a candidate memory and a list of more-recent similar memories from the same store, decide whether the candidate is still current or has been superseded.\n\n" +
323
- "Respond on the first line with exactly YES or NO.\n" +
324
- "If YES, the second line MUST be of the form `SUPERSEDED_BY: <ref>` where <ref> is the exact ref of the superseding memory from the list provided. Do NOT invent refs.\n" +
325
- "If NO, do not include any additional lines.\n" +
326
- "No prose, no preamble, no markdown.";
323
+ const SYSTEM_PROMPT = stalenessDetectSystemPrompt;
327
324
  async function askValidator(connection, candidate, allMemories, signal, timeoutMs) {
328
325
  const similar = pickSimilar(candidate, allMemories);
329
326
  if (similar.length === 0) {
@@ -65,6 +65,7 @@ export async function searchLocal(input) {
65
65
  const includeProposed = input.includeProposed === true;
66
66
  const beliefFilter = input.beliefFilter ?? "all";
67
67
  const restrictToSources = input.restrictToSources === true;
68
+ const includeExcludedTypes = input.includeExcludedTypes === true;
68
69
  const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
69
70
  const allSourceDirs = sources.map((s) => s.path);
70
71
  const rawStatus = readSemanticStatus();
@@ -114,7 +115,7 @@ export async function searchLocal(input) {
114
115
  mode: "keyword",
115
116
  };
116
117
  }
117
- const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources);
118
+ const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources, includeExcludedTypes);
118
119
  return {
119
120
  hits,
120
121
  tip: hits.length === 0
@@ -131,14 +132,19 @@ export async function searchLocal(input) {
131
132
  }
132
133
  }
133
134
  // ── Database search ─────────────────────────────────────────────────────────
134
- async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false) {
135
+ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false) {
135
136
  const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
137
+ // #627 — resolve the default type-exclusion policy. It applies ONLY on the
138
+ // untyped ('any') path and only when the caller did not opt back in via
139
+ // `includeExcludedTypes`. When the config key is ABSENT a built-in default of
140
+ // ['session'] is applied; an explicit empty list disables exclusion.
141
+ const defaultExcludes = searchType === "any" && !includeExcludedTypes ? (config.search?.defaultExcludeTypes ?? ["session"]) : [];
136
142
  // Empty queries — including ones that sanitize down to no searchable FTS
137
143
  // tokens such as "." — should enumerate matching entries instead of
138
144
  // returning an empty result set from FTS.
139
145
  if (!hasSearchableTokens) {
140
146
  const typeFilter = searchType === "any" ? undefined : searchType;
141
- const allEntries = getAllEntries(db, typeFilter);
147
+ const allEntries = getAllEntries(db, typeFilter, defaultExcludes);
142
148
  // Deduplicate by file path — multiple entries can share the same file
143
149
  const seenFilePaths = new Set();
144
150
  const uniqueEntries = allEntries.filter((ie) => {
@@ -187,7 +193,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
187
193
  const typeFilter = searchType === "any" ? undefined : searchType;
188
194
  const tEmbed0 = Date.now();
189
195
  const embeddingPromise = tryVecScores(db, query, limit * 3, config);
190
- const ftsResults = searchFts(db, query, limit * 3, typeFilter);
196
+ const ftsResults = searchFts(db, query, limit * 3, typeFilter, defaultExcludes);
191
197
  const embeddingScores = await embeddingPromise;
192
198
  const embedMs = Date.now() - tEmbed0;
193
199
  const tRank0 = Date.now();
@@ -208,6 +214,11 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
208
214
  embedScoreMap,
209
215
  getEntryById: (id) => getEntryById(db, id) ?? undefined,
210
216
  typeFilter,
217
+ // #627 — also exclude default-hidden types from the vector-only branch so a
218
+ // session asset that is a top-k vector neighbor (but not an FTS match) does
219
+ // not leak into default ('any') results. defaultExcludes is already []
220
+ // unless this is the untyped path without includeExcludedTypes.
221
+ excludeTypes: defaultExcludes,
211
222
  });
212
223
  // ── Scoring Phase ──────────────────────────────────────────────────────
213
224
  // Apply boosts as multiplicative factors (all boosts in a single phase
@@ -9,6 +9,9 @@ const TYPE_BOOST = {
9
9
  agent: 0.3,
10
10
  script: 0.2,
11
11
  knowledge: 0.22,
12
+ // Facts are authoritative, durable declarations about the stash — rank them
13
+ // alongside knowledge so they surface reliably when relevant.
14
+ fact: 0.22,
12
15
  memory: -0.02,
13
16
  };
14
17
  const MAX_BOOST_SUM = 3.0;
@@ -206,6 +209,24 @@ const lessonStrengthContributor = {
206
209
  return Math.min(0.3, 0.06 * strength);
207
210
  },
208
211
  };
212
+ /**
213
+ * Pinned-fact boost.
214
+ *
215
+ * Facts marked `pinned: true` form the small always-injected "core context"
216
+ * (see docs/design/fact-asset-type.md). The fact metadata contributor records
217
+ * a `pinned` search hint; here we give those facts a modest additive boost so
218
+ * the core outranks ordinary facts on otherwise-equal queries. Capped small so
219
+ * it cannot overpower an exact-name match.
220
+ */
221
+ const pinnedFactRankingContributor = {
222
+ name: "pinned-fact-ranking",
223
+ appliesTo(item) {
224
+ return item.entry.type === "fact" && (item.entry.searchHints?.includes("pinned") ?? false);
225
+ },
226
+ adjust() {
227
+ return 0.15;
228
+ },
229
+ };
209
230
  /**
210
231
  * Blend ratio for scoped vs. global utility signals.
211
232
  *
@@ -310,6 +331,7 @@ export const defaultRankingContributors = [
310
331
  graphRankingContributor,
311
332
  captureModeRankingContributor,
312
333
  lessonStrengthContributor,
334
+ pinnedFactRankingContributor,
313
335
  projectContextRankingContributor,
314
336
  ];
315
337
  export const defaultUtilityRankingContributors = [utilityRankingContributor];
@@ -20,6 +20,7 @@ export function normalizeFtsScores(results) {
20
20
  export function combineSearchScores(options) {
21
21
  const FTS_WEIGHT = 0.7;
22
22
  const VEC_WEIGHT = 0.3;
23
+ const excludeTypeSet = options.excludeTypes && options.excludeTypes.length > 0 ? new Set(options.excludeTypes) : null;
23
24
  const scored = [];
24
25
  const seenIds = new Set();
25
26
  for (const [id, { score: ftsScore, result }] of options.ftsScoreMap) {
@@ -42,6 +43,9 @@ export function combineSearchScores(options) {
42
43
  continue;
43
44
  if (options.typeFilter && found.entry.type !== options.typeFilter)
44
45
  continue;
46
+ // #627 — drop vector-only neighbors whose type is excluded on the default path.
47
+ if (excludeTypeSet?.has(found.entry.type))
48
+ continue;
45
49
  scored.push({
46
50
  id,
47
51
  entry: found.entry,
@@ -78,6 +78,15 @@ const DIR_TYPE_MAP = [
78
78
  type: "session",
79
79
  test: (ext) => ext === ".md",
80
80
  },
81
+ {
82
+ // Durable stash-level facts live under `facts/<category>/<name>.md`.
83
+ // classifyByDirectory walks every ancestor dir, so nested category
84
+ // subdirs still match. Without this entry a fact file would fall through
85
+ // to classifyBySmartMd and be mistyped as `knowledge`.
86
+ dir: "facts",
87
+ type: "fact",
88
+ test: (ext) => ext === ".md",
89
+ },
81
90
  ];
82
91
  const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
83
92
  // Files that should never be treated as the typed asset for the surrounding
@@ -43,6 +43,7 @@ const TYPE_HINTS = {
43
43
  script: "script assets are executable text files. Include a shebang and minimal usage comment.",
44
44
  env: "env assets are `.env` files holding a group of related CONFIGURATION for an app/service (KEY=VALUE pairs, `#` comments) — URLs, flags, and any credentials it needs. Values may or may not be sensitive; all are protected (key names discoverable, values stay on disk). Inject with `akm env run env:<name> -- <cmd>` (the safe path — values never reach stdout/your context); do NOT run `akm env export` and read its output, as that prints values. For a single sensitive value used on its own for authentication (token, key, cert) use a `secret` instead. Never echo values back to the user.",
45
45
  wiki: "wiki assets are markdown reference pages with `# Title` and structured headings.",
46
+ fact: "fact assets are durable stash-level facts (personal/team/project details, coding conventions, stash-meta). Frontmatter SHOULD include `description` and a `category` (personal|team|project|convention|meta); set `pinned: true` only for the small always-injected core. Keep each fact short, high-signal, and self-contained — it is durable context, not an episodic note.",
46
47
  };
47
48
  function hintForType(type) {
48
49
  return TYPE_HINTS[type] ?? `assets of type "${type}" — produce sensible markdown with optional frontmatter.`;
@@ -107,6 +107,16 @@ export class ClaudeCodeProvider {
107
107
  isAvailable() {
108
108
  return fs.existsSync(claudeProjectsDir());
109
109
  }
110
+ /**
111
+ * Directory holding Claude Code's per-project session JSONL files
112
+ * (`~/.claude/projects`, honoring `AKM_CLAUDE_PROJECTS_DIR`). Returns `[]`
113
+ * when the directory does not exist on this machine. See {@link
114
+ * SessionLogHarness.watchRoots}.
115
+ */
116
+ watchRoots() {
117
+ const dir = claudeProjectsDir();
118
+ return fs.existsSync(dir) ? [dir] : [];
119
+ }
110
120
  *readEvents(input) {
111
121
  try {
112
122
  for (const jsonlPath of this.#walkJsonl(claudeProjectsDir())) {
@@ -298,7 +308,7 @@ export class ClaudeCodeProvider {
298
308
  const full = path.join(dir, entry.name);
299
309
  if (entry.isDirectory())
300
310
  yield* this.#walkJsonl(full);
301
- else if (entry.name.endsWith(".jsonl"))
311
+ else if (entry.name.endsWith(".jsonl") && entry.name !== "journal.jsonl")
302
312
  yield full;
303
313
  }
304
314
  }
@@ -26,6 +26,15 @@ export class OpenCodeProvider {
26
26
  isAvailable() {
27
27
  return fs.existsSync(this.#baseDir);
28
28
  }
29
+ /**
30
+ * Directory holding opencode's per-project session metadata files
31
+ * (`<base>/storage/session`). Returns `[]` when it does not exist on this
32
+ * machine. See {@link SessionLogHarness.watchRoots}.
33
+ */
34
+ watchRoots() {
35
+ const sessionRoot = path.join(this.#baseDir, "storage", "session");
36
+ return fs.existsSync(sessionRoot) ? [sessionRoot] : [];
37
+ }
29
38
  *readEvents(input) {
30
39
  // Legacy behavior: stream raw log lines from the top-level dir and `log/`
31
40
  // subdirectory. Kept to keep `getExecutionLogCandidates` working without
@@ -28,6 +28,22 @@ const ERROR_PATTERNS = /error|failed|exception|cannot|undefined|null pointer|ENO
28
28
  export function getAvailableHarnesses() {
29
29
  return HARNESSES.filter((harness) => harness.isAvailable());
30
30
  }
31
+ /**
32
+ * Map each available harness to its `{ harnessName, roots }` watch target,
33
+ * skipping harnesses that expose no roots (absent `watchRoots()` or an empty
34
+ * result). This is the one stable entry point the watcher uses so it never
35
+ * reaches into providers directly.
36
+ */
37
+ export function getWatchTargets() {
38
+ const targets = [];
39
+ for (const harness of getAvailableHarnesses()) {
40
+ const roots = harness.watchRoots?.() ?? [];
41
+ if (roots.length === 0)
42
+ continue;
43
+ targets.push({ harnessName: harness.name, roots });
44
+ }
45
+ return targets;
46
+ }
31
47
  export function normalizeSessionTopic(text) {
32
48
  const normalized = text.replace(/\s+/g, " ").trim().toLowerCase();
33
49
  if (normalized.length < 10)
@@ -119,9 +119,23 @@ function looksLikeContextOverflow(message) {
119
119
  /**
120
120
  * Decide whether a first-attempt {@link LlmCallError} is eligible for a single
121
121
  * retry. Retryable: HTTP 5xx (`provider_error` with statusCode >= 500) and
122
- * `network_error` whose message looks like a transient connection reset
123
- * (ECONNRESET / EPIPE / "fetch failed"). NOT retryable: 4xx, `rate_limited`
124
- * (429), `timeout`, `parse_error`, and context-overflow-classified errors.
122
+ * `network_error` whose message looks like a transient connection drop.
123
+ * NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`, and
124
+ * context-overflow-classified errors.
125
+ *
126
+ * The connection-drop heuristic covers the substrings emitted across runtimes
127
+ * for a mid-flight socket close:
128
+ * - `ECONNRESET` / `EPIPE` — Node/libuv socket reset codes
129
+ * - `fetch failed` — undici's generic wrapper message
130
+ * - `socket connection was closed` — Bun's message for a dropped connection
131
+ * (e.g. "The socket connection was closed unexpectedly.")
132
+ * - `terminated` / `other side closed` — undici's phrasings for the same
133
+ *
134
+ * These all describe a transient transport failure where a second attempt can
135
+ * legitimately succeed, which is exactly the case a single bounded retry is
136
+ * meant to absorb. Before this list was widened, Bun's "socket connection was
137
+ * closed unexpectedly" fell through unretried and surfaced as a recurring
138
+ * failure in the improve/reflect and capability-probe flows.
125
139
  */
126
140
  function isRetryable(err) {
127
141
  if (looksLikeContextOverflow(err.message))
@@ -131,7 +145,12 @@ function isRetryable(err) {
131
145
  }
132
146
  if (err.code === "network_error") {
133
147
  const lower = err.message.toLowerCase();
134
- return lower.includes("econnreset") || lower.includes("epipe") || lower.includes("fetch failed");
148
+ return (lower.includes("econnreset") ||
149
+ lower.includes("epipe") ||
150
+ lower.includes("fetch failed") ||
151
+ lower.includes("socket connection was closed") ||
152
+ lower.includes("terminated") ||
153
+ lower.includes("other side closed"));
135
154
  }
136
155
  return false;
137
156
  }
@@ -2,7 +2,7 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { embedCacheKey, getCachedEmbedding, setCachedEmbedding } from "./embedders/cache.js";
5
- import { isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
5
+ import { DEFAULT_LOCAL_MODEL, isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
6
6
  import { hasRemoteEndpoint, RemoteEmbedder } from "./embedders/remote.js";
7
7
  // ── Re-exports (public API) ─────────────────────────────────────────────────
8
8
  export { clearEmbeddingCache } from "./embedders/cache.js";
@@ -52,7 +52,8 @@ export async function embed(text, embeddingConfig, signal) {
52
52
  /**
53
53
  * Generate embeddings for multiple texts in batch.
54
54
  * Uses the OpenAI-compatible batch API for remote endpoints (batches of 100).
55
- * Falls back to sequential embedding for the local transformer pipeline.
55
+ * Uses the LocalEmbedder.embedBatch path for the local transformer pipeline,
56
+ * which processes texts in chunks of 32 for genuine batched inference.
56
57
  */
57
58
  export async function embedBatch(texts, embeddingConfig, signal) {
58
59
  if (texts.length === 0)
@@ -60,8 +61,13 @@ export async function embedBatch(texts, embeddingConfig, signal) {
60
61
  if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
61
62
  return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal);
62
63
  }
63
- // Local transformer: process sequentially (pipeline handles one at a time)
64
+ // Local transformer: use the batched path (chunks of 32 via LocalEmbedder).
65
+ // When a localModel override is set we cannot share the singleton (which uses
66
+ // the default model), so fall back to per-text embedWithModel in that case.
64
67
  const localModel = embeddingConfig?.localModel;
68
+ if (!localModel) {
69
+ return getLocalEmbedder().embedBatch(texts, signal);
70
+ }
65
71
  const results = [];
66
72
  for (const text of texts) {
67
73
  if (signal?.aborted) {
@@ -77,6 +83,24 @@ export async function embedBatch(texts, embeddingConfig, signal) {
77
83
  // facade and its `@huggingface/transformers` import chain. Re-export
78
84
  // preserves the existing public API.
79
85
  export { cosineSimilarity } from "./embedders/types.js";
86
+ // ── Model ID resolution ─────────────────────────────────────────────────────
87
+ /**
88
+ * Derive a stable string identifier for the embedding model in use.
89
+ * This is the `model_id` stored in `body_embeddings` (and used for the
90
+ * drop-all-on-mismatch purge when the model changes).
91
+ *
92
+ * Rules:
93
+ * - Remote endpoint: use `config.model` (the API-level model name).
94
+ * - Local transformers: use `config.localModel ?? DEFAULT_LOCAL_MODEL`.
95
+ * - No config: use `DEFAULT_LOCAL_MODEL` (the shared singleton model).
96
+ */
97
+ export function resolveEmbeddingModelId(embeddingConfig) {
98
+ if (!embeddingConfig)
99
+ return DEFAULT_LOCAL_MODEL;
100
+ if (hasRemoteEndpoint(embeddingConfig))
101
+ return embeddingConfig.model ?? "remote";
102
+ return embeddingConfig.localModel ?? DEFAULT_LOCAL_MODEL;
103
+ }
80
104
  // ── Availability check ──────────────────────────────────────────────────────
81
105
  /**
82
106
  * Check whether embedding is available with a detailed reason on failure.