akm-cli 0.9.0-beta.1 → 0.9.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +469 -0
  2. package/dist/assets/templates/html/default.html +78 -0
  3. package/dist/assets/templates/html/health.html +732 -0
  4. package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
  5. package/dist/cli/shared.js +21 -5
  6. package/dist/cli.js +47 -5
  7. package/dist/commands/config-cli.js +0 -10
  8. package/dist/commands/feedback-cli.js +42 -37
  9. package/dist/commands/graph/graph.js +75 -71
  10. package/dist/commands/health/checks.js +48 -0
  11. package/dist/commands/health/html-report.js +666 -0
  12. package/dist/commands/health.js +186 -13
  13. package/dist/commands/improve/consolidate.js +39 -6
  14. package/dist/commands/improve/distill.js +26 -5
  15. package/dist/commands/improve/extract-prompt.js +1 -1
  16. package/dist/commands/improve/extract.js +52 -8
  17. package/dist/commands/improve/improve-auto-accept.js +33 -1
  18. package/dist/commands/improve/improve-cli.js +7 -0
  19. package/dist/commands/improve/improve-profiles.js +4 -0
  20. package/dist/commands/improve/improve.js +874 -447
  21. package/dist/commands/improve/proactive-maintenance.js +113 -0
  22. package/dist/commands/improve/reflect-noise.js +0 -0
  23. package/dist/commands/improve/reflect.js +31 -0
  24. package/dist/commands/proposal/drain.js +73 -6
  25. package/dist/commands/proposal/proposal-cli.js +22 -10
  26. package/dist/commands/proposal/proposal.js +17 -1
  27. package/dist/commands/proposal/validators/proposals.js +365 -329
  28. package/dist/commands/read/curate.js +17 -0
  29. package/dist/commands/remember.js +6 -2
  30. package/dist/commands/sources/stash-cli.js +10 -2
  31. package/dist/commands/tasks/tasks.js +32 -8
  32. package/dist/core/config/config-schema.js +30 -0
  33. package/dist/core/logs-db.js +304 -0
  34. package/dist/core/paths.js +3 -0
  35. package/dist/core/state-db.js +152 -14
  36. package/dist/indexer/db/db.js +99 -13
  37. package/dist/indexer/ensure-index.js +152 -17
  38. package/dist/indexer/index-writer-lock.js +99 -0
  39. package/dist/indexer/indexer.js +114 -111
  40. package/dist/indexer/passes/memory-inference.js +61 -22
  41. package/dist/integrations/harnesses/claude/session-log.js +17 -5
  42. package/dist/llm/client.js +38 -4
  43. package/dist/llm/usage-persist.js +77 -0
  44. package/dist/llm/usage-telemetry.js +103 -0
  45. package/dist/output/context.js +3 -2
  46. package/dist/output/html-render.js +73 -0
  47. package/dist/output/shapes/helpers.js +17 -1
  48. package/dist/output/text/helpers.js +69 -1
  49. package/dist/scripts/migrate-storage.js +153 -25
  50. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +21 -2
  51. package/dist/sources/providers/tar-utils.js +16 -8
  52. package/dist/tasks/backends/cron.js +46 -9
  53. package/dist/tasks/runner.js +99 -16
  54. package/dist/workflows/db.js +4 -0
  55. package/package.json +3 -2
  56. package/dist/commands/config-edit.js +0 -344
@@ -12,21 +12,25 @@ import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } f
12
12
  import { appendEvent, readEvents } from "../../core/events.js";
13
13
  import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../../core/file-lock.js";
14
14
  import { classifyImproveAction } from "../../core/improve-types.js";
15
+ import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
15
16
  import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
16
17
  import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
17
18
  import { info, warn } from "../../core/warn.js";
18
19
  import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
19
20
  import { ensureIndex } from "../../indexer/ensure-index.js";
20
21
  import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
22
+ import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
21
23
  import { akmIndex } from "../../indexer/indexer.js";
22
- import { runMemoryInferencePass } from "../../indexer/passes/memory-inference.js";
24
+ import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
23
25
  import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
24
26
  import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
25
27
  import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
26
28
  import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
27
29
  import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
28
30
  import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
29
- import { isLlmFeatureEnabled, isProcessEnabled } from "../../llm/feature-gate.js";
31
+ import { isProcessEnabled } from "../../llm/feature-gate.js";
32
+ import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
33
+ import { withLlmStage } from "../../llm/usage-telemetry.js";
30
34
  import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
31
35
  import { akmLint } from "../lint/index.js";
32
36
  import { drainProposals } from "../proposal/drain.js";
@@ -43,7 +47,105 @@ import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./i
43
47
  import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
44
48
  import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
45
49
  import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
50
+ import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
46
51
  import { akmReflect } from "./reflect.js";
52
+ // #607 Lock Decomposition: fine-grained per-process locks replace the single
53
+ // `improve.lock`. Three independent locks allow concurrent improve runs when
54
+ // they touch different subsystems (e.g. quick-shredder consolidate can run
55
+ // alongside daily reflect+distill).
56
+ //
57
+ // consolidate.lock — protects consolidate + memoryInference (both write index.db)
58
+ // reflect-distill.lock — protects reflect + distill (both write state.db proposals)
59
+ // triage.lock — protects triage (writes proposal promotions)
60
+ //
61
+ // Stale timeouts are per-lock, tuned to the expected runtime of the protected
62
+ // processes: consolidate is disk-bound (1h), reflect+distill is GPU-bound (2h),
63
+ // triage is fast (30min).
64
+ const PROCESS_LOCK_DEFS = {
65
+ consolidate: { fileName: "consolidate.lock", staleAfterMs: 60 * 60 * 1000 },
66
+ reflectDistill: { fileName: "reflect-distill.lock", staleAfterMs: 2 * 60 * 60 * 1000 },
67
+ triage: { fileName: "triage.lock", staleAfterMs: 30 * 60 * 1000 },
68
+ };
69
+ const heldProcessLocks = new Set();
70
+ export function resetHeldProcessLocks() {
71
+ heldProcessLocks.clear();
72
+ }
73
+ function processLockPath(lockBaseDir, lockName) {
74
+ return path.join(lockBaseDir, PROCESS_LOCK_DEFS[lockName].fileName);
75
+ }
76
+ function tryAcquireProcessLock(lockPath, staleAfterMs, skipIfLocked, lockLabel) {
77
+ fs.mkdirSync(path.dirname(lockPath), { recursive: true });
78
+ const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
79
+ if (tryAcquireLockSync(lockPath, lockPayload())) {
80
+ heldProcessLocks.add(lockPath);
81
+ return "acquired";
82
+ }
83
+ const probe = probeLock(lockPath, { staleAfterMs });
84
+ const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
85
+ const lock = rawContent
86
+ ? (() => {
87
+ try {
88
+ return JSON.parse(rawContent);
89
+ }
90
+ catch {
91
+ return null;
92
+ }
93
+ })()
94
+ : null;
95
+ if (probe.state === "stale") {
96
+ try {
97
+ appendEvent({
98
+ eventType: "improve_lock_recovered",
99
+ metadata: {
100
+ lockName: lockLabel,
101
+ stalePid: lock?.pid ?? null,
102
+ lockedAt: lock?.startedAt ?? null,
103
+ recoveredAt: new Date().toISOString(),
104
+ lockAgeMs: probe.ageMs ?? null,
105
+ reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
106
+ },
107
+ });
108
+ }
109
+ catch {
110
+ /* event emission is best-effort; never block lock recovery */
111
+ }
112
+ releaseLock(lockPath);
113
+ if (tryAcquireLockSync(lockPath, lockPayload())) {
114
+ heldProcessLocks.add(lockPath);
115
+ return "acquired";
116
+ }
117
+ if (skipIfLocked) {
118
+ warn(`[improve] ${lockLabel} lock acquired by another run during stale recovery; skipping (--skip-if-locked)`);
119
+ return "skipped";
120
+ }
121
+ throw new ConfigError(`akm improve ${lockLabel} is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
122
+ }
123
+ if (skipIfLocked) {
124
+ warn(`[improve] ${lockLabel} lock held by another run (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
125
+ return "skipped";
126
+ }
127
+ throw new ConfigError(`akm improve ${lockLabel} is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
128
+ }
129
+ function releaseProcessLock(lockPath) {
130
+ try {
131
+ fs.unlinkSync(lockPath);
132
+ }
133
+ catch {
134
+ // ignore
135
+ }
136
+ heldProcessLocks.delete(lockPath);
137
+ }
138
+ function releaseAllProcessLocks() {
139
+ for (const p of heldProcessLocks) {
140
+ try {
141
+ fs.unlinkSync(p);
142
+ }
143
+ catch {
144
+ // ignore
145
+ }
146
+ }
147
+ heldProcessLocks.clear();
148
+ }
47
149
  function resolveImproveScope(scope) {
48
150
  const trimmed = scope?.trim();
49
151
  if (!trimmed)
@@ -99,6 +201,22 @@ export function renderSyncCommitMessage(template, result, nowMs) {
99
201
  };
100
202
  return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
101
203
  }
204
+ /**
205
+ * Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
206
+ * merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
207
+ * Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
208
+ */
209
+ function dedupeRefs(refs) {
210
+ const seen = new Set();
211
+ const out = [];
212
+ for (const r of refs) {
213
+ if (seen.has(r.ref))
214
+ continue;
215
+ seen.add(r.ref);
216
+ out.push(r);
217
+ }
218
+ return out;
219
+ }
102
220
  async function collectEligibleRefs(scope, stashDir, improveProfile) {
103
221
  if (scope.mode === "ref" && scope.value) {
104
222
  const parsed = parseAssetRef(scope.value);
@@ -112,7 +230,7 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
112
230
  };
113
231
  }
114
232
  return {
115
- plannedRefs: [{ ref: scope.value, reason: "scope-ref" }],
233
+ plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
116
234
  memorySummary: {
117
235
  eligible: parsed.type === "memory" ? 1 : 0,
118
236
  derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
@@ -176,12 +294,14 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
176
294
  profileFiltered.set(ref, {
177
295
  ref,
178
296
  reason: "profile_filtered_all_passes",
297
+ filePath: indexed.filePath,
179
298
  });
180
299
  }
181
300
  else {
182
301
  planned.set(ref, {
183
302
  ref,
184
303
  reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
304
+ filePath: indexed.filePath,
185
305
  });
186
306
  }
187
307
  }
@@ -466,7 +586,9 @@ export async function akmImprove(options = {}) {
466
586
  options = {
467
587
  ...options,
468
588
  autoAccept: options.autoAccept ?? improveProfile.autoAccept,
469
- limit: options.limit ?? improveProfile.limit,
589
+ // Profile-level limit, then process-level reflect.limit as fallback.
590
+ // CLI --limit takes precedence over both.
591
+ limit: options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit,
470
592
  };
471
593
  let primaryStashDir;
472
594
  try {
@@ -484,93 +606,16 @@ export async function akmImprove(options = {}) {
484
606
  // timeout root cause). Because beforeEach runs synchronously, env is still the
485
607
  // calling test's own at this point; we capture it before yielding the loop.
486
608
  const resolvedStateDbPath = getStateDbPathInDataDir();
487
- // Phase 4 lock hoist (§7): the `improve.lock` setup is hoisted ABOVE
488
- // ensureIndex/collectEligibleRefs so the triage pre-pass (and improve's own
489
- // queue writes) run fully serialized under the lock. The dry-run early-return
490
- // below still skips the lock and triage (the lock+triage block is gated on
491
- // `!options.dryRun`); contradiction-detection and memory-cleanup analysis,
492
- // which previously ran before the lock, now sit after it for free.
493
- const resolvedLockPath = primaryStashDir
494
- ? path.join(primaryStashDir, ".akm", "improve.lock")
495
- : path.join(options.stashDir ?? ".", ".akm", "improve.lock");
496
- const MAX_LOCK_AGE_MS = 4 * 60 * 60 * 1000; // 4 hours
497
- const acquireLock = () => {
498
- fs.mkdirSync(path.dirname(resolvedLockPath), { recursive: true });
499
- const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
500
- if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
501
- return;
502
- // Lock file already exists — probe to determine whether it's still held
503
- // or whether the prior run died without cleaning up.
504
- const probe = probeLock(resolvedLockPath, { staleAfterMs: MAX_LOCK_AGE_MS });
505
- const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
506
- const lock = rawContent
507
- ? (() => {
508
- try {
509
- return JSON.parse(rawContent);
510
- }
511
- catch {
512
- return null;
513
- }
514
- })()
515
- : null;
516
- if (probe.state === "stale") {
517
- // O-7 / #394: Emit improve_lock_recovered event before recovery so the
518
- // audit trail records the abnormal prior-run exit (Temporal/Airflow pattern).
519
- try {
520
- appendEvent({
521
- eventType: "improve_lock_recovered",
522
- metadata: {
523
- stalePid: lock?.pid ?? null,
524
- lockedAt: lock?.startedAt ?? null,
525
- recoveredAt: new Date().toISOString(),
526
- lockAgeMs: probe.ageMs ?? null,
527
- reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
528
- },
529
- });
530
- }
531
- catch {
532
- /* event emission is best-effort; never block lock recovery */
533
- }
534
- releaseLock(resolvedLockPath);
535
- if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
536
- return;
537
- throw new ConfigError(`akm improve is already running. Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
538
- }
539
- throw new ConfigError(`akm improve is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
540
- };
541
- // Phase 4 lock-leak guard (§7 ordering hazard): hoisting `improve.lock` above
542
- // the pre-index region (so the triage pre-pass runs under it) means the lock is
543
- // held while ensureIndex / collectEligibleRefs / contradiction-detection /
544
- // memory-cleanup analysis run — but the main protecting `try { … } finally {
545
- // unlinkSync(resolvedLockPath) }` does not begin until after them. A throw in
546
- // any of those steps would leak the lock. We close that window by wrapping the
547
- // whole region in a try whose catch releases the lock (when held) and
548
- // re-throws. The values this region computes are declared in the outer scope so
549
- // they remain visible to the main run below. The dry-run path never sets
550
- // `lockAcquired`, so its early return releases nothing.
551
- let lockAcquired = false;
552
- const releaseLockOnError = () => {
553
- if (!lockAcquired)
554
- return;
555
- try {
556
- fs.unlinkSync(resolvedLockPath);
557
- }
558
- catch {
559
- // best-effort release on the error path
560
- }
561
- lockAcquired = false;
562
- };
563
- // Signal-safe lock release. The SIGTERM/SIGINT/SIGHUP handler in improve-cli.ts
564
- // calls `process.exit()`, which does NOT run the `finally` below that owns lock
565
- // release — so a cron-timeout SIGTERM leaked `improve.lock` every run.
566
- // `process.exit()` DOES fire `'exit'` listeners, so we release the lock from
567
- // one. `releaseLockIfOwned` only unlinks a lock still owned by this PID, so it
568
- // is safe even if a later run re-acquired it. The listener is removed in the
569
- // `finally` so the normal path stays single-release and repeated in-process
570
- // `akmImprove` calls (tests) do not accumulate listeners.
571
- const releaseLockOnExit = () => {
572
- releaseLockIfOwned(resolvedLockPath, process.pid);
573
- };
609
+ // #607 Lock decomposition: three per-process locks replace the single
610
+ // `improve.lock`. Each process acquires only the lock(s) it needs, so
611
+ // quick-shredder consolidate can run alongside daily reflect+distill.
612
+ //
613
+ // consolidate.lock — protects consolidate + memoryInference + graphExtraction (index.db writers)
614
+ // reflect-distill.lock protects reflect + distill (state.db proposal writers)
615
+ // triage.lock — protects triage pre-pass (state.db proposal promotions)
616
+ //
617
+ // Lock base directory — same `.akm/` under the primary stash dir.
618
+ const lockBaseDir = primaryStashDir ? path.join(primaryStashDir, ".akm") : path.join(options.stashDir ?? ".", ".akm");
574
619
  const preEnsureCleanupWarnings = [];
575
620
  let plannedRefs;
576
621
  let memorySummary;
@@ -579,51 +624,59 @@ export async function akmImprove(options = {}) {
579
624
  let guidance;
580
625
  let triageDrain;
581
626
  try {
582
- // Acquire the lock and run the triage pre-pass for non-dry-run executions.
583
- // The dry-run branch below produces plannedRefs/memorySummary WITHOUT the lock
584
- // or triage (decision: dry-run never mutates the queue).
627
+ // #607: Per-process lock acquisition. Each process acquires only the lock(s)
628
+ // it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
629
+ // locks (decision: dry-run never mutates the queue).
585
630
  if (!options.dryRun) {
586
- acquireLock();
587
- lockAcquired = true;
588
631
  // Backstop release on process.exit() (signal handler / budget watchdog),
589
632
  // which skips the finally below. Removed in that finally on the normal path.
590
- process.on("exit", releaseLockOnExit);
591
- // Phase 4 triage pre-pass (§7, §13): drain the standing pending backlog
592
- // BEFORE ensureIndex so improve generates fresh proposals against a cleared
593
- // queue (no `duplicate_pending` collisions) and ensureIndex absorbs triage's
594
- // promotions for free. Gated on the triage process being enabled (opt-in,
595
- // defaults off) and on a whole-stash / type-scoped run — a single-ref
596
- // `akm improve skill:x` must never drain the whole queue. Best-effort: a
597
- // triage failure is a non-fatal warning, never an abort (mirrors the
598
- // contradiction-detection pass below).
633
+ const releaseAllOnExit = () => {
634
+ for (const p of heldProcessLocks) {
635
+ releaseLockIfOwned(p, process.pid);
636
+ }
637
+ };
638
+ process.on("exit", releaseAllOnExit);
639
+ // #607 triage pre-pass: acquire triage.lock, drain the standing pending
640
+ // backlog BEFORE ensureIndex so improve generates fresh proposals against
641
+ // a cleared queue (no `duplicate_pending` collisions) and ensureIndex
642
+ // absorbs triage's promotions for free. Release immediately after —
643
+ // triage.lock is not needed again until the next improve run.
599
644
  if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
600
645
  if (scope.mode === "ref") {
601
646
  warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
602
647
  }
603
648
  else {
604
- try {
605
- const triageConfig = improveProfile.processes?.triage;
606
- const policy = resolveDrainPolicy(triageConfig?.policy);
607
- const applyMode = triageConfig?.applyMode ?? "queue";
608
- const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
609
- const judgment = triageConfig?.judgment
610
- ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
611
- : null;
612
- triageDrain = await drainProposalsFn({
613
- stashDir: primaryStashDir,
614
- policy,
615
- applyMode,
616
- maxAccepts,
617
- dryRun: false,
618
- // No fresh ids exist yet — triage runs before improve generates any.
619
- excludeIds: new Set(),
620
- ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
621
- judgment,
622
- });
649
+ const triageLPath = processLockPath(lockBaseDir, "triage");
650
+ const triageResult = tryAcquireProcessLock(triageLPath, PROCESS_LOCK_DEFS.triage.staleAfterMs, options.skipIfLocked, "triage");
651
+ if (triageResult === "skipped") {
652
+ triageDrain = undefined;
623
653
  }
624
- catch (err) {
625
- // Non-fatal: triage is a best-effort pre-pass and must never abort improve.
626
- warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
654
+ else {
655
+ try {
656
+ const triageConfig = improveProfile.processes?.triage;
657
+ const policy = resolveDrainPolicy(triageConfig?.policy);
658
+ const applyMode = triageConfig?.applyMode ?? "queue";
659
+ const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
660
+ const judgment = triageConfig?.judgment
661
+ ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
662
+ : null;
663
+ triageDrain = await drainProposalsFn({
664
+ stashDir: primaryStashDir,
665
+ policy,
666
+ applyMode,
667
+ maxAccepts,
668
+ dryRun: false,
669
+ excludeIds: new Set(),
670
+ ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
671
+ judgment,
672
+ });
673
+ }
674
+ catch (err) {
675
+ warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
676
+ }
677
+ finally {
678
+ releaseProcessLock(triageLPath);
679
+ }
627
680
  }
628
681
  }
629
682
  }
@@ -655,7 +708,7 @@ export async function akmImprove(options = {}) {
655
708
  // best-effort; leave preEnsureEntryCount undefined
656
709
  }
657
710
  try {
658
- await ensureIndexFn(primaryStashDir);
711
+ await ensureIndexFn(primaryStashDir, { mode: "blocking" });
659
712
  }
660
713
  catch (err) {
661
714
  preEnsureCleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -693,7 +746,7 @@ export async function akmImprove(options = {}) {
693
746
  if (primaryStashDir && shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)) {
694
747
  try {
695
748
  // Reuse the config resolved at the top of the run instead of a second load.
696
- await detectAndWriteContradictions(primaryStashDir, _earlyConfig);
749
+ await withLlmStage("memory-contradiction", () => detectAndWriteContradictions(primaryStashDir, _earlyConfig));
697
750
  }
698
751
  catch (err) {
699
752
  // Non-fatal: contradiction detection is a best-effort pass.
@@ -723,17 +776,14 @@ export async function akmImprove(options = {}) {
723
776
  }
724
777
  }
725
778
  catch (err) {
726
- releaseLockOnError();
779
+ releaseAllProcessLocks();
727
780
  throw err;
728
781
  }
729
- // FIX 2 (lock-leak window): everything from here on runs UNDER the lock that
730
- // `acquireLock()` just took. The single `try { } finally { unlinkSync(lock) }`
731
- // below now spans the budget-timer setup, `openStateDatabase()`, and the
732
- // `profileFilteredRefs` audit-event loop too regions that previously sat in
733
- // the gap between the lock-acquire catch (above) and the main try. A throw in
734
- // any of them used to leak the lock (blocking the next improve up to 4h);
735
- // now the finally releases it exactly once. The dry-run path already returned
736
- // above without acquiring the lock, so it never reaches this finally; the
782
+ // #607: per-process locks are acquired/released around each stage below.
783
+ // The triage pre-pass already ran under triage.lock (released). The
784
+ // preparation stage runs under consolidate.lock, the loop stage under
785
+ // reflect-distill.lock, and the post-loop stage under consolidate.lock again.
786
+ // Each stage acquires its lock just before starting and releases in finally.
737
787
  // best-effort `unlinkSync` is a no-op when no lock file exists.
738
788
  const startMs = Date.now();
739
789
  const budgetMs = options.timeoutMs ?? 2 * 60 * 60 * 1000; // default 2 hours
@@ -753,6 +803,9 @@ export async function akmImprove(options = {}) {
753
803
  // Pinned to the boundary snapshot so the fallback per-call `appendEvent`
754
804
  // opens (when the long-lived handle below fails to open) never re-read env.
755
805
  let eventsCtx = { dbPath: resolvedStateDbPath };
806
+ // #576: clears the per-run LLM usage sink. Defaults to a no-op until the sink
807
+ // is installed inside the try; the `finally` always calls it.
808
+ let disposeLlmUsageSink = () => { };
756
809
  try {
757
810
  // H7 (#566): arm the budget watchdog. `armBudgetWatchdog` captures both the
758
811
  // budget timer and the hard-kill timer it schedules on exhaustion, returning
@@ -772,19 +825,32 @@ export async function akmImprove(options = {}) {
772
825
  // still pinned to the boundary-resolved path, never a live env re-read.
773
826
  eventsCtx = { dbPath: resolvedStateDbPath };
774
827
  }
775
- // 2026-05-27: emit `improve_skipped` audit events for refs the planner
828
+ // #576: persist per-call LLM usage telemetry for this run as `llm_usage`
829
+ // events, reusing the same boundary-pinned events context (and long-lived
830
+ // handle when available). Disposed in `finally` so the sink never leaks
831
+ // across runs. Wrapping is best-effort end to end — see usage-telemetry.ts.
832
+ disposeLlmUsageSink = installLlmUsagePersistence(eventsCtx);
833
+ // 2026-05-27: emit an `improve_skipped` audit event for refs the planner
776
834
  // pre-filtered (reflect AND distill both refuse them under the active
777
- // profile). One event per ref so the existing improve_skipped histogram in
778
- // `health.ts#improveSummary.skipReasons` accumulates the right count under
779
- // the new `profile_filtered_all_passes` reason code. See
780
- // `/tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md`.
781
- for (const filtered of profileFilteredRefs) {
835
+ // profile). Emitted as a single summary event (count only) rather than one
836
+ // event per ref (#592) the per-ref loop caused O(n) sequential state.db
837
+ // writes that consumed ~500 s on a 9 000-ref stash. No downstream consumer
838
+ // needs the per-ref audit trail: health's skip histogram reads the
839
+ // `profile_filtered_all_passes` counters from `improve_completed` metadata.
840
+ if (profileFilteredRefs.length > 0) {
782
841
  appendEvent({
783
842
  eventType: "improve_skipped",
784
- ref: filtered.ref,
785
- metadata: { reason: "profile_filtered_all_passes" },
843
+ ref: undefined,
844
+ metadata: {
845
+ reason: "profile_filtered_all_passes",
846
+ count: profileFilteredRefs.length,
847
+ },
786
848
  }, eventsCtx);
787
849
  }
850
+ // #607: acquire consolidate.lock for the preparation stage (consolidate,
851
+ // ensureIndex, extract all write index.db). Released immediately after.
852
+ const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
853
+ const consolidatePrepAcquired = tryAcquireProcessLock(consolidateLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
788
854
  const preparation = await runImprovePreparationStage({
789
855
  scope,
790
856
  options,
@@ -799,6 +865,8 @@ export async function akmImprove(options = {}) {
799
865
  initialCleanupWarnings: preEnsureCleanupWarnings,
800
866
  improveProfile,
801
867
  });
868
+ if (consolidatePrepAcquired)
869
+ releaseProcessLock(consolidateLPath);
802
870
  // D6: pre-load all proposal_rejected events from the last 30 days once,
803
871
  // so the per-asset loop can use a Map lookup instead of N DB round trips.
804
872
  const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
@@ -810,6 +878,10 @@ export async function akmImprove(options = {}) {
810
878
  rejectedProposalsByRef.set(e.ref, e);
811
879
  }
812
880
  }
881
+ // #607: acquire reflect-distill.lock for the loop stage (reflect + distill
882
+ // both write proposals to state.db). Released immediately after.
883
+ const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
884
+ const reflectDistillAcquired = tryAcquireProcessLock(reflectDistillLPath, PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs, options.skipIfLocked, "reflect-distill") === "acquired";
813
885
  const { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount: loopGateCount, gateAutoAcceptFailedCount: loopGateFailedCount, } = await runImproveLoopStage({
814
886
  scope,
815
887
  options,
@@ -829,9 +901,15 @@ export async function akmImprove(options = {}) {
829
901
  eventsCtx,
830
902
  improveProfile,
831
903
  });
904
+ if (reflectDistillAcquired)
905
+ releaseProcessLock(reflectDistillLPath);
832
906
  // #551: consolidation now runs in the preparation stage (before extract);
833
907
  // its result and run-flag are read from `preparation`, not the post-loop.
834
908
  const consolidation = preparation.consolidation;
909
+ // #607: acquire consolidate.lock for the post-loop stage (memoryInference +
910
+ // graphExtraction both write index.db). Released immediately after.
911
+ const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
912
+ const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
835
913
  const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
836
914
  scope,
837
915
  options,
@@ -842,11 +920,12 @@ export async function akmImprove(options = {}) {
842
920
  memoryRefsForInference,
843
921
  reindexFn,
844
922
  eventsCtx,
845
- // O-1 (#364): propagate wall-clock budget signal to post-loop maintenance.
846
923
  budgetSignal: budgetAbortController.signal,
847
924
  improveProfile,
848
925
  consolidationRan: preparation.consolidationRan,
849
926
  });
927
+ if (consolidatePostAcquired)
928
+ releaseProcessLock(consolidatePostLPath);
850
929
  const finalActions = maintenanceActions && maintenanceActions.length > 0
851
930
  ? [...preparation.actions, ...maintenanceActions]
852
931
  : preparation.actions;
@@ -931,6 +1010,7 @@ export async function akmImprove(options = {}) {
931
1010
  },
932
1011
  }
933
1012
  : {}),
1013
+ ...(preparation.proactiveMaintenance ? { proactiveMaintenance: preparation.proactiveMaintenance } : {}),
934
1014
  ...(options.runId !== undefined ? { runId: options.runId } : {}),
935
1015
  };
936
1016
  if (!result.dryRun)
@@ -1007,18 +1087,18 @@ export async function akmImprove(options = {}) {
1007
1087
  throw err;
1008
1088
  }
1009
1089
  finally {
1090
+ // #576: clear the per-run LLM usage sink BEFORE closing `eventsDb` below, so
1091
+ // no late sink invocation can write through a closed handle.
1092
+ disposeLlmUsageSink();
1010
1093
  // O-1 (#364): Clear the budget abort timer so it does not keep the event
1011
1094
  // loop alive after the run completes.
1012
1095
  clearBudgetTimer();
1013
- try {
1014
- fs.unlinkSync(resolvedLockPath);
1015
- }
1016
- catch {
1017
- // ignore
1018
- }
1019
- // The normal path released the lock above; drop the process.exit backstop so
1020
- // it does not fire later (or accumulate across repeated in-process calls).
1021
- process.removeListener("exit", releaseLockOnExit);
1096
+ // #607: release any per-process locks still held (backstop for error paths;
1097
+ // the normal path already released each lock after its stage completed).
1098
+ releaseAllProcessLocks();
1099
+ // Drop the process.exit backstop so it does not fire later (or accumulate
1100
+ // across repeated in-process calls).
1101
+ process.removeAllListeners("exit");
1022
1102
  // I1: close the long-lived state.db connection opened at the top of the run.
1023
1103
  try {
1024
1104
  eventsDb?.close();
@@ -1131,6 +1211,11 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1131
1211
  memoryInferenceDurationMs: durations.memoryInferenceDurationMs,
1132
1212
  graphExtractionExtractedFiles: result.graphExtraction?.quality.extractedFiles ?? 0,
1133
1213
  graphExtractionDurationMs: durations.graphExtractionDurationMs,
1214
+ // Layer-2 proactive-maintenance coverage (0 when the process is disabled
1215
+ // or the run was ref-scoped) so a scheduled sweep's reach is trackable.
1216
+ proactiveSelected: result.proactiveMaintenance?.selected ?? 0,
1217
+ proactiveDueTotal: result.proactiveMaintenance?.dueTotal ?? 0,
1218
+ proactiveNeverReflected: result.proactiveMaintenance?.neverReflected ?? 0,
1134
1219
  // New metrics for tuning the improve loop.
1135
1220
  ...(durations.totalDurationMs !== undefined ? { durationMs: durations.totalDurationMs } : {}),
1136
1221
  ...(durations.warningCount !== undefined ? { warningCount: durations.warningCount } : {}),
@@ -1333,7 +1418,7 @@ async function runConsolidationPass(args) {
1333
1418
  info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
1334
1419
  }
1335
1420
  else if (!consolidationOnCooldown) {
1336
- consolidation = await akmConsolidate({
1421
+ consolidation = await withLlmStage("consolidate", () => akmConsolidate({
1337
1422
  ...options.consolidateOptions,
1338
1423
  config: consolidationConfig,
1339
1424
  stashDir: options.stashDir,
@@ -1341,16 +1426,13 @@ async function runConsolidationPass(args) {
1341
1426
  // Tie consolidate proposals back to this improve invocation so
1342
1427
  // accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
1343
1428
  sourceRun: `consolidate-${Date.now()}`,
1344
- // Incremental consolidation: pass the last-consolidation timestamp so
1345
- // akmConsolidate skips chunks with no memory changed since then. Converts
1346
- // consolidation cost from O(pool) to O(changed clusters) the fix for
1347
- // the rising p95 tail where full-pool re-judging produced 5–10 min runs
1348
- // that promoted ~0. undefined → full pass on first-ever run (bootstrap).
1349
- // volumeTriggered correctly forces the run past cooldown but must NOT
1350
- // override incrementalSince — the stash has ~1400 eligible memories so
1351
- // volumeTriggered=true on every run, permanently forcing full 12-chunk
1352
- // scans (~264s) instead of the intended 1-2 chunk incremental path (~44s).
1353
- incrementalSince: lastConsolidateTs,
1429
+ // Pass profile-configured options. incrementalSince narrows the pool to
1430
+ // recently-changed memories + graph neighbours use this for frequent
1431
+ // passes (quick-shredder). Leave absent in the nightly default profile for
1432
+ // a full-pool sweep that catches stale-but-unmerged duplicates.
1433
+ incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
1434
+ limit: improveProfile?.processes?.consolidate?.limit,
1435
+ neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
1354
1436
  maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
1355
1437
  // Honor profile.autoAccept (already merged into options.autoAccept at the
1356
1438
  // top of akmImprove). The CLI parser always supplies 90 when --auto-accept
@@ -1359,7 +1441,7 @@ async function runConsolidationPass(args) {
1359
1441
  // options.consolidateOptions.autoAccept (if explicitly provided by caller)
1360
1442
  // still wins because the spread above runs first.
1361
1443
  autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
1362
- });
1444
+ }));
1363
1445
  {
1364
1446
  const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
1365
1447
  try {
@@ -1379,7 +1461,14 @@ async function runConsolidationPass(args) {
1379
1461
  appendEvent({
1380
1462
  eventType: "consolidate_completed",
1381
1463
  ref: "memory:_consolidation",
1382
- metadata: { processed: consolidation.processed, merged: consolidation.merged },
1464
+ metadata: {
1465
+ processed: consolidation.processed,
1466
+ merged: consolidation.merged,
1467
+ deleted: consolidation.deleted,
1468
+ contradicted: consolidation.contradicted,
1469
+ failedChunks: consolidation.failedChunks ?? 0,
1470
+ durationMs: consolidation.durationMs,
1471
+ },
1383
1472
  }, eventsCtx);
1384
1473
  }
1385
1474
  }
@@ -1446,7 +1535,9 @@ async function runImprovePreparationStage(args) {
1446
1535
  // / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
1447
1536
  // hook with an on-demand pull pipeline.
1448
1537
  //
1449
- // Default-on; opt out via `profiles.improve.default.processes.extract.enabled: false`.
1538
+ // Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
1539
+ // (#593: the gate respects the resolved improve profile, not just the
1540
+ // hardcoded `default` profile path the legacy feature flag reads).
1450
1541
  // Each available harness gets one call with the default --since window;
1451
1542
  // already-seen sessions (tracked in state.db.extract_sessions_seen) are
1452
1543
  // skipped automatically so re-runs don't burn LLM calls on unchanged data.
@@ -1480,7 +1571,14 @@ async function runImprovePreparationStage(args) {
1480
1571
  const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
1481
1572
  const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
1482
1573
  const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
1483
- if (isLlmFeatureEnabled(extractConfig, "session_extraction")) {
1574
+ // #593/#594: the ACTIVE resolved improve profile is the single source of
1575
+ // truth for whether extract runs. (Previously this also ANDed in the legacy
1576
+ // `session_extraction` feature flag, which only reads
1577
+ // `profiles.improve.default.processes.extract.enabled`; that made the default
1578
+ // profile a global kill switch, so a non-default profile enabling extract was
1579
+ // silently overridden. The default profile is now just another profile.)
1580
+ // `akmExtract` re-checks the same active profile internally via `improveProfile`.
1581
+ if (resolveProcessEnabled("extract", improveProfile)) {
1484
1582
  const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
1485
1583
  // The guard engages only when minNewSessions > 0; 0 disables it entirely.
1486
1584
  let belowMinNewSessions = false;
@@ -1511,15 +1609,18 @@ async function runImprovePreparationStage(args) {
1511
1609
  extractResults = [];
1512
1610
  for (const h of availableHarnesses) {
1513
1611
  try {
1514
- const result = await akmExtract({
1612
+ const result = await withLlmStage("session-extraction", () => akmExtract({
1515
1613
  type: h.name,
1516
1614
  ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
1517
1615
  config: extractConfig,
1616
+ // Thread the ACTIVE profile so extract's internal gate + per-process
1617
+ // config read the running profile, not always `default`.
1618
+ improveProfile,
1518
1619
  dryRun: options.dryRun ?? false,
1519
1620
  ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1520
1621
  // C2: pin extract's skip-tracking state.db open to the boundary path.
1521
1622
  ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
1522
- });
1623
+ }));
1523
1624
  extractResults.push(result);
1524
1625
  {
1525
1626
  const gr = await runAutoAcceptGate(primaryStashDir
@@ -1610,7 +1711,13 @@ async function runImprovePreparationStage(args) {
1610
1711
  const validationFailures = [];
1611
1712
  for (const candidate of postCleanupRefs) {
1612
1713
  try {
1613
- const filePath = await findAssetFilePath(candidate.ref, options.stashDir);
1714
+ // #591: use the path pre-resolved at planning time when it is still on
1715
+ // disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
1716
+ // stash. Fall back to findAssetFilePath only for refs that bypassed
1717
+ // collectEligibleRefs' index scan or whose file moved since planning.
1718
+ const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
1719
+ ? candidate.filePath
1720
+ : await findAssetFilePath(candidate.ref, options.stashDir);
1614
1721
  if (!filePath) {
1615
1722
  validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
1616
1723
  continue;
@@ -1736,10 +1843,19 @@ async function runImprovePreparationStage(args) {
1736
1843
  // refs that fail the distill signal-delta gate).
1737
1844
  // distillOnlyRefs — reflect blocked but distill signal-delta passes
1738
1845
  // AND ref is a distill candidate.
1739
- // fullySkippedCount — neither gate passes synthetic skip action
1740
- // + improve_skipped event, excluded from sort.
1846
+ // noFeedbackPool — neither signal-delta gate passes *and* the ref has
1847
+ // no recent feedback signal at all. These are NOT
1848
+ // skipped here: they are handed to the high-retrieval
1849
+ // fallback (P0-A) below so frequently-retrieved but
1850
+ // never-rated assets can still be improved. Only refs
1851
+ // that P0-A declines are ultimately fully skipped.
1852
+ // fullySkippedCount — has stale feedback but no signal delta → genuine
1853
+ // skip (counted, aggregated event emitted post-loop),
1854
+ // excluded from sort.
1741
1855
  const eligibleRefs = [];
1742
1856
  const distillOnlyRefs = [];
1857
+ // Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
1858
+ const noFeedbackPool = [];
1743
1859
  let fullySkippedCount = 0;
1744
1860
  // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
1745
1861
  const scopeRefBypass = scope.mode === "ref";
@@ -1777,22 +1893,59 @@ async function runImprovePreparationStage(args) {
1777
1893
  // Reflect blocked but distill passes → distill-only bucket.
1778
1894
  distillOnlyRefs.push(r);
1779
1895
  }
1896
+ else if (!latestFeedbackTs.has(r.ref)) {
1897
+ // Neither signal-delta gate passes AND there is no recent feedback signal
1898
+ // at all. Rather than skip outright, defer to the high-retrieval fallback
1899
+ // (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
1900
+ // what that path is meant to rescue. Refs P0-A declines are skipped there.
1901
+ noFeedbackPool.push(r);
1902
+ }
1780
1903
  else {
1781
- // Neither gate passes fully skipped.
1904
+ // Has feedback on record but no signal delta since the last proposal —
1905
+ // genuinely fully skipped. Counted here; a single aggregated
1906
+ // improve_skipped event is emitted after the loop (mirrors
1907
+ // profile_filtered_all_passes) instead of one event per ref.
1782
1908
  fullySkippedCount++;
1783
1909
  actions.push({
1784
1910
  ref: r.ref,
1785
1911
  mode: "distill-skipped",
1786
1912
  result: { ok: true, reason: "no new signal since last proposal" },
1787
1913
  });
1788
- appendEvent({ eventType: "improve_skipped", ref: r.ref, metadata: { reason: "no_new_signal" } }, eventsCtx);
1789
1914
  }
1790
1915
  }
1916
+ // Emit ONE aggregated skip event for the fully-skipped bucket rather than one
1917
+ // improve_skipped event per ref (#592 pattern, mirrors
1918
+ // profile_filtered_all_passes above). The per-ref loop previously produced
1919
+ // ~11K state.db writes per run on a large stash, the dominant contributor to
1920
+ // 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
1921
+ // run summary; no downstream consumer needs a per-ref DB audit trail (health's
1922
+ // skip histogram reads the `no_new_signal` counter from the count field).
1923
+ if (fullySkippedCount > 0) {
1924
+ appendEvent({
1925
+ eventType: "improve_skipped",
1926
+ ref: undefined,
1927
+ metadata: {
1928
+ reason: "no_new_signal",
1929
+ count: fullySkippedCount,
1930
+ },
1931
+ }, eventsCtx);
1932
+ }
1791
1933
  // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
1792
- // Everything from here works only on (eligibleRefs ∪ distillOnlyRefs). The
1793
- // fully-skipped bucket has already been routed and emitted; we deliberately
1794
- // avoid spending DB/CPU on refs that cannot enter the loop.
1934
+ // Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
1935
+ // deferred noFeedbackPool that may be rescued by the high-retrieval fallback
1936
+ // (P0-A). The fully-skipped bucket has already been routed and its aggregated
1937
+ // event emitted; we deliberately avoid spending DB/CPU on refs that the
1938
+ // signal-delta gate rejected with feedback already on record.
1795
1939
  const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
1940
+ // Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
1941
+ // partition above could not place these in a reflect/distill bucket, but they
1942
+ // may still qualify if they have been retrieved often enough. Two disjoint
1943
+ // sources feed this set:
1944
+ // 1. noFeedbackPool — refs with no recent feedback that the partition loop
1945
+ // deliberately deferred here (otherwise they would never reach P0-A).
1946
+ // 2. processableRefs entries that turn out to carry no recent feedback
1947
+ // *signal* once feedbackSummary is computed below.
1948
+ // (1) is added here; (2) is folded in after feedbackSummary is built.
1796
1949
  // Gap 6: only surface feedback signals from the last 30 days so that
1797
1950
  // ancient one-off feedback events don't permanently lock an asset into
1798
1951
  // every improve run. Assets with only stale signals fall through to the
@@ -1802,8 +1955,12 @@ async function runImprovePreparationStage(args) {
1802
1955
  // Pre-compute feedback summary per ref in a single pass so we don't issue
1803
1956
  // two readEvents({type:"feedback", ref}) per asset (one for signal filtering,
1804
1957
  // one for ratio computation).
1958
+ // Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
1959
+ // ratios are available for any noFeedbackPool ref that P0-A rescues below.
1805
1960
  const feedbackSummary = new Map();
1806
- for (const candidate of processableRefs) {
1961
+ for (const candidate of [...processableRefs, ...noFeedbackPool]) {
1962
+ if (feedbackSummary.has(candidate.ref))
1963
+ continue;
1807
1964
  const { events } = readEvents({ type: "feedback", ref: candidate.ref });
1808
1965
  let hasSignal = false;
1809
1966
  let positive = 0;
@@ -1826,8 +1983,21 @@ async function runImprovePreparationStage(args) {
1826
1983
  // P0-A: also surface zero-feedback assets that have been retrieved many times.
1827
1984
  const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
1828
1985
  const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
1829
- const noFeedbackCandidates = processableRefs.filter((r) => !signalBearingSet.has(r.ref));
1986
+ // Zero-feedback candidates for P0-A: processableRefs without a recent signal,
1987
+ // plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
1988
+ // disjoint by construction, but guard against overlap defensively).
1989
+ const noFeedbackSeen = new Set();
1990
+ const noFeedbackCandidates = [];
1991
+ for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
1992
+ if (noFeedbackSeen.has(r.ref))
1993
+ continue;
1994
+ noFeedbackSeen.add(r.ref);
1995
+ noFeedbackCandidates.push(r);
1996
+ }
1830
1997
  let highRetrievalRefs = [];
1998
+ // Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
1999
+ // proactive-maintenance selector below can reuse them without a second DB pass.
2000
+ let retrievalCounts = new Map();
1831
2001
  let dbForRetrieval;
1832
2002
  try {
1833
2003
  dbForRetrieval = openExistingDatabase();
@@ -1835,15 +2005,21 @@ async function runImprovePreparationStage(args) {
1835
2005
  if (showEventCount === 0) {
1836
2006
  warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
1837
2007
  }
1838
- const retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
2008
+ retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
1839
2009
  // High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
1840
- // ref qualifies exactly once — when retrievalCount threshold AND no
1841
- // prior reflect proposal exists for it. Once a reflect proposal is on
1842
- // record, subsequent re-eligibility requires explicit feedback (which
1843
- // flows through the normal signal-delta gate above). Tracking growth in
1844
- // retrieval count would require persisting the count in proposal
1845
- // metadata; deferred to a follow-up.
1846
- highRetrievalRefs = noFeedbackCandidates.filter((r) => (retrievalCounts.get(r.ref) ?? 0) >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref));
2010
+ // ref qualifies exactly once — when it has actually been retrieved
2011
+ // (retrievalCount 1) AND retrievalCount threshold AND no prior reflect
2012
+ // proposal exists for it. Once a reflect proposal is on record, subsequent
2013
+ // re-eligibility requires explicit feedback (which flows through the normal
2014
+ // signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
2015
+ // from rescuing genuinely never-retrieved assets — the fallback is for
2016
+ // *retrieved* assets, not silent ones. Tracking growth in retrieval count
2017
+ // would require persisting the count in proposal metadata; deferred to a
2018
+ // follow-up.
2019
+ highRetrievalRefs = noFeedbackCandidates.filter((r) => {
2020
+ const count = retrievalCounts.get(r.ref) ?? 0;
2021
+ return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
2022
+ });
1847
2023
  }
1848
2024
  catch (err) {
1849
2025
  rethrowIfTestIsolationError(err);
@@ -1853,6 +2029,91 @@ async function runImprovePreparationStage(args) {
1853
2029
  if (dbForRetrieval)
1854
2030
  closeDatabase(dbForRetrieval);
1855
2031
  }
2032
+ // ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
2033
+ // The signal-delta gate and P0-A only surface assets with fresh feedback or a
2034
+ // raw-retrieval spike. Neither revisits a stable, high-value asset on a
2035
+ // schedule, so on a quiet stash useful assets drift stale and are never
2036
+ // refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
2037
+ // and the run is whole-stash / type scope, this selector ranks the eligible
2038
+ // population by a composite maintenance priority, gates on staleness ("due"),
2039
+ // bounds to top-N, and folds the winners into the SAME candidate set the other
2040
+ // two sources feed — so they flow through the existing #580 empty-diff /
2041
+ // cosmetic suppression and additive-distill gates. It adds no new mutation
2042
+ // logic of its own. The due gate doubles as the rotation cooldown: a freshly
2043
+ // reflected asset is excluded until it ages back past `dueDays`, so successive
2044
+ // runs rotate through the due pool rather than re-selecting the same heads.
2045
+ let proactiveRefs = [];
2046
+ let proactiveMaintenanceSummary;
2047
+ const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
2048
+ if (proactiveEnabled) {
2049
+ const pmCfg = improveProfile.processes?.proactiveMaintenance;
2050
+ const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
2051
+ const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
2052
+ const importanceWeights = pmCfg?.importanceWeights;
2053
+ // Candidate population: the zero-feedback / non-signal pool — exactly the
2054
+ // assets the other two sources would NOT pick this run. Exclude any P0-A
2055
+ // rescued this run so we never double-select the same ref.
2056
+ const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
2057
+ const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
2058
+ const selection = selectProactiveMaintenanceRefs({
2059
+ candidates: pmCandidates,
2060
+ lastReflectTs: lastReflectProposalTs,
2061
+ lastDistillTs: lastDistillProposalTs,
2062
+ retrievalCounts,
2063
+ sizeBytesOf: (r) => {
2064
+ const fp = r.filePath;
2065
+ if (!fp)
2066
+ return undefined;
2067
+ try {
2068
+ return fs.statSync(fp).size;
2069
+ }
2070
+ catch {
2071
+ return undefined;
2072
+ }
2073
+ },
2074
+ dueDays,
2075
+ maxPerRun,
2076
+ importanceWeights,
2077
+ });
2078
+ proactiveRefs = selection.selected;
2079
+ proactiveMaintenanceSummary = {
2080
+ selected: selection.selected.length,
2081
+ dueTotal: selection.dueTotal,
2082
+ neverReflected: selection.neverReflected,
2083
+ };
2084
+ // Aggregated observability event (never per-ref — avoids the event flood the
2085
+ // Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
2086
+ appendEvent({
2087
+ eventType: "proactive_selected",
2088
+ ref: undefined,
2089
+ metadata: {
2090
+ count: selection.selected.length,
2091
+ dueTotal: selection.dueTotal,
2092
+ neverReflected: selection.neverReflected,
2093
+ },
2094
+ }, eventsCtx);
2095
+ if (selection.selected.length > 0) {
2096
+ info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
2097
+ `(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
2098
+ }
2099
+ }
2100
+ // Record an in-memory skip action for every zero-feedback ref that the
2101
+ // partition loop deferred to P0-A but P0-A then declined (retrievalCount below
2102
+ // threshold, or a prior reflect proposal already on record). These never make
2103
+ // it into mergedRefs, so without this they would silently vanish from the run
2104
+ // summary. No DB event is written here — these refs carry no signal at all, so
2105
+ // there is nothing for the skip histogram to aggregate; the action log alone
2106
+ // preserves the per-ref audit trail (mirrors the fully-skipped action above).
2107
+ const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
2108
+ for (const r of noFeedbackPool) {
2109
+ if (rescuedSet.has(r.ref))
2110
+ continue;
2111
+ actions.push({
2112
+ ref: r.ref,
2113
+ mode: "distill-skipped",
2114
+ result: { ok: true, reason: "no new signal since last proposal" },
2115
+ });
2116
+ }
1856
2117
  // If the user explicitly scoped to a single ref, always act on it —
1857
2118
  // skip the signal/retrieval filter entirely. The filter exists to avoid
1858
2119
  // noisy "improve everything" runs; it should not gate an intentional
@@ -1862,8 +2123,48 @@ async function runImprovePreparationStage(args) {
1862
2123
  // or sufficient retrievals). A stash with no signals has 0 eligible refs —
1863
2124
  // usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
1864
2125
  // to bring them into the eligible pool.
1865
- const signalAndRetrievalRefs = [...signalFiltered, ...highRetrievalRefs];
2126
+ // Layer-2 proactive refs join the eligible set alongside feedback-signal and
2127
+ // high-retrieval (P0-A) refs. The three sources are disjoint by construction
2128
+ // (proactive draws from noFeedbackCandidates with the P0-A picks removed), but
2129
+ // dedupe defensively so a ref can never enter the loop twice. `requireFeedbackSignal`
2130
+ // still suppresses both fallback sources for callers that want feedback-only runs.
2131
+ const signalAndRetrievalRefs = dedupeRefs([...signalFiltered, ...highRetrievalRefs, ...proactiveRefs]);
1866
2132
  const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
2133
+ // ── Attribution tagging: stamp each ref with the eligibility lane that
2134
+ // selected it ──────────────────────────────────────────────────────────────
2135
+ // Every reflect/distill proposal must record WHICH lane chose its source asset
2136
+ // so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
2137
+ // (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
2138
+ // lane map here — the one place all four lanes are known — and stamp it onto
2139
+ // each ImproveEligibleRef object. Because the ref objects are shared by
2140
+ // reference across buckets, the stamp travels with the ref through the sort,
2141
+ // disk-check, and loop stages down to the reflect/distill event emit sites and
2142
+ // createProposal calls. See EligibilitySource for the lane vocabulary.
2143
+ //
2144
+ // Precedence (prefer the most specific reactive signal):
2145
+ // scope > signal-delta > high-retrieval > proactive
2146
+ // A ref with real feedback is attributed to feedback even if it was also due
2147
+ // for proactive maintenance. We apply lanes weakest-first so the strongest
2148
+ // overwrites; the explicit --scope <ref> bypass wins outright (user intent).
2149
+ const eligibilitySourceByRef = new Map();
2150
+ for (const r of proactiveRefs)
2151
+ eligibilitySourceByRef.set(r.ref, "proactive");
2152
+ for (const r of highRetrievalRefs)
2153
+ eligibilitySourceByRef.set(r.ref, "high-retrieval");
2154
+ for (const r of signalFiltered)
2155
+ eligibilitySourceByRef.set(r.ref, "signal-delta");
2156
+ if (scope.mode === "ref") {
2157
+ // O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
2158
+ // arrived via the scopeRefBypass branch, so attribute the whole set to scope.
2159
+ for (const r of processableRefs)
2160
+ eligibilitySourceByRef.set(r.ref, "scope");
2161
+ }
2162
+ for (const r of mergedRefs) {
2163
+ // "unknown" is a genuine fallback, never a silent alias for signal-delta:
2164
+ // only refs we truly cannot attribute land here (none in practice, since
2165
+ // mergedRefs is always a subset of the four lanes above).
2166
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2167
+ }
1867
2168
  const utilityMap = buildUtilityMap(mergedRefs);
1868
2169
  // Load feedback ratio per ref from the pre-computed summary (no extra DB pass).
1869
2170
  const feedbackRatios = new Map();
@@ -1918,15 +2219,32 @@ async function runImprovePreparationStage(args) {
1918
2219
  const assetMissingOnDisk = [];
1919
2220
  const existsCheckedActionable = [];
1920
2221
  for (const candidate of sorted) {
1921
- const filePath = await findAssetFilePath(candidate.ref, options.stashDir);
2222
+ // #591: prefer the path pre-resolved at planning time (synchronous
2223
+ // existsSync) over a serial async DB lookup per ref.
2224
+ const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
2225
+ ? candidate.filePath
2226
+ : await findAssetFilePath(candidate.ref, options.stashDir);
1922
2227
  if (filePath && fs.existsSync(filePath)) {
1923
2228
  existsCheckedActionable.push(candidate);
1924
2229
  }
1925
2230
  else {
1926
2231
  assetMissingOnDisk.push(candidate.ref);
1927
- appendEvent({ eventType: "improve_skipped", ref: candidate.ref, metadata: { reason: "asset_missing_on_disk" } }, eventsCtx);
1928
2232
  }
1929
2233
  }
2234
+ // #592 audit: one summary event instead of one per missing ref. Normally
2235
+ // tiny, but a stash deletion racing the run could make this O(n) sequential
2236
+ // state.db writes. `refs` is capped so the metadata row stays bounded.
2237
+ if (assetMissingOnDisk.length > 0) {
2238
+ appendEvent({
2239
+ eventType: "improve_skipped",
2240
+ ref: undefined,
2241
+ metadata: {
2242
+ reason: "asset_missing_on_disk",
2243
+ count: assetMissingOnDisk.length,
2244
+ refs: assetMissingOnDisk.slice(0, 50),
2245
+ },
2246
+ }, eventsCtx);
2247
+ }
1930
2248
  const actionableRefs = existsCheckedActionable;
1931
2249
  // Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
1932
2250
  // preserving sort order. distillOnlyRefs participate in the sort so --limit
@@ -1987,6 +2305,7 @@ async function runImprovePreparationStage(args) {
1987
2305
  gateAutoAcceptFailedCount,
1988
2306
  consolidation: consolidationPass.consolidation,
1989
2307
  consolidationRan: consolidationPass.consolidationRan,
2308
+ ...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
1990
2309
  };
1991
2310
  }
1992
2311
  async function runImproveLoopStage(args) {
@@ -1995,6 +2314,14 @@ async function runImproveLoopStage(args) {
1995
2314
  // receives only its fair share of the wall-clock budget.
1996
2315
  const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
1997
2316
  const RECENT_ERRORS_CAP = 3;
2317
+ // requirePlannedRefs guard: when the distill profile sets this flag, skip
2318
+ // distill for distill-only refs if the reflect phase produced no planned refs.
2319
+ // Prevents the distill loop from generating hundreds of distill-skipped events
2320
+ // on quiet passes (all refs on reflect cooldown, no new signal to distill).
2321
+ const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
2322
+ const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
2323
+ const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
2324
+ const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
1998
2325
  // R-2 / #389: Self-Consistency multi-sample voting helpers.
1999
2326
  // Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
2000
2327
  const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
@@ -2155,6 +2482,9 @@ async function runImproveLoopStage(args) {
2155
2482
  eventSource: "improve",
2156
2483
  ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
2157
2484
  ...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
2485
+ // Attribution: carry the eligibility lane so reflect stamps it on
2486
+ // the reflect_invoked event and the persisted proposal.
2487
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2158
2488
  };
2159
2489
  // R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
2160
2490
  // Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
@@ -2167,9 +2497,11 @@ async function runImproveLoopStage(args) {
2167
2497
  if (remainingBudgetMs() <= 0)
2168
2498
  break;
2169
2499
  // draftMode: skip DB write so each sample doesn't create a proposal.
2170
- samples.push(await reflectFn({ ...reflectCallArgs, draftMode: true }));
2500
+ samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
2171
2501
  }
2172
- const winner = pickMajorityVote(samples.length > 0 ? samples : [await reflectFn({ ...reflectCallArgs, draftMode: true })]);
2502
+ const winner = pickMajorityVote(samples.length > 0
2503
+ ? samples
2504
+ : [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
2173
2505
  // Persist only the majority-vote winner as a single real proposal.
2174
2506
  if (winner.ok && primaryStashDir) {
2175
2507
  const persistResult = createProposal(primaryStashDir, {
@@ -2177,6 +2509,9 @@ async function runImproveLoopStage(args) {
2177
2509
  source: "reflect",
2178
2510
  sourceRun: `reflect-sc-${Date.now()}`,
2179
2511
  payload: winner.proposal.payload,
2512
+ // Attribution: the self-consistency path persists the winner here
2513
+ // (draftMode skips reflect's own createProposal), so stamp the lane.
2514
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2180
2515
  });
2181
2516
  reflectResult = isProposalSkipped(persistResult)
2182
2517
  ? {
@@ -2194,7 +2529,7 @@ async function runImproveLoopStage(args) {
2194
2529
  }
2195
2530
  }
2196
2531
  else {
2197
- reflectResult = await reflectFn(reflectCallArgs);
2532
+ reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
2198
2533
  }
2199
2534
  const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
2200
2535
  // Content-policy guard hits (reflect size-rail rejections) are NOT
@@ -2211,6 +2546,12 @@ async function runImproveLoopStage(args) {
2211
2546
  // user's stack were this case; see review §1a row "Reflect refused
2212
2547
  // asset type".
2213
2548
  const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
2549
+ // Noise-gate suppression (#580): the candidate edit was an empty
2550
+ // diff or a cosmetic-only reformat of the current asset. Like
2551
+ // `unsupported_type`, this is a deterministic skip — not an LLM
2552
+ // fault — so it routes to the `reflect-skipped` bucket and stays
2553
+ // out of recentErrors/avoidPatterns.
2554
+ const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
2214
2555
  actions.push({
2215
2556
  ref: planned.ref,
2216
2557
  mode: reflectResult.ok
@@ -2219,18 +2560,19 @@ async function runImproveLoopStage(args) {
2219
2560
  ? "reflect-cooldown"
2220
2561
  : isGuardReject
2221
2562
  ? "reflect-guard-rejected"
2222
- : isTypeRefused
2563
+ : isTypeRefused || isNoChange
2223
2564
  ? "reflect-skipped"
2224
2565
  : "reflect-failed",
2225
2566
  result: reflectResult,
2226
2567
  });
2227
- // Cooldown skips, guard rejects, and type-refused skips are not
2228
- // failures — do not pollute recentErrors with them (those get
2229
- // injected as `avoidPatterns` into the next reflect prompt). Guard
2230
- // rejects ARE worth showing the LLM as a learn-signal so the next
2231
- // iteration sees "your last expansion was too large"; type-refused
2232
- // is deterministic and adds no learning signal.
2233
- if (!reflectResult.ok && !isCooldown && !isTypeRefused) {
2568
+ // Cooldown skips, guard rejects, type-refused skips, and noise-gate
2569
+ // skips are not failures — do not pollute recentErrors with them
2570
+ // (those get injected as `avoidPatterns` into the next reflect
2571
+ // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
2572
+ // so the next iteration sees "your last expansion was too large";
2573
+ // type-refused and no-change are deterministic and add no learning
2574
+ // signal.
2575
+ if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
2234
2576
  const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
2235
2577
  pushRecentError("reflect", errMsg);
2236
2578
  }
@@ -2283,6 +2625,18 @@ async function runImproveLoopStage(args) {
2283
2625
  info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2284
2626
  continue;
2285
2627
  }
2628
+ // requirePlannedRefs guard: skip distill for distill-only refs when no
2629
+ // reflect-eligible refs were planned this run, preventing mass skip events.
2630
+ if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
2631
+ actions.push({
2632
+ ref: planned.ref,
2633
+ mode: "distill-skipped",
2634
+ result: { ok: true, reason: "require_planned_refs" },
2635
+ });
2636
+ completedCount++;
2637
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2638
+ continue;
2639
+ }
2286
2640
  // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
2287
2641
  // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
2288
2642
  // it will refuse.
@@ -2352,11 +2706,14 @@ async function runImproveLoopStage(args) {
2352
2706
  }
2353
2707
  }
2354
2708
  }
2355
- const distillResult = await distillFn({
2709
+ const distillResult = await withLlmStage("distill", () => distillFn({
2356
2710
  ref: planned.ref,
2357
2711
  ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
2358
2712
  ...(options.stashDir ? { stashDir: options.stashDir } : {}),
2359
- });
2713
+ // Attribution: carry the eligibility lane so distill stamps it on the
2714
+ // distill_invoked event and the persisted proposal.
2715
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2716
+ }));
2360
2717
  actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
2361
2718
  if (distillResult.outcome === "queued" && distillResult.proposal) {
2362
2719
  const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
@@ -2497,7 +2854,9 @@ async function runImprovePostLoopStage(args) {
2497
2854
  };
2498
2855
  }
2499
2856
  // TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
2500
- async function runImproveMaintenancePasses(args) {
2857
+ // Exported for tests (#584/#585 DB-locking regression coverage); production
2858
+ // callers reach it only through akmImprove → runImprovePostLoopStage.
2859
+ export async function runImproveMaintenancePasses(args) {
2501
2860
  const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
2502
2861
  if (!primaryStashDir)
2503
2862
  return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
@@ -2516,276 +2875,344 @@ async function runImproveMaintenancePasses(args) {
2516
2875
  let graphExtractionDurationMs = 0;
2517
2876
  let orphansPurged = 0;
2518
2877
  let proposalsExpired = 0;
2519
- try {
2520
- db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
2521
- // Memory inference candidate-discovery (post-Item 9 fix from
2522
- // memory:akm-improve-critical-review-2026-05-20). Previously this pass
2523
- // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
2524
- // candidateRefs filter. But memoryRefsForInference is populated from refs
2525
- // distilled THIS RUN by the time that happens, those parents are
2526
- // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
2527
- // them. The genuinely-pending parents in the stash never entered the
2528
- // filter. Result: 0/0/0 for 25 consecutive runs.
2529
- //
2530
- // Fix: always run the pass when the feature is enabled; let the pass's
2531
- // own `collectPendingMemories` + `isPendingMemory` predicate find
2532
- // candidates from the filesystem-of-truth. The this-run set is still
2533
- // logged as a hint but no longer used as a filter.
2534
- const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
2535
- if (memoryInferenceDisabledByProfile) {
2536
- info("[improve] memory inference skipped (disabled by improve profile)");
2878
+ const openIndexDb = () => openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
2879
+ // #584: reindexFn opens its own write handle on the same index.db WAL file.
2880
+ // Holding our handle across that call produced SQLITE_BUSY / "database is
2881
+ // locked" failures in production, so the handle is closed BEFORE every
2882
+ // reindex and reopened after the fresh handle also sees the post-reindex
2883
+ // state that graph extraction and staleness detection below rely on. The
2884
+ // reopen runs in `finally` so a failed reindex still leaves a usable handle.
2885
+ const reindexWithIndexDbReleased = async (stashDir) => {
2886
+ if (db) {
2887
+ closeDatabase(db);
2888
+ db = undefined;
2537
2889
  }
2538
- else {
2539
- const hintRefs = memoryRefsForInference.size;
2540
- info(hintRefs > 0
2541
- ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
2542
- : "[improve] memory inference starting (discovering pending parents)");
2543
- const inferenceStart = Date.now();
2544
- try {
2545
- // O-1 (#364): pass budget signal so a hung inference call is cancelled.
2546
- memoryInference = await memoryInferenceFn({
2547
- config,
2548
- sources,
2549
- signal: budgetSignal,
2550
- db,
2551
- reEnrich: false,
2552
- onProgress: (event) => {
2553
- const current = event.currentRef ? ` ${event.currentRef}` : "";
2554
- info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
2555
- },
2556
- });
2557
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2558
- actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
2559
- info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2560
- }
2561
- catch (err) {
2562
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2563
- allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2564
- }
2890
+ try {
2891
+ await reindexFn({ stashDir });
2565
2892
  }
2566
- if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
2567
- info("[improve] reindexing after memory inference writes");
2568
- try {
2569
- await reindexFn({ stashDir: primaryStashDir });
2570
- reindexedAfterInference = true;
2571
- info("[improve] reindex after memory inference complete");
2572
- }
2573
- catch (err) {
2574
- allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2575
- }
2893
+ finally {
2894
+ db = openIndexDb();
2576
2895
  }
2577
- const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
2578
- const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
2579
- const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
2580
- // Build the set of refs actually touched this run.
2581
- const touchedRefs = new Set();
2582
- for (const r of args.actionableRefs)
2583
- touchedRefs.add(r.ref);
2584
- for (const r of memoryRefsForInference)
2585
- touchedRefs.add(r);
2586
- // INVARIANT: graph extraction normally runs only on files touched by
2587
- // actionable refs (candidatePaths). Full-corpus scans are opt-in via
2588
- // profile.processes.graphExtraction.fullScan = true (used by the
2589
- // `graph-refresh` built-in profile and its weekly scheduled task).
2590
- // The empty-Set fallback is intentional when no refs were touched
2591
- // the extractor's filter rejects every file and returns empty, keeping
2592
- // the pass invoked so the action is recorded and tests stay exercised.
2593
- if (graphExtractionDisabledByProfile) {
2594
- info("[improve] graph extraction skipped (disabled by improve profile)");
2595
- }
2596
- else if (sources.length > 0 && graphEnabled) {
2597
- info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
2598
- const extractionStart = Date.now();
2599
- try {
2600
- // D9: if consolidation ran but memory inference did not reindex, force a reindex
2601
- // so graph extraction sees current DB state after consolidation writes.
2602
- if (consolidationRan && !reindexedAfterInference) {
2603
- info("[improve] reindexing after consolidation (graph extraction needs current state)");
2604
- try {
2605
- await reindexFn({ stashDir: primaryStashDir });
2606
- reindexedAfterInference = true;
2607
- info("[improve] reindex after consolidation complete");
2608
- }
2609
- catch (err) {
2610
- allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
2611
- }
2896
+ };
2897
+ await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
2898
+ try {
2899
+ db = openIndexDb();
2900
+ // Memory inference candidate-discovery (post-Item 9 fix from
2901
+ // memory:akm-improve-critical-review-2026-05-20). Previously this pass
2902
+ // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
2903
+ // candidateRefs filter. But memoryRefsForInference is populated from refs
2904
+ // distilled THIS RUN — by the time that happens, those parents are
2905
+ // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
2906
+ // them. The genuinely-pending parents in the stash never entered the
2907
+ // filter. Result: 0/0/0 for 25 consecutive runs.
2908
+ //
2909
+ // Fix: always run the pass when the feature is enabled; let the pass's
2910
+ // own `collectPendingMemories` + `isPendingMemory` predicate find
2911
+ // candidates from the filesystem-of-truth. The this-run set is still
2912
+ // logged as a hint but no longer used as a filter.
2913
+ const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
2914
+ const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
2915
+ const pendingBelowMinCount = (() => {
2916
+ if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
2917
+ return false;
2918
+ const pending = collectPendingMemories(primaryStashDir).length;
2919
+ if (pending < minPendingCount) {
2920
+ info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
2921
+ return true;
2612
2922
  }
2613
- if (db && reindexedAfterInference) {
2614
- closeDatabase(db);
2615
- db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
2923
+ return false;
2924
+ })();
2925
+ if (memoryInferenceDisabledByProfile) {
2926
+ info("[improve] memory inference skipped (disabled by improve profile)");
2927
+ }
2928
+ else if (pendingBelowMinCount) {
2929
+ // skipped — message already emitted above
2930
+ }
2931
+ else {
2932
+ const hintRefs = memoryRefsForInference.size;
2933
+ info(hintRefs > 0
2934
+ ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
2935
+ : "[improve] memory inference starting (discovering pending parents)");
2936
+ const inferenceStart = Date.now();
2937
+ try {
2938
+ // O-1 (#364): pass budget signal so a hung inference call is cancelled.
2939
+ memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
2940
+ config,
2941
+ sources,
2942
+ signal: budgetSignal,
2943
+ db,
2944
+ reEnrich: false,
2945
+ onProgress: (event) => {
2946
+ const current = event.currentRef ? ` ${event.currentRef}` : "";
2947
+ info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
2948
+ },
2949
+ }));
2950
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
2951
+ actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
2952
+ info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2616
2953
  }
2617
- // Resolve touched refs to absolute file paths. Skipped for fullScan
2618
- // (candidatePaths stays undefined → extractor processes all files).
2619
- let candidatePaths;
2620
- if (!graphExtractionFullScan) {
2621
- candidatePaths = new Set();
2622
- if (primaryStashDir && touchedRefs.size > 0) {
2623
- const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
2624
- const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
2625
- for (const p of resolved) {
2626
- if (typeof p === "string" && p.length > 0)
2627
- candidatePaths.add(p);
2628
- }
2629
- }
2954
+ catch (err) {
2955
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
2956
+ allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2630
2957
  }
2631
- const progressHandler = (event) => {
2632
- const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
2633
- info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
2634
- };
2635
- // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
2636
- graphExtraction = await graphExtractionFn({
2637
- config,
2638
- sources,
2639
- signal: budgetSignal,
2640
- db,
2641
- reEnrich: false,
2642
- onProgress: progressHandler,
2643
- options: { candidatePaths },
2644
- });
2645
- graphExtractionDurationMs = Date.now() - extractionStart;
2646
- actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
2647
- info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
2648
2958
  }
2649
- catch (err) {
2650
- graphExtractionDurationMs = Date.now() - extractionStart;
2651
- allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
2652
- }
2653
- }
2654
- else if (sources.length > 0 && !graphEnabled) {
2655
- info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
2656
- }
2657
- // Orphan proposal purge reject pending reflect proposals whose target
2658
- // asset no longer exists on disk. Runs after graph extraction so newly
2659
- // promoted assets from accept flows during this run are already present.
2660
- if (primaryStashDir) {
2661
- try {
2662
- const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
2663
- orphansPurged = purgeResult.rejected;
2664
- if (purgeResult.rejected > 0) {
2665
- info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
2959
+ if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
2960
+ info("[improve] reindexing after memory inference writes");
2961
+ try {
2962
+ await reindexWithIndexDbReleased(primaryStashDir);
2963
+ reindexedAfterInference = true;
2964
+ info("[improve] reindex after memory inference complete");
2965
+ }
2966
+ catch (err) {
2967
+ allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2666
2968
  }
2667
- appendEvent({
2668
- eventType: "proposal_orphan_purge",
2669
- ref: "proposals:_orphan-purge",
2670
- metadata: {
2671
- checked: purgeResult.checked,
2672
- rejected: purgeResult.rejected,
2673
- durationMs: purgeResult.durationMs,
2674
- byType: purgeResult.byType,
2675
- orphans: purgeResult.orphans.map((o) => o.ref),
2676
- },
2677
- }, eventsCtx);
2678
2969
  }
2679
- catch (err) {
2680
- allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
2970
+ const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
2971
+ const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
2972
+ const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
2973
+ // Build the set of refs actually touched this run.
2974
+ const touchedRefs = new Set();
2975
+ for (const r of args.actionableRefs)
2976
+ touchedRefs.add(r.ref);
2977
+ for (const r of memoryRefsForInference)
2978
+ touchedRefs.add(r);
2979
+ // INVARIANT: graph extraction normally runs only on files touched by
2980
+ // actionable refs (candidatePaths). Full-corpus scans are opt-in via
2981
+ // profile.processes.graphExtraction.fullScan = true (used by the
2982
+ // `graph-refresh` built-in profile and its weekly scheduled task).
2983
+ // The empty-Set fallback is intentional when no refs were touched —
2984
+ // the extractor's filter rejects every file and returns empty, keeping
2985
+ // the pass invoked so the action is recorded and tests stay exercised.
2986
+ if (graphExtractionDisabledByProfile) {
2987
+ info("[improve] graph extraction skipped (disabled by improve profile)");
2681
2988
  }
2682
- // Phase 6B (Advantage D6b): expire pending proposals that have aged past
2683
- // the retention window. Runs AFTER orphan purge so we never double-archive
2684
- // a proposal that orphan-purge already moved. `expireStaleProposals` emits
2685
- // its own per-proposal `proposal_expired` events; we additionally emit a
2686
- // single roll-up event here for parity with the orphan-purge surface.
2687
- try {
2688
- const expireResult = expireStaleProposals(primaryStashDir, config);
2689
- proposalsExpired = expireResult.expired;
2690
- if (expireResult.expired > 0) {
2691
- info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
2692
- `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
2989
+ else if (sources.length > 0 && graphEnabled) {
2990
+ info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
2991
+ const extractionStart = Date.now();
2992
+ try {
2993
+ // D9: if consolidation ran but memory inference did not reindex, force a reindex
2994
+ // so graph extraction sees current DB state after consolidation writes.
2995
+ if (consolidationRan && !reindexedAfterInference) {
2996
+ info("[improve] reindexing after consolidation (graph extraction needs current state)");
2997
+ try {
2998
+ await reindexWithIndexDbReleased(primaryStashDir);
2999
+ reindexedAfterInference = true;
3000
+ info("[improve] reindex after consolidation complete");
3001
+ }
3002
+ catch (err) {
3003
+ allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
3004
+ }
3005
+ }
3006
+ // #584: no close/reopen needed here — reindexWithIndexDbReleased
3007
+ // already swapped in a fresh post-reindex handle.
3008
+ // Resolve touched refs to absolute file paths. Skipped for fullScan
3009
+ // (candidatePaths stays undefined → extractor processes all files).
3010
+ let candidatePaths;
3011
+ if (!graphExtractionFullScan) {
3012
+ candidatePaths = new Set();
3013
+ if (primaryStashDir && touchedRefs.size > 0) {
3014
+ const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
3015
+ const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
3016
+ for (const p of resolved) {
3017
+ if (typeof p === "string" && p.length > 0)
3018
+ candidatePaths.add(p);
3019
+ }
3020
+ }
3021
+ }
3022
+ const progressHandler = (event) => {
3023
+ const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
3024
+ info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
3025
+ };
3026
+ // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
3027
+ graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
3028
+ config,
3029
+ sources,
3030
+ signal: budgetSignal,
3031
+ db,
3032
+ reEnrich: false,
3033
+ onProgress: progressHandler,
3034
+ options: { candidatePaths },
3035
+ }));
3036
+ graphExtractionDurationMs = Date.now() - extractionStart;
3037
+ actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
3038
+ info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
3039
+ }
3040
+ catch (err) {
3041
+ graphExtractionDurationMs = Date.now() - extractionStart;
3042
+ allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
2693
3043
  }
2694
- appendEvent({
2695
- eventType: "proposal_expiration_pass",
2696
- ref: "proposals:_expiration",
2697
- metadata: {
2698
- checked: expireResult.checked,
2699
- expired: expireResult.expired,
2700
- durationMs: expireResult.durationMs,
2701
- retentionDays: expireResult.retentionDays,
2702
- expiredProposals: expireResult.expiredProposals,
2703
- },
2704
- }, eventsCtx);
2705
3044
  }
2706
- catch (err) {
2707
- allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
3045
+ else if (sources.length > 0 && !graphEnabled) {
3046
+ info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
2708
3047
  }
2709
- }
2710
- // Fix #2 (observability 0.8.0): trim the events table in state.db so it
2711
- // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
2712
- // invocation, and every command surface emits at least one event besides —
2713
- // without this trim, state.db is a permanent append-only log. Config key
2714
- // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
2715
- // window. `purgeOldEvents()` opens its own state.db handle separate from
2716
- // the index `db` above (different SQLite file).
2717
- {
2718
- const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
2719
- if (retentionDays > 0) {
2720
- let stateDb;
3048
+ // Orphan proposal purge — reject pending reflect proposals whose target
3049
+ // asset no longer exists on disk. Runs after graph extraction so newly
3050
+ // promoted assets from accept flows during this run are already present.
3051
+ if (primaryStashDir) {
2721
3052
  try {
2722
- // C2: reuse the boundary-pinned state.db path carried on eventsCtx so
2723
- // this purge open never re-reads `process.env` live mid-run. The path
2724
- // is always set by akmImprove; openStateDatabase() falls back to the
2725
- // env-derived default only if a caller omitted it entirely.
2726
- stateDb = openStateDatabase(eventsCtx?.dbPath);
2727
- const purgedCount = purgeOldEvents(stateDb, retentionDays);
2728
- if (purgedCount > 0) {
2729
- info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
3053
+ const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
3054
+ orphansPurged = purgeResult.rejected;
3055
+ if (purgeResult.rejected > 0) {
3056
+ info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
2730
3057
  }
2731
3058
  appendEvent({
2732
- eventType: "events_purged",
2733
- ref: "events:_purge",
2734
- metadata: { purgedCount, retentionDays },
3059
+ eventType: "proposal_orphan_purge",
3060
+ ref: "proposals:_orphan-purge",
3061
+ metadata: {
3062
+ checked: purgeResult.checked,
3063
+ rejected: purgeResult.rejected,
3064
+ durationMs: purgeResult.durationMs,
3065
+ byType: purgeResult.byType,
3066
+ orphans: purgeResult.orphans.map((o) => o.ref),
3067
+ },
2735
3068
  }, eventsCtx);
2736
- // improve_runs uses the same retention window as events — both are
2737
- // observability/audit data, both grow append-only, both have a
2738
- // dedicated purge helper. Mirroring the events purge here means a
2739
- // single retention knob (improve.eventRetentionDays) governs both.
2740
- const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
2741
- if (improveRunsPurged > 0) {
2742
- info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
3069
+ }
3070
+ catch (err) {
3071
+ allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
3072
+ }
3073
+ // Phase 6B (Advantage D6b): expire pending proposals that have aged past
3074
+ // the retention window. Runs AFTER orphan purge so we never double-archive
3075
+ // a proposal that orphan-purge already moved. `expireStaleProposals` emits
3076
+ // its own per-proposal `proposal_expired` events; we additionally emit a
3077
+ // single roll-up event here for parity with the orphan-purge surface.
3078
+ try {
3079
+ const expireResult = expireStaleProposals(primaryStashDir, config);
3080
+ proposalsExpired = expireResult.expired;
3081
+ if (expireResult.expired > 0) {
3082
+ info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
3083
+ `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
2743
3084
  }
2744
3085
  appendEvent({
2745
- eventType: "improve_runs_purged",
2746
- ref: "improve_runs:_purge",
2747
- metadata: { purgedCount: improveRunsPurged, retentionDays },
3086
+ eventType: "proposal_expiration_pass",
3087
+ ref: "proposals:_expiration",
3088
+ metadata: {
3089
+ checked: expireResult.checked,
3090
+ expired: expireResult.expired,
3091
+ durationMs: expireResult.durationMs,
3092
+ retentionDays: expireResult.retentionDays,
3093
+ expiredProposals: expireResult.expiredProposals,
3094
+ },
2748
3095
  }, eventsCtx);
2749
3096
  }
2750
3097
  catch (err) {
2751
- allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
3098
+ allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
2752
3099
  }
2753
- finally {
2754
- if (stateDb) {
2755
- try {
2756
- stateDb.close();
3100
+ }
3101
+ // Fix #2 (observability 0.8.0): trim the events table in state.db so it
3102
+ // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
3103
+ // invocation, and every command surface emits at least one event besides —
3104
+ // without this trim, state.db is a permanent append-only log. Config key
3105
+ // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
3106
+ // window. The purge runs against state.db (a different SQLite file from
3107
+ // the index `db` above).
3108
+ {
3109
+ const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
3110
+ if (retentionDays > 0) {
3111
+ // #585: reuse the long-lived eventsCtx.db connection when akmImprove
3112
+ // opened one — opening a second state.db write connection while
3113
+ // eventsDb is still live made two simultaneous writers contend on the
3114
+ // same WAL file ("database is locked"). Only the eventsCtx.dbPath
3115
+ // fallback path (state.db failed to open up-front) opens — and then
3116
+ // owns and closes — its own handle. C2 still holds: the fallback uses
3117
+ // the boundary-pinned path, never a live `process.env` re-read.
3118
+ const ownsStateDb = !eventsCtx?.db;
3119
+ let stateDb;
3120
+ try {
3121
+ stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
3122
+ const purgedCount = purgeOldEvents(stateDb, retentionDays);
3123
+ if (purgedCount > 0) {
3124
+ info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
2757
3125
  }
2758
- catch {
2759
- // best-effort
3126
+ appendEvent({
3127
+ eventType: "events_purged",
3128
+ ref: "events:_purge",
3129
+ metadata: { purgedCount, retentionDays },
3130
+ }, eventsCtx);
3131
+ // improve_runs uses the same retention window as events — both are
3132
+ // observability/audit data, both grow append-only, both have a
3133
+ // dedicated purge helper. Mirroring the events purge here means a
3134
+ // single retention knob (improve.eventRetentionDays) governs both.
3135
+ const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
3136
+ if (improveRunsPurged > 0) {
3137
+ info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
3138
+ }
3139
+ appendEvent({
3140
+ eventType: "improve_runs_purged",
3141
+ ref: "improve_runs:_purge",
3142
+ metadata: { purgedCount: improveRunsPurged, retentionDays },
3143
+ }, eventsCtx);
3144
+ }
3145
+ catch (err) {
3146
+ allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
3147
+ }
3148
+ finally {
3149
+ if (ownsStateDb && stateDb) {
3150
+ try {
3151
+ stateDb.close();
3152
+ }
3153
+ catch {
3154
+ // best-effort
3155
+ }
3156
+ }
3157
+ }
3158
+ // task_logs in logs.db (#579) shares the same retention window as
3159
+ // events/improve_runs — all three are observability data governed by
3160
+ // the single improve.eventRetentionDays knob. Separate try/finally
3161
+ // because logs.db is a different file: a locked/missing logs.db must
3162
+ // not block the state.db purges above.
3163
+ let logsDb;
3164
+ try {
3165
+ logsDb = openLogsDatabase();
3166
+ const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
3167
+ if (taskLogsPurged > 0) {
3168
+ info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
3169
+ }
3170
+ appendEvent({
3171
+ eventType: "task_logs_purged",
3172
+ ref: "task_logs:_purge",
3173
+ metadata: { purgedCount: taskLogsPurged, retentionDays },
3174
+ }, eventsCtx);
3175
+ }
3176
+ catch (err) {
3177
+ allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
3178
+ }
3179
+ finally {
3180
+ if (logsDb) {
3181
+ try {
3182
+ logsDb.close();
3183
+ }
3184
+ catch {
3185
+ // best-effort
3186
+ }
2760
3187
  }
2761
3188
  }
2762
3189
  }
2763
3190
  }
2764
- }
2765
- // Phase 4A (staleness detection). Activates the `deprecated` belief-state
2766
- // machinery shipped in Phase 1A. Default OFF gated by
2767
- // `features.index.staleness_detection.enabled`. Runs after orphan purge
2768
- // and before the URL check (which lives in the outer caller).
2769
- if (sources.length > 0) {
2770
- try {
2771
- stalenessDetection = await stalenessDetectionFn({ config, sources, signal: budgetSignal, db });
2772
- if (stalenessDetection.considered > 0) {
2773
- info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
2774
- `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
2775
- `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
3191
+ // Phase 4A (staleness detection). Activates the `deprecated` belief-state
3192
+ // machinery shipped in Phase 1A. Default OFF gated by
3193
+ // `features.index.staleness_detection.enabled`. Runs after orphan purge
3194
+ // and before the URL check (which lives in the outer caller).
3195
+ if (sources.length > 0) {
3196
+ try {
3197
+ stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
3198
+ if (stalenessDetection.considered > 0) {
3199
+ info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
3200
+ `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
3201
+ `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
3202
+ }
3203
+ for (const w of stalenessDetection.warnings)
3204
+ allWarnings.push(`[improve] staleness detection: ${w}`);
3205
+ }
3206
+ catch (err) {
3207
+ allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
2776
3208
  }
2777
- for (const w of stalenessDetection.warnings)
2778
- allWarnings.push(`[improve] staleness detection: ${w}`);
2779
- }
2780
- catch (err) {
2781
- allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
2782
3209
  }
2783
3210
  }
2784
- }
2785
- finally {
2786
- if (db)
2787
- closeDatabase(db);
2788
- }
3211
+ finally {
3212
+ if (db)
3213
+ closeDatabase(db);
3214
+ }
3215
+ });
2789
3216
  return {
2790
3217
  ...(memoryInference ? { memoryInference } : {}),
2791
3218
  ...(graphExtraction ? { graphExtraction } : {}),