akm-cli 0.9.0-beta.2 → 0.9.0-beta.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CHANGELOG.md +614 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +5 -1
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/templates/html/default.html +78 -0
  16. package/dist/assets/templates/html/health.html +730 -0
  17. package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
  18. package/dist/cli/shared.js +21 -5
  19. package/dist/cli.js +47 -5
  20. package/dist/commands/agent/contribute-cli.js +16 -3
  21. package/dist/commands/feedback-cli.js +15 -6
  22. package/dist/commands/graph/graph.js +75 -71
  23. package/dist/commands/health/checks.js +48 -0
  24. package/dist/commands/health/html-report.js +790 -0
  25. package/dist/commands/health.js +478 -15
  26. package/dist/commands/improve/calibration.js +161 -0
  27. package/dist/commands/improve/consolidate.js +634 -111
  28. package/dist/commands/improve/dedup.js +482 -0
  29. package/dist/commands/improve/distill.js +145 -69
  30. package/dist/commands/improve/encoding-salience.js +205 -0
  31. package/dist/commands/improve/extract-cli.js +115 -1
  32. package/dist/commands/improve/extract-prompt.js +33 -2
  33. package/dist/commands/improve/extract-watch.js +140 -0
  34. package/dist/commands/improve/extract.js +280 -35
  35. package/dist/commands/improve/feedback-valence.js +54 -0
  36. package/dist/commands/improve/homeostatic.js +467 -0
  37. package/dist/commands/improve/improve-auto-accept.js +139 -6
  38. package/dist/commands/improve/improve-profiles.js +12 -0
  39. package/dist/commands/improve/improve.js +1851 -515
  40. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  41. package/dist/commands/improve/outcome-loop.js +256 -0
  42. package/dist/commands/improve/proactive-maintenance.js +87 -0
  43. package/dist/commands/improve/procedural.js +409 -0
  44. package/dist/commands/improve/recombine.js +488 -0
  45. package/dist/commands/improve/reflect-noise.js +0 -0
  46. package/dist/commands/improve/reflect.js +51 -1
  47. package/dist/commands/improve/related-sessions.js +120 -0
  48. package/dist/commands/improve/salience.js +386 -0
  49. package/dist/commands/improve/triage.js +95 -0
  50. package/dist/commands/lint/agent-linter.js +19 -24
  51. package/dist/commands/lint/base-linter.js +173 -60
  52. package/dist/commands/lint/command-linter.js +19 -24
  53. package/dist/commands/lint/env-key-rules.js +34 -1
  54. package/dist/commands/lint/index.js +30 -13
  55. package/dist/commands/lint/memory-linter.js +1 -1
  56. package/dist/commands/lint/registry.js +5 -2
  57. package/dist/commands/lint/task-linter.js +3 -3
  58. package/dist/commands/lint/workflow-linter.js +26 -1
  59. package/dist/commands/proposal/drain.js +73 -6
  60. package/dist/commands/proposal/proposal-cli.js +22 -10
  61. package/dist/commands/proposal/proposal.js +17 -1
  62. package/dist/commands/proposal/validators/proposals.js +369 -329
  63. package/dist/commands/read/curate.js +294 -79
  64. package/dist/commands/read/search-cli.js +7 -0
  65. package/dist/commands/read/search.js +1 -0
  66. package/dist/commands/remember.js +6 -2
  67. package/dist/commands/sources/installed-stashes.js +5 -1
  68. package/dist/commands/sources/stash-cli.js +10 -2
  69. package/dist/core/asset/frontmatter.js +166 -167
  70. package/dist/core/asset/markdown.js +8 -0
  71. package/dist/core/config/config-schema.js +241 -0
  72. package/dist/core/config/config.js +2 -2
  73. package/dist/core/logs-db.js +305 -0
  74. package/dist/core/paths.js +3 -0
  75. package/dist/core/state-db.js +706 -42
  76. package/dist/indexer/db/db.js +347 -38
  77. package/dist/indexer/db/graph-db.js +81 -86
  78. package/dist/indexer/ensure-index.js +152 -17
  79. package/dist/indexer/graph/graph-boost.js +51 -41
  80. package/dist/indexer/index-writer-lock.js +99 -0
  81. package/dist/indexer/indexer.js +114 -111
  82. package/dist/indexer/passes/memory-inference.js +71 -25
  83. package/dist/indexer/passes/staleness-detect.js +2 -5
  84. package/dist/indexer/search/db-search.js +15 -4
  85. package/dist/indexer/search/ranking.js +4 -0
  86. package/dist/integrations/harnesses/claude/session-log.js +27 -5
  87. package/dist/integrations/harnesses/opencode/session-log.js +9 -0
  88. package/dist/integrations/session-logs/index.js +16 -0
  89. package/dist/llm/client.js +38 -4
  90. package/dist/llm/embedder.js +27 -3
  91. package/dist/llm/embedders/local.js +66 -2
  92. package/dist/llm/graph-extract.js +2 -1
  93. package/dist/llm/memory-infer.js +4 -8
  94. package/dist/llm/metadata-enhance.js +9 -1
  95. package/dist/llm/usage-persist.js +77 -0
  96. package/dist/llm/usage-telemetry.js +103 -0
  97. package/dist/output/context.js +3 -2
  98. package/dist/output/html-render.js +73 -0
  99. package/dist/output/shapes/curate.js +14 -2
  100. package/dist/output/shapes/helpers.js +17 -1
  101. package/dist/output/text/helpers.js +78 -1
  102. package/dist/runtime.js +25 -1
  103. package/dist/scripts/migrate-storage.js +1194 -607
  104. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +455 -270
  105. package/dist/sources/providers/tar-utils.js +16 -8
  106. package/dist/storage/sqlite-pragmas.js +146 -0
  107. package/dist/tasks/runner.js +99 -16
  108. package/dist/workflows/db.js +5 -2
  109. package/dist/workflows/validate-summary.js +2 -7
  110. package/docs/data-and-telemetry.md +1 -0
  111. package/package.json +7 -5
@@ -12,21 +12,25 @@ import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } f
12
12
  import { appendEvent, readEvents } from "../../core/events.js";
13
13
  import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../../core/file-lock.js";
14
14
  import { classifyImproveAction } from "../../core/improve-types.js";
15
+ import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
15
16
  import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
16
- import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
17
+ import { listProposalGateDecisions, listStateProposals, openStateDatabase, persistPhaseThreshold, purgeOldEvents, purgeOldImproveRuns, } from "../../core/state-db.js";
17
18
  import { info, warn } from "../../core/warn.js";
18
19
  import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
19
20
  import { ensureIndex } from "../../indexer/ensure-index.js";
20
21
  import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
22
+ import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
21
23
  import { akmIndex } from "../../indexer/indexer.js";
22
- import { runMemoryInferencePass } from "../../indexer/passes/memory-inference.js";
24
+ import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
23
25
  import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
24
26
  import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
25
27
  import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
26
28
  import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
27
29
  import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
28
30
  import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
29
- import { isLlmFeatureEnabled, isProcessEnabled } from "../../llm/feature-gate.js";
31
+ import { isProcessEnabled } from "../../llm/feature-gate.js";
32
+ import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
33
+ import { withLlmStage } from "../../llm/usage-telemetry.js";
30
34
  import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
31
35
  import { akmLint } from "../lint/index.js";
32
36
  import { drainProposals } from "../proposal/drain.js";
@@ -34,16 +38,120 @@ import { resolveDrainPolicy } from "../proposal/drain-policies.js";
34
38
  import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
35
39
  import { runSchemaRepairPass } from "../sources/schema-repair.js";
36
40
  import { checkDeadUrls } from "../url-checker.js";
41
+ import { computeThresholdAutoTune, gateDecisionsToSamples, summarizeCalibration, } from "./calibration.js";
37
42
  import { akmConsolidate } from "./consolidate.js";
38
43
  import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
39
44
  import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
40
45
  import { countEvalCases, writeEvalCase } from "./eval-cases.js";
41
46
  import { akmExtract, countNewExtractCandidates } from "./extract.js";
47
+ import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
42
48
  import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
43
49
  import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
44
50
  import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
45
51
  import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
52
+ import { computeProxyAdequacy, getAllAssetOutcomes, getOutcomeScoresByRef, outcomeScoreToSalience, updateAssetOutcome, } from "./outcome-loop.js";
53
+ import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
54
+ import { akmProcedural } from "./procedural.js";
55
+ import { akmRecombine } from "./recombine.js";
46
56
  import { akmReflect } from "./reflect.js";
57
+ import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getConsecutiveNoOps, getLastUseMsByRef, recordNoOp, resetConsecutiveNoOps, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
58
+ // #607 Lock Decomposition: fine-grained per-process locks replace the single
59
+ // `improve.lock`. Three independent locks allow concurrent improve runs when
60
+ // they touch different subsystems (e.g. quick-shredder consolidate can run
61
+ // alongside daily reflect+distill).
62
+ //
63
+ // consolidate.lock — protects consolidate + memoryInference (both write index.db)
64
+ // reflect-distill.lock — protects reflect + distill (both write state.db proposals)
65
+ // triage.lock — protects triage (writes proposal promotions)
66
+ //
67
+ // Stale timeouts are per-lock, tuned to the expected runtime of the protected
68
+ // processes: consolidate is disk-bound (1h), reflect+distill is GPU-bound (2h),
69
+ // triage is fast (30min).
70
+ const PROCESS_LOCK_DEFS = {
71
+ consolidate: { fileName: "consolidate.lock", staleAfterMs: 60 * 60 * 1000 },
72
+ reflectDistill: { fileName: "reflect-distill.lock", staleAfterMs: 2 * 60 * 60 * 1000 },
73
+ triage: { fileName: "triage.lock", staleAfterMs: 30 * 60 * 1000 },
74
+ };
75
+ const heldProcessLocks = new Set();
76
+ export function resetHeldProcessLocks() {
77
+ heldProcessLocks.clear();
78
+ }
79
+ function processLockPath(lockBaseDir, lockName) {
80
+ return path.join(lockBaseDir, PROCESS_LOCK_DEFS[lockName].fileName);
81
+ }
82
+ function tryAcquireProcessLock(lockPath, staleAfterMs, skipIfLocked, lockLabel) {
83
+ fs.mkdirSync(path.dirname(lockPath), { recursive: true });
84
+ const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
85
+ if (tryAcquireLockSync(lockPath, lockPayload())) {
86
+ heldProcessLocks.add(lockPath);
87
+ return "acquired";
88
+ }
89
+ const probe = probeLock(lockPath, { staleAfterMs });
90
+ const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
91
+ const lock = rawContent
92
+ ? (() => {
93
+ try {
94
+ return JSON.parse(rawContent);
95
+ }
96
+ catch {
97
+ return null;
98
+ }
99
+ })()
100
+ : null;
101
+ if (probe.state === "stale") {
102
+ try {
103
+ appendEvent({
104
+ eventType: "improve_lock_recovered",
105
+ metadata: {
106
+ lockName: lockLabel,
107
+ stalePid: lock?.pid ?? null,
108
+ lockedAt: lock?.startedAt ?? null,
109
+ recoveredAt: new Date().toISOString(),
110
+ lockAgeMs: probe.ageMs ?? null,
111
+ reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
112
+ },
113
+ });
114
+ }
115
+ catch {
116
+ /* event emission is best-effort; never block lock recovery */
117
+ }
118
+ releaseLock(lockPath);
119
+ if (tryAcquireLockSync(lockPath, lockPayload())) {
120
+ heldProcessLocks.add(lockPath);
121
+ return "acquired";
122
+ }
123
+ if (skipIfLocked) {
124
+ warn(`[improve] ${lockLabel} lock acquired by another run during stale recovery; skipping (--skip-if-locked)`);
125
+ return "skipped";
126
+ }
127
+ throw new ConfigError(`akm improve ${lockLabel} is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
128
+ }
129
+ if (skipIfLocked) {
130
+ warn(`[improve] ${lockLabel} lock held by another run (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
131
+ return "skipped";
132
+ }
133
+ throw new ConfigError(`akm improve ${lockLabel} is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
134
+ }
135
+ function releaseProcessLock(lockPath) {
136
+ try {
137
+ fs.unlinkSync(lockPath);
138
+ }
139
+ catch {
140
+ // ignore
141
+ }
142
+ heldProcessLocks.delete(lockPath);
143
+ }
144
+ function releaseAllProcessLocks() {
145
+ for (const p of heldProcessLocks) {
146
+ try {
147
+ fs.unlinkSync(p);
148
+ }
149
+ catch {
150
+ // ignore
151
+ }
152
+ }
153
+ heldProcessLocks.clear();
154
+ }
47
155
  function resolveImproveScope(scope) {
48
156
  const trimmed = scope?.trim();
49
157
  if (!trimmed)
@@ -99,6 +207,22 @@ export function renderSyncCommitMessage(template, result, nowMs) {
99
207
  };
100
208
  return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
101
209
  }
210
+ /**
211
+ * Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
212
+ * merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
213
+ * Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
214
+ */
215
+ function dedupeRefs(refs) {
216
+ const seen = new Set();
217
+ const out = [];
218
+ for (const r of refs) {
219
+ if (seen.has(r.ref))
220
+ continue;
221
+ seen.add(r.ref);
222
+ out.push(r);
223
+ }
224
+ return out;
225
+ }
102
226
  async function collectEligibleRefs(scope, stashDir, improveProfile) {
103
227
  if (scope.mode === "ref" && scope.value) {
104
228
  const parsed = parseAssetRef(scope.value);
@@ -112,7 +236,7 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
112
236
  };
113
237
  }
114
238
  return {
115
- plannedRefs: [{ ref: scope.value, reason: "scope-ref" }],
239
+ plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
116
240
  memorySummary: {
117
241
  eligible: parsed.type === "memory" ? 1 : 0,
118
242
  derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
@@ -176,12 +300,14 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
176
300
  profileFiltered.set(ref, {
177
301
  ref,
178
302
  reason: "profile_filtered_all_passes",
303
+ filePath: indexed.filePath,
179
304
  });
180
305
  }
181
306
  else {
182
307
  planned.set(ref, {
183
308
  ref,
184
309
  reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
310
+ filePath: indexed.filePath,
185
311
  });
186
312
  }
187
313
  }
@@ -449,6 +575,107 @@ export function armBudgetWatchdog(budgetMs, controller, deps) {
449
575
  }
450
576
  };
451
577
  }
578
+ /**
579
+ * #612 / WS-4 — bounded, opt-in per-phase auto-accept threshold auto-tune.
580
+ *
581
+ * Reads `improve.calibration` from config. When `autoTune` is enabled, computes
582
+ * the calibration of recent gate decisions, derives a bounded threshold
583
+ * adjustment (clamped into the configured band, capped per step), logs it, and
584
+ * records a `calibration_autotune` event. Returns the new threshold (integer
585
+ * 0-100) when an adjustment was made, or `undefined` to leave the caller's
586
+ * threshold unchanged.
587
+ *
588
+ * WS-4 change: accepts an optional `phase` parameter. When provided, the tuned
589
+ * threshold is persisted to `improve_gate_thresholds` (state.db Migration 012)
590
+ * keyed by phase so `makeGateConfig` can read it back on the next run and each
591
+ * phase maintains its own calibrated threshold rather than a shared global.
592
+ *
593
+ * WS-4 ceiling: `maxThreshold` defaults to 85 (not 100) to prevent the gate
594
+ * converging to pure exploitation and shutting down Gap-3/4 novelty throughput.
595
+ *
596
+ * DEFAULT OFF: with no `improve.calibration` block (or `autoTune: false`) this
597
+ * returns `undefined` immediately, so the gate threshold is unchanged and
598
+ * behaviour is byte-identical to today.
599
+ */
600
+ export function maybeAutoTuneThreshold(currentThreshold, config, stateDbPath, ctx, phase) {
601
+ const cal = config.improve?.calibration;
602
+ if (!cal?.autoTune)
603
+ return undefined;
604
+ // WS-4: default maxThreshold is now 85 (ceiling to prevent pure exploitation).
605
+ // Callers that explicitly set maxThreshold in config override this default.
606
+ const tuneConfig = {
607
+ autoTune: true,
608
+ minThreshold: cal.minThreshold ?? 0,
609
+ maxThreshold: cal.maxThreshold ?? 85,
610
+ maxStep: cal.maxStep ?? 5,
611
+ minSamples: cal.minSamples ?? 20,
612
+ targetAcceptRate: cal.targetAcceptRate ?? 0.9,
613
+ };
614
+ // Defensive: an inverted band disables tuning rather than clamping to nonsense.
615
+ if (tuneConfig.minThreshold > tuneConfig.maxThreshold)
616
+ return undefined;
617
+ const db = openStateDatabase(stateDbPath);
618
+ let summary;
619
+ try {
620
+ const allDecisions = listProposalGateDecisions(db);
621
+ // WS-4 fix: when called with a phase label, restrict calibration to that
622
+ // phase's decision pool so a reflect-dominated run cannot tighten the
623
+ // consolidate gate (or vice-versa). The gate field is `improve:<phase>`,
624
+ // matching what improve-auto-accept.ts stamps at line ~163.
625
+ const gateLabel = phase ? `improve:${phase}` : undefined;
626
+ const decisions = gateLabel ? allDecisions.filter((d) => d.gate === gateLabel) : allDecisions;
627
+ summary = summarizeCalibration(gateDecisionsToSamples(decisions));
628
+ }
629
+ finally {
630
+ db.close();
631
+ }
632
+ const result = computeThresholdAutoTune(currentThreshold, summary, tuneConfig);
633
+ if (!result.adjusted)
634
+ return undefined;
635
+ const appendEventFn = ctx?.appendEventFn ?? appendEvent;
636
+ const phaseLabel = phase ?? "global";
637
+ info(`[improve] calibration auto-tune (${phaseLabel}): threshold ${result.previousThreshold} -> ${result.newThreshold} ` +
638
+ `(${result.reason}; samples=${summary.samples}, acceptRate=${summary.overallAcceptRate}, ` +
639
+ `gap=${summary.calibrationGap}, band=[${tuneConfig.minThreshold},${tuneConfig.maxThreshold}])`);
640
+ try {
641
+ appendEventFn({
642
+ eventType: "calibration_autotune",
643
+ ref: "improve:calibration",
644
+ metadata: {
645
+ phase: phaseLabel,
646
+ previousThreshold: result.previousThreshold,
647
+ newThreshold: result.newThreshold,
648
+ delta: result.delta,
649
+ reason: result.reason,
650
+ samples: summary.samples,
651
+ overallAcceptRate: summary.overallAcceptRate,
652
+ calibrationGap: summary.calibrationGap,
653
+ minThreshold: tuneConfig.minThreshold,
654
+ maxThreshold: tuneConfig.maxThreshold,
655
+ },
656
+ });
657
+ }
658
+ catch (err) {
659
+ warn(`[improve] calibration auto-tune event not recorded: ${err instanceof Error ? err.message : String(err)}`);
660
+ }
661
+ // WS-4: Persist the per-phase threshold so makeGateConfig reads it on the
662
+ // next run. Best-effort — a write failure must not abort the improve run.
663
+ if (phase) {
664
+ try {
665
+ const persistDb = openStateDatabase(stateDbPath);
666
+ try {
667
+ persistPhaseThreshold(persistDb, phase, result.newThreshold);
668
+ }
669
+ finally {
670
+ persistDb.close();
671
+ }
672
+ }
673
+ catch (err) {
674
+ warn(`[improve] calibration auto-tune: failed to persist phase threshold for ${phase}: ${err instanceof Error ? err.message : String(err)}`);
675
+ }
676
+ }
677
+ return result.newThreshold;
678
+ }
452
679
  export async function akmImprove(options = {}) {
453
680
  const scope = resolveImproveScope(options.scope);
454
681
  const reflectFn = options.reflectFn ?? akmReflect;
@@ -466,7 +693,9 @@ export async function akmImprove(options = {}) {
466
693
  options = {
467
694
  ...options,
468
695
  autoAccept: options.autoAccept ?? improveProfile.autoAccept,
469
- limit: options.limit ?? improveProfile.limit,
696
+ // Profile-level limit, then process-level reflect.limit as fallback.
697
+ // CLI --limit takes precedence over both.
698
+ limit: options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit,
470
699
  };
471
700
  let primaryStashDir;
472
701
  try {
@@ -484,103 +713,33 @@ export async function akmImprove(options = {}) {
484
713
  // timeout root cause). Because beforeEach runs synchronously, env is still the
485
714
  // calling test's own at this point; we capture it before yielding the loop.
486
715
  const resolvedStateDbPath = getStateDbPathInDataDir();
487
- // Phase 4 lock hoist (§7): the `improve.lock` setup is hoisted ABOVE
488
- // ensureIndex/collectEligibleRefs so the triage pre-pass (and improve's own
489
- // queue writes) run fully serialized under the lock. The dry-run early-return
490
- // below still skips the lock and triage (the lock+triage block is gated on
491
- // `!options.dryRun`); contradiction-detection and memory-cleanup analysis,
492
- // which previously ran before the lock, now sit after it for free.
493
- const resolvedLockPath = primaryStashDir
494
- ? path.join(primaryStashDir, ".akm", "improve.lock")
495
- : path.join(options.stashDir ?? ".", ".akm", "improve.lock");
496
- const MAX_LOCK_AGE_MS = 4 * 60 * 60 * 1000; // 4 hours
497
- const acquireLock = () => {
498
- fs.mkdirSync(path.dirname(resolvedLockPath), { recursive: true });
499
- const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
500
- if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
501
- return "acquired";
502
- // Lock file already exists probe to determine whether it's still held
503
- // or whether the prior run died without cleaning up.
504
- const probe = probeLock(resolvedLockPath, { staleAfterMs: MAX_LOCK_AGE_MS });
505
- const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
506
- const lock = rawContent
507
- ? (() => {
508
- try {
509
- return JSON.parse(rawContent);
510
- }
511
- catch {
512
- return null;
513
- }
514
- })()
515
- : null;
516
- if (probe.state === "stale") {
517
- // O-7 / #394: Emit improve_lock_recovered event before recovery so the
518
- // audit trail records the abnormal prior-run exit (Temporal/Airflow pattern).
519
- try {
520
- appendEvent({
521
- eventType: "improve_lock_recovered",
522
- metadata: {
523
- stalePid: lock?.pid ?? null,
524
- lockedAt: lock?.startedAt ?? null,
525
- recoveredAt: new Date().toISOString(),
526
- lockAgeMs: probe.ageMs ?? null,
527
- reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
528
- },
529
- });
530
- }
531
- catch {
532
- /* event emission is best-effort; never block lock recovery */
533
- }
534
- releaseLock(resolvedLockPath);
535
- if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
536
- return "acquired";
537
- // Lost the race to another run that grabbed the freed stale lock.
538
- if (options.skipIfLocked) {
539
- warn("[improve] another run acquired the lock during stale recovery; skipping (--skip-if-locked)");
540
- return "skipped";
541
- }
542
- throw new ConfigError(`akm improve is already running. Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
543
- }
544
- // Lock is held by a live run within the staleness window.
545
- if (options.skipIfLocked) {
546
- warn(`[improve] another improve run holds the lock (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
547
- return "skipped";
548
- }
549
- throw new ConfigError(`akm improve is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
550
- };
551
- // Phase 4 lock-leak guard (§7 ordering hazard): hoisting `improve.lock` above
552
- // the pre-index region (so the triage pre-pass runs under it) means the lock is
553
- // held while ensureIndex / collectEligibleRefs / contradiction-detection /
554
- // memory-cleanup analysis run — but the main protecting `try { … } finally {
555
- // unlinkSync(resolvedLockPath) }` does not begin until after them. A throw in
556
- // any of those steps would leak the lock. We close that window by wrapping the
557
- // whole region in a try whose catch releases the lock (when held) and
558
- // re-throws. The values this region computes are declared in the outer scope so
559
- // they remain visible to the main run below. The dry-run path never sets
560
- // `lockAcquired`, so its early return releases nothing.
561
- let lockAcquired = false;
562
- const releaseLockOnError = () => {
563
- if (!lockAcquired)
564
- return;
565
- try {
566
- fs.unlinkSync(resolvedLockPath);
567
- }
568
- catch {
569
- // best-effort release on the error path
570
- }
571
- lockAcquired = false;
572
- };
573
- // Signal-safe lock release. The SIGTERM/SIGINT/SIGHUP handler in improve-cli.ts
574
- // calls `process.exit()`, which does NOT run the `finally` below that owns lock
575
- // release — so a cron-timeout SIGTERM leaked `improve.lock` every run.
576
- // `process.exit()` DOES fire `'exit'` listeners, so we release the lock from
577
- // one. `releaseLockIfOwned` only unlinks a lock still owned by this PID, so it
578
- // is safe even if a later run re-acquired it. The listener is removed in the
579
- // `finally` so the normal path stays single-release and repeated in-process
580
- // `akmImprove` calls (tests) do not accumulate listeners.
581
- const releaseLockOnExit = () => {
582
- releaseLockIfOwned(resolvedLockPath, process.pid);
583
- };
716
+ // #612 / WS-4 bounded, OPT-IN per-phase auto-accept threshold auto-tune.
717
+ // DEFAULT OFF: `autoTune: false` (or absent) is a complete no-op.
718
+ //
719
+ // WS-4 change: thresholds are now PER PHASE. The old single global mutation
720
+ // of `options.autoAccept` is retired it caused every phase to share one
721
+ // calibration signal, so a reflect-dominated run could tighten the consolidate
722
+ // gate (or vice-versa). Instead:
723
+ // - Each `makeGateConfig` call reads the phase's stored threshold from
724
+ // state.db (Migration 012) and uses it as `phaseThreshold`, overriding
725
+ // the `globalThreshold` (= options.autoAccept) for that phase.
726
+ // - Per-phase `maybeAutoTuneThreshold` calls fire AFTER each phase's gate
727
+ // has run and persist the new threshold to state.db for the NEXT run.
728
+ // - `options.autoAccept` stays unchanged (it is the operator-supplied
729
+ // baseline, not a mutable run-time state).
730
+ //
731
+ // The global tune call is intentionally removed here. See per-phase calls
732
+ // below (near each makeGateConfig / runAutoAcceptGate block).
733
+ // #607 Lock decomposition: three per-process locks replace the single
734
+ // `improve.lock`. Each process acquires only the lock(s) it needs, so
735
+ // quick-shredder consolidate can run alongside daily reflect+distill.
736
+ //
737
+ // consolidate.lock — protects consolidate + memoryInference + graphExtraction (index.db writers)
738
+ // reflect-distill.lock — protects reflect + distill (state.db proposal writers)
739
+ // triage.lock — protects triage pre-pass (state.db proposal promotions)
740
+ //
741
+ // Lock base directory — same `.akm/` under the primary stash dir.
742
+ const lockBaseDir = primaryStashDir ? path.join(primaryStashDir, ".akm") : path.join(options.stashDir ?? ".", ".akm");
584
743
  const preEnsureCleanupWarnings = [];
585
744
  let plannedRefs;
586
745
  let memorySummary;
@@ -589,65 +748,59 @@ export async function akmImprove(options = {}) {
589
748
  let guidance;
590
749
  let triageDrain;
591
750
  try {
592
- // Acquire the lock and run the triage pre-pass for non-dry-run executions.
593
- // The dry-run branch below produces plannedRefs/memorySummary WITHOUT the lock
594
- // or triage (decision: dry-run never mutates the queue).
751
+ // #607: Per-process lock acquisition. Each process acquires only the lock(s)
752
+ // it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
753
+ // locks (decision: dry-run never mutates the queue).
595
754
  if (!options.dryRun) {
596
- if (acquireLock() === "skipped") {
597
- // Another improve holds the lock and the caller asked to skip rather
598
- // than fail. Return a clean no-op result (exit 0) before any index/DB
599
- // work — never registered the exit listener, never set lockAcquired,
600
- // so we release nothing belonging to the run that owns the lock.
601
- return {
602
- schemaVersion: 1,
603
- ok: true,
604
- scope,
605
- dryRun: false,
606
- skipped: { reason: "lock-held" },
607
- memorySummary: { eligible: 0, derived: 0 },
608
- plannedRefs: [],
609
- };
610
- }
611
- lockAcquired = true;
612
755
  // Backstop release on process.exit() (signal handler / budget watchdog),
613
756
  // which skips the finally below. Removed in that finally on the normal path.
614
- process.on("exit", releaseLockOnExit);
615
- // Phase 4 triage pre-pass (§7, §13): drain the standing pending backlog
616
- // BEFORE ensureIndex so improve generates fresh proposals against a cleared
617
- // queue (no `duplicate_pending` collisions) and ensureIndex absorbs triage's
618
- // promotions for free. Gated on the triage process being enabled (opt-in,
619
- // defaults off) and on a whole-stash / type-scoped run — a single-ref
620
- // `akm improve skill:x` must never drain the whole queue. Best-effort: a
621
- // triage failure is a non-fatal warning, never an abort (mirrors the
622
- // contradiction-detection pass below).
757
+ const releaseAllOnExit = () => {
758
+ for (const p of heldProcessLocks) {
759
+ releaseLockIfOwned(p, process.pid);
760
+ }
761
+ };
762
+ process.on("exit", releaseAllOnExit);
763
+ // #607 triage pre-pass: acquire triage.lock, drain the standing pending
764
+ // backlog BEFORE ensureIndex so improve generates fresh proposals against
765
+ // a cleared queue (no `duplicate_pending` collisions) and ensureIndex
766
+ // absorbs triage's promotions for free. Release immediately after —
767
+ // triage.lock is not needed again until the next improve run.
623
768
  if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
624
769
  if (scope.mode === "ref") {
625
770
  warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
626
771
  }
627
772
  else {
628
- try {
629
- const triageConfig = improveProfile.processes?.triage;
630
- const policy = resolveDrainPolicy(triageConfig?.policy);
631
- const applyMode = triageConfig?.applyMode ?? "queue";
632
- const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
633
- const judgment = triageConfig?.judgment
634
- ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
635
- : null;
636
- triageDrain = await drainProposalsFn({
637
- stashDir: primaryStashDir,
638
- policy,
639
- applyMode,
640
- maxAccepts,
641
- dryRun: false,
642
- // No fresh ids exist yet — triage runs before improve generates any.
643
- excludeIds: new Set(),
644
- ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
645
- judgment,
646
- });
773
+ const triageLPath = processLockPath(lockBaseDir, "triage");
774
+ const triageResult = tryAcquireProcessLock(triageLPath, PROCESS_LOCK_DEFS.triage.staleAfterMs, options.skipIfLocked, "triage");
775
+ if (triageResult === "skipped") {
776
+ triageDrain = undefined;
647
777
  }
648
- catch (err) {
649
- // Non-fatal: triage is a best-effort pre-pass and must never abort improve.
650
- warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
778
+ else {
779
+ try {
780
+ const triageConfig = improveProfile.processes?.triage;
781
+ const policy = resolveDrainPolicy(triageConfig?.policy);
782
+ const applyMode = triageConfig?.applyMode ?? "queue";
783
+ const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
784
+ const judgment = triageConfig?.judgment
785
+ ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
786
+ : null;
787
+ triageDrain = await drainProposalsFn({
788
+ stashDir: primaryStashDir,
789
+ policy,
790
+ applyMode,
791
+ maxAccepts,
792
+ dryRun: false,
793
+ excludeIds: new Set(),
794
+ ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
795
+ judgment,
796
+ });
797
+ }
798
+ catch (err) {
799
+ warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
800
+ }
801
+ finally {
802
+ releaseProcessLock(triageLPath);
803
+ }
651
804
  }
652
805
  }
653
806
  }
@@ -679,7 +832,7 @@ export async function akmImprove(options = {}) {
679
832
  // best-effort; leave preEnsureEntryCount undefined
680
833
  }
681
834
  try {
682
- await ensureIndexFn(primaryStashDir);
835
+ await ensureIndexFn(primaryStashDir, { mode: "blocking" });
683
836
  }
684
837
  catch (err) {
685
838
  preEnsureCleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -717,7 +870,7 @@ export async function akmImprove(options = {}) {
717
870
  if (primaryStashDir && shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)) {
718
871
  try {
719
872
  // Reuse the config resolved at the top of the run instead of a second load.
720
- await detectAndWriteContradictions(primaryStashDir, _earlyConfig);
873
+ await withLlmStage("memory-contradiction", () => detectAndWriteContradictions(primaryStashDir, _earlyConfig));
721
874
  }
722
875
  catch (err) {
723
876
  // Non-fatal: contradiction detection is a best-effort pass.
@@ -747,17 +900,14 @@ export async function akmImprove(options = {}) {
747
900
  }
748
901
  }
749
902
  catch (err) {
750
- releaseLockOnError();
903
+ releaseAllProcessLocks();
751
904
  throw err;
752
905
  }
753
- // FIX 2 (lock-leak window): everything from here on runs UNDER the lock that
754
- // `acquireLock()` just took. The single `try { } finally { unlinkSync(lock) }`
755
- // below now spans the budget-timer setup, `openStateDatabase()`, and the
756
- // `profileFilteredRefs` audit-event loop too regions that previously sat in
757
- // the gap between the lock-acquire catch (above) and the main try. A throw in
758
- // any of them used to leak the lock (blocking the next improve up to 4h);
759
- // now the finally releases it exactly once. The dry-run path already returned
760
- // above without acquiring the lock, so it never reaches this finally; the
906
+ // #607: per-process locks are acquired/released around each stage below.
907
+ // The triage pre-pass already ran under triage.lock (released). The
908
+ // preparation stage runs under consolidate.lock, the loop stage under
909
+ // reflect-distill.lock, and the post-loop stage under consolidate.lock again.
910
+ // Each stage acquires its lock just before starting and releases in finally.
761
911
  // best-effort `unlinkSync` is a no-op when no lock file exists.
762
912
  const startMs = Date.now();
763
913
  const budgetMs = options.timeoutMs ?? 2 * 60 * 60 * 1000; // default 2 hours
@@ -766,6 +916,16 @@ export async function akmImprove(options = {}) {
766
916
  // run past the declared budget.
767
917
  // References: Anthropic *Building Effective Agents* (2024); CoALA §5 (arXiv:2309.02427).
768
918
  const budgetAbortController = new AbortController();
919
+ // Attach a live `remainingBudgetMs` getter to the signal so sub-callers
920
+ // (e.g. consolidate.ts cold-start budget estimation) can read the remaining
921
+ // wall-clock budget without needing an extra plumbing parameter. The property
922
+ // is computed at access time via a getter so it always reflects the actual
923
+ // elapsed time rather than a stale snapshot taken at arm time.
924
+ Object.defineProperty(budgetAbortController.signal, "remainingBudgetMs", {
925
+ get: () => Math.max(0, budgetMs - (Date.now() - startMs)),
926
+ enumerable: false,
927
+ configurable: true,
928
+ });
769
929
  // Declared in the outer scope so the `finally` can clear the timer even if a
770
930
  // throw occurs before/after it is armed. Defaults to a no-op until armed.
771
931
  let clearBudgetTimer = () => { };
@@ -777,6 +937,9 @@ export async function akmImprove(options = {}) {
777
937
  // Pinned to the boundary snapshot so the fallback per-call `appendEvent`
778
938
  // opens (when the long-lived handle below fails to open) never re-read env.
779
939
  let eventsCtx = { dbPath: resolvedStateDbPath };
940
+ // #576: clears the per-run LLM usage sink. Defaults to a no-op until the sink
941
+ // is installed inside the try; the `finally` always calls it.
942
+ let disposeLlmUsageSink = () => { };
780
943
  try {
781
944
  // H7 (#566): arm the budget watchdog. `armBudgetWatchdog` captures both the
782
945
  // budget timer and the hard-kill timer it schedules on exhaustion, returning
@@ -796,19 +959,32 @@ export async function akmImprove(options = {}) {
796
959
  // still pinned to the boundary-resolved path, never a live env re-read.
797
960
  eventsCtx = { dbPath: resolvedStateDbPath };
798
961
  }
799
- // 2026-05-27: emit `improve_skipped` audit events for refs the planner
962
+ // #576: persist per-call LLM usage telemetry for this run as `llm_usage`
963
+ // events, reusing the same boundary-pinned events context (and long-lived
964
+ // handle when available). Disposed in `finally` so the sink never leaks
965
+ // across runs. Wrapping is best-effort end to end — see usage-telemetry.ts.
966
+ disposeLlmUsageSink = installLlmUsagePersistence(eventsCtx);
967
+ // 2026-05-27: emit an `improve_skipped` audit event for refs the planner
800
968
  // pre-filtered (reflect AND distill both refuse them under the active
801
- // profile). One event per ref so the existing improve_skipped histogram in
802
- // `health.ts#improveSummary.skipReasons` accumulates the right count under
803
- // the new `profile_filtered_all_passes` reason code. See
804
- // `/tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md`.
805
- for (const filtered of profileFilteredRefs) {
969
+ // profile). Emitted as a single summary event (count only) rather than one
970
+ // event per ref (#592) the per-ref loop caused O(n) sequential state.db
971
+ // writes that consumed ~500 s on a 9 000-ref stash. No downstream consumer
972
+ // needs the per-ref audit trail: health's skip histogram reads the
973
+ // `profile_filtered_all_passes` counters from `improve_completed` metadata.
974
+ if (profileFilteredRefs.length > 0) {
806
975
  appendEvent({
807
976
  eventType: "improve_skipped",
808
- ref: filtered.ref,
809
- metadata: { reason: "profile_filtered_all_passes" },
977
+ ref: undefined,
978
+ metadata: {
979
+ reason: "profile_filtered_all_passes",
980
+ count: profileFilteredRefs.length,
981
+ },
810
982
  }, eventsCtx);
811
983
  }
984
+ // #607: acquire consolidate.lock for the preparation stage (consolidate,
985
+ // ensureIndex, extract all write index.db). Released immediately after.
986
+ const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
987
+ const consolidatePrepAcquired = tryAcquireProcessLock(consolidateLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
812
988
  const preparation = await runImprovePreparationStage({
813
989
  scope,
814
990
  options,
@@ -822,7 +998,10 @@ export async function akmImprove(options = {}) {
822
998
  eventsCtx,
823
999
  initialCleanupWarnings: preEnsureCleanupWarnings,
824
1000
  improveProfile,
1001
+ budgetSignal: budgetAbortController.signal,
825
1002
  });
1003
+ if (consolidatePrepAcquired)
1004
+ releaseProcessLock(consolidateLPath);
826
1005
  // D6: pre-load all proposal_rejected events from the last 30 days once,
827
1006
  // so the per-asset loop can use a Map lookup instead of N DB round trips.
828
1007
  const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
@@ -834,6 +1013,10 @@ export async function akmImprove(options = {}) {
834
1013
  rejectedProposalsByRef.set(e.ref, e);
835
1014
  }
836
1015
  }
1016
+ // #607: acquire reflect-distill.lock for the loop stage (reflect + distill
1017
+ // both write proposals to state.db). Released immediately after.
1018
+ const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
1019
+ const reflectDistillAcquired = tryAcquireProcessLock(reflectDistillLPath, PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs, options.skipIfLocked, "reflect-distill") === "acquired";
837
1020
  const { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount: loopGateCount, gateAutoAcceptFailedCount: loopGateFailedCount, } = await runImproveLoopStage({
838
1021
  scope,
839
1022
  options,
@@ -853,10 +1036,16 @@ export async function akmImprove(options = {}) {
853
1036
  eventsCtx,
854
1037
  improveProfile,
855
1038
  });
1039
+ if (reflectDistillAcquired)
1040
+ releaseProcessLock(reflectDistillLPath);
856
1041
  // #551: consolidation now runs in the preparation stage (before extract);
857
1042
  // its result and run-flag are read from `preparation`, not the post-loop.
858
1043
  const consolidation = preparation.consolidation;
859
- const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
1044
+ // #607: acquire consolidate.lock for the post-loop stage (memoryInference +
1045
+ // graphExtraction both write index.db). Released immediately after.
1046
+ const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
1047
+ const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
1048
+ const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, recombination, proceduralCompilation, } = await runImprovePostLoopStage({
860
1049
  scope,
861
1050
  options,
862
1051
  primaryStashDir,
@@ -866,11 +1055,12 @@ export async function akmImprove(options = {}) {
866
1055
  memoryRefsForInference,
867
1056
  reindexFn,
868
1057
  eventsCtx,
869
- // O-1 (#364): propagate wall-clock budget signal to post-loop maintenance.
870
1058
  budgetSignal: budgetAbortController.signal,
871
1059
  improveProfile,
872
1060
  consolidationRan: preparation.consolidationRan,
873
1061
  });
1062
+ if (consolidatePostAcquired)
1063
+ releaseProcessLock(consolidatePostLPath);
874
1064
  const finalActions = maintenanceActions && maintenanceActions.length > 0
875
1065
  ? [...preparation.actions, ...maintenanceActions]
876
1066
  : preparation.actions;
@@ -932,6 +1122,8 @@ export async function akmImprove(options = {}) {
932
1122
  ...(memoryInferenceDurationMs > 0 ? { memoryInferenceDurationMs } : {}),
933
1123
  ...(graphExtractionDurationMs > 0 ? { graphExtractionDurationMs } : {}),
934
1124
  ...(stalenessDetection ? { stalenessDetection } : {}),
1125
+ ...(recombination ? { recombination } : {}),
1126
+ ...(proceduralCompilation ? { proceduralCompilation } : {}),
935
1127
  ...(orphansPurged !== undefined ? { orphansPurged } : {}),
936
1128
  ...(proposalsExpired !== undefined && proposalsExpired > 0 ? { proposalsExpired } : {}),
937
1129
  reflectCooldownActions: finalActions.filter((a) => a.mode === "reflect-cooldown").length,
@@ -955,6 +1147,7 @@ export async function akmImprove(options = {}) {
955
1147
  },
956
1148
  }
957
1149
  : {}),
1150
+ ...(preparation.proactiveMaintenance ? { proactiveMaintenance: preparation.proactiveMaintenance } : {}),
958
1151
  ...(options.runId !== undefined ? { runId: options.runId } : {}),
959
1152
  };
960
1153
  if (!result.dryRun)
@@ -1031,18 +1224,18 @@ export async function akmImprove(options = {}) {
1031
1224
  throw err;
1032
1225
  }
1033
1226
  finally {
1227
+ // #576: clear the per-run LLM usage sink BEFORE closing `eventsDb` below, so
1228
+ // no late sink invocation can write through a closed handle.
1229
+ disposeLlmUsageSink();
1034
1230
  // O-1 (#364): Clear the budget abort timer so it does not keep the event
1035
1231
  // loop alive after the run completes.
1036
1232
  clearBudgetTimer();
1037
- try {
1038
- fs.unlinkSync(resolvedLockPath);
1039
- }
1040
- catch {
1041
- // ignore
1042
- }
1043
- // The normal path released the lock above; drop the process.exit backstop so
1044
- // it does not fire later (or accumulate across repeated in-process calls).
1045
- process.removeListener("exit", releaseLockOnExit);
1233
+ // #607: release any per-process locks still held (backstop for error paths;
1234
+ // the normal path already released each lock after its stage completed).
1235
+ releaseAllProcessLocks();
1236
+ // Drop the process.exit backstop so it does not fire later (or accumulate
1237
+ // across repeated in-process calls).
1238
+ process.removeAllListeners("exit");
1046
1239
  // I1: close the long-lived state.db connection opened at the top of the run.
1047
1240
  try {
1048
1241
  eventsDb?.close();
@@ -1155,6 +1348,11 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1155
1348
  memoryInferenceDurationMs: durations.memoryInferenceDurationMs,
1156
1349
  graphExtractionExtractedFiles: result.graphExtraction?.quality.extractedFiles ?? 0,
1157
1350
  graphExtractionDurationMs: durations.graphExtractionDurationMs,
1351
+ // Layer-2 proactive-maintenance coverage (0 when the process is disabled
1352
+ // or the run was ref-scoped) so a scheduled sweep's reach is trackable.
1353
+ proactiveSelected: result.proactiveMaintenance?.selected ?? 0,
1354
+ proactiveDueTotal: result.proactiveMaintenance?.dueTotal ?? 0,
1355
+ proactiveNeverReflected: result.proactiveMaintenance?.neverReflected ?? 0,
1158
1356
  // New metrics for tuning the improve loop.
1159
1357
  ...(durations.totalDurationMs !== undefined ? { durationMs: durations.totalDurationMs } : {}),
1160
1358
  ...(durations.warningCount !== undefined ? { warningCount: durations.warningCount } : {}),
@@ -1193,7 +1391,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1193
1391
  * need, so the fix is non-invasive and provably correct.
1194
1392
  */
1195
1393
  async function runConsolidationPass(args) {
1196
- const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx } = args;
1394
+ const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx, budgetSignal, runBudgetMs } = args;
1197
1395
  const baseConfig = options.config ?? loadConfig();
1198
1396
  const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
1199
1397
  const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
@@ -1337,6 +1535,7 @@ async function runConsolidationPass(args) {
1337
1535
  stashDir: primaryStashDir,
1338
1536
  config: consolidationConfig,
1339
1537
  eventsCtx,
1538
+ stateDbPath: eventsCtx?.dbPath,
1340
1539
  }, { minimumThreshold: 95 });
1341
1540
  if (consolidateDisabledByProfile) {
1342
1541
  info("[improve] consolidation skipped (disabled by improve profile)");
@@ -1357,7 +1556,7 @@ async function runConsolidationPass(args) {
1357
1556
  info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
1358
1557
  }
1359
1558
  else if (!consolidationOnCooldown) {
1360
- consolidation = await akmConsolidate({
1559
+ consolidation = await withLlmStage("consolidate", () => akmConsolidate({
1361
1560
  ...options.consolidateOptions,
1362
1561
  config: consolidationConfig,
1363
1562
  stashDir: options.stashDir,
@@ -1365,14 +1564,22 @@ async function runConsolidationPass(args) {
1365
1564
  // Tie consolidate proposals back to this improve invocation so
1366
1565
  // accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
1367
1566
  sourceRun: `consolidate-${Date.now()}`,
1368
- // Full-pool sweep: consolidation only runs on the nightly default-profile
1369
- // pass (quick/frequent disable it), so a complete re-cluster is correct and
1370
- // affordable here. Do NOT pass incrementalSince the time-window narrowing
1371
- // it triggers permanently excludes stale-but-unmerged duplicate clusters,
1372
- // starving merge recall and letting the pool grow unbounded. (The narrowing
1373
- // was a band-aid for an every-30-min consolidation cadence that the profile
1374
- // split has since eliminated.) lastConsolidateTs still gates whether we run.
1567
+ // Pass profile-configured options. incrementalSince narrows the pool to
1568
+ // recently-changed memories + graph neighbours use this for frequent
1569
+ // passes (quick-shredder). Leave absent in the nightly default profile for
1570
+ // a full-pool sweep that catches stale-but-unmerged duplicates.
1571
+ incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
1572
+ limit: improveProfile?.processes?.consolidate?.limit,
1573
+ neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
1375
1574
  maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
1575
+ // #617 — deterministic near-duplicate dedup pre-pass. DEFAULT OFF; only
1576
+ // runs when the profile explicitly sets `consolidate.dedup.enabled`.
1577
+ dedup: improveProfile?.processes?.consolidate?.dedup,
1578
+ // #581 — judged-state cache. DEFAULT OFF; only engages when the profile
1579
+ // explicitly sets `consolidate.judgedCache.enabled`. Skips memories
1580
+ // judged-unchanged since their last judge so one run sweeps the full
1581
+ // corpus instead of narrowing to a time-window slice.
1582
+ judgedCache: improveProfile?.processes?.consolidate?.judgedCache,
1376
1583
  // Honor profile.autoAccept (already merged into options.autoAccept at the
1377
1584
  // top of akmImprove). The CLI parser always supplies 90 when --auto-accept
1378
1585
  // is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
@@ -1380,7 +1587,14 @@ async function runConsolidationPass(args) {
1380
1587
  // options.consolidateOptions.autoAccept (if explicitly provided by caller)
1381
1588
  // still wins because the spread above runs first.
1382
1589
  autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
1383
- });
1590
+ // WS-3a: forward budget signal for graceful abort on timeout, and pass
1591
+ // the profile's p90 estimate for cold-start budget reduction.
1592
+ signal: budgetSignal,
1593
+ p90ChunkSecondsDefault: improveProfile?.processes?.consolidate?.p90ChunkSecondsDefault,
1594
+ // WS-5: pass total run budget so perfTelemetry.estimatedBudgetFractionUsed
1595
+ // can flag when consolidation alone exceeded the budget.
1596
+ runBudgetMs,
1597
+ }));
1384
1598
  {
1385
1599
  const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
1386
1600
  try {
@@ -1400,7 +1614,14 @@ async function runConsolidationPass(args) {
1400
1614
  appendEvent({
1401
1615
  eventType: "consolidate_completed",
1402
1616
  ref: "memory:_consolidation",
1403
- metadata: { processed: consolidation.processed, merged: consolidation.merged },
1617
+ metadata: {
1618
+ processed: consolidation.processed,
1619
+ merged: consolidation.merged,
1620
+ deleted: consolidation.deleted,
1621
+ contradicted: consolidation.contradicted,
1622
+ failedChunks: consolidation.failedChunks ?? 0,
1623
+ durationMs: consolidation.durationMs,
1624
+ },
1404
1625
  }, eventsCtx);
1405
1626
  }
1406
1627
  }
@@ -1417,10 +1638,21 @@ async function runConsolidationPass(args) {
1417
1638
  }
1418
1639
  // D9: track whether consolidation wrote any data so graph extraction can reindex if needed
1419
1640
  const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
1641
+ // WS-4: Per-phase threshold auto-tune for the consolidate phase.
1642
+ // Persists result for the NEXT run's makeGateConfig to read.
1643
+ const consolidateTuneDbPath = eventsCtx?.dbPath;
1644
+ if (options.autoAccept !== undefined && consolidateTuneDbPath) {
1645
+ try {
1646
+ maybeAutoTuneThreshold(consolidateGateCfg.phaseThreshold ?? options.autoAccept, consolidationConfig, consolidateTuneDbPath, undefined, "consolidate");
1647
+ }
1648
+ catch (err) {
1649
+ warn(`[improve] calibration auto-tune (consolidate) skipped: ${err instanceof Error ? err.message : String(err)}`);
1650
+ }
1651
+ }
1420
1652
  return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
1421
1653
  }
1422
1654
  async function runImprovePreparationStage(args) {
1423
- const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, } = args;
1655
+ const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, budgetSignal, } = args;
1424
1656
  const actions = [];
1425
1657
  const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
1426
1658
  // Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
@@ -1457,6 +1689,8 @@ async function runImprovePreparationStage(args) {
1457
1689
  memorySummary,
1458
1690
  improveProfile,
1459
1691
  eventsCtx,
1692
+ budgetSignal,
1693
+ runBudgetMs: budgetMs,
1460
1694
  });
1461
1695
  // Phase 0.4 — session-extract pass.
1462
1696
  //
@@ -1467,7 +1701,9 @@ async function runImprovePreparationStage(args) {
1467
1701
  // / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
1468
1702
  // hook with an on-demand pull pipeline.
1469
1703
  //
1470
- // Default-on; opt out via `profiles.improve.default.processes.extract.enabled: false`.
1704
+ // Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
1705
+ // (#593: the gate respects the resolved improve profile, not just the
1706
+ // hardcoded `default` profile path the legacy feature flag reads).
1471
1707
  // Each available harness gets one call with the default --since window;
1472
1708
  // already-seen sessions (tracked in state.db.extract_sessions_seen) are
1473
1709
  // skipped automatically so re-runs don't burn LLM calls on unchanged data.
@@ -1487,6 +1723,7 @@ async function runImprovePreparationStage(args) {
1487
1723
  stashDir: primaryStashDir,
1488
1724
  config: extractConfig,
1489
1725
  eventsCtx,
1726
+ stateDbPath: eventsCtx?.dbPath,
1490
1727
  });
1491
1728
  // #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
1492
1729
  // already done upstream; here we elide every akmExtract/processSession call)
@@ -1501,7 +1738,14 @@ async function runImprovePreparationStage(args) {
1501
1738
  const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
1502
1739
  const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
1503
1740
  const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
1504
- if (isLlmFeatureEnabled(extractConfig, "session_extraction")) {
1741
+ // #593/#594: the ACTIVE resolved improve profile is the single source of
1742
+ // truth for whether extract runs. (Previously this also ANDed in the legacy
1743
+ // `session_extraction` feature flag, which only reads
1744
+ // `profiles.improve.default.processes.extract.enabled`; that made the default
1745
+ // profile a global kill switch, so a non-default profile enabling extract was
1746
+ // silently overridden. The default profile is now just another profile.)
1747
+ // `akmExtract` re-checks the same active profile internally via `improveProfile`.
1748
+ if (resolveProcessEnabled("extract", improveProfile)) {
1505
1749
  const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
1506
1750
  // The guard engages only when minNewSessions > 0; 0 disables it entirely.
1507
1751
  let belowMinNewSessions = false;
@@ -1532,15 +1776,18 @@ async function runImprovePreparationStage(args) {
1532
1776
  extractResults = [];
1533
1777
  for (const h of availableHarnesses) {
1534
1778
  try {
1535
- const result = await akmExtract({
1779
+ const result = await withLlmStage("session-extraction", () => akmExtract({
1536
1780
  type: h.name,
1537
1781
  ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
1538
1782
  config: extractConfig,
1783
+ // Thread the ACTIVE profile so extract's internal gate + per-process
1784
+ // config read the running profile, not always `default`.
1785
+ improveProfile,
1539
1786
  dryRun: options.dryRun ?? false,
1540
1787
  ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1541
1788
  // C2: pin extract's skip-tracking state.db open to the boundary path.
1542
1789
  ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
1543
- });
1790
+ }));
1544
1791
  extractResults.push(result);
1545
1792
  {
1546
1793
  const gr = await runAutoAcceptGate(primaryStashDir
@@ -1631,7 +1878,13 @@ async function runImprovePreparationStage(args) {
1631
1878
  const validationFailures = [];
1632
1879
  for (const candidate of postCleanupRefs) {
1633
1880
  try {
1634
- const filePath = await findAssetFilePath(candidate.ref, options.stashDir);
1881
+ // #591: use the path pre-resolved at planning time when it is still on
1882
+ // disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
1883
+ // stash. Fall back to findAssetFilePath only for refs that bypassed
1884
+ // collectEligibleRefs' index scan or whose file moved since planning.
1885
+ const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
1886
+ ? candidate.filePath
1887
+ : await findAssetFilePath(candidate.ref, options.stashDir);
1635
1888
  if (!filePath) {
1636
1889
  validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
1637
1890
  continue;
@@ -1757,10 +2010,19 @@ async function runImprovePreparationStage(args) {
1757
2010
  // refs that fail the distill signal-delta gate).
1758
2011
  // distillOnlyRefs — reflect blocked but distill signal-delta passes
1759
2012
  // AND ref is a distill candidate.
1760
- // fullySkippedCount — neither gate passes synthetic skip action
1761
- // + improve_skipped event, excluded from sort.
2013
+ // noFeedbackPool — neither signal-delta gate passes *and* the ref has
2014
+ // no recent feedback signal at all. These are NOT
2015
+ // skipped here: they are handed to the high-retrieval
2016
+ // fallback (P0-A) below so frequently-retrieved but
2017
+ // never-rated assets can still be improved. Only refs
2018
+ // that P0-A declines are ultimately fully skipped.
2019
+ // fullySkippedCount — has stale feedback but no signal delta → genuine
2020
+ // skip (counted, aggregated event emitted post-loop),
2021
+ // excluded from sort.
1762
2022
  const eligibleRefs = [];
1763
2023
  const distillOnlyRefs = [];
2024
+ // Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
2025
+ const noFeedbackPool = [];
1764
2026
  let fullySkippedCount = 0;
1765
2027
  // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
1766
2028
  const scopeRefBypass = scope.mode === "ref";
@@ -1798,57 +2060,134 @@ async function runImprovePreparationStage(args) {
1798
2060
  // Reflect blocked but distill passes → distill-only bucket.
1799
2061
  distillOnlyRefs.push(r);
1800
2062
  }
2063
+ else if (!latestFeedbackTs.has(r.ref)) {
2064
+ // Neither signal-delta gate passes AND there is no recent feedback signal
2065
+ // at all. Rather than skip outright, defer to the high-retrieval fallback
2066
+ // (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
2067
+ // what that path is meant to rescue. Refs P0-A declines are skipped there.
2068
+ noFeedbackPool.push(r);
2069
+ }
1801
2070
  else {
1802
- // Neither gate passes fully skipped.
2071
+ // Has feedback on record but no signal delta since the last proposal —
2072
+ // genuinely fully skipped. Counted here; a single aggregated
2073
+ // improve_skipped event is emitted after the loop (mirrors
2074
+ // profile_filtered_all_passes) instead of one event per ref.
1803
2075
  fullySkippedCount++;
1804
2076
  actions.push({
1805
2077
  ref: r.ref,
1806
2078
  mode: "distill-skipped",
1807
2079
  result: { ok: true, reason: "no new signal since last proposal" },
1808
2080
  });
1809
- appendEvent({ eventType: "improve_skipped", ref: r.ref, metadata: { reason: "no_new_signal" } }, eventsCtx);
1810
2081
  }
1811
2082
  }
2083
+ // Emit ONE aggregated skip event for the fully-skipped bucket rather than one
2084
+ // improve_skipped event per ref (#592 pattern, mirrors
2085
+ // profile_filtered_all_passes above). The per-ref loop previously produced
2086
+ // ~11K state.db writes per run on a large stash, the dominant contributor to
2087
+ // 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
2088
+ // run summary; no downstream consumer needs a per-ref DB audit trail (health's
2089
+ // skip histogram reads the `no_new_signal` counter from the count field).
2090
+ if (fullySkippedCount > 0) {
2091
+ appendEvent({
2092
+ eventType: "improve_skipped",
2093
+ ref: undefined,
2094
+ metadata: {
2095
+ reason: "no_new_signal",
2096
+ count: fullySkippedCount,
2097
+ },
2098
+ }, eventsCtx);
2099
+ }
1812
2100
  // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
1813
- // Everything from here works only on (eligibleRefs ∪ distillOnlyRefs). The
1814
- // fully-skipped bucket has already been routed and emitted; we deliberately
1815
- // avoid spending DB/CPU on refs that cannot enter the loop.
2101
+ // Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
2102
+ // deferred noFeedbackPool that may be rescued by the high-retrieval fallback
2103
+ // (P0-A). The fully-skipped bucket has already been routed and its aggregated
2104
+ // event emitted; we deliberately avoid spending DB/CPU on refs that the
2105
+ // signal-delta gate rejected with feedback already on record.
1816
2106
  const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
2107
+ // Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
2108
+ // partition above could not place these in a reflect/distill bucket, but they
2109
+ // may still qualify if they have been retrieved often enough. Two disjoint
2110
+ // sources feed this set:
2111
+ // 1. noFeedbackPool — refs with no recent feedback that the partition loop
2112
+ // deliberately deferred here (otherwise they would never reach P0-A).
2113
+ // 2. processableRefs entries that turn out to carry no recent feedback
2114
+ // *signal* once feedbackSummary is computed below.
2115
+ // (1) is added here; (2) is folded in after feedbackSummary is built.
1817
2116
  // Gap 6: only surface feedback signals from the last 30 days so that
1818
2117
  // ancient one-off feedback events don't permanently lock an asset into
1819
2118
  // every improve run. Assets with only stale signals fall through to the
1820
2119
  // high-retrieval path (P0-A) or are skipped until new signals arrive.
1821
2120
  // (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
1822
2121
  // Phase 2 above for the signal-delta gate; we reuse them here.)
1823
- // Pre-compute feedback summary per ref in a single pass so we don't issue
1824
- // two readEvents({type:"feedback", ref}) per asset (one for signal filtering,
1825
- // one for ratio computation).
2122
+ // Pre-compute feedback summary per ref in a SINGLE bulk read so we don't
2123
+ // open state.db once per asset (which caused 5000+ accumulated FDs and a
2124
+ // 2-hour runaway on a 13K-asset stash). Pattern mirrors buildLatestFeedbackTsMap
2125
+ // above: one readEvents() call fetches ALL feedback events, then we aggregate
2126
+ // in-memory by ref — O(1) DB opens regardless of candidate set size.
2127
+ // Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
2128
+ // ratios are available for any noFeedbackPool ref that P0-A rescues below.
2129
+ //
2130
+ // Behavioral note: positive/negative COUNTS are all-time (same as the old
2131
+ // per-ref readEvents call which had no `since` filter); hasSignal is bounded
2132
+ // to feedbackSinceCutoff (same as the old inline `(e.ts ?? "") >= cutoff` guard).
1826
2133
  const feedbackSummary = new Map();
1827
- for (const candidate of processableRefs) {
1828
- const { events } = readEvents({ type: "feedback", ref: candidate.ref });
1829
- let hasSignal = false;
1830
- let positive = 0;
1831
- let negative = 0;
1832
- for (const e of events) {
1833
- if (!hasSignal &&
1834
- (e.ts ?? "") >= feedbackSinceCutoff &&
1835
- e.metadata !== undefined &&
1836
- (typeof e.metadata.signal === "string" || typeof e.metadata.note === "string")) {
1837
- hasSignal = true;
1838
- }
1839
- if (e.metadata?.signal === "positive")
1840
- positive++;
1841
- else if (e.metadata?.signal === "negative")
1842
- negative++;
1843
- }
1844
- feedbackSummary.set(candidate.ref, { hasSignal, positive, negative });
2134
+ {
2135
+ const feedbackCandidateSet = new Set([...processableRefs, ...noFeedbackPool].map((r) => r.ref));
2136
+ if (feedbackCandidateSet.size > 0) {
2137
+ // Fetch ALL feedback events in one query (no ref filter, no since filter =
2138
+ // single full table scan). Filtering per-ref in memory avoids N sequential
2139
+ // state.db opens the dominant FD-leak path on large stashes.
2140
+ const { events: allFeedbackEvents } = readEvents({ type: "feedback" }, eventsCtx);
2141
+ for (const e of allFeedbackEvents) {
2142
+ const ref = e.ref;
2143
+ if (!ref || !feedbackCandidateSet.has(ref))
2144
+ continue;
2145
+ const entry = feedbackSummary.get(ref) ?? { hasSignal: false, positive: 0, negative: 0 };
2146
+ const meta = e.metadata;
2147
+ // hasSignal: only count feedback events within the 30-day window.
2148
+ if (!entry.hasSignal &&
2149
+ (e.ts ?? "") >= feedbackSinceCutoff &&
2150
+ meta !== undefined &&
2151
+ (typeof meta.signal === "string" || typeof meta.note === "string")) {
2152
+ entry.hasSignal = true;
2153
+ }
2154
+ // positive/negative: all-time counts (no since filter, matching prior behaviour).
2155
+ if (meta?.signal === "positive")
2156
+ entry.positive++;
2157
+ else if (meta?.signal === "negative")
2158
+ entry.negative++;
2159
+ feedbackSummary.set(ref, entry);
2160
+ }
2161
+ // Ensure every candidate has an entry (even refs with zero feedback events).
2162
+ for (const ref of feedbackCandidateSet) {
2163
+ if (!feedbackSummary.has(ref)) {
2164
+ feedbackSummary.set(ref, { hasSignal: false, positive: 0, negative: 0 });
2165
+ }
2166
+ }
2167
+ }
1845
2168
  }
1846
2169
  const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
1847
2170
  // P0-A: also surface zero-feedback assets that have been retrieved many times.
1848
2171
  const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
1849
2172
  const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
1850
- const noFeedbackCandidates = processableRefs.filter((r) => !signalBearingSet.has(r.ref));
2173
+ // Zero-feedback candidates for P0-A: processableRefs without a recent signal,
2174
+ // plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
2175
+ // disjoint by construction, but guard against overlap defensively).
2176
+ const noFeedbackSeen = new Set();
2177
+ const noFeedbackCandidates = [];
2178
+ for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
2179
+ if (noFeedbackSeen.has(r.ref))
2180
+ continue;
2181
+ noFeedbackSeen.add(r.ref);
2182
+ noFeedbackCandidates.push(r);
2183
+ }
1851
2184
  let highRetrievalRefs = [];
2185
+ // Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
2186
+ // proactive-maintenance selector below can reuse them without a second DB pass.
2187
+ // Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
2188
+ // step 2: recency is MANDATORY — never pinned to floor).
2189
+ let retrievalCounts = new Map();
2190
+ let lastUseMsForProactive = new Map();
1852
2191
  let dbForRetrieval;
1853
2192
  try {
1854
2193
  dbForRetrieval = openExistingDatabase();
@@ -1856,15 +2195,29 @@ async function runImprovePreparationStage(args) {
1856
2195
  if (showEventCount === 0) {
1857
2196
  warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
1858
2197
  }
1859
- const retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
2198
+ // Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
2199
+ // Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
2200
+ // had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
2201
+ // to 0 regardless of actual use. Two assets of the same type — one
2202
+ // heavily-retrieved, one never-touched — would receive identical rankScores.
2203
+ // Fix (WS-1 blocker 3): union the feedback pool into the lookup.
2204
+ const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
2205
+ retrievalCounts = getRetrievalCounts(dbForRetrieval, allCandidateRefs);
2206
+ lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
1860
2207
  // High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
1861
- // ref qualifies exactly once — when retrievalCount threshold AND no
1862
- // prior reflect proposal exists for it. Once a reflect proposal is on
1863
- // record, subsequent re-eligibility requires explicit feedback (which
1864
- // flows through the normal signal-delta gate above). Tracking growth in
1865
- // retrieval count would require persisting the count in proposal
1866
- // metadata; deferred to a follow-up.
1867
- highRetrievalRefs = noFeedbackCandidates.filter((r) => (retrievalCounts.get(r.ref) ?? 0) >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref));
2208
+ // ref qualifies exactly once — when it has actually been retrieved
2209
+ // (retrievalCount 1) AND retrievalCount threshold AND no prior reflect
2210
+ // proposal exists for it. Once a reflect proposal is on record, subsequent
2211
+ // re-eligibility requires explicit feedback (which flows through the normal
2212
+ // signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
2213
+ // from rescuing genuinely never-retrieved assets — the fallback is for
2214
+ // *retrieved* assets, not silent ones. Tracking growth in retrieval count
2215
+ // would require persisting the count in proposal metadata; deferred to a
2216
+ // follow-up.
2217
+ highRetrievalRefs = noFeedbackCandidates.filter((r) => {
2218
+ const count = retrievalCounts.get(r.ref) ?? 0;
2219
+ return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
2220
+ });
1868
2221
  }
1869
2222
  catch (err) {
1870
2223
  rethrowIfTestIsolationError(err);
@@ -1874,6 +2227,129 @@ async function runImprovePreparationStage(args) {
1874
2227
  if (dbForRetrieval)
1875
2228
  closeDatabase(dbForRetrieval);
1876
2229
  }
2230
+ // ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
2231
+ // The signal-delta gate and P0-A only surface assets with fresh feedback or a
2232
+ // raw-retrieval spike. Neither revisits a stable, high-value asset on a
2233
+ // schedule, so on a quiet stash useful assets drift stale and are never
2234
+ // refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
2235
+ // and the run is whole-stash / type scope, this selector ranks the eligible
2236
+ // population by a composite maintenance priority, gates on staleness ("due"),
2237
+ // bounds to top-N, and folds the winners into the SAME candidate set the other
2238
+ // two sources feed — so they flow through the existing #580 empty-diff /
2239
+ // cosmetic suppression and additive-distill gates. It adds no new mutation
2240
+ // logic of its own. The due gate doubles as the rotation cooldown: a freshly
2241
+ // reflected asset is excluded until it ages back past `dueDays`, so successive
2242
+ // runs rotate through the due pool rather than re-selecting the same heads.
2243
+ let proactiveRefs = [];
2244
+ let proactiveMaintenanceSummary;
2245
+ const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
2246
+ if (proactiveEnabled) {
2247
+ const pmCfg = improveProfile.processes?.proactiveMaintenance;
2248
+ const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
2249
+ const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
2250
+ // Candidate population: the zero-feedback / non-signal pool — exactly the
2251
+ // assets the other two sources would NOT pick this run. Exclude any P0-A
2252
+ // rescued this run so we never double-select the same ref.
2253
+ const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
2254
+ const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
2255
+ const selection = selectProactiveMaintenanceRefs({
2256
+ candidates: pmCandidates,
2257
+ lastReflectTs: lastReflectProposalTs,
2258
+ lastDistillTs: lastDistillProposalTs,
2259
+ retrievalCounts,
2260
+ // WS-1: wire lastUseMs so the recency decay term is genuine (plan §step 2).
2261
+ lastUseMs: lastUseMsForProactive,
2262
+ sizeBytesOf: (r) => {
2263
+ const fp = r.filePath;
2264
+ if (!fp)
2265
+ return undefined;
2266
+ try {
2267
+ return fs.statSync(fp).size;
2268
+ }
2269
+ catch {
2270
+ return undefined;
2271
+ }
2272
+ },
2273
+ dueDays,
2274
+ maxPerRun,
2275
+ });
2276
+ proactiveRefs = selection.selected;
2277
+ proactiveMaintenanceSummary = {
2278
+ selected: selection.selected.length,
2279
+ dueTotal: selection.dueTotal,
2280
+ neverReflected: selection.neverReflected,
2281
+ };
2282
+ // Aggregated observability event (never per-ref — avoids the event flood the
2283
+ // Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
2284
+ appendEvent({
2285
+ eventType: "proactive_selected",
2286
+ ref: undefined,
2287
+ metadata: {
2288
+ count: selection.selected.length,
2289
+ dueTotal: selection.dueTotal,
2290
+ neverReflected: selection.neverReflected,
2291
+ },
2292
+ }, eventsCtx);
2293
+ if (selection.selected.length > 0) {
2294
+ info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
2295
+ `(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
2296
+ }
2297
+ }
2298
+ // ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
2299
+ // Zero-feedback refs whose encoding_salience (set at distill time by
2300
+ // scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
2301
+ // into the improve run even without retrieval or feedback signal. This rescues
2302
+ // newly distilled assets that the stash has not yet surfaced to users.
2303
+ //
2304
+ // Cap: at most 10% of the effective run limit so the lane cannot crowd out
2305
+ // reactive feedback. Requires state.db to have an asset_salience row — refs
2306
+ // without a row (pre-#608 assets still on the type-weight stub) are skipped.
2307
+ const highSalienceRefs = [];
2308
+ const salienceCfg = (options.config ?? loadConfig()).improve?.salience;
2309
+ const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
2310
+ const proactiveAndRetrievalSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
2311
+ {
2312
+ let dbForHighSalience;
2313
+ try {
2314
+ dbForHighSalience = openStateDatabase(eventsCtx?.dbPath);
2315
+ const effectiveLimit = options.limit ?? 10;
2316
+ const highSalienceCap = Math.max(1, Math.floor(effectiveLimit * 0.1));
2317
+ const candidates = noFeedbackCandidates.filter((r) => !proactiveAndRetrievalSet.has(r.ref));
2318
+ for (const r of candidates) {
2319
+ if (highSalienceRefs.length >= highSalienceCap)
2320
+ break;
2321
+ const row = getAssetSalience(dbForHighSalience, r.ref);
2322
+ if (row && row.encoding_salience >= salienceThreshold) {
2323
+ highSalienceRefs.push(r);
2324
+ }
2325
+ }
2326
+ }
2327
+ catch (err) {
2328
+ rethrowIfTestIsolationError(err);
2329
+ // best-effort: if DB unavailable, highSalienceRefs stays empty
2330
+ }
2331
+ finally {
2332
+ if (dbForHighSalience)
2333
+ dbForHighSalience.close();
2334
+ }
2335
+ }
2336
+ // Record an in-memory skip action for every zero-feedback ref that the
2337
+ // partition loop deferred to P0-A but P0-A then declined (retrievalCount below
2338
+ // threshold, or a prior reflect proposal already on record). These never make
2339
+ // it into mergedRefs, so without this they would silently vanish from the run
2340
+ // summary. No DB event is written here — these refs carry no signal at all, so
2341
+ // there is nothing for the skip histogram to aggregate; the action log alone
2342
+ // preserves the per-ref audit trail (mirrors the fully-skipped action above).
2343
+ const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs, ...highSalienceRefs].map((r) => r.ref));
2344
+ for (const r of noFeedbackPool) {
2345
+ if (rescuedSet.has(r.ref))
2346
+ continue;
2347
+ actions.push({
2348
+ ref: r.ref,
2349
+ mode: "distill-skipped",
2350
+ result: { ok: true, reason: "no new signal since last proposal" },
2351
+ });
2352
+ }
1877
2353
  // If the user explicitly scoped to a single ref, always act on it —
1878
2354
  // skip the signal/retrieval filter entirely. The filter exists to avoid
1879
2355
  // noisy "improve everything" runs; it should not gate an intentional
@@ -1883,29 +2359,612 @@ async function runImprovePreparationStage(args) {
1883
2359
  // or sufficient retrievals). A stash with no signals has 0 eligible refs —
1884
2360
  // usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
1885
2361
  // to bring them into the eligible pool.
1886
- const signalAndRetrievalRefs = [...signalFiltered, ...highRetrievalRefs];
1887
- const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
2362
+ // Layer-2 proactive refs join the eligible set alongside feedback-signal and
2363
+ // high-retrieval (P0-A) refs. The four sources are disjoint by construction
2364
+ // (proactive draws from noFeedbackCandidates with the P0-A picks removed, and
2365
+ // high-salience draws from the remainder), but dedupe defensively so a ref can
2366
+ // never enter the loop twice. `requireFeedbackSignal` still suppresses all
2367
+ // fallback sources for callers that want feedback-only runs.
2368
+ const signalAndRetrievalRefs = dedupeRefs([
2369
+ ...signalFiltered,
2370
+ ...highRetrievalRefs,
2371
+ ...proactiveRefs,
2372
+ ...highSalienceRefs,
2373
+ ]);
2374
+ let mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
2375
+ // ── Attribution tagging: stamp each ref with the eligibility lane that
2376
+ // selected it ──────────────────────────────────────────────────────────────
2377
+ // Every reflect/distill proposal must record WHICH lane chose its source asset
2378
+ // so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
2379
+ // (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
2380
+ // lane map here — the one place all four lanes are known — and stamp it onto
2381
+ // each ImproveEligibleRef object. Because the ref objects are shared by
2382
+ // reference across buckets, the stamp travels with the ref through the sort,
2383
+ // disk-check, and loop stages down to the reflect/distill event emit sites and
2384
+ // createProposal calls. See EligibilitySource for the lane vocabulary.
2385
+ //
2386
+ // Precedence (prefer the most specific reactive signal):
2387
+ // scope > signal-delta > high-retrieval > proactive > high-salience
2388
+ // A ref with real feedback is attributed to feedback even if it was also due
2389
+ // for proactive maintenance or had high encoding salience. We apply lanes
2390
+ // weakest-first so the strongest overwrites; the explicit --scope <ref> bypass
2391
+ // wins outright (user intent).
2392
+ const eligibilitySourceByRef = new Map();
2393
+ for (const r of highSalienceRefs)
2394
+ eligibilitySourceByRef.set(r.ref, "high-salience");
2395
+ for (const r of proactiveRefs)
2396
+ eligibilitySourceByRef.set(r.ref, "proactive");
2397
+ for (const r of highRetrievalRefs)
2398
+ eligibilitySourceByRef.set(r.ref, "high-retrieval");
2399
+ for (const r of signalFiltered)
2400
+ eligibilitySourceByRef.set(r.ref, "signal-delta");
2401
+ if (scope.mode === "ref") {
2402
+ // O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
2403
+ // arrived via the scopeRefBypass branch, so attribute the whole set to scope.
2404
+ for (const r of processableRefs)
2405
+ eligibilitySourceByRef.set(r.ref, "scope");
2406
+ }
2407
+ for (const r of mergedRefs) {
2408
+ // "unknown" is a genuine fallback, never a silent alias for signal-delta:
2409
+ // only refs we truly cannot attribute land here (none in practice, since
2410
+ // mergedRefs is always a subset of the four lanes above).
2411
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2412
+ }
2413
+ // WS-1 — Unified salience vector (S1 seam).
2414
+ //
2415
+ // The legacy sort combined three independent formulas (utility EMA, negative-only
2416
+ // ratio / symmetric-valence magnitude, and the proactive-maintenance priority
2417
+ // formula). WS-1 converges them into one `computeSalience()` call per ref, with
2418
+ // three independently-stored sub-scores and one documented rankScore projection.
2419
+ //
2420
+ // Migration note: if a profile still has `symmetricValence` set, emit a one-time
2421
+ // warning — its behaviour (symmetric |valence| attention) is now always-on as
2422
+ // part of the salience vector, so the knob is a no-op and will be removed in 0.10.
2423
+ if (improveProfile.symmetricValence === true) {
2424
+ warn("[improve] Profile option 'symmetricValence' is deprecated (WS-1 salience vector). " +
2425
+ "Symmetric valence is now always active; remove the option from your improve profile.");
2426
+ }
2427
+ // Fetch last-use timestamps from the index DB for the full merged set so the
2428
+ // recency term in retrievalSalience is genuinely decayable (plan §WS-1 step 2).
2429
+ // This reuses the index DB opened earlier for retrieval counts; a separate
2430
+ // lightweight open is used here to avoid holding the connection longer than needed.
2431
+ let lastUseMsByRef = new Map();
2432
+ // utilityMap is kept for backward-compatible observability (health report reads it).
1888
2433
  const utilityMap = buildUtilityMap(mergedRefs);
1889
- // Load feedback ratio per ref from the pre-computed summary (no extra DB pass).
1890
- const feedbackRatios = new Map();
1891
- for (const ref of mergedRefs) {
1892
- const summary = feedbackSummary.get(ref.ref);
1893
- const positive = summary?.positive ?? 0;
1894
- const negative = summary?.negative ?? 0;
1895
- const total = positive + negative;
1896
- // ratio = negative proportion (high = needs more improvement)
1897
- feedbackRatios.set(ref.ref, total > 0 ? negative / total : 0);
1898
- }
1899
- // Sort: combine utility (desc) with feedback negativity (desc) — high-negative assets rank higher
2434
+ let dbForSalience;
2435
+ try {
2436
+ dbForSalience = openExistingDatabase();
2437
+ lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs.map((r) => r.ref));
2438
+ }
2439
+ catch (err) {
2440
+ rethrowIfTestIsolationError(err);
2441
+ // best-effort: if DB unavailable, recency term stays at floor (lastUseMs=0)
2442
+ }
2443
+ finally {
2444
+ if (dbForSalience)
2445
+ closeDatabase(dbForSalience);
2446
+ }
2447
+ // ── WS-2 Outcome loop ─────────────────────────────────────────────────────
2448
+ //
2449
+ // Update asset_outcome for every ref in the merged set BEFORE computing the
2450
+ // salience vector so the updated outcome_score feeds outcomeSalience this run.
2451
+ //
2452
+ // Inputs per ref:
2453
+ // - currentRetrievalCount: from retrievalCounts (index DB)
2454
+ // - lastRetrievedAt: from lastUseMsByRef (utility_scores.last_used_at)
2455
+ // - negativeFeedbackCount: cumulative negatives from feedbackSummary
2456
+ // - acceptedChangeCount: accepted proposals for this ref (state.db)
2457
+ // - valence: net valence from computeValenceScore(feedbackSummary.get(ref))
2458
+ // - utilityScore: from utilityMap (for warm-start seed on new rows)
2459
+ //
2460
+ // Best-effort: outcome failures never block the salience or ranking pass.
2461
+ const outcomeSalienceByRef = new Map();
2462
+ try {
2463
+ const outcomeDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
2464
+ const ownsOutcomeDb = !eventsCtx?.db;
2465
+ try {
2466
+ // Count accepted proposals per ref in one pass (avoid N separate queries).
2467
+ // Scoped to primaryStashDir when available so multi-stash installs don't
2468
+ // inflate counts with proposals from other stashes.
2469
+ const acceptedCountByRef = new Map();
2470
+ try {
2471
+ const acceptedProposals = listStateProposals(outcomeDb, {
2472
+ status: "accepted",
2473
+ ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
2474
+ });
2475
+ for (const p of acceptedProposals) {
2476
+ acceptedCountByRef.set(p.ref, (acceptedCountByRef.get(p.ref) ?? 0) + 1);
2477
+ }
2478
+ }
2479
+ catch {
2480
+ // best-effort: if proposals query fails, accepted counts stay at 0
2481
+ }
2482
+ // Update each ref's outcome row and collect the resulting outcome scores.
2483
+ const rawOutcomeScores = new Map();
2484
+ const nowForOutcome = Date.now();
2485
+ for (const r of mergedRefs) {
2486
+ const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
2487
+ const valenceResult = computeValenceScore(fb);
2488
+ try {
2489
+ const result = updateAssetOutcome(outcomeDb, {
2490
+ ref: r.ref,
2491
+ currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
2492
+ lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
2493
+ acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
2494
+ negativeFeedbackCount: fb.negative,
2495
+ valence: valenceResult.valence,
2496
+ utilityScore: utilityMap.get(r.ref),
2497
+ now: nowForOutcome,
2498
+ });
2499
+ rawOutcomeScores.set(r.ref, result.outcomeScore);
2500
+ }
2501
+ catch {
2502
+ // best-effort per-ref: skip this ref's outcome update on failure
2503
+ }
2504
+ }
2505
+ // Compute stash-wide max outcome_score for normalisation (diversity floor).
2506
+ // Read ALL rows (not just this run's batch) so the normalisation is
2507
+ // stash-relative, not pool-relative.
2508
+ let maxOutcomeScore = 0;
2509
+ try {
2510
+ const allOutcomes = getAllAssetOutcomes(outcomeDb);
2511
+ for (const row of allOutcomes) {
2512
+ if (row.outcome_score > maxOutcomeScore)
2513
+ maxOutcomeScore = row.outcome_score;
2514
+ }
2515
+ // Proxy-adequacy tripwire: emit a health event if outcome_score is
2516
+ // negatively correlated with accepted_change_rate (inverted proxy).
2517
+ const adequacy = computeProxyAdequacy(allOutcomes);
2518
+ if (adequacy.isInverted) {
2519
+ appendEvent({
2520
+ eventType: "outcome_proxy_inverted",
2521
+ ref: undefined,
2522
+ metadata: {
2523
+ correlation: adequacy.correlation,
2524
+ n: adequacy.n,
2525
+ note: "corr(outcome_score, accepted_change_rate) < −0.3: high-outcome_score assets have LOW accepted-change rates — the proxy's 'doing well' signal is inverted, so the coarse retrieval-delta signal is no longer trustworthy and the 0.10+ rich in-session signal is no longer deferrable. See plan §WS-2 proxy-adequacy tripwire.",
2526
+ },
2527
+ }, eventsCtx);
2528
+ }
2529
+ }
2530
+ catch {
2531
+ // best-effort: tripwire failure never blocks ranking
2532
+ }
2533
+ // Convert raw outcome scores → normalised outcomeSalience values in [0,1].
2534
+ for (const [ref, score] of rawOutcomeScores) {
2535
+ const normalised = outcomeScoreToSalience(score, maxOutcomeScore);
2536
+ outcomeSalienceByRef.set(ref, normalised);
2537
+ }
2538
+ // Also fetch outcome scores for refs NOT updated this run (stale or absent)
2539
+ // so the outcomeSalience read path works for all refs in the batch.
2540
+ const missingRefs = mergedRefs.map((r) => r.ref).filter((ref) => !rawOutcomeScores.has(ref));
2541
+ if (missingRefs.length > 0) {
2542
+ const storedScores = getOutcomeScoresByRef(outcomeDb, missingRefs);
2543
+ for (const [ref, score] of storedScores) {
2544
+ outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
2545
+ }
2546
+ }
2547
+ }
2548
+ finally {
2549
+ if (ownsOutcomeDb)
2550
+ outcomeDb.close();
2551
+ }
2552
+ }
2553
+ catch (err) {
2554
+ rethrowIfTestIsolationError(err);
2555
+ // best-effort: outcome failures never block salience computation
2556
+ }
2557
+ // Compute the salience vector for every ref in the merged set.
2558
+ // retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
2559
+ // so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
2560
+ // outcomeSalienceByRef is populated by WS-2 above (or empty on first run).
2561
+ //
2562
+ // Part-V gate: read the operator opt-in flag from config. Default false
2563
+ // (WS-1 parity weights) until the maintainer runs scripts/akm-eval and sets
2564
+ // improve.salience.outcomeWeightEnabled: true in the config.
2565
+ const salienceConfig = (options.config ?? loadConfig()).improve?.salience;
2566
+ const outcomeWeightEnabled = salienceConfig?.outcomeWeightEnabled === true;
2567
+ const salienceMap = new Map();
2568
+ const nowForSalience = Date.now();
2569
+ for (const r of mergedRefs) {
2570
+ const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
2571
+ const sizeBytes = (() => {
2572
+ const fp = r.filePath;
2573
+ if (!fp)
2574
+ return undefined;
2575
+ try {
2576
+ return fs.statSync(fp).size;
2577
+ }
2578
+ catch {
2579
+ return undefined;
2580
+ }
2581
+ })();
2582
+ const vector = computeSalience({
2583
+ ref: r.ref,
2584
+ type,
2585
+ retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
2586
+ lastUseMs: lastUseMsByRef.get(r.ref),
2587
+ utilityScore: utilityMap.get(r.ref),
2588
+ outcomeSalience: outcomeSalienceByRef.get(r.ref),
2589
+ sizeBytes,
2590
+ now: nowForSalience,
2591
+ outcomeWeightEnabled,
2592
+ });
2593
+ salienceMap.set(r.ref, vector);
2594
+ }
2595
+ // Persist salience vectors to state.db (best-effort, non-blocking).
2596
+ // The canonical store enables WS-3 homeostatic demotion and WS-2 outcome reads.
2597
+ //
2598
+ // Forgetting-safety report (plan §WS-1 step 7) — stash-wide rank comparison:
2599
+ //
2600
+ // BEFORE persisting the new rankScores, read ALL existing rows from state.db
2601
+ // (not just the per-run candidate pool). This gives stash-wide rank positions so
2602
+ // the top-200/below-500 thresholds are meaningful.
2603
+ //
2604
+ // Two distinct scenarios:
2605
+ //
2606
+ // A. First WS-1 run (table empty): the old stash-wide combinedEligibilityScore
2607
+ // ordering was never persisted in state.db (asset_salience is a new WS-1 table).
2608
+ // However, the old formula's inputs are available in-scope for every candidate
2609
+ // in the current pool: utility comes from utilityMap and the attention term
2610
+ // from feedbackSummary (positive/negative counts). We reconstruct the old
2611
+ // combinedEligibilityScore = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT
2612
+ // for every ref in salienceMap and rank them, giving a candidate-pool-scoped
2613
+ // old ordering. This is a partial reconstruction (only current-pool refs, not
2614
+ // stash-wide), but it is the most faithful comparison possible at cutover and
2615
+ // allows the top-200→below-500 forgetting guard to fire if the formula change
2616
+ // dramatically reorders the candidate pool.
2617
+ // See docs/design/improve-reconciliation-plan.md §WS-1 step 7 — the stash-wide
2618
+ // ordering was unreconstructable (no prior state.db snapshot), so this candidate-
2619
+ // pool partial reconstruction is the documented resolution for the first-run case.
2620
+ // Emit `improve_salience_first_run` to mark the cutover moment and include the
2621
+ // reconstructed comparison result in the metadata.
2622
+ //
2623
+ // B. Subsequent runs (table has rows): use ALL existing rows as old ranks, merge
2624
+ // them with the current run's salienceMap updates for new ranks, and call
2625
+ // buildRankChangeReport with stash-wide positions. This detects real rank drift
2626
+ // — e.g. a retrieval-pattern shift causing a previously top-200 asset to slip
2627
+ // below position 500.
2628
+ //
2629
+ // Measurement-protocol deferral (plan §269, Part-V):
2630
+ // The Part-V T0 baseline (scripts/akm-eval + health report) and the throughput/
2631
+ // quality gate are deferred pending owner sign-off. Full measurement requires a
2632
+ // before/after `akm health` report. Owner-acknowledged deferral: WS-2 landing
2633
+ // will re-introduce outcome salience and trigger the full re-tuning pass at that
2634
+ // time. salience.ts already accepts outcomeSalience directly as an input
2635
+ // (see SalienceInputs.outcomeSalience); no separate hook is needed.
2636
+ //
2637
+ // Forgetting-safety collection: populated inside scenario B below, consumed
2638
+ // after the try/catch to union candidates into mergedRefs before the sort.
2639
+ // Only refs from a real pre-existing ordering (scenario B) are collected;
2640
+ // empty on scenario A or when no candidates dropped below the threshold.
2641
+ let pendingForgettingRefs = [];
2642
+ try {
2643
+ const stateDb = openStateDatabase(eventsCtx?.dbPath);
2644
+ try {
2645
+ // Step 7: stash-wide rank-change report BEFORE overwriting the table.
2646
+ //
2647
+ // Load ALL existing rows so rank positions are stash-relative, not pool-relative.
2648
+ const existingAllScores = getAllRankScores(stateDb);
2649
+ if (existingAllScores.size === 0) {
2650
+ // Scenario A: first WS-1 run — table empty.
2651
+ //
2652
+ // Reconstruct the old combinedEligibilityScore ordering for the current
2653
+ // candidate pool using inputs that are already in-scope: utility from
2654
+ // utilityMap and the attention term from feedbackSummary (positive/negative
2655
+ // counts). Old formula: score = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT.
2656
+ //
2657
+ // Limitation: this covers only the current-run candidate pool, not the full
2658
+ // stash. The stash-wide ordering was never persisted (asset_salience is a new
2659
+ // WS-1 table), so this is the most faithful comparison possible at cutover.
2660
+ // See docs/design/improve-reconciliation-plan.md §WS-1 step 7.
2661
+ const reconstructedOldScores = new Map();
2662
+ for (const ref of salienceMap.keys()) {
2663
+ const utility = utilityMap.get(ref) ?? 0;
2664
+ const fb = feedbackSummary.get(ref) ?? { positive: 0, negative: 0 };
2665
+ const attention = computeValenceScore(fb).attention;
2666
+ reconstructedOldScores.set(ref, utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT);
2667
+ }
2668
+ // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2669
+ const toRanks = (scores) => {
2670
+ const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
2671
+ return new Map(sorted.map(([ref], i) => [ref, i + 1]));
2672
+ };
2673
+ const oldRanks = toRanks(reconstructedOldScores);
2674
+ const newRanks = toRanks(new Map([...salienceMap.entries()].map(([ref, v]) => [ref, v.rankScore])));
2675
+ const firstRunReport = buildRankChangeReport(oldRanks, newRanks);
2676
+ if (firstRunReport.forgettingCandidates.length > 0) {
2677
+ warn(`[improve/salience] WS-1 first-run rank-change report: ${firstRunReport.forgettingCandidates.length} asset(s) fell from top-200 to below position 500 (cutover formula change). ` +
2678
+ `Top drops: ${firstRunReport.forgettingCandidates
2679
+ .slice(0, 5)
2680
+ .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2681
+ .join(", ")}`);
2682
+ pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
2683
+ }
2684
+ appendEvent({
2685
+ eventType: "improve_salience_first_run",
2686
+ ref: undefined,
2687
+ metadata: {
2688
+ candidateCount: salienceMap.size,
2689
+ note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); see improve-reconciliation-plan.md §WS-1 step 7",
2690
+ forgettingCandidates: firstRunReport.forgettingCandidates.length,
2691
+ topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
2692
+ ref: e.ref,
2693
+ oldRank: e.oldRank,
2694
+ newRank: e.newRank,
2695
+ })),
2696
+ },
2697
+ }, eventsCtx);
2698
+ }
2699
+ else {
2700
+ // Scenario B: subsequent run — compare stash-wide old vs. new ranks.
2701
+ //
2702
+ // Build new scores by merging the full table with this run's updates.
2703
+ // Refs in salienceMap override their stored value; refs not in this run
2704
+ // retain their stored value unchanged. This gives a complete stash-wide
2705
+ // picture of what the new ordering looks like after this run.
2706
+ const mergedNewScores = new Map(existingAllScores);
2707
+ for (const [ref, vector] of salienceMap) {
2708
+ mergedNewScores.set(ref, vector.rankScore);
2709
+ }
2710
+ // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2711
+ const toRanks = (scores) => {
2712
+ const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
2713
+ return new Map(sorted.map(([ref], i) => [ref, i + 1]));
2714
+ };
2715
+ const oldRanks = toRanks(existingAllScores);
2716
+ const newRanks = toRanks(mergedNewScores);
2717
+ const report = buildRankChangeReport(oldRanks, newRanks);
2718
+ if (report.forgettingCandidates.length > 0) {
2719
+ warn(`[improve/salience] WS-1 rank-change report: ${report.forgettingCandidates.length} asset(s) fell from top-200 to below position 500. ` +
2720
+ `Top drops: ${report.forgettingCandidates
2721
+ .slice(0, 5)
2722
+ .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2723
+ .join(", ")}`);
2724
+ // Collect refs for protective consolidation pass (plan §WS-1 step 7).
2725
+ // These are force-included in the candidate pool (mergedRefs) after
2726
+ // this try block, bypassing cooldown/signal-delta gating.
2727
+ pendingForgettingRefs = report.forgettingCandidates.map((e) => e.ref);
2728
+ }
2729
+ appendEvent({
2730
+ eventType: "improve_salience_rank_change",
2731
+ ref: undefined,
2732
+ metadata: {
2733
+ stashSize: existingAllScores.size,
2734
+ totalChanged: report.allChanges.length,
2735
+ forgettingCandidates: report.forgettingCandidates.length,
2736
+ topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
2737
+ ref: e.ref,
2738
+ oldRank: e.oldRank,
2739
+ newRank: e.newRank,
2740
+ })),
2741
+ },
2742
+ }, eventsCtx);
2743
+ }
2744
+ for (const [ref, vector] of salienceMap) {
2745
+ upsertAssetSalience(stateDb, ref, vector, nowForSalience);
2746
+ }
2747
+ }
2748
+ finally {
2749
+ stateDb.close();
2750
+ }
2751
+ }
2752
+ catch (err) {
2753
+ rethrowIfTestIsolationError(err);
2754
+ // best-effort: salience persistence failure never blocks ranking
2755
+ }
2756
+ // ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
2757
+ // Forgetting candidates detected in scenario B are force-injected into
2758
+ // mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
2759
+ // signal-delta gating. Any ref already present in mergedRefs keeps its
2760
+ // existing eligibilitySource (stronger reactive signals win); refs not yet in
2761
+ // the pool are synthesised as minimal ImproveEligibleRef stubs and labelled
2762
+ // 'forgetting-safety' so S5/WS-5 can slice by lane. The dedupeRefs call
2763
+ // ensures no ref can enter the loop twice.
2764
+ if (pendingForgettingRefs.length > 0 && scope.mode !== "ref") {
2765
+ const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
2766
+ const newForgettingRefs = [];
2767
+ for (const ref of pendingForgettingRefs) {
2768
+ if (!existingRefSet.has(ref)) {
2769
+ // Ref not already in the candidate pool — synthesise a stub so it
2770
+ // participates in the reflect/distill loop with proper attribution.
2771
+ newForgettingRefs.push({ ref, reason: "scope-type", eligibilitySource: "forgetting-safety" });
2772
+ }
2773
+ // Always stamp the lane in the attribution map (overwrites weaker lanes;
2774
+ // stronger reactive signals — scope/signal-delta/high-retrieval/proactive
2775
+ // — are written after this block so they take precedence).
2776
+ eligibilitySourceByRef.set(ref, "forgetting-safety");
2777
+ }
2778
+ if (newForgettingRefs.length > 0) {
2779
+ mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
2780
+ }
2781
+ // Re-stamp attribution for any refs whose lane needs updating.
2782
+ // Precedence (weakest → strongest, each overwrites the previous):
2783
+ // proactive < high-retrieval < forgetting-safety < signal-delta
2784
+ // Scope mode is already excluded by the outer guard (`scope.mode !== "ref"`).
2785
+ // forgetting-safety sits above proactive and high-retrieval so that a ref
2786
+ // flagged as a forgetting candidate is always visible to S5/WS-5 as such,
2787
+ // even when it was also due for a proactive maintenance run. signal-delta
2788
+ // overrides forgetting-safety because a ref with fresh feedback is reactive
2789
+ // and doesn't need the protective pass label for measurement purposes.
2790
+ for (const r of highSalienceRefs)
2791
+ eligibilitySourceByRef.set(r.ref, "high-salience");
2792
+ for (const r of proactiveRefs)
2793
+ eligibilitySourceByRef.set(r.ref, "proactive");
2794
+ for (const r of highRetrievalRefs)
2795
+ eligibilitySourceByRef.set(r.ref, "high-retrieval");
2796
+ // Apply forgetting-safety OVER proactive, high-retrieval, and high-salience
2797
+ // (already stamped in the loop above via
2798
+ // `eligibilitySourceByRef.set(ref, "forgetting-safety")`). No-op here: the
2799
+ // set() calls above for proactive/high-retrieval/high-salience overwrite the
2800
+ // earlier forgetting-safety stamp — so we re-apply forgetting-safety now for
2801
+ // those refs that are both forgetting candidates AND in another fallback lane.
2802
+ for (const ref of pendingForgettingRefs) {
2803
+ eligibilitySourceByRef.set(ref, "forgetting-safety");
2804
+ }
2805
+ // signal-delta is the strongest reactive signal and overrides forgetting-safety.
2806
+ for (const r of signalFiltered)
2807
+ eligibilitySourceByRef.set(r.ref, "signal-delta");
2808
+ // Update eligibilitySource on the ref objects themselves for any refs whose
2809
+ // lane changed (covers both new stubs and pre-existing refs).
2810
+ for (const r of mergedRefs) {
2811
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2812
+ }
2813
+ }
2814
+ // ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
2815
+ // Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
2816
+ // revisited even with zero reactive signal (no feedback, no retrieval) and
2817
+ // regardless of cooldown — exactly like the forgetting-safety lane, replay is
2818
+ // injected AFTER cooldown/signal-delta partitioning so it bypasses those gates.
2819
+ //
2820
+ // Strictly additive: the replay slice is appended AFTER the --limit fresh slice
2821
+ // (see the loopRefs partition below), so it can never shrink the fresh-ref set.
2822
+ // Replay is the WEAKEST lane — it only stamps refs no other lane already claimed,
2823
+ // and budget is spent only on refs not already in mergedRefs (so a stronger lane
2824
+ // never has its budget wasted or its label overwritten).
2825
+ //
2826
+ // Default replayBudget=0 ⇒ this whole block is a no-op (no DB open, no event,
2827
+ // no mergedRefs mutation), preserving byte-identical pre-#610 selection behavior.
2828
+ const replayBudget = (options.config ?? loadConfig()).improve?.salience?.replayBudget ?? 0;
2829
+ const replayRefSet = new Set();
2830
+ if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
2831
+ let replayDb;
2832
+ try {
2833
+ replayDb = openStateDatabase(eventsCtx?.dbPath);
2834
+ const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
2835
+ const allRankScores = getAllRankScores(replayDb);
2836
+ // Candidate universe = every salience row NOT already in the pool, ordered by
2837
+ // rank_score desc with a deterministic ref-string tie-break (mirrors the main
2838
+ // sort). Converged refs (consecutive_no_ops >= dampener threshold) are fully
2839
+ // EXCLUDED — a stronger skip than the dampener (which only halves order).
2840
+ let convergedSkipped = 0;
2841
+ const candidates = [];
2842
+ for (const [ref, rankScore] of allRankScores) {
2843
+ if (alreadyInPool.has(ref))
2844
+ continue;
2845
+ const noOps = getConsecutiveNoOps(replayDb, ref);
2846
+ if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
2847
+ convergedSkipped++;
2848
+ continue;
2849
+ }
2850
+ candidates.push({ ref, rankScore });
2851
+ }
2852
+ candidates.sort((a, b) => b.rankScore !== a.rankScore ? b.rankScore - a.rankScore : a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0);
2853
+ const candidatePool = candidates.length;
2854
+ const selected = candidates.slice(0, replayBudget);
2855
+ const newReplayRefs = [];
2856
+ for (const { ref } of selected) {
2857
+ replayRefSet.add(ref);
2858
+ // Synthesise a stub (mirror the forgetting-safety stub). Resolve the
2859
+ // backing file from the planned-ref pool so the downstream existsSync
2860
+ // guard keeps the ref (a replay candidate from asset_salience whose file
2861
+ // is gone correctly drops out). Only refs present in the indexed pool can
2862
+ // be revisited — refs without a planned entry get no filePath and are
2863
+ // dropped by the disk check, which is the desired behavior.
2864
+ const planned = plannedRefs.find((p) => p.ref === ref);
2865
+ newReplayRefs.push({
2866
+ ref,
2867
+ reason: "scope-type",
2868
+ eligibilitySource: "replay",
2869
+ ...(planned?.filePath ? { filePath: planned.filePath } : {}),
2870
+ });
2871
+ // Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
2872
+ if (!salienceMap.has(ref)) {
2873
+ salienceMap.set(ref, {
2874
+ encoding: 0,
2875
+ outcome: 0,
2876
+ retrieval: 0,
2877
+ rankScore: allRankScores.get(ref) ?? 0,
2878
+ });
2879
+ }
2880
+ }
2881
+ if (newReplayRefs.length > 0) {
2882
+ mergedRefs = dedupeRefs([...mergedRefs, ...newReplayRefs]);
2883
+ // Replay is the WEAKEST lane: stamp 'replay' ONLY for refs not already
2884
+ // keyed by a stronger lane.
2885
+ for (const ref of replayRefSet) {
2886
+ if (!eligibilitySourceByRef.has(ref))
2887
+ eligibilitySourceByRef.set(ref, "replay");
2888
+ }
2889
+ for (const r of mergedRefs) {
2890
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2891
+ }
2892
+ }
2893
+ // Aggregated observability event (never per-ref).
2894
+ appendEvent({
2895
+ eventType: "improve_replay_selected",
2896
+ ref: undefined,
2897
+ metadata: {
2898
+ count: newReplayRefs.length,
2899
+ budget: replayBudget,
2900
+ convergedSkipped,
2901
+ candidatePool,
2902
+ },
2903
+ }, eventsCtx);
2904
+ }
2905
+ catch (err) {
2906
+ rethrowIfTestIsolationError(err);
2907
+ // best-effort: if DB unavailable, replayRefSet stays empty
2908
+ }
2909
+ finally {
2910
+ if (replayDb)
2911
+ replayDb.close();
2912
+ }
2913
+ }
2914
+ // Build no-op map for consolidation-selection dampener (plan §WS-1 step 8).
2915
+ // Reads consecutive_no_ops from the SAME pinned db handle used elsewhere in
2916
+ // this function. The effective score is used ONLY for processing/selection
2917
+ // order — the persisted rank_score in asset_salience is never mutated here.
2918
+ const noOpMap = new Map();
2919
+ try {
2920
+ const noOpDb = eventsCtx?.db ?? (eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
2921
+ if (noOpDb) {
2922
+ const ownsNoOpDb = !eventsCtx?.db;
2923
+ try {
2924
+ for (const r of mergedRefs) {
2925
+ noOpMap.set(r.ref, getConsecutiveNoOps(noOpDb, r.ref));
2926
+ }
2927
+ }
2928
+ finally {
2929
+ if (ownsNoOpDb)
2930
+ noOpDb.close();
2931
+ }
2932
+ }
2933
+ }
2934
+ catch {
2935
+ // best-effort: dampener failure never blocks selection
2936
+ }
2937
+ // Sort by effective selection score (desc), with explicit ref-string tie-break
2938
+ // for determinism. The effective score applies the consolidation-selection
2939
+ // dampener: assets that have been repeatedly skipped (consecutive_no_ops >=
2940
+ // THRESHOLD) are penalised by FACTOR so they sort after peers with similar
2941
+ // rankScore. The persisted rank_score is left unchanged — this is the whole
2942
+ // point of the dampener (stable assets stay fully retrievable).
2943
+ //
2944
+ // WIRING NOTE (plan §WS-1 step 8 / "consolidation-selection" disambiguation):
2945
+ // "consolidation-selection" in the plan refers to THIS reflect/distill
2946
+ // eligibility ordering — i.e. which assets are chosen for the reflect/distill
2947
+ // LLM pass — NOT to akmConsolidate (the cluster-merge phase at ~line 1994,
2948
+ // which runs earlier and never reads noOpMap). The no-op counter originates
2949
+ // from no-change reflect / quality-rejected distill outcomes; the dampener
2950
+ // suppresses repeated LLM attempts on those same assets without touching their
2951
+ // persisted rank_score (so they remain fully retrievable).
2952
+ //
2953
+ // This is the ONLY ranking path — negativeOnlyRatio and the legacy
2954
+ // symmetricValence branch are replaced. The three eligibilitySource lanes
2955
+ // (signal-delta / high-retrieval / proactive) survive as labels (set above).
2956
+ const effectiveScore = (ref) => {
2957
+ const rankScore = salienceMap.get(ref)?.rankScore ?? 0;
2958
+ const noOps = noOpMap.get(ref) ?? 0;
2959
+ return noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rankScore * SALIENCE_NO_OP_DAMPEN_FACTOR : rankScore;
2960
+ };
1900
2961
  const sorted = [...mergedRefs].sort((a, b) => {
1901
- const utilA = utilityMap.get(a.ref) ?? 0;
1902
- const utilB = utilityMap.get(b.ref) ?? 0;
1903
- const ratioA = feedbackRatios.get(a.ref) ?? 0;
1904
- const ratioB = feedbackRatios.get(b.ref) ?? 0;
1905
- // Combined score: 70% utility, 30% negative ratio
1906
- const scoreA = utilA * 0.7 + ratioA * 0.3;
1907
- const scoreB = utilB * 0.7 + ratioB * 0.3;
1908
- return scoreB - scoreA;
2962
+ const scoreA = effectiveScore(a.ref);
2963
+ const scoreB = effectiveScore(b.ref);
2964
+ if (scoreB !== scoreA)
2965
+ return scoreB - scoreA;
2966
+ // Stable tie-break: deterministic regardless of input ordering.
2967
+ return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
1909
2968
  });
1910
2969
  // Phase 0: surface coverage gaps from zero-result search queries
1911
2970
  let coverageGaps = [];
@@ -1939,15 +2998,32 @@ async function runImprovePreparationStage(args) {
1939
2998
  const assetMissingOnDisk = [];
1940
2999
  const existsCheckedActionable = [];
1941
3000
  for (const candidate of sorted) {
1942
- const filePath = await findAssetFilePath(candidate.ref, options.stashDir);
3001
+ // #591: prefer the path pre-resolved at planning time (synchronous
3002
+ // existsSync) over a serial async DB lookup per ref.
3003
+ const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
3004
+ ? candidate.filePath
3005
+ : await findAssetFilePath(candidate.ref, options.stashDir);
1943
3006
  if (filePath && fs.existsSync(filePath)) {
1944
3007
  existsCheckedActionable.push(candidate);
1945
3008
  }
1946
3009
  else {
1947
3010
  assetMissingOnDisk.push(candidate.ref);
1948
- appendEvent({ eventType: "improve_skipped", ref: candidate.ref, metadata: { reason: "asset_missing_on_disk" } }, eventsCtx);
1949
3011
  }
1950
3012
  }
3013
+ // #592 audit: one summary event instead of one per missing ref. Normally
3014
+ // tiny, but a stash deletion racing the run could make this O(n) sequential
3015
+ // state.db writes. `refs` is capped so the metadata row stays bounded.
3016
+ if (assetMissingOnDisk.length > 0) {
3017
+ appendEvent({
3018
+ eventType: "improve_skipped",
3019
+ ref: undefined,
3020
+ metadata: {
3021
+ reason: "asset_missing_on_disk",
3022
+ count: assetMissingOnDisk.length,
3023
+ refs: assetMissingOnDisk.slice(0, 50),
3024
+ },
3025
+ }, eventsCtx);
3026
+ }
1951
3027
  const actionableRefs = existsCheckedActionable;
1952
3028
  // Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
1953
3029
  // preserving sort order. distillOnlyRefs participate in the sort so --limit
@@ -1964,8 +3040,22 @@ async function runImprovePreparationStage(args) {
1964
3040
  }
1965
3041
  }
1966
3042
  // ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
3043
+ //
3044
+ // #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
3045
+ // fresh slice. Without this split, a high-rankScore replay ref could sort above
3046
+ // a fresh ref in the single combined slice and STEAL its slot (violating AC2).
3047
+ // We partition into the replay lane vs the rest, apply --limit to the
3048
+ // non-replay (fresh) refs only, then APPEND up to `replayBudget` replay refs
3049
+ // after the fresh slice. Sort order within each partition is preserved.
3050
+ //
3051
+ // Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
3052
+ // replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
3053
+ // `replayLoop.slice(0, 0) === []` — byte-identical.
1967
3054
  const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
1968
- const loopRefs = options.limit ? allLoopRefs.slice(0, options.limit) : allLoopRefs;
3055
+ const replayLoop = allLoopRefs.filter((r) => r.eligibilitySource === "replay");
3056
+ const nonReplayLoop = allLoopRefs.filter((r) => r.eligibilitySource !== "replay");
3057
+ const baseLoop = options.limit ? nonReplayLoop.slice(0, options.limit) : nonReplayLoop;
3058
+ const loopRefs = [...baseLoop, ...replayLoop.slice(0, replayBudget)];
1969
3059
  // Update the returned distillOnlyRefs to the sorted order so callers see the
1970
3060
  // ranked view (loop stage uses it as a Set so order is irrelevant, but the
1971
3061
  // shape change keeps downstream consumers consistent).
@@ -1976,7 +3066,7 @@ async function runImprovePreparationStage(args) {
1976
3066
  `(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
1977
3067
  }
1978
3068
  if (signalAndRetrievalRefs.length > 0) {
1979
- info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval)`);
3069
+ info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
1980
3070
  }
1981
3071
  if (validationFailureRefs.size > 0) {
1982
3072
  info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
@@ -1987,6 +3077,17 @@ async function runImprovePreparationStage(args) {
1987
3077
  const deferredCount = actionableRefs.length - loopRefs.length;
1988
3078
  info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
1989
3079
  (options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
3080
+ // WS-4: Per-phase threshold auto-tune for the extract phase.
3081
+ // Persists result for the NEXT run's makeGateConfig to read.
3082
+ const extractTuneDbPath = eventsCtx?.dbPath;
3083
+ if (options.autoAccept !== undefined && extractTuneDbPath) {
3084
+ try {
3085
+ maybeAutoTuneThreshold(extractGateCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), extractTuneDbPath, undefined, "extract");
3086
+ }
3087
+ catch (err) {
3088
+ warn(`[improve] calibration auto-tune (extract) skipped: ${err instanceof Error ? err.message : String(err)}`);
3089
+ }
3090
+ }
1990
3091
  return {
1991
3092
  actions,
1992
3093
  cleanupWarnings,
@@ -2008,6 +3109,7 @@ async function runImprovePreparationStage(args) {
2008
3109
  gateAutoAcceptFailedCount,
2009
3110
  consolidation: consolidationPass.consolidation,
2010
3111
  consolidationRan: consolidationPass.consolidationRan,
3112
+ ...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
2011
3113
  };
2012
3114
  }
2013
3115
  async function runImproveLoopStage(args) {
@@ -2016,6 +3118,14 @@ async function runImproveLoopStage(args) {
2016
3118
  // receives only its fair share of the wall-clock budget.
2017
3119
  const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
2018
3120
  const RECENT_ERRORS_CAP = 3;
3121
+ // requirePlannedRefs guard: when the distill profile sets this flag, skip
3122
+ // distill for distill-only refs if the reflect phase produced no planned refs.
3123
+ // Prevents the distill loop from generating hundreds of distill-skipped events
3124
+ // on quiet passes (all refs on reflect cooldown, no new signal to distill).
3125
+ const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
3126
+ const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
3127
+ const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
3128
+ const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
2019
3129
  // R-2 / #389: Self-Consistency multi-sample voting helpers.
2020
3130
  // Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
2021
3131
  const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
@@ -2097,6 +3207,10 @@ async function runImproveLoopStage(args) {
2097
3207
  stashDir: primaryStashDir,
2098
3208
  config: options.config ?? loadConfig(),
2099
3209
  eventsCtx,
3210
+ stateDbPath: eventsCtx?.dbPath,
3211
+ // candidateCount drives the exploration budget. loopRefs is the per-phase
3212
+ // set for reflect/distill; pass it so exploration budget is proportional.
3213
+ candidateCount: loopRefs.length,
2100
3214
  });
2101
3215
  const distillGateCfg = makeGateConfig("distill", {
2102
3216
  globalThreshold: options.autoAccept,
@@ -2104,6 +3218,8 @@ async function runImproveLoopStage(args) {
2104
3218
  stashDir: primaryStashDir,
2105
3219
  config: options.config ?? loadConfig(),
2106
3220
  eventsCtx,
3221
+ stateDbPath: eventsCtx?.dbPath,
3222
+ candidateCount: loopRefs.length,
2107
3223
  });
2108
3224
  for (const planned of loopRefs) {
2109
3225
  if (Date.now() - startMs >= budgetMs) {
@@ -2176,6 +3292,9 @@ async function runImproveLoopStage(args) {
2176
3292
  eventSource: "improve",
2177
3293
  ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
2178
3294
  ...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
3295
+ // Attribution: carry the eligibility lane so reflect stamps it on
3296
+ // the reflect_invoked event and the persisted proposal.
3297
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2179
3298
  };
2180
3299
  // R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
2181
3300
  // Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
@@ -2188,9 +3307,11 @@ async function runImproveLoopStage(args) {
2188
3307
  if (remainingBudgetMs() <= 0)
2189
3308
  break;
2190
3309
  // draftMode: skip DB write so each sample doesn't create a proposal.
2191
- samples.push(await reflectFn({ ...reflectCallArgs, draftMode: true }));
3310
+ samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
2192
3311
  }
2193
- const winner = pickMajorityVote(samples.length > 0 ? samples : [await reflectFn({ ...reflectCallArgs, draftMode: true })]);
3312
+ const winner = pickMajorityVote(samples.length > 0
3313
+ ? samples
3314
+ : [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
2194
3315
  // Persist only the majority-vote winner as a single real proposal.
2195
3316
  if (winner.ok && primaryStashDir) {
2196
3317
  const persistResult = createProposal(primaryStashDir, {
@@ -2198,6 +3319,9 @@ async function runImproveLoopStage(args) {
2198
3319
  source: "reflect",
2199
3320
  sourceRun: `reflect-sc-${Date.now()}`,
2200
3321
  payload: winner.proposal.payload,
3322
+ // Attribution: the self-consistency path persists the winner here
3323
+ // (draftMode skips reflect's own createProposal), so stamp the lane.
3324
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2201
3325
  });
2202
3326
  reflectResult = isProposalSkipped(persistResult)
2203
3327
  ? {
@@ -2215,7 +3339,7 @@ async function runImproveLoopStage(args) {
2215
3339
  }
2216
3340
  }
2217
3341
  else {
2218
- reflectResult = await reflectFn(reflectCallArgs);
3342
+ reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
2219
3343
  }
2220
3344
  const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
2221
3345
  // Content-policy guard hits (reflect size-rail rejections) are NOT
@@ -2232,6 +3356,12 @@ async function runImproveLoopStage(args) {
2232
3356
  // user's stack were this case; see review §1a row "Reflect refused
2233
3357
  // asset type".
2234
3358
  const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
3359
+ // Noise-gate suppression (#580): the candidate edit was an empty
3360
+ // diff or a cosmetic-only reformat of the current asset. Like
3361
+ // `unsupported_type`, this is a deterministic skip — not an LLM
3362
+ // fault — so it routes to the `reflect-skipped` bucket and stays
3363
+ // out of recentErrors/avoidPatterns.
3364
+ const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
2235
3365
  actions.push({
2236
3366
  ref: planned.ref,
2237
3367
  mode: reflectResult.ok
@@ -2240,18 +3370,19 @@ async function runImproveLoopStage(args) {
2240
3370
  ? "reflect-cooldown"
2241
3371
  : isGuardReject
2242
3372
  ? "reflect-guard-rejected"
2243
- : isTypeRefused
3373
+ : isTypeRefused || isNoChange
2244
3374
  ? "reflect-skipped"
2245
3375
  : "reflect-failed",
2246
3376
  result: reflectResult,
2247
3377
  });
2248
- // Cooldown skips, guard rejects, and type-refused skips are not
2249
- // failures — do not pollute recentErrors with them (those get
2250
- // injected as `avoidPatterns` into the next reflect prompt). Guard
2251
- // rejects ARE worth showing the LLM as a learn-signal so the next
2252
- // iteration sees "your last expansion was too large"; type-refused
2253
- // is deterministic and adds no learning signal.
2254
- if (!reflectResult.ok && !isCooldown && !isTypeRefused) {
3378
+ // Cooldown skips, guard rejects, type-refused skips, and noise-gate
3379
+ // skips are not failures — do not pollute recentErrors with them
3380
+ // (those get injected as `avoidPatterns` into the next reflect
3381
+ // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
3382
+ // so the next iteration sees "your last expansion was too large";
3383
+ // type-refused and no-change are deterministic and add no learning
3384
+ // signal.
3385
+ if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
2255
3386
  const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
2256
3387
  pushRecentError("reflect", errMsg);
2257
3388
  }
@@ -2266,6 +3397,28 @@ async function runImproveLoopStage(args) {
2266
3397
  reason: reflectResult.ok ? undefined : reflectResult.reason,
2267
3398
  },
2268
3399
  }, eventsCtx);
3400
+ // Plasticity counter (plan §WS-1 step 8): record no-ops so the
3401
+ // WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
3402
+ // repeatedly-silent assets during consolidation-selection.
3403
+ // A no_change reflect means the LLM was invoked but found nothing to
3404
+ // improve — the asset is stable. Track it. A successful reflect means
3405
+ // the asset changed; reset the counter so the dampener lifts.
3406
+ if (isNoChange && eventsCtx?.db) {
3407
+ try {
3408
+ recordNoOp(eventsCtx.db, planned.ref);
3409
+ }
3410
+ catch {
3411
+ // best-effort: plasticity counter failure never blocks the run
3412
+ }
3413
+ }
3414
+ else if (reflectResult.ok && eventsCtx?.db) {
3415
+ try {
3416
+ resetConsecutiveNoOps(eventsCtx.db, planned.ref);
3417
+ }
3418
+ catch {
3419
+ // best-effort
3420
+ }
3421
+ }
2269
3422
  if (reflectResult.ok) {
2270
3423
  const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
2271
3424
  gateAutoAcceptedCount += reflectGr.promoted.length;
@@ -2304,6 +3457,18 @@ async function runImproveLoopStage(args) {
2304
3457
  info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2305
3458
  continue;
2306
3459
  }
3460
+ // requirePlannedRefs guard: skip distill for distill-only refs when no
3461
+ // reflect-eligible refs were planned this run, preventing mass skip events.
3462
+ if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
3463
+ actions.push({
3464
+ ref: planned.ref,
3465
+ mode: "distill-skipped",
3466
+ result: { ok: true, reason: "require_planned_refs" },
3467
+ });
3468
+ completedCount++;
3469
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
3470
+ continue;
3471
+ }
2307
3472
  // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
2308
3473
  // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
2309
3474
  // it will refuse.
@@ -2373,11 +3538,14 @@ async function runImproveLoopStage(args) {
2373
3538
  }
2374
3539
  }
2375
3540
  }
2376
- const distillResult = await distillFn({
3541
+ const distillResult = await withLlmStage("distill", () => distillFn({
2377
3542
  ref: planned.ref,
2378
3543
  ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
2379
3544
  ...(options.stashDir ? { stashDir: options.stashDir } : {}),
2380
- });
3545
+ // Attribution: carry the eligibility lane so distill stamps it on the
3546
+ // distill_invoked event and the persisted proposal.
3547
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
3548
+ }));
2381
3549
  actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
2382
3550
  if (distillResult.outcome === "queued" && distillResult.proposal) {
2383
3551
  const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
@@ -2389,6 +3557,23 @@ async function runImproveLoopStage(args) {
2389
3557
  if (!promotedToKnowledge)
2390
3558
  memoryRefsForInference.add(planned.ref);
2391
3559
  }
3560
+ // Plasticity counter (plan §WS-1 step 8) for the distill path.
3561
+ // quality_rejected: the LLM ran but produced output that didn't pass the
3562
+ // quality gate — the asset is not yielding useful distill output.
3563
+ // queued: a proposal was produced; reset the no-op counter.
3564
+ if (eventsCtx?.db) {
3565
+ try {
3566
+ if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
3567
+ recordNoOp(eventsCtx.db, planned.ref);
3568
+ }
3569
+ else if (distillResult.outcome === "queued") {
3570
+ resetConsecutiveNoOps(eventsCtx.db, planned.ref);
3571
+ }
3572
+ }
3573
+ catch {
3574
+ // best-effort: plasticity counter failure never blocks the run
3575
+ }
3576
+ }
2392
3577
  if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
2393
3578
  const slug = planned.ref
2394
3579
  .replace(/[^a-z0-9]/gi, "-")
@@ -2455,6 +3640,26 @@ async function runImproveLoopStage(args) {
2455
3640
  completedCount++;
2456
3641
  info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2457
3642
  }
3643
+ // WS-4: Per-phase threshold auto-tune — runs AFTER the loop so the gate
3644
+ // has processed all candidates for this run. Persists each phase's tuned
3645
+ // threshold to state.db for the NEXT run's makeGateConfig to read.
3646
+ // Best-effort: a tune failure must never fail the improve run.
3647
+ const stateDbPathForTune = eventsCtx?.dbPath;
3648
+ if (options.autoAccept !== undefined && stateDbPathForTune) {
3649
+ const phaseGateCfgMap = {
3650
+ reflect: reflectGateCfg,
3651
+ distill: distillGateCfg,
3652
+ };
3653
+ for (const phase of ["reflect", "distill"]) {
3654
+ const phaseCfg = phaseGateCfgMap[phase];
3655
+ try {
3656
+ maybeAutoTuneThreshold(phaseCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), stateDbPathForTune, undefined, phase);
3657
+ }
3658
+ catch (err) {
3659
+ warn(`[improve] calibration auto-tune (${phase}) skipped: ${err instanceof Error ? err.message : String(err)}`);
3660
+ }
3661
+ }
3662
+ }
2458
3663
  return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
2459
3664
  }
2460
3665
  async function runImprovePostLoopStage(args) {
@@ -2498,9 +3703,70 @@ async function runImprovePostLoopStage(args) {
2498
3703
  // best-effort
2499
3704
  }
2500
3705
  }
3706
+ // #609 — recombine / synthesize pass. Whole-corpus cross-episodic
3707
+ // generalization. Runs in the post-loop stage under consolidate.lock (it
3708
+ // reads the consolidated corpus and writes proposals). Opt-in: gated on the
3709
+ // `recombine` process being enabled, whole-stash / type scope (never `ref`),
3710
+ // and not a dry run. Mirrors the proactiveMaintenance opt-in wiring.
3711
+ let recombination;
3712
+ if (primaryStashDir &&
3713
+ improveProfile &&
3714
+ resolveProcessEnabled("recombine", improveProfile) &&
3715
+ scope.mode !== "ref" &&
3716
+ !options.dryRun) {
3717
+ const recombineFn = options.recombineFn ?? akmRecombine;
3718
+ try {
3719
+ recombination = await recombineFn({
3720
+ stashDir: primaryStashDir,
3721
+ config: options.config ?? loadConfig(),
3722
+ ...(options.runId ? { sourceRun: options.runId } : {}),
3723
+ ...(budgetSignal ? { signal: budgetSignal } : {}),
3724
+ ...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
3725
+ eligibilitySource: "recombine",
3726
+ ...(eventsCtx ? { ctx: eventsCtx } : {}),
3727
+ minClusterSize: improveProfile.processes?.recombine?.minClusterSize,
3728
+ maxClustersPerRun: improveProfile.processes?.recombine?.maxClustersPerRun,
3729
+ relatednessSource: improveProfile.processes?.recombine?.relatednessSource,
3730
+ confirmThreshold: improveProfile.processes?.recombine?.confirmThreshold,
3731
+ });
3732
+ }
3733
+ catch (e) {
3734
+ allWarnings.push(`recombine: ${String(e)}`);
3735
+ }
3736
+ }
3737
+ // #615 — procedural-compilation pass. Detects recurring successful ordered
3738
+ // action sequences and compiles them into workflow proposals. Opt-in: gated
3739
+ // on the `procedural` process being enabled, whole-stash / type scope (never
3740
+ // `ref`), and not a dry run. Mirrors the recombine opt-in wiring.
3741
+ let proceduralCompilation;
3742
+ if (primaryStashDir &&
3743
+ improveProfile &&
3744
+ resolveProcessEnabled("procedural", improveProfile) &&
3745
+ scope.mode !== "ref" &&
3746
+ !options.dryRun) {
3747
+ const proceduralFn = options.proceduralFn ?? akmProcedural;
3748
+ try {
3749
+ proceduralCompilation = await proceduralFn({
3750
+ stashDir: primaryStashDir,
3751
+ config: options.config ?? loadConfig(),
3752
+ ...(options.runId ? { sourceRun: options.runId } : {}),
3753
+ ...(budgetSignal ? { signal: budgetSignal } : {}),
3754
+ ...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
3755
+ eligibilitySource: "procedural",
3756
+ ...(eventsCtx ? { ctx: eventsCtx } : {}),
3757
+ minRecurrence: improveProfile.processes?.procedural?.minRecurrence,
3758
+ maxProposalsPerRun: improveProfile.processes?.procedural?.maxProposalsPerRun,
3759
+ });
3760
+ }
3761
+ catch (e) {
3762
+ allWarnings.push(`procedural: ${String(e)}`);
3763
+ }
3764
+ }
2501
3765
  return {
2502
3766
  allWarnings,
2503
3767
  deadUrls,
3768
+ ...(recombination ? { recombination } : {}),
3769
+ ...(proceduralCompilation ? { proceduralCompilation } : {}),
2504
3770
  ...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
2505
3771
  ...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
2506
3772
  ...(maintenanceResult.stalenessDetection ? { stalenessDetection: maintenanceResult.stalenessDetection } : {}),
@@ -2518,7 +3784,9 @@ async function runImprovePostLoopStage(args) {
2518
3784
  };
2519
3785
  }
2520
3786
  // TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
2521
- async function runImproveMaintenancePasses(args) {
3787
+ // Exported for tests (#584/#585 DB-locking regression coverage); production
3788
+ // callers reach it only through akmImprove → runImprovePostLoopStage.
3789
+ export async function runImproveMaintenancePasses(args) {
2522
3790
  const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
2523
3791
  if (!primaryStashDir)
2524
3792
  return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
@@ -2537,276 +3805,344 @@ async function runImproveMaintenancePasses(args) {
2537
3805
  let graphExtractionDurationMs = 0;
2538
3806
  let orphansPurged = 0;
2539
3807
  let proposalsExpired = 0;
2540
- try {
2541
- db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
2542
- // Memory inference candidate-discovery (post-Item 9 fix from
2543
- // memory:akm-improve-critical-review-2026-05-20). Previously this pass
2544
- // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
2545
- // candidateRefs filter. But memoryRefsForInference is populated from refs
2546
- // distilled THIS RUN by the time that happens, those parents are
2547
- // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
2548
- // them. The genuinely-pending parents in the stash never entered the
2549
- // filter. Result: 0/0/0 for 25 consecutive runs.
2550
- //
2551
- // Fix: always run the pass when the feature is enabled; let the pass's
2552
- // own `collectPendingMemories` + `isPendingMemory` predicate find
2553
- // candidates from the filesystem-of-truth. The this-run set is still
2554
- // logged as a hint but no longer used as a filter.
2555
- const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
2556
- if (memoryInferenceDisabledByProfile) {
2557
- info("[improve] memory inference skipped (disabled by improve profile)");
3808
+ const openIndexDb = () => openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
3809
+ // #584: reindexFn opens its own write handle on the same index.db WAL file.
3810
+ // Holding our handle across that call produced SQLITE_BUSY / "database is
3811
+ // locked" failures in production, so the handle is closed BEFORE every
3812
+ // reindex and reopened after the fresh handle also sees the post-reindex
3813
+ // state that graph extraction and staleness detection below rely on. The
3814
+ // reopen runs in `finally` so a failed reindex still leaves a usable handle.
3815
+ const reindexWithIndexDbReleased = async (stashDir) => {
3816
+ if (db) {
3817
+ closeDatabase(db);
3818
+ db = undefined;
2558
3819
  }
2559
- else {
2560
- const hintRefs = memoryRefsForInference.size;
2561
- info(hintRefs > 0
2562
- ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
2563
- : "[improve] memory inference starting (discovering pending parents)");
2564
- const inferenceStart = Date.now();
2565
- try {
2566
- // O-1 (#364): pass budget signal so a hung inference call is cancelled.
2567
- memoryInference = await memoryInferenceFn({
2568
- config,
2569
- sources,
2570
- signal: budgetSignal,
2571
- db,
2572
- reEnrich: false,
2573
- onProgress: (event) => {
2574
- const current = event.currentRef ? ` ${event.currentRef}` : "";
2575
- info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
2576
- },
2577
- });
2578
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2579
- actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
2580
- info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2581
- }
2582
- catch (err) {
2583
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2584
- allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2585
- }
3820
+ try {
3821
+ await reindexFn({ stashDir });
2586
3822
  }
2587
- if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
2588
- info("[improve] reindexing after memory inference writes");
2589
- try {
2590
- await reindexFn({ stashDir: primaryStashDir });
2591
- reindexedAfterInference = true;
2592
- info("[improve] reindex after memory inference complete");
2593
- }
2594
- catch (err) {
2595
- allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2596
- }
2597
- }
2598
- const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
2599
- const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
2600
- const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
2601
- // Build the set of refs actually touched this run.
2602
- const touchedRefs = new Set();
2603
- for (const r of args.actionableRefs)
2604
- touchedRefs.add(r.ref);
2605
- for (const r of memoryRefsForInference)
2606
- touchedRefs.add(r);
2607
- // INVARIANT: graph extraction normally runs only on files touched by
2608
- // actionable refs (candidatePaths). Full-corpus scans are opt-in via
2609
- // profile.processes.graphExtraction.fullScan = true (used by the
2610
- // `graph-refresh` built-in profile and its weekly scheduled task).
2611
- // The empty-Set fallback is intentional when no refs were touched —
2612
- // the extractor's filter rejects every file and returns empty, keeping
2613
- // the pass invoked so the action is recorded and tests stay exercised.
2614
- if (graphExtractionDisabledByProfile) {
2615
- info("[improve] graph extraction skipped (disabled by improve profile)");
2616
- }
2617
- else if (sources.length > 0 && graphEnabled) {
2618
- info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
2619
- const extractionStart = Date.now();
2620
- try {
2621
- // D9: if consolidation ran but memory inference did not reindex, force a reindex
2622
- // so graph extraction sees current DB state after consolidation writes.
2623
- if (consolidationRan && !reindexedAfterInference) {
2624
- info("[improve] reindexing after consolidation (graph extraction needs current state)");
2625
- try {
2626
- await reindexFn({ stashDir: primaryStashDir });
2627
- reindexedAfterInference = true;
2628
- info("[improve] reindex after consolidation complete");
2629
- }
2630
- catch (err) {
2631
- allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
2632
- }
3823
+ finally {
3824
+ db = openIndexDb();
3825
+ }
3826
+ };
3827
+ await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
3828
+ try {
3829
+ db = openIndexDb();
3830
+ // Memory inference candidate-discovery (post-Item 9 fix from
3831
+ // memory:akm-improve-critical-review-2026-05-20). Previously this pass
3832
+ // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
3833
+ // candidateRefs filter. But memoryRefsForInference is populated from refs
3834
+ // distilled THIS RUN — by the time that happens, those parents are
3835
+ // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
3836
+ // them. The genuinely-pending parents in the stash never entered the
3837
+ // filter. Result: 0/0/0 for 25 consecutive runs.
3838
+ //
3839
+ // Fix: always run the pass when the feature is enabled; let the pass's
3840
+ // own `collectPendingMemories` + `isPendingMemory` predicate find
3841
+ // candidates from the filesystem-of-truth. The this-run set is still
3842
+ // logged as a hint but no longer used as a filter.
3843
+ const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
3844
+ const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
3845
+ const pendingBelowMinCount = (() => {
3846
+ if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
3847
+ return false;
3848
+ const pending = collectPendingMemories(primaryStashDir).length;
3849
+ if (pending < minPendingCount) {
3850
+ info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
3851
+ return true;
2633
3852
  }
2634
- if (db && reindexedAfterInference) {
2635
- closeDatabase(db);
2636
- db = openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
3853
+ return false;
3854
+ })();
3855
+ if (memoryInferenceDisabledByProfile) {
3856
+ info("[improve] memory inference skipped (disabled by improve profile)");
3857
+ }
3858
+ else if (pendingBelowMinCount) {
3859
+ // skipped — message already emitted above
3860
+ }
3861
+ else {
3862
+ const hintRefs = memoryRefsForInference.size;
3863
+ info(hintRefs > 0
3864
+ ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
3865
+ : "[improve] memory inference starting (discovering pending parents)");
3866
+ const inferenceStart = Date.now();
3867
+ try {
3868
+ // O-1 (#364): pass budget signal so a hung inference call is cancelled.
3869
+ memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
3870
+ config,
3871
+ sources,
3872
+ signal: budgetSignal,
3873
+ db,
3874
+ reEnrich: false,
3875
+ onProgress: (event) => {
3876
+ const current = event.currentRef ? ` ${event.currentRef}` : "";
3877
+ info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
3878
+ },
3879
+ }));
3880
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
3881
+ actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
3882
+ info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2637
3883
  }
2638
- // Resolve touched refs to absolute file paths. Skipped for fullScan
2639
- // (candidatePaths stays undefined → extractor processes all files).
2640
- let candidatePaths;
2641
- if (!graphExtractionFullScan) {
2642
- candidatePaths = new Set();
2643
- if (primaryStashDir && touchedRefs.size > 0) {
2644
- const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
2645
- const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
2646
- for (const p of resolved) {
2647
- if (typeof p === "string" && p.length > 0)
2648
- candidatePaths.add(p);
2649
- }
2650
- }
3884
+ catch (err) {
3885
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
3886
+ allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2651
3887
  }
2652
- const progressHandler = (event) => {
2653
- const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
2654
- info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
2655
- };
2656
- // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
2657
- graphExtraction = await graphExtractionFn({
2658
- config,
2659
- sources,
2660
- signal: budgetSignal,
2661
- db,
2662
- reEnrich: false,
2663
- onProgress: progressHandler,
2664
- options: { candidatePaths },
2665
- });
2666
- graphExtractionDurationMs = Date.now() - extractionStart;
2667
- actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
2668
- info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
2669
- }
2670
- catch (err) {
2671
- graphExtractionDurationMs = Date.now() - extractionStart;
2672
- allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
2673
3888
  }
2674
- }
2675
- else if (sources.length > 0 && !graphEnabled) {
2676
- info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
2677
- }
2678
- // Orphan proposal purge — reject pending reflect proposals whose target
2679
- // asset no longer exists on disk. Runs after graph extraction so newly
2680
- // promoted assets from accept flows during this run are already present.
2681
- if (primaryStashDir) {
2682
- try {
2683
- const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
2684
- orphansPurged = purgeResult.rejected;
2685
- if (purgeResult.rejected > 0) {
2686
- info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
3889
+ if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
3890
+ info("[improve] reindexing after memory inference writes");
3891
+ try {
3892
+ await reindexWithIndexDbReleased(primaryStashDir);
3893
+ reindexedAfterInference = true;
3894
+ info("[improve] reindex after memory inference complete");
3895
+ }
3896
+ catch (err) {
3897
+ allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2687
3898
  }
2688
- appendEvent({
2689
- eventType: "proposal_orphan_purge",
2690
- ref: "proposals:_orphan-purge",
2691
- metadata: {
2692
- checked: purgeResult.checked,
2693
- rejected: purgeResult.rejected,
2694
- durationMs: purgeResult.durationMs,
2695
- byType: purgeResult.byType,
2696
- orphans: purgeResult.orphans.map((o) => o.ref),
2697
- },
2698
- }, eventsCtx);
2699
3899
  }
2700
- catch (err) {
2701
- allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
3900
+ const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
3901
+ const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
3902
+ const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
3903
+ // Build the set of refs actually touched this run.
3904
+ const touchedRefs = new Set();
3905
+ for (const r of args.actionableRefs)
3906
+ touchedRefs.add(r.ref);
3907
+ for (const r of memoryRefsForInference)
3908
+ touchedRefs.add(r);
3909
+ // INVARIANT: graph extraction normally runs only on files touched by
3910
+ // actionable refs (candidatePaths). Full-corpus scans are opt-in via
3911
+ // profile.processes.graphExtraction.fullScan = true (used by the
3912
+ // `graph-refresh` built-in profile and its weekly scheduled task).
3913
+ // The empty-Set fallback is intentional when no refs were touched —
3914
+ // the extractor's filter rejects every file and returns empty, keeping
3915
+ // the pass invoked so the action is recorded and tests stay exercised.
3916
+ if (graphExtractionDisabledByProfile) {
3917
+ info("[improve] graph extraction skipped (disabled by improve profile)");
2702
3918
  }
2703
- // Phase 6B (Advantage D6b): expire pending proposals that have aged past
2704
- // the retention window. Runs AFTER orphan purge so we never double-archive
2705
- // a proposal that orphan-purge already moved. `expireStaleProposals` emits
2706
- // its own per-proposal `proposal_expired` events; we additionally emit a
2707
- // single roll-up event here for parity with the orphan-purge surface.
2708
- try {
2709
- const expireResult = expireStaleProposals(primaryStashDir, config);
2710
- proposalsExpired = expireResult.expired;
2711
- if (expireResult.expired > 0) {
2712
- info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
2713
- `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
3919
+ else if (sources.length > 0 && graphEnabled) {
3920
+ info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
3921
+ const extractionStart = Date.now();
3922
+ try {
3923
+ // D9: if consolidation ran but memory inference did not reindex, force a reindex
3924
+ // so graph extraction sees current DB state after consolidation writes.
3925
+ if (consolidationRan && !reindexedAfterInference) {
3926
+ info("[improve] reindexing after consolidation (graph extraction needs current state)");
3927
+ try {
3928
+ await reindexWithIndexDbReleased(primaryStashDir);
3929
+ reindexedAfterInference = true;
3930
+ info("[improve] reindex after consolidation complete");
3931
+ }
3932
+ catch (err) {
3933
+ allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
3934
+ }
3935
+ }
3936
+ // #584: no close/reopen needed here — reindexWithIndexDbReleased
3937
+ // already swapped in a fresh post-reindex handle.
3938
+ // Resolve touched refs to absolute file paths. Skipped for fullScan
3939
+ // (candidatePaths stays undefined → extractor processes all files).
3940
+ let candidatePaths;
3941
+ if (!graphExtractionFullScan) {
3942
+ candidatePaths = new Set();
3943
+ if (primaryStashDir && touchedRefs.size > 0) {
3944
+ const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
3945
+ const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
3946
+ for (const p of resolved) {
3947
+ if (typeof p === "string" && p.length > 0)
3948
+ candidatePaths.add(p);
3949
+ }
3950
+ }
3951
+ }
3952
+ const progressHandler = (event) => {
3953
+ const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
3954
+ info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
3955
+ };
3956
+ // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
3957
+ graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
3958
+ config,
3959
+ sources,
3960
+ signal: budgetSignal,
3961
+ db,
3962
+ reEnrich: false,
3963
+ onProgress: progressHandler,
3964
+ options: { candidatePaths },
3965
+ }));
3966
+ graphExtractionDurationMs = Date.now() - extractionStart;
3967
+ actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
3968
+ info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
3969
+ }
3970
+ catch (err) {
3971
+ graphExtractionDurationMs = Date.now() - extractionStart;
3972
+ allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
2714
3973
  }
2715
- appendEvent({
2716
- eventType: "proposal_expiration_pass",
2717
- ref: "proposals:_expiration",
2718
- metadata: {
2719
- checked: expireResult.checked,
2720
- expired: expireResult.expired,
2721
- durationMs: expireResult.durationMs,
2722
- retentionDays: expireResult.retentionDays,
2723
- expiredProposals: expireResult.expiredProposals,
2724
- },
2725
- }, eventsCtx);
2726
3974
  }
2727
- catch (err) {
2728
- allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
3975
+ else if (sources.length > 0 && !graphEnabled) {
3976
+ info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
2729
3977
  }
2730
- }
2731
- // Fix #2 (observability 0.8.0): trim the events table in state.db so it
2732
- // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
2733
- // invocation, and every command surface emits at least one event besides —
2734
- // without this trim, state.db is a permanent append-only log. Config key
2735
- // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
2736
- // window. `purgeOldEvents()` opens its own state.db handle separate from
2737
- // the index `db` above (different SQLite file).
2738
- {
2739
- const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
2740
- if (retentionDays > 0) {
2741
- let stateDb;
3978
+ // Orphan proposal purge — reject pending reflect proposals whose target
3979
+ // asset no longer exists on disk. Runs after graph extraction so newly
3980
+ // promoted assets from accept flows during this run are already present.
3981
+ if (primaryStashDir) {
2742
3982
  try {
2743
- // C2: reuse the boundary-pinned state.db path carried on eventsCtx so
2744
- // this purge open never re-reads `process.env` live mid-run. The path
2745
- // is always set by akmImprove; openStateDatabase() falls back to the
2746
- // env-derived default only if a caller omitted it entirely.
2747
- stateDb = openStateDatabase(eventsCtx?.dbPath);
2748
- const purgedCount = purgeOldEvents(stateDb, retentionDays);
2749
- if (purgedCount > 0) {
2750
- info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
3983
+ const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
3984
+ orphansPurged = purgeResult.rejected;
3985
+ if (purgeResult.rejected > 0) {
3986
+ info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
2751
3987
  }
2752
3988
  appendEvent({
2753
- eventType: "events_purged",
2754
- ref: "events:_purge",
2755
- metadata: { purgedCount, retentionDays },
3989
+ eventType: "proposal_orphan_purge",
3990
+ ref: "proposals:_orphan-purge",
3991
+ metadata: {
3992
+ checked: purgeResult.checked,
3993
+ rejected: purgeResult.rejected,
3994
+ durationMs: purgeResult.durationMs,
3995
+ byType: purgeResult.byType,
3996
+ orphans: purgeResult.orphans.map((o) => o.ref),
3997
+ },
2756
3998
  }, eventsCtx);
2757
- // improve_runs uses the same retention window as events — both are
2758
- // observability/audit data, both grow append-only, both have a
2759
- // dedicated purge helper. Mirroring the events purge here means a
2760
- // single retention knob (improve.eventRetentionDays) governs both.
2761
- const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
2762
- if (improveRunsPurged > 0) {
2763
- info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
3999
+ }
4000
+ catch (err) {
4001
+ allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
4002
+ }
4003
+ // Phase 6B (Advantage D6b): expire pending proposals that have aged past
4004
+ // the retention window. Runs AFTER orphan purge so we never double-archive
4005
+ // a proposal that orphan-purge already moved. `expireStaleProposals` emits
4006
+ // its own per-proposal `proposal_expired` events; we additionally emit a
4007
+ // single roll-up event here for parity with the orphan-purge surface.
4008
+ try {
4009
+ const expireResult = expireStaleProposals(primaryStashDir, config);
4010
+ proposalsExpired = expireResult.expired;
4011
+ if (expireResult.expired > 0) {
4012
+ info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
4013
+ `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
2764
4014
  }
2765
4015
  appendEvent({
2766
- eventType: "improve_runs_purged",
2767
- ref: "improve_runs:_purge",
2768
- metadata: { purgedCount: improveRunsPurged, retentionDays },
4016
+ eventType: "proposal_expiration_pass",
4017
+ ref: "proposals:_expiration",
4018
+ metadata: {
4019
+ checked: expireResult.checked,
4020
+ expired: expireResult.expired,
4021
+ durationMs: expireResult.durationMs,
4022
+ retentionDays: expireResult.retentionDays,
4023
+ expiredProposals: expireResult.expiredProposals,
4024
+ },
2769
4025
  }, eventsCtx);
2770
4026
  }
2771
4027
  catch (err) {
2772
- allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
4028
+ allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
2773
4029
  }
2774
- finally {
2775
- if (stateDb) {
2776
- try {
2777
- stateDb.close();
4030
+ }
4031
+ // Fix #2 (observability 0.8.0): trim the events table in state.db so it
4032
+ // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
4033
+ // invocation, and every command surface emits at least one event besides —
4034
+ // without this trim, state.db is a permanent append-only log. Config key
4035
+ // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
4036
+ // window. The purge runs against state.db (a different SQLite file from
4037
+ // the index `db` above).
4038
+ {
4039
+ const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
4040
+ if (retentionDays > 0) {
4041
+ // #585: reuse the long-lived eventsCtx.db connection when akmImprove
4042
+ // opened one — opening a second state.db write connection while
4043
+ // eventsDb is still live made two simultaneous writers contend on the
4044
+ // same WAL file ("database is locked"). Only the eventsCtx.dbPath
4045
+ // fallback path (state.db failed to open up-front) opens — and then
4046
+ // owns and closes — its own handle. C2 still holds: the fallback uses
4047
+ // the boundary-pinned path, never a live `process.env` re-read.
4048
+ const ownsStateDb = !eventsCtx?.db;
4049
+ let stateDb;
4050
+ try {
4051
+ stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
4052
+ const purgedCount = purgeOldEvents(stateDb, retentionDays);
4053
+ if (purgedCount > 0) {
4054
+ info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
2778
4055
  }
2779
- catch {
2780
- // best-effort
4056
+ appendEvent({
4057
+ eventType: "events_purged",
4058
+ ref: "events:_purge",
4059
+ metadata: { purgedCount, retentionDays },
4060
+ }, eventsCtx);
4061
+ // improve_runs uses the same retention window as events — both are
4062
+ // observability/audit data, both grow append-only, both have a
4063
+ // dedicated purge helper. Mirroring the events purge here means a
4064
+ // single retention knob (improve.eventRetentionDays) governs both.
4065
+ const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
4066
+ if (improveRunsPurged > 0) {
4067
+ info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
4068
+ }
4069
+ appendEvent({
4070
+ eventType: "improve_runs_purged",
4071
+ ref: "improve_runs:_purge",
4072
+ metadata: { purgedCount: improveRunsPurged, retentionDays },
4073
+ }, eventsCtx);
4074
+ }
4075
+ catch (err) {
4076
+ allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
4077
+ }
4078
+ finally {
4079
+ if (ownsStateDb && stateDb) {
4080
+ try {
4081
+ stateDb.close();
4082
+ }
4083
+ catch {
4084
+ // best-effort
4085
+ }
4086
+ }
4087
+ }
4088
+ // task_logs in logs.db (#579) shares the same retention window as
4089
+ // events/improve_runs — all three are observability data governed by
4090
+ // the single improve.eventRetentionDays knob. Separate try/finally
4091
+ // because logs.db is a different file: a locked/missing logs.db must
4092
+ // not block the state.db purges above.
4093
+ let logsDb;
4094
+ try {
4095
+ logsDb = openLogsDatabase();
4096
+ const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
4097
+ if (taskLogsPurged > 0) {
4098
+ info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
4099
+ }
4100
+ appendEvent({
4101
+ eventType: "task_logs_purged",
4102
+ ref: "task_logs:_purge",
4103
+ metadata: { purgedCount: taskLogsPurged, retentionDays },
4104
+ }, eventsCtx);
4105
+ }
4106
+ catch (err) {
4107
+ allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
4108
+ }
4109
+ finally {
4110
+ if (logsDb) {
4111
+ try {
4112
+ logsDb.close();
4113
+ }
4114
+ catch {
4115
+ // best-effort
4116
+ }
2781
4117
  }
2782
4118
  }
2783
4119
  }
2784
4120
  }
2785
- }
2786
- // Phase 4A (staleness detection). Activates the `deprecated` belief-state
2787
- // machinery shipped in Phase 1A. Default OFF gated by
2788
- // `features.index.staleness_detection.enabled`. Runs after orphan purge
2789
- // and before the URL check (which lives in the outer caller).
2790
- if (sources.length > 0) {
2791
- try {
2792
- stalenessDetection = await stalenessDetectionFn({ config, sources, signal: budgetSignal, db });
2793
- if (stalenessDetection.considered > 0) {
2794
- info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
2795
- `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
2796
- `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
4121
+ // Phase 4A (staleness detection). Activates the `deprecated` belief-state
4122
+ // machinery shipped in Phase 1A. Default OFF gated by
4123
+ // `features.index.staleness_detection.enabled`. Runs after orphan purge
4124
+ // and before the URL check (which lives in the outer caller).
4125
+ if (sources.length > 0) {
4126
+ try {
4127
+ stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
4128
+ if (stalenessDetection.considered > 0) {
4129
+ info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
4130
+ `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
4131
+ `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
4132
+ }
4133
+ for (const w of stalenessDetection.warnings)
4134
+ allWarnings.push(`[improve] staleness detection: ${w}`);
4135
+ }
4136
+ catch (err) {
4137
+ allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
2797
4138
  }
2798
- for (const w of stalenessDetection.warnings)
2799
- allWarnings.push(`[improve] staleness detection: ${w}`);
2800
- }
2801
- catch (err) {
2802
- allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
2803
4139
  }
2804
4140
  }
2805
- }
2806
- finally {
2807
- if (db)
2808
- closeDatabase(db);
2809
- }
4141
+ finally {
4142
+ if (db)
4143
+ closeDatabase(db);
4144
+ }
4145
+ });
2810
4146
  return {
2811
4147
  ...(memoryInference ? { memoryInference } : {}),
2812
4148
  ...(graphExtraction ? { graphExtraction } : {}),