akm-cli 0.9.0-beta.4 → 0.9.0-beta.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +646 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +6 -2
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +22 -0
  16. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +22 -0
  17. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +24 -0
  18. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +22 -0
  19. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +25 -0
  20. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +21 -0
  21. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +21 -0
  22. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +23 -0
  23. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +22 -0
  24. package/dist/assets/templates/html/health.html +281 -111
  25. package/dist/cli.js +14 -3
  26. package/dist/commands/agent/contribute-cli.js +16 -3
  27. package/dist/commands/feedback-cli.js +15 -6
  28. package/dist/commands/graph/graph.js +75 -71
  29. package/dist/commands/health/checks.js +48 -0
  30. package/dist/commands/health/html-report.js +422 -80
  31. package/dist/commands/health.js +381 -9
  32. package/dist/commands/improve/calibration.js +161 -0
  33. package/dist/commands/improve/consolidate.js +631 -111
  34. package/dist/commands/improve/dedup.js +482 -0
  35. package/dist/commands/improve/distill.js +163 -69
  36. package/dist/commands/improve/encoding-salience.js +205 -0
  37. package/dist/commands/improve/extract-cli.js +115 -1
  38. package/dist/commands/improve/extract-prompt.js +39 -2
  39. package/dist/commands/improve/extract-watch.js +140 -0
  40. package/dist/commands/improve/extract.js +403 -40
  41. package/dist/commands/improve/feedback-valence.js +54 -0
  42. package/dist/commands/improve/homeostatic.js +467 -0
  43. package/dist/commands/improve/improve-auto-accept.js +113 -6
  44. package/dist/commands/improve/improve-profiles.js +12 -0
  45. package/dist/commands/improve/improve.js +2042 -612
  46. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  47. package/dist/commands/improve/outcome-loop.js +256 -0
  48. package/dist/commands/improve/proactive-maintenance.js +115 -0
  49. package/dist/commands/improve/procedural.js +418 -0
  50. package/dist/commands/improve/recombine.js +602 -0
  51. package/dist/commands/improve/reflect-noise.js +0 -0
  52. package/dist/commands/improve/reflect.js +46 -4
  53. package/dist/commands/improve/related-sessions.js +120 -0
  54. package/dist/commands/improve/salience.js +438 -0
  55. package/dist/commands/improve/triage.js +93 -0
  56. package/dist/commands/lint/agent-linter.js +19 -24
  57. package/dist/commands/lint/base-linter.js +173 -60
  58. package/dist/commands/lint/command-linter.js +19 -24
  59. package/dist/commands/lint/env-key-rules.js +34 -1
  60. package/dist/commands/lint/fact-linter.js +39 -0
  61. package/dist/commands/lint/index.js +31 -13
  62. package/dist/commands/lint/memory-linter.js +1 -1
  63. package/dist/commands/lint/registry.js +7 -2
  64. package/dist/commands/lint/task-linter.js +3 -3
  65. package/dist/commands/lint/workflow-linter.js +26 -1
  66. package/dist/commands/proposal/drain-policies.js +5 -0
  67. package/dist/commands/proposal/drain.js +17 -1
  68. package/dist/commands/proposal/proposal.js +5 -0
  69. package/dist/commands/proposal/propose.js +5 -0
  70. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  71. package/dist/commands/proposal/validators/proposals.js +187 -57
  72. package/dist/commands/read/curate.js +344 -80
  73. package/dist/commands/read/search-cli.js +7 -0
  74. package/dist/commands/read/search.js +1 -0
  75. package/dist/commands/read/show.js +67 -2
  76. package/dist/commands/sources/init.js +36 -9
  77. package/dist/commands/sources/installed-stashes.js +5 -1
  78. package/dist/commands/sources/schema-repair.js +13 -1
  79. package/dist/commands/sources/stash-cli.js +19 -3
  80. package/dist/commands/sources/stash-skeleton.js +23 -8
  81. package/dist/core/asset/asset-registry.js +2 -0
  82. package/dist/core/asset/asset-spec.js +14 -0
  83. package/dist/core/asset/frontmatter.js +166 -167
  84. package/dist/core/asset/markdown.js +8 -0
  85. package/dist/core/authoring-rules.js +83 -0
  86. package/dist/core/config/config-schema.js +274 -2
  87. package/dist/core/config/config.js +2 -2
  88. package/dist/core/logs-db.js +4 -3
  89. package/dist/core/paths.js +3 -0
  90. package/dist/core/standards/resolve-standards-context.js +87 -0
  91. package/dist/core/standards/resolve-stash-standards.js +99 -0
  92. package/dist/core/standards/resolve-type-conventions.js +66 -0
  93. package/dist/core/state-db.js +691 -30
  94. package/dist/indexer/db/db.js +364 -38
  95. package/dist/indexer/db/graph-db.js +129 -86
  96. package/dist/indexer/ensure-index.js +152 -17
  97. package/dist/indexer/graph/graph-boost.js +51 -41
  98. package/dist/indexer/graph/graph-extraction.js +203 -3
  99. package/dist/indexer/index-writer-lock.js +99 -0
  100. package/dist/indexer/indexer.js +114 -111
  101. package/dist/indexer/passes/memory-inference.js +10 -3
  102. package/dist/indexer/passes/staleness-detect.js +2 -5
  103. package/dist/indexer/search/db-search.js +15 -4
  104. package/dist/indexer/search/ranking-contributors.js +22 -0
  105. package/dist/indexer/search/ranking.js +4 -0
  106. package/dist/indexer/walk/matchers.js +9 -0
  107. package/dist/integrations/agent/prompts.js +33 -0
  108. package/dist/integrations/harnesses/claude/session-log.js +11 -1
  109. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  110. package/dist/integrations/session-logs/index.js +16 -0
  111. package/dist/llm/client.js +23 -4
  112. package/dist/llm/embedder.js +27 -3
  113. package/dist/llm/embedders/local.js +66 -2
  114. package/dist/llm/feature-gate.js +8 -4
  115. package/dist/llm/graph-extract.js +2 -1
  116. package/dist/llm/memory-infer.js +4 -8
  117. package/dist/llm/metadata-enhance.js +9 -1
  118. package/dist/output/renderers.js +73 -1
  119. package/dist/output/shapes/curate.js +14 -2
  120. package/dist/output/text/helpers.js +16 -1
  121. package/dist/runtime.js +25 -1
  122. package/dist/scripts/migrate-storage.js +1378 -599
  123. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +479 -270
  124. package/dist/setup/setup.js +3 -3
  125. package/dist/sources/providers/git.js +71 -61
  126. package/dist/sources/providers/tar-utils.js +16 -8
  127. package/dist/storage/sqlite-pragmas.js +146 -0
  128. package/dist/wiki/wiki.js +37 -0
  129. package/dist/workflows/db.js +3 -4
  130. package/dist/workflows/validate-summary.js +2 -7
  131. package/docs/data-and-telemetry.md +1 -0
  132. package/package.json +8 -6
@@ -14,20 +14,21 @@ import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "
14
14
  import { classifyImproveAction } from "../../core/improve-types.js";
15
15
  import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
16
16
  import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
17
- import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
17
+ import { listProposalGateDecisions, listStateProposals, openStateDatabase, persistPhaseThreshold, purgeOldEvents, purgeOldImproveRuns, } from "../../core/state-db.js";
18
18
  import { info, warn } from "../../core/warn.js";
19
19
  import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
20
20
  import { ensureIndex } from "../../indexer/ensure-index.js";
21
21
  import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
22
+ import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
22
23
  import { akmIndex } from "../../indexer/indexer.js";
23
- import { runMemoryInferencePass } from "../../indexer/passes/memory-inference.js";
24
+ import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
24
25
  import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
25
26
  import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
26
27
  import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
27
28
  import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
28
29
  import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
29
30
  import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
30
- import { isLlmFeatureEnabled, isProcessEnabled } from "../../llm/feature-gate.js";
31
+ import { isProcessEnabled } from "../../llm/feature-gate.js";
31
32
  import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
32
33
  import { withLlmStage } from "../../llm/usage-telemetry.js";
33
34
  import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
@@ -37,16 +38,120 @@ import { resolveDrainPolicy } from "../proposal/drain-policies.js";
37
38
  import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
38
39
  import { runSchemaRepairPass } from "../sources/schema-repair.js";
39
40
  import { checkDeadUrls } from "../url-checker.js";
41
+ import { computeThresholdAutoTune, gateDecisionsToSamples, summarizeCalibration, } from "./calibration.js";
40
42
  import { akmConsolidate } from "./consolidate.js";
41
43
  import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
42
44
  import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
43
45
  import { countEvalCases, writeEvalCase } from "./eval-cases.js";
44
46
  import { akmExtract, countNewExtractCandidates } from "./extract.js";
47
+ import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
45
48
  import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
46
49
  import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
47
50
  import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
48
51
  import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
52
+ import { computeProxyAdequacy, getAllAssetOutcomes, getOutcomeScoresByRef, outcomeScoreToSalience, updateAssetOutcome, } from "./outcome-loop.js";
53
+ import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, filterProactiveDue, selectProactiveMaintenanceRefs, } from "./proactive-maintenance.js";
54
+ import { akmProcedural } from "./procedural.js";
55
+ import { akmRecombine } from "./recombine.js";
49
56
  import { akmReflect } from "./reflect.js";
57
+ import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getConsecutiveNoOps, getLastUseMsByRef, isContentEncodingRow, recordNoOp, resetConsecutiveNoOps, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
58
+ // #607 Lock Decomposition: fine-grained per-process locks replace the single
59
+ // `improve.lock`. Three independent locks allow concurrent improve runs when
60
+ // they touch different subsystems (e.g. quick-shredder consolidate can run
61
+ // alongside daily reflect+distill).
62
+ //
63
+ // consolidate.lock — protects consolidate + memoryInference (both write index.db)
64
+ // reflect-distill.lock — protects reflect + distill (both write state.db proposals)
65
+ // triage.lock — protects triage (writes proposal promotions)
66
+ //
67
+ // Stale timeouts are per-lock, tuned to the expected runtime of the protected
68
+ // processes: consolidate is disk-bound (1h), reflect+distill is GPU-bound (2h),
69
+ // triage is fast (30min).
70
+ const PROCESS_LOCK_DEFS = {
71
+ consolidate: { fileName: "consolidate.lock", staleAfterMs: 60 * 60 * 1000 },
72
+ reflectDistill: { fileName: "reflect-distill.lock", staleAfterMs: 2 * 60 * 60 * 1000 },
73
+ triage: { fileName: "triage.lock", staleAfterMs: 30 * 60 * 1000 },
74
+ };
75
+ const heldProcessLocks = new Set();
76
+ export function resetHeldProcessLocks() {
77
+ heldProcessLocks.clear();
78
+ }
79
+ function processLockPath(lockBaseDir, lockName) {
80
+ return path.join(lockBaseDir, PROCESS_LOCK_DEFS[lockName].fileName);
81
+ }
82
+ function tryAcquireProcessLock(lockPath, staleAfterMs, skipIfLocked, lockLabel) {
83
+ fs.mkdirSync(path.dirname(lockPath), { recursive: true });
84
+ const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
85
+ if (tryAcquireLockSync(lockPath, lockPayload())) {
86
+ heldProcessLocks.add(lockPath);
87
+ return "acquired";
88
+ }
89
+ const probe = probeLock(lockPath, { staleAfterMs });
90
+ const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
91
+ const lock = rawContent
92
+ ? (() => {
93
+ try {
94
+ return JSON.parse(rawContent);
95
+ }
96
+ catch {
97
+ return null;
98
+ }
99
+ })()
100
+ : null;
101
+ if (probe.state === "stale") {
102
+ try {
103
+ appendEvent({
104
+ eventType: "improve_lock_recovered",
105
+ metadata: {
106
+ lockName: lockLabel,
107
+ stalePid: lock?.pid ?? null,
108
+ lockedAt: lock?.startedAt ?? null,
109
+ recoveredAt: new Date().toISOString(),
110
+ lockAgeMs: probe.ageMs ?? null,
111
+ reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
112
+ },
113
+ });
114
+ }
115
+ catch {
116
+ /* event emission is best-effort; never block lock recovery */
117
+ }
118
+ releaseLock(lockPath);
119
+ if (tryAcquireLockSync(lockPath, lockPayload())) {
120
+ heldProcessLocks.add(lockPath);
121
+ return "acquired";
122
+ }
123
+ if (skipIfLocked) {
124
+ warn(`[improve] ${lockLabel} lock acquired by another run during stale recovery; skipping (--skip-if-locked)`);
125
+ return "skipped";
126
+ }
127
+ throw new ConfigError(`akm improve ${lockLabel} is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
128
+ }
129
+ if (skipIfLocked) {
130
+ warn(`[improve] ${lockLabel} lock held by another run (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
131
+ return "skipped";
132
+ }
133
+ throw new ConfigError(`akm improve ${lockLabel} is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
134
+ }
135
+ function releaseProcessLock(lockPath) {
136
+ try {
137
+ fs.unlinkSync(lockPath);
138
+ }
139
+ catch {
140
+ // ignore
141
+ }
142
+ heldProcessLocks.delete(lockPath);
143
+ }
144
+ function releaseAllProcessLocks() {
145
+ for (const p of heldProcessLocks) {
146
+ try {
147
+ fs.unlinkSync(p);
148
+ }
149
+ catch {
150
+ // ignore
151
+ }
152
+ }
153
+ heldProcessLocks.clear();
154
+ }
50
155
  function resolveImproveScope(scope) {
51
156
  const trimmed = scope?.trim();
52
157
  if (!trimmed)
@@ -102,6 +207,22 @@ export function renderSyncCommitMessage(template, result, nowMs) {
102
207
  };
103
208
  return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
104
209
  }
210
+ /**
211
+ * Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
212
+ * merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
213
+ * Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
214
+ */
215
+ function dedupeRefs(refs) {
216
+ const seen = new Set();
217
+ const out = [];
218
+ for (const r of refs) {
219
+ if (seen.has(r.ref))
220
+ continue;
221
+ seen.add(r.ref);
222
+ out.push(r);
223
+ }
224
+ return out;
225
+ }
105
226
  async function collectEligibleRefs(scope, stashDir, improveProfile) {
106
227
  if (scope.mode === "ref" && scope.value) {
107
228
  const parsed = parseAssetRef(scope.value);
@@ -454,6 +575,107 @@ export function armBudgetWatchdog(budgetMs, controller, deps) {
454
575
  }
455
576
  };
456
577
  }
578
+ /**
579
+ * #612 / WS-4 — bounded, opt-in per-phase auto-accept threshold auto-tune.
580
+ *
581
+ * Reads `improve.calibration` from config. When `autoTune` is enabled, computes
582
+ * the calibration of recent gate decisions, derives a bounded threshold
583
+ * adjustment (clamped into the configured band, capped per step), logs it, and
584
+ * records a `calibration_autotune` event. Returns the new threshold (integer
585
+ * 0-100) when an adjustment was made, or `undefined` to leave the caller's
586
+ * threshold unchanged.
587
+ *
588
+ * WS-4 change: accepts an optional `phase` parameter. When provided, the tuned
589
+ * threshold is persisted to `improve_gate_thresholds` (state.db Migration 012)
590
+ * keyed by phase so `makeGateConfig` can read it back on the next run and each
591
+ * phase maintains its own calibrated threshold rather than a shared global.
592
+ *
593
+ * WS-4 ceiling: `maxThreshold` defaults to 85 (not 100) to prevent the gate
594
+ * converging to pure exploitation and shutting down Gap-3/4 novelty throughput.
595
+ *
596
+ * DEFAULT OFF: with no `improve.calibration` block (or `autoTune: false`) this
597
+ * returns `undefined` immediately, so the gate threshold is unchanged and
598
+ * behaviour is byte-identical to today.
599
+ */
600
+ export function maybeAutoTuneThreshold(currentThreshold, config, stateDbPath, ctx, phase) {
601
+ const cal = config.improve?.calibration;
602
+ if (!cal?.autoTune)
603
+ return undefined;
604
+ // WS-4: default maxThreshold is now 85 (ceiling to prevent pure exploitation).
605
+ // Callers that explicitly set maxThreshold in config override this default.
606
+ const tuneConfig = {
607
+ autoTune: true,
608
+ minThreshold: cal.minThreshold ?? 0,
609
+ maxThreshold: cal.maxThreshold ?? 85,
610
+ maxStep: cal.maxStep ?? 5,
611
+ minSamples: cal.minSamples ?? 20,
612
+ targetAcceptRate: cal.targetAcceptRate ?? 0.9,
613
+ };
614
+ // Defensive: an inverted band disables tuning rather than clamping to nonsense.
615
+ if (tuneConfig.minThreshold > tuneConfig.maxThreshold)
616
+ return undefined;
617
+ const db = openStateDatabase(stateDbPath);
618
+ let summary;
619
+ try {
620
+ const allDecisions = listProposalGateDecisions(db);
621
+ // WS-4 fix: when called with a phase label, restrict calibration to that
622
+ // phase's decision pool so a reflect-dominated run cannot tighten the
623
+ // consolidate gate (or vice-versa). The gate field is `improve:<phase>`,
624
+ // matching what improve-auto-accept.ts stamps at line ~163.
625
+ const gateLabel = phase ? `improve:${phase}` : undefined;
626
+ const decisions = gateLabel ? allDecisions.filter((d) => d.gate === gateLabel) : allDecisions;
627
+ summary = summarizeCalibration(gateDecisionsToSamples(decisions));
628
+ }
629
+ finally {
630
+ db.close();
631
+ }
632
+ const result = computeThresholdAutoTune(currentThreshold, summary, tuneConfig);
633
+ if (!result.adjusted)
634
+ return undefined;
635
+ const appendEventFn = ctx?.appendEventFn ?? appendEvent;
636
+ const phaseLabel = phase ?? "global";
637
+ info(`[improve] calibration auto-tune (${phaseLabel}): threshold ${result.previousThreshold} -> ${result.newThreshold} ` +
638
+ `(${result.reason}; samples=${summary.samples}, acceptRate=${summary.overallAcceptRate}, ` +
639
+ `gap=${summary.calibrationGap}, band=[${tuneConfig.minThreshold},${tuneConfig.maxThreshold}])`);
640
+ try {
641
+ appendEventFn({
642
+ eventType: "calibration_autotune",
643
+ ref: "improve:calibration",
644
+ metadata: {
645
+ phase: phaseLabel,
646
+ previousThreshold: result.previousThreshold,
647
+ newThreshold: result.newThreshold,
648
+ delta: result.delta,
649
+ reason: result.reason,
650
+ samples: summary.samples,
651
+ overallAcceptRate: summary.overallAcceptRate,
652
+ calibrationGap: summary.calibrationGap,
653
+ minThreshold: tuneConfig.minThreshold,
654
+ maxThreshold: tuneConfig.maxThreshold,
655
+ },
656
+ });
657
+ }
658
+ catch (err) {
659
+ warn(`[improve] calibration auto-tune event not recorded: ${err instanceof Error ? err.message : String(err)}`);
660
+ }
661
+ // WS-4: Persist the per-phase threshold so makeGateConfig reads it on the
662
+ // next run. Best-effort — a write failure must not abort the improve run.
663
+ if (phase) {
664
+ try {
665
+ const persistDb = openStateDatabase(stateDbPath);
666
+ try {
667
+ persistPhaseThreshold(persistDb, phase, result.newThreshold);
668
+ }
669
+ finally {
670
+ persistDb.close();
671
+ }
672
+ }
673
+ catch (err) {
674
+ warn(`[improve] calibration auto-tune: failed to persist phase threshold for ${phase}: ${err instanceof Error ? err.message : String(err)}`);
675
+ }
676
+ }
677
+ return result.newThreshold;
678
+ }
457
679
  export async function akmImprove(options = {}) {
458
680
  const scope = resolveImproveScope(options.scope);
459
681
  const reflectFn = options.reflectFn ?? akmReflect;
@@ -461,6 +683,11 @@ export async function akmImprove(options = {}) {
461
683
  const ensureIndexFn = options.ensureIndexFn ?? ensureIndex;
462
684
  const reindexFn = options.reindexFn ?? akmIndex;
463
685
  const drainProposalsFn = options.drainProposalsFn ?? drainProposals;
686
+ // #616 multi-cycle test seams. Default to the real module-local fns.
687
+ const collectEligibleRefsImpl = options.collectEligibleRefsFn ?? collectEligibleRefs;
688
+ const runImprovePreparationStageImpl = options.runImprovePreparationStageFn ?? runImprovePreparationStage;
689
+ const runImproveLoopStageImpl = options.runImproveLoopStageFn ?? runImproveLoopStage;
690
+ const runImprovePostLoopStageImpl = options.runImprovePostLoopStageFn ?? runImprovePostLoopStage;
464
691
  // Resolve the improve profile for this run. Profile drives type filtering,
465
692
  // process gating, and default autoAccept/limit values.
466
693
  const _earlyConfig = options.config ?? loadConfig();
@@ -471,8 +698,13 @@ export async function akmImprove(options = {}) {
471
698
  options = {
472
699
  ...options,
473
700
  autoAccept: options.autoAccept ?? improveProfile.autoAccept,
474
- limit: options.limit ?? improveProfile.limit,
701
+ // Profile-level limit, then process-level reflect.limit as fallback.
702
+ // CLI --limit takes precedence over both.
703
+ limit: options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit,
475
704
  };
705
+ // #616 — bounded multi-cycle phasing. CLI/programmatic override wins over
706
+ // profile.maxCycles; default 1 => single pass (byte-identical to pre-#616).
707
+ const maxCycles = Math.max(1, Math.trunc(options.maxCycles ?? improveProfile.maxCycles ?? 1));
476
708
  let primaryStashDir;
477
709
  try {
478
710
  primaryStashDir = resolveSourceEntries(options.stashDir)[0]?.path;
@@ -489,174 +721,51 @@ export async function akmImprove(options = {}) {
489
721
  // timeout root cause). Because beforeEach runs synchronously, env is still the
490
722
  // calling test's own at this point; we capture it before yielding the loop.
491
723
  const resolvedStateDbPath = getStateDbPathInDataDir();
492
- // Phase 4 lock hoist (§7): the `improve.lock` setup is hoisted ABOVE
493
- // ensureIndex/collectEligibleRefs so the triage pre-pass (and improve's own
494
- // queue writes) run fully serialized under the lock. The dry-run early-return
495
- // below still skips the lock and triage (the lock+triage block is gated on
496
- // `!options.dryRun`); contradiction-detection and memory-cleanup analysis,
497
- // which previously ran before the lock, now sit after it for free.
498
- const resolvedLockPath = primaryStashDir
499
- ? path.join(primaryStashDir, ".akm", "improve.lock")
500
- : path.join(options.stashDir ?? ".", ".akm", "improve.lock");
501
- const MAX_LOCK_AGE_MS = 4 * 60 * 60 * 1000; // 4 hours
502
- const acquireLock = () => {
503
- fs.mkdirSync(path.dirname(resolvedLockPath), { recursive: true });
504
- const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
505
- if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
506
- return "acquired";
507
- // Lock file already exists probe to determine whether it's still held
508
- // or whether the prior run died without cleaning up.
509
- const probe = probeLock(resolvedLockPath, { staleAfterMs: MAX_LOCK_AGE_MS });
510
- const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
511
- const lock = rawContent
512
- ? (() => {
513
- try {
514
- return JSON.parse(rawContent);
515
- }
516
- catch {
517
- return null;
518
- }
519
- })()
520
- : null;
521
- if (probe.state === "stale") {
522
- // O-7 / #394: Emit improve_lock_recovered event before recovery so the
523
- // audit trail records the abnormal prior-run exit (Temporal/Airflow pattern).
524
- try {
525
- appendEvent({
526
- eventType: "improve_lock_recovered",
527
- metadata: {
528
- stalePid: lock?.pid ?? null,
529
- lockedAt: lock?.startedAt ?? null,
530
- recoveredAt: new Date().toISOString(),
531
- lockAgeMs: probe.ageMs ?? null,
532
- reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
533
- },
534
- });
535
- }
536
- catch {
537
- /* event emission is best-effort; never block lock recovery */
538
- }
539
- releaseLock(resolvedLockPath);
540
- if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
541
- return "acquired";
542
- // Lost the race to another run that grabbed the freed stale lock.
543
- if (options.skipIfLocked) {
544
- warn("[improve] another run acquired the lock during stale recovery; skipping (--skip-if-locked)");
545
- return "skipped";
546
- }
547
- throw new ConfigError(`akm improve is already running. Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
548
- }
549
- // Lock is held by a live run within the staleness window.
550
- if (options.skipIfLocked) {
551
- warn(`[improve] another improve run holds the lock (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
552
- return "skipped";
553
- }
554
- throw new ConfigError(`akm improve is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
555
- };
556
- // Phase 4 lock-leak guard (§7 ordering hazard): hoisting `improve.lock` above
557
- // the pre-index region (so the triage pre-pass runs under it) means the lock is
558
- // held while ensureIndex / collectEligibleRefs / contradiction-detection /
559
- // memory-cleanup analysis run — but the main protecting `try { … } finally {
560
- // unlinkSync(resolvedLockPath) }` does not begin until after them. A throw in
561
- // any of those steps would leak the lock. We close that window by wrapping the
562
- // whole region in a try whose catch releases the lock (when held) and
563
- // re-throws. The values this region computes are declared in the outer scope so
564
- // they remain visible to the main run below. The dry-run path never sets
565
- // `lockAcquired`, so its early return releases nothing.
566
- let lockAcquired = false;
567
- const releaseLockOnError = () => {
568
- if (!lockAcquired)
569
- return;
570
- try {
571
- fs.unlinkSync(resolvedLockPath);
572
- }
573
- catch {
574
- // best-effort release on the error path
575
- }
576
- lockAcquired = false;
577
- };
578
- // Signal-safe lock release. The SIGTERM/SIGINT/SIGHUP handler in improve-cli.ts
579
- // calls `process.exit()`, which does NOT run the `finally` below that owns lock
580
- // release — so a cron-timeout SIGTERM leaked `improve.lock` every run.
581
- // `process.exit()` DOES fire `'exit'` listeners, so we release the lock from
582
- // one. `releaseLockIfOwned` only unlinks a lock still owned by this PID, so it
583
- // is safe even if a later run re-acquired it. The listener is removed in the
584
- // `finally` so the normal path stays single-release and repeated in-process
585
- // `akmImprove` calls (tests) do not accumulate listeners.
586
- const releaseLockOnExit = () => {
587
- releaseLockIfOwned(resolvedLockPath, process.pid);
588
- };
724
+ // #612 / WS-4 bounded, OPT-IN per-phase auto-accept threshold auto-tune.
725
+ // DEFAULT OFF: `autoTune: false` (or absent) is a complete no-op.
726
+ //
727
+ // WS-4 change: thresholds are now PER PHASE. The old single global mutation
728
+ // of `options.autoAccept` is retired it caused every phase to share one
729
+ // calibration signal, so a reflect-dominated run could tighten the consolidate
730
+ // gate (or vice-versa). Instead:
731
+ // - Each `makeGateConfig` call reads the phase's stored threshold from
732
+ // state.db (Migration 012) and uses it as `phaseThreshold`, overriding
733
+ // the `globalThreshold` (= options.autoAccept) for that phase.
734
+ // - Per-phase `maybeAutoTuneThreshold` calls fire AFTER each phase's gate
735
+ // has run and persist the new threshold to state.db for the NEXT run.
736
+ // - `options.autoAccept` stays unchanged (it is the operator-supplied
737
+ // baseline, not a mutable run-time state).
738
+ //
739
+ // The global tune call is intentionally removed here. See per-phase calls
740
+ // below (near each makeGateConfig / runAutoAcceptGate block).
741
+ // #607 Lock decomposition: three per-process locks replace the single
742
+ // `improve.lock`. Each process acquires only the lock(s) it needs, so
743
+ // quick-shredder consolidate can run alongside daily reflect+distill.
744
+ //
745
+ // consolidate.lock — protects consolidate + memoryInference + graphExtraction (index.db writers)
746
+ // reflect-distill.lock — protects reflect + distill (state.db proposal writers)
747
+ // triage.lock — protects triage pre-pass (state.db proposal promotions)
748
+ //
749
+ // Lock base directory — same `.akm/` under the primary stash dir.
750
+ const lockBaseDir = primaryStashDir ? path.join(primaryStashDir, ".akm") : path.join(options.stashDir ?? ".", ".akm");
589
751
  const preEnsureCleanupWarnings = [];
590
- let plannedRefs;
591
- let memorySummary;
592
- let profileFilteredRefs;
752
+ // #616: assigned by runIndexAndCollect() (closure) so TS cannot prove definite
753
+ // assignment — seed with empty values; the first runIndexAndCollect() call
754
+ // (cycle 1, in the first try) always overwrites them before any read.
755
+ let plannedRefs = [];
756
+ let memorySummary = { eligible: 0, derived: 0 };
757
+ let profileFilteredRefs = [];
593
758
  let memoryCleanupPlan;
594
759
  let guidance;
595
760
  let triageDrain;
596
- try {
597
- // Acquire the lock and run the triage pre-pass for non-dry-run executions.
598
- // The dry-run branch below produces plannedRefs/memorySummary WITHOUT the lock
599
- // or triage (decision: dry-run never mutates the queue).
600
- if (!options.dryRun) {
601
- if (acquireLock() === "skipped") {
602
- // Another improve holds the lock and the caller asked to skip rather
603
- // than fail. Return a clean no-op result (exit 0) before any index/DB
604
- // work — never registered the exit listener, never set lockAcquired,
605
- // so we release nothing belonging to the run that owns the lock.
606
- return {
607
- schemaVersion: 1,
608
- ok: true,
609
- scope,
610
- dryRun: false,
611
- skipped: { reason: "lock-held" },
612
- memorySummary: { eligible: 0, derived: 0 },
613
- plannedRefs: [],
614
- };
615
- }
616
- lockAcquired = true;
617
- // Backstop release on process.exit() (signal handler / budget watchdog),
618
- // which skips the finally below. Removed in that finally on the normal path.
619
- process.on("exit", releaseLockOnExit);
620
- // Phase 4 triage pre-pass (§7, §13): drain the standing pending backlog
621
- // BEFORE ensureIndex so improve generates fresh proposals against a cleared
622
- // queue (no `duplicate_pending` collisions) and ensureIndex absorbs triage's
623
- // promotions for free. Gated on the triage process being enabled (opt-in,
624
- // defaults off) and on a whole-stash / type-scoped run — a single-ref
625
- // `akm improve skill:x` must never drain the whole queue. Best-effort: a
626
- // triage failure is a non-fatal warning, never an abort (mirrors the
627
- // contradiction-detection pass below).
628
- if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
629
- if (scope.mode === "ref") {
630
- warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
631
- }
632
- else {
633
- try {
634
- const triageConfig = improveProfile.processes?.triage;
635
- const policy = resolveDrainPolicy(triageConfig?.policy);
636
- const applyMode = triageConfig?.applyMode ?? "queue";
637
- const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
638
- const judgment = triageConfig?.judgment
639
- ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
640
- : null;
641
- triageDrain = await drainProposalsFn({
642
- stashDir: primaryStashDir,
643
- policy,
644
- applyMode,
645
- maxAccepts,
646
- dryRun: false,
647
- // No fresh ids exist yet — triage runs before improve generates any.
648
- excludeIds: new Set(),
649
- ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
650
- judgment,
651
- });
652
- }
653
- catch (err) {
654
- // Non-fatal: triage is a best-effort pre-pass and must never abort improve.
655
- warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
656
- }
657
- }
658
- }
659
- }
761
+ // #616 — ensureIndex + collectEligibleRefs + memory-cleanup recompute, lifted
762
+ // into a helper so the SAME sequence runs once for cycle 1 (below, in the
763
+ // first try) and is re-run at the top of each subsequent multi-cycle cycle.
764
+ // Re-running ensureIndex between cycles makes cycle N's gate-promoted
765
+ // proposals visible to cycle N+1's collectEligibleRefs. Mutates the
766
+ // outer-scope plannedRefs/memorySummary/profileFilteredRefs/memoryCleanupPlan/
767
+ // guidance so for maxCycles:1 the body is byte-identical to pre-#616.
768
+ const runIndexAndCollect = async () => {
660
769
  // #339 fix: ensureIndex MUST run BEFORE collectEligibleRefs. The eligible-ref
661
770
  // query reads the `entries` table; if a DB version upgrade just dropped that
662
771
  // table (or the index is otherwise empty), the prior run order silently
@@ -684,7 +793,7 @@ export async function akmImprove(options = {}) {
684
793
  // best-effort; leave preEnsureEntryCount undefined
685
794
  }
686
795
  try {
687
- await ensureIndexFn(primaryStashDir);
796
+ await ensureIndexFn(primaryStashDir, { mode: "blocking" });
688
797
  }
689
798
  catch (err) {
690
799
  preEnsureCleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -714,7 +823,7 @@ export async function akmImprove(options = {}) {
714
823
  }
715
824
  }
716
825
  }
717
- ({ plannedRefs, memorySummary, profileFilteredRefs } = await collectEligibleRefs(scope, options.stashDir, improveProfile));
826
+ ({ plannedRefs, memorySummary, profileFilteredRefs } = await collectEligibleRefsImpl(scope, options.stashDir, improveProfile));
718
827
  const cleanupParentRef = memoryCleanupParentRef(scope, options.stashDir);
719
828
  // M-1 (#367): Run contradiction-detection BEFORE analyzeMemoryCleanup so
720
829
  // the SCC resolver in resolveFamilyContradictions has edges to work on.
@@ -736,6 +845,69 @@ export async function akmImprove(options = {}) {
736
845
  memorySummary.eligible > 0
737
846
  ? "Improve folds memory cleanup into the same proposal queue: speculative promotions still go through reflect/distill proposals, while high-confidence redundant derived memories are moved into a recoverable cleanup archive instead of being left active in the stash."
738
847
  : undefined;
848
+ };
849
+ try {
850
+ // #607: Per-process lock acquisition. Each process acquires only the lock(s)
851
+ // it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
852
+ // locks (decision: dry-run never mutates the queue).
853
+ if (!options.dryRun) {
854
+ // Backstop release on process.exit() (signal handler / budget watchdog),
855
+ // which skips the finally below. Removed in that finally on the normal path.
856
+ const releaseAllOnExit = () => {
857
+ for (const p of heldProcessLocks) {
858
+ releaseLockIfOwned(p, process.pid);
859
+ }
860
+ };
861
+ process.on("exit", releaseAllOnExit);
862
+ // #607 triage pre-pass: acquire triage.lock, drain the standing pending
863
+ // backlog BEFORE ensureIndex so improve generates fresh proposals against
864
+ // a cleared queue (no `duplicate_pending` collisions) and ensureIndex
865
+ // absorbs triage's promotions for free. Release immediately after —
866
+ // triage.lock is not needed again until the next improve run.
867
+ if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
868
+ if (scope.mode === "ref") {
869
+ warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
870
+ }
871
+ else {
872
+ const triageLPath = processLockPath(lockBaseDir, "triage");
873
+ const triageResult = tryAcquireProcessLock(triageLPath, PROCESS_LOCK_DEFS.triage.staleAfterMs, options.skipIfLocked, "triage");
874
+ if (triageResult === "skipped") {
875
+ triageDrain = undefined;
876
+ }
877
+ else {
878
+ try {
879
+ const triageConfig = improveProfile.processes?.triage;
880
+ const policy = resolveDrainPolicy(triageConfig?.policy);
881
+ const applyMode = triageConfig?.applyMode ?? "queue";
882
+ const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
883
+ const judgment = triageConfig?.judgment
884
+ ? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
885
+ : null;
886
+ triageDrain = await drainProposalsFn({
887
+ stashDir: primaryStashDir,
888
+ policy,
889
+ applyMode,
890
+ maxAccepts,
891
+ dryRun: false,
892
+ excludeIds: new Set(),
893
+ ...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
894
+ judgment,
895
+ });
896
+ }
897
+ catch (err) {
898
+ warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
899
+ }
900
+ finally {
901
+ releaseProcessLock(triageLPath);
902
+ }
903
+ }
904
+ }
905
+ }
906
+ }
907
+ // #339 fix: ensureIndex MUST run BEFORE collectEligibleRefs (now inside the
908
+ // helper). Cycle 1 runs it here; subsequent multi-cycle cycles re-run it via
909
+ // the same helper at the top of each cycle below.
910
+ await runIndexAndCollect();
739
911
  if (options.dryRun) {
740
912
  const result = {
741
913
  schemaVersion: 1,
@@ -752,17 +924,14 @@ export async function akmImprove(options = {}) {
752
924
  }
753
925
  }
754
926
  catch (err) {
755
- releaseLockOnError();
927
+ releaseAllProcessLocks();
756
928
  throw err;
757
929
  }
758
- // FIX 2 (lock-leak window): everything from here on runs UNDER the lock that
759
- // `acquireLock()` just took. The single `try { } finally { unlinkSync(lock) }`
760
- // below now spans the budget-timer setup, `openStateDatabase()`, and the
761
- // `profileFilteredRefs` audit-event loop too regions that previously sat in
762
- // the gap between the lock-acquire catch (above) and the main try. A throw in
763
- // any of them used to leak the lock (blocking the next improve up to 4h);
764
- // now the finally releases it exactly once. The dry-run path already returned
765
- // above without acquiring the lock, so it never reaches this finally; the
930
+ // #607: per-process locks are acquired/released around each stage below.
931
+ // The triage pre-pass already ran under triage.lock (released). The
932
+ // preparation stage runs under consolidate.lock, the loop stage under
933
+ // reflect-distill.lock, and the post-loop stage under consolidate.lock again.
934
+ // Each stage acquires its lock just before starting and releases in finally.
766
935
  // best-effort `unlinkSync` is a no-op when no lock file exists.
767
936
  const startMs = Date.now();
768
937
  const budgetMs = options.timeoutMs ?? 2 * 60 * 60 * 1000; // default 2 hours
@@ -771,6 +940,16 @@ export async function akmImprove(options = {}) {
771
940
  // run past the declared budget.
772
941
  // References: Anthropic *Building Effective Agents* (2024); CoALA §5 (arXiv:2309.02427).
773
942
  const budgetAbortController = new AbortController();
943
+ // Attach a live `remainingBudgetMs` getter to the signal so sub-callers
944
+ // (e.g. consolidate.ts cold-start budget estimation) can read the remaining
945
+ // wall-clock budget without needing an extra plumbing parameter. The property
946
+ // is computed at access time via a getter so it always reflects the actual
947
+ // elapsed time rather than a stale snapshot taken at arm time.
948
+ Object.defineProperty(budgetAbortController.signal, "remainingBudgetMs", {
949
+ get: () => Math.max(0, budgetMs - (Date.now() - startMs)),
950
+ enumerable: false,
951
+ configurable: true,
952
+ });
774
953
  // Declared in the outer scope so the `finally` can clear the timer even if a
775
954
  // throw occurs before/after it is armed. Defaults to a no-op until armed.
776
955
  let clearBudgetTimer = () => { };
@@ -826,71 +1005,213 @@ export async function akmImprove(options = {}) {
826
1005
  },
827
1006
  }, eventsCtx);
828
1007
  }
829
- const preparation = await runImprovePreparationStage({
830
- scope,
831
- options,
832
- plannedRefs,
833
- memoryCleanupPlan,
834
- primaryStashDir,
835
- memorySummary,
836
- reindexFn,
837
- startMs,
838
- budgetMs,
839
- eventsCtx,
840
- initialCleanupWarnings: preEnsureCleanupWarnings,
841
- improveProfile,
842
- });
843
- // D6: pre-load all proposal_rejected events from the last 30 days once,
844
- // so the per-asset loop can use a Map lookup instead of N DB round trips.
845
- const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
846
- const rejectedProposalSince = new Date(Date.now() - REJECTED_PROPOSAL_WINDOW_MS).toISOString();
847
- const allRejectedProposalEvents = readEvents({ type: "proposal_rejected", since: rejectedProposalSince }).events;
848
- const rejectedProposalsByRef = new Map();
849
- for (const e of allRejectedProposalEvents) {
850
- if (e.ref && (!rejectedProposalsByRef.has(e.ref) || e.ts > (rejectedProposalsByRef.get(e.ref)?.ts ?? ""))) {
851
- rejectedProposalsByRef.set(e.ref, e);
852
- }
853
- }
854
- const { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount: loopGateCount, gateAutoAcceptFailedCount: loopGateFailedCount, } = await runImproveLoopStage({
855
- scope,
856
- options,
857
- primaryStashDir,
858
- reflectFn,
859
- distillFn,
860
- loopRefs: preparation.loopRefs,
861
- actions: preparation.actions,
862
- signalBearingSet: preparation.signalBearingSet,
863
- distillCooledRefs: preparation.distillCooledRefs,
864
- distillOnlyRefs: preparation.distillOnlyRefs,
865
- recentErrors: preparation.recentErrors,
866
- rejectedProposalsByRef,
867
- utilityMap: preparation.utilityMap,
868
- startMs,
869
- budgetMs,
870
- eventsCtx,
871
- improveProfile,
872
- });
873
- // #551: consolidation now runs in the preparation stage (before extract);
874
- // its result and run-flag are read from `preparation`, not the post-loop.
875
- const consolidation = preparation.consolidation;
876
- const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
877
- scope,
878
- options,
879
- primaryStashDir,
880
- actionableRefs: preparation.actionableRefs,
881
- appliedCleanup: preparation.appliedCleanup,
882
- cleanupWarnings: preparation.cleanupWarnings,
883
- memoryRefsForInference,
884
- reindexFn,
885
- eventsCtx,
886
- // O-1 (#364): propagate wall-clock budget signal to post-loop maintenance.
887
- budgetSignal: budgetAbortController.signal,
888
- improveProfile,
889
- consolidationRan: preparation.consolidationRan,
890
- });
891
- const finalActions = maintenanceActions && maintenanceActions.length > 0
892
- ? [...preparation.actions, ...maintenanceActions]
893
- : preparation.actions;
1008
+ // #616 bounded multi-cycle phasing. The prep->loop->post-loop sequence is
1009
+ // wrapped in an N-cycle loop. Each cycle re-runs ensureIndex +
1010
+ // collectEligibleRefs (via runIndexAndCollect) so gate-accepted output of
1011
+ // cycle N becomes selectable input to cycle N+1. The per-stage process locks
1012
+ // (consolidate / reflect-distill) are acquired+released INSIDE each cycle,
1013
+ // exactly as the single-pass path did. For maxCycles:1 the loop runs once and
1014
+ // every accumulator below collapses to the single-cycle value (sum-of-one,
1015
+ // concat-of-one, last==only) => BYTE-IDENTICAL to pre-#616.
1016
+ //
1017
+ // Accumulators (see CONSTRAINTS / aggregation plan in #616): SUM the count
1018
+ // fields and durations; CONCAT the array fields; LAST-WINS for point-in-time
1019
+ // objects (the final cycle's value reflects the converged state).
1020
+ let cyclesRun = 0;
1021
+ // Last-wins point-in-time values (assigned every cycle; the final cycle wins).
1022
+ let preparation;
1023
+ let memoryRefsForInference = new Set();
1024
+ let consolidation;
1025
+ let memoryInference;
1026
+ let graphExtraction;
1027
+ let stalenessDetection;
1028
+ let recombination;
1029
+ let proceduralCompilation;
1030
+ // Summed counters/durations.
1031
+ let prepGateCount = 0;
1032
+ let prepGateFailedCount = 0;
1033
+ let reflectsWithErrorContext = 0;
1034
+ let loopGateCount = 0;
1035
+ let loopGateFailedCount = 0;
1036
+ let postLoopGateCount = 0;
1037
+ let postLoopGateFailedCount = 0;
1038
+ let memoryInferenceDurationMs = 0;
1039
+ let graphExtractionDurationMs = 0;
1040
+ let orphansPurged;
1041
+ let proposalsExpired;
1042
+ // Concatenated arrays.
1043
+ const allWarnings = [];
1044
+ let deadUrls;
1045
+ const finalActions = [];
1046
+ for (let cycleIndex = 0; cycleIndex < maxCycles; cycleIndex++) {
1047
+ // #616 budget gate: never start a NEW cycle once the run's wall-clock
1048
+ // budget is exhausted (or the run was aborted). Cycle 0 ALWAYS runs so
1049
+ // maxCycles:1 is byte-identical regardless of budget.
1050
+ if (cycleIndex > 0) {
1051
+ const remaining = budgetAbortController.signal.remainingBudgetMs;
1052
+ if (budgetAbortController.signal.aborted || (remaining !== undefined && remaining <= 0)) {
1053
+ break;
1054
+ }
1055
+ }
1056
+ // Re-run ensureIndex + collectEligibleRefs + memory-cleanup recompute for
1057
+ // cycles 2+ (cycle 1 already ran them in the first try above). This makes
1058
+ // cycle N's gate-promoted proposals visible to this cycle's ref selection.
1059
+ if (cycleIndex > 0) {
1060
+ await runIndexAndCollect();
1061
+ // Re-emit the profile-filtered audit summary for this cycle's selection.
1062
+ if (profileFilteredRefs.length > 0) {
1063
+ appendEvent({
1064
+ eventType: "improve_skipped",
1065
+ ref: undefined,
1066
+ metadata: { reason: "profile_filtered_all_passes", count: profileFilteredRefs.length },
1067
+ }, eventsCtx);
1068
+ }
1069
+ }
1070
+ // #607: acquire consolidate.lock for the preparation stage (consolidate,
1071
+ // ensureIndex, extract all write index.db). Released immediately after.
1072
+ const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
1073
+ const consolidatePrepAcquired = tryAcquireProcessLock(consolidateLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
1074
+ preparation = await runImprovePreparationStageImpl({
1075
+ scope,
1076
+ options,
1077
+ plannedRefs,
1078
+ memoryCleanupPlan,
1079
+ primaryStashDir,
1080
+ memorySummary,
1081
+ reindexFn,
1082
+ startMs,
1083
+ budgetMs,
1084
+ eventsCtx,
1085
+ initialCleanupWarnings: preEnsureCleanupWarnings,
1086
+ improveProfile,
1087
+ budgetSignal: budgetAbortController.signal,
1088
+ });
1089
+ if (consolidatePrepAcquired)
1090
+ releaseProcessLock(consolidateLPath);
1091
+ prepGateCount += preparation.gateAutoAcceptedCount;
1092
+ prepGateFailedCount += preparation.gateAutoAcceptFailedCount;
1093
+ // D6: pre-load all proposal_rejected events from the last 30 days once,
1094
+ // so the per-asset loop can use a Map lookup instead of N DB round trips.
1095
+ const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
1096
+ const rejectedProposalSince = new Date(Date.now() - REJECTED_PROPOSAL_WINDOW_MS).toISOString();
1097
+ const allRejectedProposalEvents = readEvents({ type: "proposal_rejected", since: rejectedProposalSince }).events;
1098
+ const rejectedProposalsByRef = new Map();
1099
+ for (const e of allRejectedProposalEvents) {
1100
+ if (e.ref && (!rejectedProposalsByRef.has(e.ref) || e.ts > (rejectedProposalsByRef.get(e.ref)?.ts ?? ""))) {
1101
+ rejectedProposalsByRef.set(e.ref, e);
1102
+ }
1103
+ }
1104
+ // #607: acquire reflect-distill.lock for the loop stage (reflect + distill
1105
+ // both write proposals to state.db). Released immediately after.
1106
+ const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
1107
+ const reflectDistillAcquired = tryAcquireProcessLock(reflectDistillLPath, PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs, options.skipIfLocked, "reflect-distill") === "acquired";
1108
+ // Post-lock cooldown re-filter for proactive refs (#SELECT-TIME-LEAK).
1109
+ // Planning built `lastReflectProposalTs` BEFORE acquiring this lock, so a
1110
+ // concurrent run's `reflect_invoked` writes are invisible to it. Now that
1111
+ // we hold the lock, re-read fresh timestamp maps for the proactive subset
1112
+ // and drop any ref whose cooldown has been consumed by the concurrent run.
1113
+ const proactiveLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource === "proactive");
1114
+ let postLockLoopRefs = preparation.loopRefs;
1115
+ if (proactiveLoopRefs.length > 0) {
1116
+ const proactiveRefStrs = proactiveLoopRefs.map((r) => r.ref);
1117
+ const freshReflectTs = buildLatestProposalTsMap(proactiveRefStrs, "reflect");
1118
+ const freshDistillTs = buildLatestProposalTsMap(proactiveRefStrs, "distill");
1119
+ const pmDueDays = improveProfile.processes?.proactiveMaintenance?.dueDays ?? DEFAULT_DUE_DAYS;
1120
+ const stillDue = new Set(filterProactiveDue(proactiveLoopRefs, freshReflectTs, freshDistillTs, pmDueDays, Date.now()).map((r) => r.ref));
1121
+ const dropped = proactiveLoopRefs.filter((r) => !stillDue.has(r.ref));
1122
+ if (dropped.length > 0) {
1123
+ info(`[improve] post-lock cooldown re-filter: dropped ${dropped.length} proactive ref(s) claimed by concurrent run (${dropped.map((r) => r.ref).join(", ")})`);
1124
+ postLockLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource !== "proactive" || stillDue.has(r.ref));
1125
+ }
1126
+ }
1127
+ const loopResult = await runImproveLoopStageImpl({
1128
+ scope,
1129
+ options,
1130
+ primaryStashDir,
1131
+ reflectFn,
1132
+ distillFn,
1133
+ loopRefs: postLockLoopRefs,
1134
+ actions: preparation.actions,
1135
+ signalBearingSet: preparation.signalBearingSet,
1136
+ distillCooledRefs: preparation.distillCooledRefs,
1137
+ distillOnlyRefs: preparation.distillOnlyRefs,
1138
+ recentErrors: preparation.recentErrors,
1139
+ rejectedProposalsByRef,
1140
+ utilityMap: preparation.utilityMap,
1141
+ startMs,
1142
+ budgetMs,
1143
+ eventsCtx,
1144
+ improveProfile,
1145
+ budgetSignal: budgetAbortController.signal,
1146
+ });
1147
+ if (reflectDistillAcquired)
1148
+ releaseProcessLock(reflectDistillLPath);
1149
+ const loopGateCountThisCycle = loopResult.gateAutoAcceptedCount;
1150
+ reflectsWithErrorContext += loopResult.reflectsWithErrorContext;
1151
+ loopGateCount += loopResult.gateAutoAcceptedCount;
1152
+ loopGateFailedCount += loopResult.gateAutoAcceptFailedCount;
1153
+ memoryRefsForInference = loopResult.memoryRefsForInference;
1154
+ // #551: consolidation now runs in the preparation stage (before extract);
1155
+ // its result and run-flag are read from `preparation`, not the post-loop.
1156
+ consolidation = preparation.consolidation;
1157
+ // #607: acquire consolidate.lock for the post-loop stage (memoryInference +
1158
+ // graphExtraction both write index.db). Released immediately after.
1159
+ const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
1160
+ const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
1161
+ const postLoopResult = await runImprovePostLoopStageImpl({
1162
+ scope,
1163
+ options,
1164
+ primaryStashDir,
1165
+ actionableRefs: preparation.actionableRefs,
1166
+ appliedCleanup: preparation.appliedCleanup,
1167
+ cleanupWarnings: preparation.cleanupWarnings,
1168
+ memoryRefsForInference,
1169
+ reindexFn,
1170
+ eventsCtx,
1171
+ budgetSignal: budgetAbortController.signal,
1172
+ improveProfile,
1173
+ consolidationRan: preparation.consolidationRan,
1174
+ });
1175
+ if (consolidatePostAcquired)
1176
+ releaseProcessLock(consolidatePostLPath);
1177
+ const postLoopGateCountThisCycle = postLoopResult.gateAutoAcceptedCount;
1178
+ // Last-wins point-in-time objects.
1179
+ memoryInference = postLoopResult.memoryInference;
1180
+ graphExtraction = postLoopResult.graphExtraction;
1181
+ stalenessDetection = postLoopResult.stalenessDetection;
1182
+ recombination = postLoopResult.recombination;
1183
+ proceduralCompilation = postLoopResult.proceduralCompilation;
1184
+ // Summed counters/durations.
1185
+ postLoopGateCount += postLoopResult.gateAutoAcceptedCount;
1186
+ postLoopGateFailedCount += postLoopResult.gateAutoAcceptFailedCount;
1187
+ memoryInferenceDurationMs += postLoopResult.memoryInferenceDurationMs;
1188
+ graphExtractionDurationMs += postLoopResult.graphExtractionDurationMs;
1189
+ if (postLoopResult.orphansPurged !== undefined) {
1190
+ orphansPurged = (orphansPurged ?? 0) + postLoopResult.orphansPurged;
1191
+ }
1192
+ if (postLoopResult.proposalsExpired !== undefined) {
1193
+ proposalsExpired = (proposalsExpired ?? 0) + postLoopResult.proposalsExpired;
1194
+ }
1195
+ // Concatenated arrays.
1196
+ allWarnings.push(...postLoopResult.allWarnings);
1197
+ if (postLoopResult.deadUrls !== undefined) {
1198
+ deadUrls = [...(deadUrls ?? []), ...postLoopResult.deadUrls];
1199
+ }
1200
+ const maintenanceActions = postLoopResult.maintenanceActions;
1201
+ if (maintenanceActions && maintenanceActions.length > 0) {
1202
+ finalActions.push(...preparation.actions, ...maintenanceActions);
1203
+ }
1204
+ else {
1205
+ finalActions.push(...preparation.actions);
1206
+ }
1207
+ cyclesRun++;
1208
+ // #616 fixed-point stop: a cycle that produced ZERO gate-accepted proposals
1209
+ // (summed across prep + loop + post-loop) would feed cycle N+1 an identical
1210
+ // ref set, so end the loop here rather than spin a pointless next cycle.
1211
+ const gateAcceptedThisCycle = preparation.gateAutoAcceptedCount + loopGateCountThisCycle + postLoopGateCountThisCycle;
1212
+ if (gateAcceptedThisCycle === 0)
1213
+ break;
1214
+ }
894
1215
  const result = {
895
1216
  schemaVersion: 1,
896
1217
  ok: true,
@@ -949,17 +1270,19 @@ export async function akmImprove(options = {}) {
949
1270
  ...(memoryInferenceDurationMs > 0 ? { memoryInferenceDurationMs } : {}),
950
1271
  ...(graphExtractionDurationMs > 0 ? { graphExtractionDurationMs } : {}),
951
1272
  ...(stalenessDetection ? { stalenessDetection } : {}),
1273
+ ...(recombination ? { recombination } : {}),
1274
+ ...(proceduralCompilation ? { proceduralCompilation } : {}),
952
1275
  ...(orphansPurged !== undefined ? { orphansPurged } : {}),
953
1276
  ...(proposalsExpired !== undefined && proposalsExpired > 0 ? { proposalsExpired } : {}),
954
1277
  reflectCooldownActions: finalActions.filter((a) => a.mode === "reflect-cooldown").length,
955
1278
  reflectSkippedActions: finalActions.filter((a) => a.mode === "reflect-skipped").length,
956
1279
  reflectGuardRejectedActions: finalActions.filter((a) => a.mode === "reflect-guard-rejected").length,
957
1280
  ...(() => {
958
- const t = preparation.gateAutoAcceptedCount + loopGateCount + postLoopGateCount;
1281
+ const t = prepGateCount + loopGateCount + postLoopGateCount;
959
1282
  return t > 0 ? { gateAutoAcceptedCount: t } : {};
960
1283
  })(),
961
1284
  ...(() => {
962
- const f = preparation.gateAutoAcceptFailedCount + loopGateFailedCount + postLoopGateFailedCount;
1285
+ const f = prepGateFailedCount + loopGateFailedCount + postLoopGateFailedCount;
963
1286
  return f > 0 ? { gateAutoAcceptFailedCount: f } : {};
964
1287
  })(),
965
1288
  ...(triageDrain
@@ -972,6 +1295,10 @@ export async function akmImprove(options = {}) {
972
1295
  },
973
1296
  }
974
1297
  : {}),
1298
+ ...(preparation.proactiveMaintenance ? { proactiveMaintenance: preparation.proactiveMaintenance } : {}),
1299
+ // #616 — report cycles run only when >1 so the default single-pass
1300
+ // serialized envelope stays byte-identical to pre-#616 (AC1).
1301
+ ...(cyclesRun > 1 ? { cyclesRun } : {}),
975
1302
  ...(options.runId !== undefined ? { runId: options.runId } : {}),
976
1303
  };
977
1304
  if (!result.dryRun)
@@ -1054,15 +1381,12 @@ export async function akmImprove(options = {}) {
1054
1381
  // O-1 (#364): Clear the budget abort timer so it does not keep the event
1055
1382
  // loop alive after the run completes.
1056
1383
  clearBudgetTimer();
1057
- try {
1058
- fs.unlinkSync(resolvedLockPath);
1059
- }
1060
- catch {
1061
- // ignore
1062
- }
1063
- // The normal path released the lock above; drop the process.exit backstop so
1064
- // it does not fire later (or accumulate across repeated in-process calls).
1065
- process.removeListener("exit", releaseLockOnExit);
1384
+ // #607: release any per-process locks still held (backstop for error paths;
1385
+ // the normal path already released each lock after its stage completed).
1386
+ releaseAllProcessLocks();
1387
+ // Drop the process.exit backstop so it does not fire later (or accumulate
1388
+ // across repeated in-process calls).
1389
+ process.removeAllListeners("exit");
1066
1390
  // I1: close the long-lived state.db connection opened at the top of the run.
1067
1391
  try {
1068
1392
  eventsDb?.close();
@@ -1175,6 +1499,11 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1175
1499
  memoryInferenceDurationMs: durations.memoryInferenceDurationMs,
1176
1500
  graphExtractionExtractedFiles: result.graphExtraction?.quality.extractedFiles ?? 0,
1177
1501
  graphExtractionDurationMs: durations.graphExtractionDurationMs,
1502
+ // Layer-2 proactive-maintenance coverage (0 when the process is disabled
1503
+ // or the run was ref-scoped) so a scheduled sweep's reach is trackable.
1504
+ proactiveSelected: result.proactiveMaintenance?.selected ?? 0,
1505
+ proactiveDueTotal: result.proactiveMaintenance?.dueTotal ?? 0,
1506
+ proactiveNeverReflected: result.proactiveMaintenance?.neverReflected ?? 0,
1178
1507
  // New metrics for tuning the improve loop.
1179
1508
  ...(durations.totalDurationMs !== undefined ? { durationMs: durations.totalDurationMs } : {}),
1180
1509
  ...(durations.warningCount !== undefined ? { warningCount: durations.warningCount } : {}),
@@ -1213,7 +1542,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1213
1542
  * need, so the fix is non-invasive and provably correct.
1214
1543
  */
1215
1544
  async function runConsolidationPass(args) {
1216
- const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx } = args;
1545
+ const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx, budgetSignal, runBudgetMs } = args;
1217
1546
  const baseConfig = options.config ?? loadConfig();
1218
1547
  const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
1219
1548
  const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
@@ -1357,6 +1686,7 @@ async function runConsolidationPass(args) {
1357
1686
  stashDir: primaryStashDir,
1358
1687
  config: consolidationConfig,
1359
1688
  eventsCtx,
1689
+ stateDbPath: eventsCtx?.dbPath,
1360
1690
  }, { minimumThreshold: 95 });
1361
1691
  if (consolidateDisabledByProfile) {
1362
1692
  info("[improve] consolidation skipped (disabled by improve profile)");
@@ -1385,14 +1715,22 @@ async function runConsolidationPass(args) {
1385
1715
  // Tie consolidate proposals back to this improve invocation so
1386
1716
  // accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
1387
1717
  sourceRun: `consolidate-${Date.now()}`,
1388
- // Full-pool sweep: consolidation only runs on the nightly default-profile
1389
- // pass (quick/frequent disable it), so a complete re-cluster is correct and
1390
- // affordable here. Do NOT pass incrementalSince the time-window narrowing
1391
- // it triggers permanently excludes stale-but-unmerged duplicate clusters,
1392
- // starving merge recall and letting the pool grow unbounded. (The narrowing
1393
- // was a band-aid for an every-30-min consolidation cadence that the profile
1394
- // split has since eliminated.) lastConsolidateTs still gates whether we run.
1718
+ // Pass profile-configured options. incrementalSince narrows the pool to
1719
+ // recently-changed memories + graph neighbours use this for frequent
1720
+ // passes (quick-shredder). Leave absent in the nightly default profile for
1721
+ // a full-pool sweep that catches stale-but-unmerged duplicates.
1722
+ incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
1723
+ limit: improveProfile?.processes?.consolidate?.limit,
1724
+ neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
1395
1725
  maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
1726
+ // #617 — deterministic near-duplicate dedup pre-pass. DEFAULT OFF; only
1727
+ // runs when the profile explicitly sets `consolidate.dedup.enabled`.
1728
+ dedup: improveProfile?.processes?.consolidate?.dedup,
1729
+ // #581 — judged-state cache. DEFAULT OFF; only engages when the profile
1730
+ // explicitly sets `consolidate.judgedCache.enabled`. Skips memories
1731
+ // judged-unchanged since their last judge so one run sweeps the full
1732
+ // corpus instead of narrowing to a time-window slice.
1733
+ judgedCache: improveProfile?.processes?.consolidate?.judgedCache,
1396
1734
  // Honor profile.autoAccept (already merged into options.autoAccept at the
1397
1735
  // top of akmImprove). The CLI parser always supplies 90 when --auto-accept
1398
1736
  // is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
@@ -1400,6 +1738,13 @@ async function runConsolidationPass(args) {
1400
1738
  // options.consolidateOptions.autoAccept (if explicitly provided by caller)
1401
1739
  // still wins because the spread above runs first.
1402
1740
  autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
1741
+ // WS-3a: forward budget signal for graceful abort on timeout, and pass
1742
+ // the profile's p90 estimate for cold-start budget reduction.
1743
+ signal: budgetSignal,
1744
+ p90ChunkSecondsDefault: improveProfile?.processes?.consolidate?.p90ChunkSecondsDefault,
1745
+ // WS-5: pass total run budget so perfTelemetry.estimatedBudgetFractionUsed
1746
+ // can flag when consolidation alone exceeded the budget.
1747
+ runBudgetMs,
1403
1748
  }));
1404
1749
  {
1405
1750
  const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
@@ -1420,7 +1765,14 @@ async function runConsolidationPass(args) {
1420
1765
  appendEvent({
1421
1766
  eventType: "consolidate_completed",
1422
1767
  ref: "memory:_consolidation",
1423
- metadata: { processed: consolidation.processed, merged: consolidation.merged },
1768
+ metadata: {
1769
+ processed: consolidation.processed,
1770
+ merged: consolidation.merged,
1771
+ deleted: consolidation.deleted,
1772
+ contradicted: consolidation.contradicted,
1773
+ failedChunks: consolidation.failedChunks ?? 0,
1774
+ durationMs: consolidation.durationMs,
1775
+ },
1424
1776
  }, eventsCtx);
1425
1777
  }
1426
1778
  }
@@ -1437,10 +1789,21 @@ async function runConsolidationPass(args) {
1437
1789
  }
1438
1790
  // D9: track whether consolidation wrote any data so graph extraction can reindex if needed
1439
1791
  const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
1792
+ // WS-4: Per-phase threshold auto-tune for the consolidate phase.
1793
+ // Persists result for the NEXT run's makeGateConfig to read.
1794
+ const consolidateTuneDbPath = eventsCtx?.dbPath;
1795
+ if (options.autoAccept !== undefined && consolidateTuneDbPath) {
1796
+ try {
1797
+ maybeAutoTuneThreshold(consolidateGateCfg.phaseThreshold ?? options.autoAccept, consolidationConfig, consolidateTuneDbPath, undefined, "consolidate");
1798
+ }
1799
+ catch (err) {
1800
+ warn(`[improve] calibration auto-tune (consolidate) skipped: ${err instanceof Error ? err.message : String(err)}`);
1801
+ }
1802
+ }
1440
1803
  return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
1441
1804
  }
1442
1805
  async function runImprovePreparationStage(args) {
1443
- const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, } = args;
1806
+ const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, budgetSignal, } = args;
1444
1807
  const actions = [];
1445
1808
  const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
1446
1809
  // Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
@@ -1477,6 +1840,8 @@ async function runImprovePreparationStage(args) {
1477
1840
  memorySummary,
1478
1841
  improveProfile,
1479
1842
  eventsCtx,
1843
+ budgetSignal,
1844
+ runBudgetMs: budgetMs,
1480
1845
  });
1481
1846
  // Phase 0.4 — session-extract pass.
1482
1847
  //
@@ -1509,6 +1874,7 @@ async function runImprovePreparationStage(args) {
1509
1874
  stashDir: primaryStashDir,
1510
1875
  config: extractConfig,
1511
1876
  eventsCtx,
1877
+ stateDbPath: eventsCtx?.dbPath,
1512
1878
  });
1513
1879
  // #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
1514
1880
  // already done upstream; here we elide every akmExtract/processSession call)
@@ -1521,15 +1887,19 @@ async function runImprovePreparationStage(args) {
1521
1887
  // memory mtimes, so a skipped extract never flags work for the NEXT run's
1522
1888
  // consolidation mtime-gate (the downstream trigger #554 asks us to suppress).
1523
1889
  const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
1524
- const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
1890
+ // Read from the ACTIVE resolved profile (not always `default`), matching how
1891
+ // `extract.enabled` resolves — otherwise a non-default profile (e.g.
1892
+ // `frequent`) setting `minNewSessions` was silently ignored.
1893
+ const configuredMinNewSessions = improveProfile.processes?.extract?.minNewSessions;
1525
1894
  const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
1526
- // #593: gate on BOTH the legacy feature flag (which only reads
1527
- // `profiles.improve.default.processes.extract.enabled` kept for back-compat
1528
- // with users who disable extract via the default-profile path) AND the active
1529
- // resolved profile. Without the second check a non-default profile setting
1530
- // `extract.enabled: false` (e.g. the built-in `quick`) was silently ignored
1531
- // and extract ran on every improve call regardless.
1532
- if (isLlmFeatureEnabled(extractConfig, "session_extraction") && resolveProcessEnabled("extract", improveProfile)) {
1895
+ // #593/#594: the ACTIVE resolved improve profile is the single source of
1896
+ // truth for whether extract runs. (Previously this also ANDed in the legacy
1897
+ // `session_extraction` feature flag, which only reads
1898
+ // `profiles.improve.default.processes.extract.enabled`; that made the default
1899
+ // profile a global kill switch, so a non-default profile enabling extract was
1900
+ // silently overridden. The default profile is now just another profile.)
1901
+ // `akmExtract` re-checks the same active profile internally via `improveProfile`.
1902
+ if (resolveProcessEnabled("extract", improveProfile)) {
1533
1903
  const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
1534
1904
  // The guard engages only when minNewSessions > 0; 0 disables it entirely.
1535
1905
  let belowMinNewSessions = false;
@@ -1537,6 +1907,11 @@ async function runImprovePreparationStage(args) {
1537
1907
  const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
1538
1908
  const newCandidateCount = countFn(extractConfig, {
1539
1909
  ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1910
+ // Use the ACTIVE profile's discovery window so the gate counts over the
1911
+ // same window akmExtract will scan (not always `default`).
1912
+ ...(improveProfile.processes?.extract?.defaultSince
1913
+ ? { since: improveProfile.processes.extract.defaultSince }
1914
+ : {}),
1540
1915
  // C2: pin the candidate-count state.db open to the boundary-resolved path.
1541
1916
  ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
1542
1917
  });
@@ -1564,6 +1939,9 @@ async function runImprovePreparationStage(args) {
1564
1939
  type: h.name,
1565
1940
  ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
1566
1941
  config: extractConfig,
1942
+ // Thread the ACTIVE profile so extract's internal gate + per-process
1943
+ // config read the running profile, not always `default`.
1944
+ improveProfile,
1567
1945
  dryRun: options.dryRun ?? false,
1568
1946
  ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1569
1947
  // C2: pin extract's skip-tracking state.db open to the boundary path.
@@ -1791,10 +2169,19 @@ async function runImprovePreparationStage(args) {
1791
2169
  // refs that fail the distill signal-delta gate).
1792
2170
  // distillOnlyRefs — reflect blocked but distill signal-delta passes
1793
2171
  // AND ref is a distill candidate.
1794
- // fullySkippedCount — neither gate passes synthetic skip action
1795
- // + improve_skipped event, excluded from sort.
2172
+ // noFeedbackPool — neither signal-delta gate passes *and* the ref has
2173
+ // no recent feedback signal at all. These are NOT
2174
+ // skipped here: they are handed to the high-retrieval
2175
+ // fallback (P0-A) below so frequently-retrieved but
2176
+ // never-rated assets can still be improved. Only refs
2177
+ // that P0-A declines are ultimately fully skipped.
2178
+ // fullySkippedCount — has stale feedback but no signal delta → genuine
2179
+ // skip (counted, aggregated event emitted post-loop),
2180
+ // excluded from sort.
1796
2181
  const eligibleRefs = [];
1797
2182
  const distillOnlyRefs = [];
2183
+ // Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
2184
+ const noFeedbackPool = [];
1798
2185
  let fullySkippedCount = 0;
1799
2186
  // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
1800
2187
  const scopeRefBypass = scope.mode === "ref";
@@ -1832,57 +2219,134 @@ async function runImprovePreparationStage(args) {
1832
2219
  // Reflect blocked but distill passes → distill-only bucket.
1833
2220
  distillOnlyRefs.push(r);
1834
2221
  }
2222
+ else if (!latestFeedbackTs.has(r.ref)) {
2223
+ // Neither signal-delta gate passes AND there is no recent feedback signal
2224
+ // at all. Rather than skip outright, defer to the high-retrieval fallback
2225
+ // (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
2226
+ // what that path is meant to rescue. Refs P0-A declines are skipped there.
2227
+ noFeedbackPool.push(r);
2228
+ }
1835
2229
  else {
1836
- // Neither gate passes fully skipped.
2230
+ // Has feedback on record but no signal delta since the last proposal —
2231
+ // genuinely fully skipped. Counted here; a single aggregated
2232
+ // improve_skipped event is emitted after the loop (mirrors
2233
+ // profile_filtered_all_passes) instead of one event per ref.
1837
2234
  fullySkippedCount++;
1838
2235
  actions.push({
1839
2236
  ref: r.ref,
1840
2237
  mode: "distill-skipped",
1841
2238
  result: { ok: true, reason: "no new signal since last proposal" },
1842
2239
  });
1843
- appendEvent({ eventType: "improve_skipped", ref: r.ref, metadata: { reason: "no_new_signal" } }, eventsCtx);
1844
2240
  }
1845
2241
  }
2242
+ // Emit ONE aggregated skip event for the fully-skipped bucket rather than one
2243
+ // improve_skipped event per ref (#592 pattern, mirrors
2244
+ // profile_filtered_all_passes above). The per-ref loop previously produced
2245
+ // ~11K state.db writes per run on a large stash, the dominant contributor to
2246
+ // 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
2247
+ // run summary; no downstream consumer needs a per-ref DB audit trail (health's
2248
+ // skip histogram reads the `no_new_signal` counter from the count field).
2249
+ if (fullySkippedCount > 0) {
2250
+ appendEvent({
2251
+ eventType: "improve_skipped",
2252
+ ref: undefined,
2253
+ metadata: {
2254
+ reason: "no_new_signal",
2255
+ count: fullySkippedCount,
2256
+ },
2257
+ }, eventsCtx);
2258
+ }
1846
2259
  // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
1847
- // Everything from here works only on (eligibleRefs ∪ distillOnlyRefs). The
1848
- // fully-skipped bucket has already been routed and emitted; we deliberately
1849
- // avoid spending DB/CPU on refs that cannot enter the loop.
2260
+ // Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
2261
+ // deferred noFeedbackPool that may be rescued by the high-retrieval fallback
2262
+ // (P0-A). The fully-skipped bucket has already been routed and its aggregated
2263
+ // event emitted; we deliberately avoid spending DB/CPU on refs that the
2264
+ // signal-delta gate rejected with feedback already on record.
1850
2265
  const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
2266
+ // Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
2267
+ // partition above could not place these in a reflect/distill bucket, but they
2268
+ // may still qualify if they have been retrieved often enough. Two disjoint
2269
+ // sources feed this set:
2270
+ // 1. noFeedbackPool — refs with no recent feedback that the partition loop
2271
+ // deliberately deferred here (otherwise they would never reach P0-A).
2272
+ // 2. processableRefs entries that turn out to carry no recent feedback
2273
+ // *signal* once feedbackSummary is computed below.
2274
+ // (1) is added here; (2) is folded in after feedbackSummary is built.
1851
2275
  // Gap 6: only surface feedback signals from the last 30 days so that
1852
2276
  // ancient one-off feedback events don't permanently lock an asset into
1853
2277
  // every improve run. Assets with only stale signals fall through to the
1854
2278
  // high-retrieval path (P0-A) or are skipped until new signals arrive.
1855
2279
  // (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
1856
2280
  // Phase 2 above for the signal-delta gate; we reuse them here.)
1857
- // Pre-compute feedback summary per ref in a single pass so we don't issue
1858
- // two readEvents({type:"feedback", ref}) per asset (one for signal filtering,
1859
- // one for ratio computation).
2281
+ // Pre-compute feedback summary per ref in a SINGLE bulk read so we don't
2282
+ // open state.db once per asset (which caused 5000+ accumulated FDs and a
2283
+ // 2-hour runaway on a 13K-asset stash). Pattern mirrors buildLatestFeedbackTsMap
2284
+ // above: one readEvents() call fetches ALL feedback events, then we aggregate
2285
+ // in-memory by ref — O(1) DB opens regardless of candidate set size.
2286
+ // Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
2287
+ // ratios are available for any noFeedbackPool ref that P0-A rescues below.
2288
+ //
2289
+ // Behavioral note: positive/negative COUNTS are all-time (same as the old
2290
+ // per-ref readEvents call which had no `since` filter); hasSignal is bounded
2291
+ // to feedbackSinceCutoff (same as the old inline `(e.ts ?? "") >= cutoff` guard).
1860
2292
  const feedbackSummary = new Map();
1861
- for (const candidate of processableRefs) {
1862
- const { events } = readEvents({ type: "feedback", ref: candidate.ref });
1863
- let hasSignal = false;
1864
- let positive = 0;
1865
- let negative = 0;
1866
- for (const e of events) {
1867
- if (!hasSignal &&
1868
- (e.ts ?? "") >= feedbackSinceCutoff &&
1869
- e.metadata !== undefined &&
1870
- (typeof e.metadata.signal === "string" || typeof e.metadata.note === "string")) {
1871
- hasSignal = true;
1872
- }
1873
- if (e.metadata?.signal === "positive")
1874
- positive++;
1875
- else if (e.metadata?.signal === "negative")
1876
- negative++;
1877
- }
1878
- feedbackSummary.set(candidate.ref, { hasSignal, positive, negative });
2293
+ {
2294
+ const feedbackCandidateSet = new Set([...processableRefs, ...noFeedbackPool].map((r) => r.ref));
2295
+ if (feedbackCandidateSet.size > 0) {
2296
+ // Fetch ALL feedback events in one query (no ref filter, no since filter =
2297
+ // single full table scan). Filtering per-ref in memory avoids N sequential
2298
+ // state.db opens the dominant FD-leak path on large stashes.
2299
+ const { events: allFeedbackEvents } = readEvents({ type: "feedback" }, eventsCtx);
2300
+ for (const e of allFeedbackEvents) {
2301
+ const ref = e.ref;
2302
+ if (!ref || !feedbackCandidateSet.has(ref))
2303
+ continue;
2304
+ const entry = feedbackSummary.get(ref) ?? { hasSignal: false, positive: 0, negative: 0 };
2305
+ const meta = e.metadata;
2306
+ // hasSignal: only count feedback events within the 30-day window.
2307
+ if (!entry.hasSignal &&
2308
+ (e.ts ?? "") >= feedbackSinceCutoff &&
2309
+ meta !== undefined &&
2310
+ (typeof meta.signal === "string" || typeof meta.note === "string")) {
2311
+ entry.hasSignal = true;
2312
+ }
2313
+ // positive/negative: all-time counts (no since filter, matching prior behaviour).
2314
+ if (meta?.signal === "positive")
2315
+ entry.positive++;
2316
+ else if (meta?.signal === "negative")
2317
+ entry.negative++;
2318
+ feedbackSummary.set(ref, entry);
2319
+ }
2320
+ // Ensure every candidate has an entry (even refs with zero feedback events).
2321
+ for (const ref of feedbackCandidateSet) {
2322
+ if (!feedbackSummary.has(ref)) {
2323
+ feedbackSummary.set(ref, { hasSignal: false, positive: 0, negative: 0 });
2324
+ }
2325
+ }
2326
+ }
1879
2327
  }
1880
2328
  const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
1881
2329
  // P0-A: also surface zero-feedback assets that have been retrieved many times.
1882
2330
  const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
1883
2331
  const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
1884
- const noFeedbackCandidates = processableRefs.filter((r) => !signalBearingSet.has(r.ref));
2332
+ // Zero-feedback candidates for P0-A: processableRefs without a recent signal,
2333
+ // plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
2334
+ // disjoint by construction, but guard against overlap defensively).
2335
+ const noFeedbackSeen = new Set();
2336
+ const noFeedbackCandidates = [];
2337
+ for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
2338
+ if (noFeedbackSeen.has(r.ref))
2339
+ continue;
2340
+ noFeedbackSeen.add(r.ref);
2341
+ noFeedbackCandidates.push(r);
2342
+ }
1885
2343
  let highRetrievalRefs = [];
2344
+ // Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
2345
+ // proactive-maintenance selector below can reuse them without a second DB pass.
2346
+ // Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
2347
+ // step 2: recency is MANDATORY — never pinned to floor).
2348
+ let retrievalCounts = new Map();
2349
+ let lastUseMsForProactive = new Map();
1886
2350
  let dbForRetrieval;
1887
2351
  try {
1888
2352
  dbForRetrieval = openExistingDatabase();
@@ -1890,15 +2354,29 @@ async function runImprovePreparationStage(args) {
1890
2354
  if (showEventCount === 0) {
1891
2355
  warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
1892
2356
  }
1893
- const retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
2357
+ // Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
2358
+ // Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
2359
+ // had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
2360
+ // to 0 regardless of actual use. Two assets of the same type — one
2361
+ // heavily-retrieved, one never-touched — would receive identical rankScores.
2362
+ // Fix (WS-1 blocker 3): union the feedback pool into the lookup.
2363
+ const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
2364
+ retrievalCounts = getRetrievalCounts(dbForRetrieval, allCandidateRefs);
2365
+ lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
1894
2366
  // High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
1895
- // ref qualifies exactly once — when retrievalCount threshold AND no
1896
- // prior reflect proposal exists for it. Once a reflect proposal is on
1897
- // record, subsequent re-eligibility requires explicit feedback (which
1898
- // flows through the normal signal-delta gate above). Tracking growth in
1899
- // retrieval count would require persisting the count in proposal
1900
- // metadata; deferred to a follow-up.
1901
- highRetrievalRefs = noFeedbackCandidates.filter((r) => (retrievalCounts.get(r.ref) ?? 0) >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref));
2367
+ // ref qualifies exactly once — when it has actually been retrieved
2368
+ // (retrievalCount 1) AND retrievalCount threshold AND no prior reflect
2369
+ // proposal exists for it. Once a reflect proposal is on record, subsequent
2370
+ // re-eligibility requires explicit feedback (which flows through the normal
2371
+ // signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
2372
+ // from rescuing genuinely never-retrieved assets — the fallback is for
2373
+ // *retrieved* assets, not silent ones. Tracking growth in retrieval count
2374
+ // would require persisting the count in proposal metadata; deferred to a
2375
+ // follow-up.
2376
+ highRetrievalRefs = noFeedbackCandidates.filter((r) => {
2377
+ const count = retrievalCounts.get(r.ref) ?? 0;
2378
+ return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
2379
+ });
1902
2380
  }
1903
2381
  catch (err) {
1904
2382
  rethrowIfTestIsolationError(err);
@@ -1908,6 +2386,137 @@ async function runImprovePreparationStage(args) {
1908
2386
  if (dbForRetrieval)
1909
2387
  closeDatabase(dbForRetrieval);
1910
2388
  }
2389
+ // ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
2390
+ // The signal-delta gate and P0-A only surface assets with fresh feedback or a
2391
+ // raw-retrieval spike. Neither revisits a stable, high-value asset on a
2392
+ // schedule, so on a quiet stash useful assets drift stale and are never
2393
+ // refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
2394
+ // and the run is whole-stash / type scope, this selector ranks the eligible
2395
+ // population by a composite maintenance priority, gates on staleness ("due"),
2396
+ // bounds to top-N, and folds the winners into the SAME candidate set the other
2397
+ // two sources feed — so they flow through the existing #580 empty-diff /
2398
+ // cosmetic suppression and additive-distill gates. It adds no new mutation
2399
+ // logic of its own. The due gate doubles as the rotation cooldown: a freshly
2400
+ // reflected asset is excluded until it ages back past `dueDays`, so successive
2401
+ // runs rotate through the due pool rather than re-selecting the same heads.
2402
+ let proactiveRefs = [];
2403
+ let proactiveMaintenanceSummary;
2404
+ const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
2405
+ if (proactiveEnabled) {
2406
+ const pmCfg = improveProfile.processes?.proactiveMaintenance;
2407
+ const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
2408
+ const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
2409
+ // Candidate population: the zero-feedback / non-signal pool — exactly the
2410
+ // assets the other two sources would NOT pick this run. Exclude any P0-A
2411
+ // rescued this run so we never double-select the same ref.
2412
+ const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
2413
+ const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
2414
+ const selection = selectProactiveMaintenanceRefs({
2415
+ candidates: pmCandidates,
2416
+ lastReflectTs: lastReflectProposalTs,
2417
+ lastDistillTs: lastDistillProposalTs,
2418
+ retrievalCounts,
2419
+ // WS-1: wire lastUseMs so the recency decay term is genuine (plan §step 2).
2420
+ lastUseMs: lastUseMsForProactive,
2421
+ sizeBytesOf: (r) => {
2422
+ const fp = r.filePath;
2423
+ if (!fp)
2424
+ return undefined;
2425
+ try {
2426
+ return fs.statSync(fp).size;
2427
+ }
2428
+ catch {
2429
+ return undefined;
2430
+ }
2431
+ },
2432
+ dueDays,
2433
+ maxPerRun,
2434
+ });
2435
+ proactiveRefs = selection.selected;
2436
+ proactiveMaintenanceSummary = {
2437
+ selected: selection.selected.length,
2438
+ dueTotal: selection.dueTotal,
2439
+ neverReflected: selection.neverReflected,
2440
+ };
2441
+ // Aggregated observability event (never per-ref — avoids the event flood the
2442
+ // Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
2443
+ appendEvent({
2444
+ eventType: "proactive_selected",
2445
+ ref: undefined,
2446
+ metadata: {
2447
+ count: selection.selected.length,
2448
+ dueTotal: selection.dueTotal,
2449
+ neverReflected: selection.neverReflected,
2450
+ },
2451
+ }, eventsCtx);
2452
+ if (selection.selected.length > 0) {
2453
+ info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
2454
+ `(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
2455
+ }
2456
+ }
2457
+ // ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
2458
+ // Zero-feedback refs whose encoding_salience (set at distill time by
2459
+ // scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
2460
+ // into the improve run even without retrieval or feedback signal. This rescues
2461
+ // newly distilled assets that the stash has not yet surfaced to users.
2462
+ //
2463
+ // Cap: at most 10% of the effective run limit so the lane cannot crowd out
2464
+ // reactive feedback. Requires state.db to have an asset_salience row — refs
2465
+ // without a row (pre-#608 assets still on the type-weight stub) are skipped.
2466
+ //
2467
+ // Cooldown: a ref qualifies at most once — when no prior reflect proposal
2468
+ // exists for it (`!lastReflectProposalTs.has`). Without this guard the lane
2469
+ // re-selects the same high-salience refs on EVERY run (auto-accept emits a
2470
+ // `promoted` event, not `feedback`, so the ref never leaves
2471
+ // noFeedbackCandidates), burning LLM calls and churning the asset. This
2472
+ // mirrors the P0-A high-retrieval gate's `!lastReflectProposalTs.has(r.ref)`
2473
+ // guard above so all "rescue" lanes share the same once-per-asset semantics.
2474
+ const highSalienceRefs = [];
2475
+ const salienceCfg = (options.config ?? loadConfig()).improve?.salience;
2476
+ const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
2477
+ const proactiveAndRetrievalSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
2478
+ {
2479
+ let dbForHighSalience;
2480
+ try {
2481
+ dbForHighSalience = openStateDatabase(eventsCtx?.dbPath);
2482
+ const effectiveLimit = options.limit ?? 10;
2483
+ const highSalienceCap = Math.max(1, Math.floor(effectiveLimit * 0.1));
2484
+ const candidates = noFeedbackCandidates.filter((r) => !proactiveAndRetrievalSet.has(r.ref));
2485
+ for (const r of candidates) {
2486
+ if (highSalienceRefs.length >= highSalienceCap)
2487
+ break;
2488
+ const row = getAssetSalience(dbForHighSalience, r.ref);
2489
+ if (row && row.encoding_salience >= salienceThreshold && !lastReflectProposalTs.has(r.ref)) {
2490
+ highSalienceRefs.push(r);
2491
+ }
2492
+ }
2493
+ }
2494
+ catch (err) {
2495
+ rethrowIfTestIsolationError(err);
2496
+ // best-effort: if DB unavailable, highSalienceRefs stays empty
2497
+ }
2498
+ finally {
2499
+ if (dbForHighSalience)
2500
+ dbForHighSalience.close();
2501
+ }
2502
+ }
2503
+ // Record an in-memory skip action for every zero-feedback ref that the
2504
+ // partition loop deferred to P0-A but P0-A then declined (retrievalCount below
2505
+ // threshold, or a prior reflect proposal already on record). These never make
2506
+ // it into mergedRefs, so without this they would silently vanish from the run
2507
+ // summary. No DB event is written here — these refs carry no signal at all, so
2508
+ // there is nothing for the skip histogram to aggregate; the action log alone
2509
+ // preserves the per-ref audit trail (mirrors the fully-skipped action above).
2510
+ const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs, ...highSalienceRefs].map((r) => r.ref));
2511
+ for (const r of noFeedbackPool) {
2512
+ if (rescuedSet.has(r.ref))
2513
+ continue;
2514
+ actions.push({
2515
+ ref: r.ref,
2516
+ mode: "distill-skipped",
2517
+ result: { ok: true, reason: "no new signal since last proposal" },
2518
+ });
2519
+ }
1911
2520
  // If the user explicitly scoped to a single ref, always act on it —
1912
2521
  // skip the signal/retrieval filter entirely. The filter exists to avoid
1913
2522
  // noisy "improve everything" runs; it should not gate an intentional
@@ -1917,38 +2526,656 @@ async function runImprovePreparationStage(args) {
1917
2526
  // or sufficient retrievals). A stash with no signals has 0 eligible refs —
1918
2527
  // usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
1919
2528
  // to bring them into the eligible pool.
1920
- const signalAndRetrievalRefs = [...signalFiltered, ...highRetrievalRefs];
1921
- const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
2529
+ // Layer-2 proactive refs join the eligible set alongside feedback-signal and
2530
+ // high-retrieval (P0-A) refs. The four sources are disjoint by construction
2531
+ // (proactive draws from noFeedbackCandidates with the P0-A picks removed, and
2532
+ // high-salience draws from the remainder), but dedupe defensively so a ref can
2533
+ // never enter the loop twice. `requireFeedbackSignal` still suppresses all
2534
+ // fallback sources for callers that want feedback-only runs.
2535
+ const signalAndRetrievalRefs = dedupeRefs([
2536
+ ...signalFiltered,
2537
+ ...highRetrievalRefs,
2538
+ ...proactiveRefs,
2539
+ ...highSalienceRefs,
2540
+ ]);
2541
+ let mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
2542
+ // ── Attribution tagging: stamp each ref with the eligibility lane that
2543
+ // selected it ──────────────────────────────────────────────────────────────
2544
+ // Every reflect/distill proposal must record WHICH lane chose its source asset
2545
+ // so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
2546
+ // (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
2547
+ // lane map here — the one place all four lanes are known — and stamp it onto
2548
+ // each ImproveEligibleRef object. Because the ref objects are shared by
2549
+ // reference across buckets, the stamp travels with the ref through the sort,
2550
+ // disk-check, and loop stages down to the reflect/distill event emit sites and
2551
+ // createProposal calls. See EligibilitySource for the lane vocabulary.
2552
+ //
2553
+ // Precedence (prefer the most specific reactive signal):
2554
+ // scope > signal-delta > high-retrieval > proactive > high-salience
2555
+ // A ref with real feedback is attributed to feedback even if it was also due
2556
+ // for proactive maintenance or had high encoding salience. We apply lanes
2557
+ // weakest-first so the strongest overwrites; the explicit --scope <ref> bypass
2558
+ // wins outright (user intent).
2559
+ const eligibilitySourceByRef = new Map();
2560
+ for (const r of highSalienceRefs)
2561
+ eligibilitySourceByRef.set(r.ref, "high-salience");
2562
+ for (const r of proactiveRefs)
2563
+ eligibilitySourceByRef.set(r.ref, "proactive");
2564
+ for (const r of highRetrievalRefs)
2565
+ eligibilitySourceByRef.set(r.ref, "high-retrieval");
2566
+ for (const r of signalFiltered)
2567
+ eligibilitySourceByRef.set(r.ref, "signal-delta");
2568
+ if (scope.mode === "ref") {
2569
+ // O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
2570
+ // arrived via the scopeRefBypass branch, so attribute the whole set to scope.
2571
+ for (const r of processableRefs)
2572
+ eligibilitySourceByRef.set(r.ref, "scope");
2573
+ }
2574
+ for (const r of mergedRefs) {
2575
+ // "unknown" is a genuine fallback, never a silent alias for signal-delta:
2576
+ // only refs we truly cannot attribute land here (none in practice, since
2577
+ // mergedRefs is always a subset of the four lanes above).
2578
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2579
+ }
2580
+ // WS-1 — Unified salience vector (S1 seam).
2581
+ //
2582
+ // The legacy sort combined three independent formulas (utility EMA, negative-only
2583
+ // ratio / symmetric-valence magnitude, and the proactive-maintenance priority
2584
+ // formula). WS-1 converges them into one `computeSalience()` call per ref, with
2585
+ // three independently-stored sub-scores and one documented rankScore projection.
2586
+ //
2587
+ // Migration note: if a profile still has `symmetricValence` set, emit a one-time
2588
+ // warning — its behaviour (symmetric |valence| attention) is now always-on as
2589
+ // part of the salience vector, so the knob is a no-op and will be removed in 0.10.
2590
+ if (improveProfile.symmetricValence === true) {
2591
+ warn("[improve] Profile option 'symmetricValence' is deprecated (WS-1 salience vector). " +
2592
+ "Symmetric valence is now always active; remove the option from your improve profile.");
2593
+ }
2594
+ // Fetch last-use timestamps from the index DB for the full merged set so the
2595
+ // recency term in retrievalSalience is genuinely decayable (plan §WS-1 step 2).
2596
+ // This reuses the index DB opened earlier for retrieval counts; a separate
2597
+ // lightweight open is used here to avoid holding the connection longer than needed.
2598
+ let lastUseMsByRef = new Map();
2599
+ // utilityMap is kept for backward-compatible observability (health report reads it).
1922
2600
  const utilityMap = buildUtilityMap(mergedRefs);
1923
- // Load feedback ratio per ref from the pre-computed summary (no extra DB pass).
1924
- const feedbackRatios = new Map();
1925
- for (const ref of mergedRefs) {
1926
- const summary = feedbackSummary.get(ref.ref);
1927
- const positive = summary?.positive ?? 0;
1928
- const negative = summary?.negative ?? 0;
1929
- const total = positive + negative;
1930
- // ratio = negative proportion (high = needs more improvement)
1931
- feedbackRatios.set(ref.ref, total > 0 ? negative / total : 0);
1932
- }
1933
- // Sort: combine utility (desc) with feedback negativity (desc) — high-negative assets rank higher
1934
- const sorted = [...mergedRefs].sort((a, b) => {
1935
- const utilA = utilityMap.get(a.ref) ?? 0;
1936
- const utilB = utilityMap.get(b.ref) ?? 0;
1937
- const ratioA = feedbackRatios.get(a.ref) ?? 0;
1938
- const ratioB = feedbackRatios.get(b.ref) ?? 0;
1939
- // Combined score: 70% utility, 30% negative ratio
1940
- const scoreA = utilA * 0.7 + ratioA * 0.3;
1941
- const scoreB = utilB * 0.7 + ratioB * 0.3;
1942
- return scoreB - scoreA;
1943
- });
1944
- // Phase 0: surface coverage gaps from zero-result search queries
1945
- let coverageGaps = [];
2601
+ let dbForSalience;
1946
2602
  try {
1947
- const dbForGaps = openExistingDatabase();
1948
- try {
1949
- coverageGaps = getZeroResultSearches(dbForGaps);
1950
- }
1951
- finally {
2603
+ dbForSalience = openExistingDatabase();
2604
+ lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs.map((r) => r.ref));
2605
+ }
2606
+ catch (err) {
2607
+ rethrowIfTestIsolationError(err);
2608
+ // best-effort: if DB unavailable, recency term stays at floor (lastUseMs=0)
2609
+ }
2610
+ finally {
2611
+ if (dbForSalience)
2612
+ closeDatabase(dbForSalience);
2613
+ }
2614
+ // ── WS-2 Outcome loop ─────────────────────────────────────────────────────
2615
+ //
2616
+ // Update asset_outcome for every ref in the merged set BEFORE computing the
2617
+ // salience vector so the updated outcome_score feeds outcomeSalience this run.
2618
+ //
2619
+ // Inputs per ref:
2620
+ // - currentRetrievalCount: from retrievalCounts (index DB)
2621
+ // - lastRetrievedAt: from lastUseMsByRef (utility_scores.last_used_at)
2622
+ // - negativeFeedbackCount: cumulative negatives from feedbackSummary
2623
+ // - acceptedChangeCount: accepted proposals for this ref (state.db)
2624
+ // - valence: net valence from computeValenceScore(feedbackSummary.get(ref))
2625
+ // - utilityScore: from utilityMap (for warm-start seed on new rows)
2626
+ //
2627
+ // Best-effort: outcome failures never block the salience or ranking pass.
2628
+ const outcomeSalienceByRef = new Map();
2629
+ try {
2630
+ const outcomeDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
2631
+ const ownsOutcomeDb = !eventsCtx?.db;
2632
+ try {
2633
+ // Count accepted proposals per ref in one pass (avoid N separate queries).
2634
+ // Scoped to primaryStashDir when available so multi-stash installs don't
2635
+ // inflate counts with proposals from other stashes.
2636
+ const acceptedCountByRef = new Map();
2637
+ try {
2638
+ const acceptedProposals = listStateProposals(outcomeDb, {
2639
+ status: "accepted",
2640
+ ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
2641
+ });
2642
+ for (const p of acceptedProposals) {
2643
+ acceptedCountByRef.set(p.ref, (acceptedCountByRef.get(p.ref) ?? 0) + 1);
2644
+ }
2645
+ }
2646
+ catch {
2647
+ // best-effort: if proposals query fails, accepted counts stay at 0
2648
+ }
2649
+ // Update each ref's outcome row and collect the resulting outcome scores.
2650
+ const rawOutcomeScores = new Map();
2651
+ const nowForOutcome = Date.now();
2652
+ for (const r of mergedRefs) {
2653
+ const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
2654
+ const valenceResult = computeValenceScore(fb);
2655
+ try {
2656
+ const result = updateAssetOutcome(outcomeDb, {
2657
+ ref: r.ref,
2658
+ currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
2659
+ lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
2660
+ acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
2661
+ negativeFeedbackCount: fb.negative,
2662
+ valence: valenceResult.valence,
2663
+ utilityScore: utilityMap.get(r.ref),
2664
+ now: nowForOutcome,
2665
+ });
2666
+ rawOutcomeScores.set(r.ref, result.outcomeScore);
2667
+ }
2668
+ catch {
2669
+ // best-effort per-ref: skip this ref's outcome update on failure
2670
+ }
2671
+ }
2672
+ // Compute stash-wide max outcome_score for normalisation (diversity floor).
2673
+ // Read ALL rows (not just this run's batch) so the normalisation is
2674
+ // stash-relative, not pool-relative.
2675
+ let maxOutcomeScore = 0;
2676
+ try {
2677
+ const allOutcomes = getAllAssetOutcomes(outcomeDb);
2678
+ for (const row of allOutcomes) {
2679
+ if (row.outcome_score > maxOutcomeScore)
2680
+ maxOutcomeScore = row.outcome_score;
2681
+ }
2682
+ // Proxy-adequacy tripwire: emit a health event if outcome_score is
2683
+ // negatively correlated with accepted_change_rate (inverted proxy).
2684
+ const adequacy = computeProxyAdequacy(allOutcomes);
2685
+ if (adequacy.isInverted) {
2686
+ appendEvent({
2687
+ eventType: "outcome_proxy_inverted",
2688
+ ref: undefined,
2689
+ metadata: {
2690
+ correlation: adequacy.correlation,
2691
+ n: adequacy.n,
2692
+ note: "corr(outcome_score, accepted_change_rate) < −0.3: high-outcome_score assets have LOW accepted-change rates — the proxy's 'doing well' signal is inverted, so the coarse retrieval-delta signal is no longer trustworthy and the 0.10+ rich in-session signal is no longer deferrable. See plan §WS-2 proxy-adequacy tripwire.",
2693
+ },
2694
+ }, eventsCtx);
2695
+ }
2696
+ }
2697
+ catch {
2698
+ // best-effort: tripwire failure never blocks ranking
2699
+ }
2700
+ // Convert raw outcome scores → normalised outcomeSalience values in [0,1].
2701
+ for (const [ref, score] of rawOutcomeScores) {
2702
+ const normalised = outcomeScoreToSalience(score, maxOutcomeScore);
2703
+ outcomeSalienceByRef.set(ref, normalised);
2704
+ }
2705
+ // Also fetch outcome scores for refs NOT updated this run (stale or absent)
2706
+ // so the outcomeSalience read path works for all refs in the batch.
2707
+ const missingRefs = mergedRefs.map((r) => r.ref).filter((ref) => !rawOutcomeScores.has(ref));
2708
+ if (missingRefs.length > 0) {
2709
+ const storedScores = getOutcomeScoresByRef(outcomeDb, missingRefs);
2710
+ for (const [ref, score] of storedScores) {
2711
+ outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
2712
+ }
2713
+ }
2714
+ }
2715
+ finally {
2716
+ if (ownsOutcomeDb)
2717
+ outcomeDb.close();
2718
+ }
2719
+ }
2720
+ catch (err) {
2721
+ rethrowIfTestIsolationError(err);
2722
+ // best-effort: outcome failures never block salience computation
2723
+ }
2724
+ // Compute the salience vector for every ref in the merged set.
2725
+ // retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
2726
+ // so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
2727
+ // outcomeSalienceByRef is populated by WS-2 above (or empty on first run).
2728
+ //
2729
+ // Part-V gate: read the operator opt-in flag from config. Default false
2730
+ // (WS-1 parity weights) until the maintainer runs scripts/akm-eval and sets
2731
+ // improve.salience.outcomeWeightEnabled: true in the config.
2732
+ const salienceConfig = (options.config ?? loadConfig()).improve?.salience;
2733
+ const outcomeWeightEnabled = salienceConfig?.outcomeWeightEnabled === true;
2734
+ const salienceMap = new Map();
2735
+ const nowForSalience = Date.now();
2736
+ // #644 — preserve content-derived encoding scores across runs.
2737
+ //
2738
+ // Before computing the salience vector, load each ref's stored encoding score
2739
+ // and its provenance. When the stored row carries a genuine content-derived
2740
+ // score (written by the distill path via `scoreEncodingSalience`), pass that
2741
+ // value back in as `inputs.encodingSalience` so `computeSalience` does NOT fall
2742
+ // back to the type-weight stub — keeping both the persisted `encoding_salience`
2743
+ // AND the derived `rank_score` keyed on real novelty/magnitude/prediction-error.
2744
+ // Refs that have never been content-scored keep the type-weight stub fallback.
2745
+ const storedEncodingByRef = new Map();
2746
+ {
2747
+ let dbForStoredEncoding;
2748
+ try {
2749
+ dbForStoredEncoding = openStateDatabase(eventsCtx?.dbPath);
2750
+ for (const r of mergedRefs) {
2751
+ const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
2752
+ const row = getAssetSalience(dbForStoredEncoding, r.ref);
2753
+ if (row && isContentEncodingRow(row, type)) {
2754
+ storedEncodingByRef.set(r.ref, row.encoding_salience);
2755
+ }
2756
+ }
2757
+ }
2758
+ catch (err) {
2759
+ rethrowIfTestIsolationError(err);
2760
+ // best-effort: if DB unavailable, fall back to type-weight stub (prior behaviour)
2761
+ }
2762
+ finally {
2763
+ if (dbForStoredEncoding)
2764
+ dbForStoredEncoding.close();
2765
+ }
2766
+ }
2767
+ for (const r of mergedRefs) {
2768
+ const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
2769
+ const sizeBytes = (() => {
2770
+ const fp = r.filePath;
2771
+ if (!fp)
2772
+ return undefined;
2773
+ try {
2774
+ return fs.statSync(fp).size;
2775
+ }
2776
+ catch {
2777
+ return undefined;
2778
+ }
2779
+ })();
2780
+ const storedEncoding = storedEncodingByRef.get(r.ref);
2781
+ const vector = computeSalience({
2782
+ ref: r.ref,
2783
+ type,
2784
+ // #644: pass the stored content-derived score (if any) so the type-weight
2785
+ // stub is NOT re-asserted over a real distill-written encoding score.
2786
+ ...(storedEncoding !== undefined ? { encodingSalience: storedEncoding } : {}),
2787
+ retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
2788
+ lastUseMs: lastUseMsByRef.get(r.ref),
2789
+ utilityScore: utilityMap.get(r.ref),
2790
+ outcomeSalience: outcomeSalienceByRef.get(r.ref),
2791
+ sizeBytes,
2792
+ now: nowForSalience,
2793
+ outcomeWeightEnabled,
2794
+ });
2795
+ salienceMap.set(r.ref, vector);
2796
+ }
2797
+ // Persist salience vectors to state.db (best-effort, non-blocking).
2798
+ // The canonical store enables WS-3 homeostatic demotion and WS-2 outcome reads.
2799
+ //
2800
+ // Forgetting-safety report (plan §WS-1 step 7) — stash-wide rank comparison:
2801
+ //
2802
+ // BEFORE persisting the new rankScores, read ALL existing rows from state.db
2803
+ // (not just the per-run candidate pool). This gives stash-wide rank positions so
2804
+ // the top-200/below-500 thresholds are meaningful.
2805
+ //
2806
+ // Two distinct scenarios:
2807
+ //
2808
+ // A. First WS-1 run (table empty): the old stash-wide combinedEligibilityScore
2809
+ // ordering was never persisted in state.db (asset_salience is a new WS-1 table).
2810
+ // However, the old formula's inputs are available in-scope for every candidate
2811
+ // in the current pool: utility comes from utilityMap and the attention term
2812
+ // from feedbackSummary (positive/negative counts). We reconstruct the old
2813
+ // combinedEligibilityScore = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT
2814
+ // for every ref in salienceMap and rank them, giving a candidate-pool-scoped
2815
+ // old ordering. This is a partial reconstruction (only current-pool refs, not
2816
+ // stash-wide), but it is the most faithful comparison possible at cutover and
2817
+ // allows the top-200→below-500 forgetting guard to fire if the formula change
2818
+ // dramatically reorders the candidate pool.
2819
+ // See docs/design/improve-reconciliation-plan.md §WS-1 step 7 — the stash-wide
2820
+ // ordering was unreconstructable (no prior state.db snapshot), so this candidate-
2821
+ // pool partial reconstruction is the documented resolution for the first-run case.
2822
+ // Emit `improve_salience_first_run` to mark the cutover moment and include the
2823
+ // reconstructed comparison result in the metadata.
2824
+ //
2825
+ // B. Subsequent runs (table has rows): use ALL existing rows as old ranks, merge
2826
+ // them with the current run's salienceMap updates for new ranks, and call
2827
+ // buildRankChangeReport with stash-wide positions. This detects real rank drift
2828
+ // — e.g. a retrieval-pattern shift causing a previously top-200 asset to slip
2829
+ // below position 500.
2830
+ //
2831
+ // Measurement-protocol deferral (plan §269, Part-V):
2832
+ // The Part-V T0 baseline (scripts/akm-eval + health report) and the throughput/
2833
+ // quality gate are deferred pending owner sign-off. Full measurement requires a
2834
+ // before/after `akm health` report. Owner-acknowledged deferral: WS-2 landing
2835
+ // will re-introduce outcome salience and trigger the full re-tuning pass at that
2836
+ // time. salience.ts already accepts outcomeSalience directly as an input
2837
+ // (see SalienceInputs.outcomeSalience); no separate hook is needed.
2838
+ //
2839
+ // Forgetting-safety collection: populated inside scenario B below, consumed
2840
+ // after the try/catch to union candidates into mergedRefs before the sort.
2841
+ // Only refs from a real pre-existing ordering (scenario B) are collected;
2842
+ // empty on scenario A or when no candidates dropped below the threshold.
2843
+ let pendingForgettingRefs = [];
2844
+ try {
2845
+ const stateDb = openStateDatabase(eventsCtx?.dbPath);
2846
+ try {
2847
+ // Step 7: stash-wide rank-change report BEFORE overwriting the table.
2848
+ //
2849
+ // Load ALL existing rows so rank positions are stash-relative, not pool-relative.
2850
+ const existingAllScores = getAllRankScores(stateDb);
2851
+ if (existingAllScores.size === 0) {
2852
+ // Scenario A: first WS-1 run — table empty.
2853
+ //
2854
+ // Reconstruct the old combinedEligibilityScore ordering for the current
2855
+ // candidate pool using inputs that are already in-scope: utility from
2856
+ // utilityMap and the attention term from feedbackSummary (positive/negative
2857
+ // counts). Old formula: score = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT.
2858
+ //
2859
+ // Limitation: this covers only the current-run candidate pool, not the full
2860
+ // stash. The stash-wide ordering was never persisted (asset_salience is a new
2861
+ // WS-1 table), so this is the most faithful comparison possible at cutover.
2862
+ // See docs/design/improve-reconciliation-plan.md §WS-1 step 7.
2863
+ const reconstructedOldScores = new Map();
2864
+ for (const ref of salienceMap.keys()) {
2865
+ const utility = utilityMap.get(ref) ?? 0;
2866
+ const fb = feedbackSummary.get(ref) ?? { positive: 0, negative: 0 };
2867
+ const attention = computeValenceScore(fb).attention;
2868
+ reconstructedOldScores.set(ref, utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT);
2869
+ }
2870
+ // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2871
+ const toRanks = (scores) => {
2872
+ const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
2873
+ return new Map(sorted.map(([ref], i) => [ref, i + 1]));
2874
+ };
2875
+ const oldRanks = toRanks(reconstructedOldScores);
2876
+ const newRanks = toRanks(new Map([...salienceMap.entries()].map(([ref, v]) => [ref, v.rankScore])));
2877
+ const firstRunReport = buildRankChangeReport(oldRanks, newRanks);
2878
+ if (firstRunReport.forgettingCandidates.length > 0) {
2879
+ warn(`[improve/salience] WS-1 first-run rank-change report: ${firstRunReport.forgettingCandidates.length} asset(s) fell from top-200 to below position 500 (cutover formula change). ` +
2880
+ `Top drops: ${firstRunReport.forgettingCandidates
2881
+ .slice(0, 5)
2882
+ .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2883
+ .join(", ")}`);
2884
+ pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
2885
+ }
2886
+ appendEvent({
2887
+ eventType: "improve_salience_first_run",
2888
+ ref: undefined,
2889
+ metadata: {
2890
+ candidateCount: salienceMap.size,
2891
+ note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); see improve-reconciliation-plan.md §WS-1 step 7",
2892
+ forgettingCandidates: firstRunReport.forgettingCandidates.length,
2893
+ topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
2894
+ ref: e.ref,
2895
+ oldRank: e.oldRank,
2896
+ newRank: e.newRank,
2897
+ })),
2898
+ },
2899
+ }, eventsCtx);
2900
+ }
2901
+ else {
2902
+ // Scenario B: subsequent run — compare stash-wide old vs. new ranks.
2903
+ //
2904
+ // Build new scores by merging the full table with this run's updates.
2905
+ // Refs in salienceMap override their stored value; refs not in this run
2906
+ // retain their stored value unchanged. This gives a complete stash-wide
2907
+ // picture of what the new ordering looks like after this run.
2908
+ const mergedNewScores = new Map(existingAllScores);
2909
+ for (const [ref, vector] of salienceMap) {
2910
+ mergedNewScores.set(ref, vector.rankScore);
2911
+ }
2912
+ // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2913
+ const toRanks = (scores) => {
2914
+ const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
2915
+ return new Map(sorted.map(([ref], i) => [ref, i + 1]));
2916
+ };
2917
+ const oldRanks = toRanks(existingAllScores);
2918
+ const newRanks = toRanks(mergedNewScores);
2919
+ const report = buildRankChangeReport(oldRanks, newRanks);
2920
+ if (report.forgettingCandidates.length > 0) {
2921
+ warn(`[improve/salience] WS-1 rank-change report: ${report.forgettingCandidates.length} asset(s) fell from top-200 to below position 500. ` +
2922
+ `Top drops: ${report.forgettingCandidates
2923
+ .slice(0, 5)
2924
+ .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2925
+ .join(", ")}`);
2926
+ // Collect refs for protective consolidation pass (plan §WS-1 step 7).
2927
+ // These are force-included in the candidate pool (mergedRefs) after
2928
+ // this try block, bypassing cooldown/signal-delta gating.
2929
+ pendingForgettingRefs = report.forgettingCandidates.map((e) => e.ref);
2930
+ }
2931
+ appendEvent({
2932
+ eventType: "improve_salience_rank_change",
2933
+ ref: undefined,
2934
+ metadata: {
2935
+ stashSize: existingAllScores.size,
2936
+ totalChanged: report.allChanges.length,
2937
+ forgettingCandidates: report.forgettingCandidates.length,
2938
+ topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
2939
+ ref: e.ref,
2940
+ oldRank: e.oldRank,
2941
+ newRank: e.newRank,
2942
+ })),
2943
+ },
2944
+ }, eventsCtx);
2945
+ }
2946
+ for (const [ref, vector] of salienceMap) {
2947
+ upsertAssetSalience(stateDb, ref, vector, nowForSalience);
2948
+ }
2949
+ }
2950
+ finally {
2951
+ stateDb.close();
2952
+ }
2953
+ }
2954
+ catch (err) {
2955
+ rethrowIfTestIsolationError(err);
2956
+ // best-effort: salience persistence failure never blocks ranking
2957
+ }
2958
+ // ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
2959
+ // Forgetting candidates detected in scenario B are force-injected into
2960
+ // mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
2961
+ // signal-delta gating. Any ref already present in mergedRefs keeps its
2962
+ // existing eligibilitySource (stronger reactive signals win); refs not yet in
2963
+ // the pool are synthesised as minimal ImproveEligibleRef stubs and labelled
2964
+ // 'forgetting-safety' so S5/WS-5 can slice by lane. The dedupeRefs call
2965
+ // ensures no ref can enter the loop twice.
2966
+ if (pendingForgettingRefs.length > 0 && scope.mode !== "ref") {
2967
+ const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
2968
+ const newForgettingRefs = [];
2969
+ for (const ref of pendingForgettingRefs) {
2970
+ if (!existingRefSet.has(ref)) {
2971
+ // Ref not already in the candidate pool — synthesise a stub so it
2972
+ // participates in the reflect/distill loop with proper attribution.
2973
+ newForgettingRefs.push({ ref, reason: "scope-type", eligibilitySource: "forgetting-safety" });
2974
+ }
2975
+ // Always stamp the lane in the attribution map (overwrites weaker lanes;
2976
+ // stronger reactive signals — scope/signal-delta/high-retrieval/proactive
2977
+ // — are written after this block so they take precedence).
2978
+ eligibilitySourceByRef.set(ref, "forgetting-safety");
2979
+ }
2980
+ if (newForgettingRefs.length > 0) {
2981
+ mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
2982
+ }
2983
+ // Re-stamp attribution for any refs whose lane needs updating.
2984
+ // Precedence (weakest → strongest, each overwrites the previous):
2985
+ // proactive < high-retrieval < forgetting-safety < signal-delta
2986
+ // Scope mode is already excluded by the outer guard (`scope.mode !== "ref"`).
2987
+ // forgetting-safety sits above proactive and high-retrieval so that a ref
2988
+ // flagged as a forgetting candidate is always visible to S5/WS-5 as such,
2989
+ // even when it was also due for a proactive maintenance run. signal-delta
2990
+ // overrides forgetting-safety because a ref with fresh feedback is reactive
2991
+ // and doesn't need the protective pass label for measurement purposes.
2992
+ for (const r of highSalienceRefs)
2993
+ eligibilitySourceByRef.set(r.ref, "high-salience");
2994
+ for (const r of proactiveRefs)
2995
+ eligibilitySourceByRef.set(r.ref, "proactive");
2996
+ for (const r of highRetrievalRefs)
2997
+ eligibilitySourceByRef.set(r.ref, "high-retrieval");
2998
+ // Apply forgetting-safety OVER proactive, high-retrieval, and high-salience
2999
+ // (already stamped in the loop above via
3000
+ // `eligibilitySourceByRef.set(ref, "forgetting-safety")`). No-op here: the
3001
+ // set() calls above for proactive/high-retrieval/high-salience overwrite the
3002
+ // earlier forgetting-safety stamp — so we re-apply forgetting-safety now for
3003
+ // those refs that are both forgetting candidates AND in another fallback lane.
3004
+ for (const ref of pendingForgettingRefs) {
3005
+ eligibilitySourceByRef.set(ref, "forgetting-safety");
3006
+ }
3007
+ // signal-delta is the strongest reactive signal and overrides forgetting-safety.
3008
+ for (const r of signalFiltered)
3009
+ eligibilitySourceByRef.set(r.ref, "signal-delta");
3010
+ // Update eligibilitySource on the ref objects themselves for any refs whose
3011
+ // lane changed (covers both new stubs and pre-existing refs).
3012
+ for (const r of mergedRefs) {
3013
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
3014
+ }
3015
+ }
3016
+ // ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
3017
+ // Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
3018
+ // revisited even with zero reactive signal (no feedback, no retrieval) and
3019
+ // regardless of cooldown — exactly like the forgetting-safety lane, replay is
3020
+ // injected AFTER cooldown/signal-delta partitioning so it bypasses those gates.
3021
+ //
3022
+ // Strictly additive: the replay slice is appended AFTER the --limit fresh slice
3023
+ // (see the loopRefs partition below), so it can never shrink the fresh-ref set.
3024
+ // Replay is the WEAKEST lane — it only stamps refs no other lane already claimed,
3025
+ // and budget is spent only on refs not already in mergedRefs (so a stronger lane
3026
+ // never has its budget wasted or its label overwritten).
3027
+ //
3028
+ // Default replayBudget=0 ⇒ this whole block is a no-op (no DB open, no event,
3029
+ // no mergedRefs mutation), preserving byte-identical pre-#610 selection behavior.
3030
+ const replayBudget = (options.config ?? loadConfig()).improve?.salience?.replayBudget ?? 0;
3031
+ const replayRefSet = new Set();
3032
+ if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
3033
+ let replayDb;
3034
+ try {
3035
+ replayDb = openStateDatabase(eventsCtx?.dbPath);
3036
+ const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
3037
+ const allRankScores = getAllRankScores(replayDb);
3038
+ // Candidate universe = every salience row NOT already in the pool, ordered by
3039
+ // rank_score desc with a deterministic ref-string tie-break (mirrors the main
3040
+ // sort). Converged refs (consecutive_no_ops >= dampener threshold) are fully
3041
+ // EXCLUDED — a stronger skip than the dampener (which only halves order).
3042
+ let convergedSkipped = 0;
3043
+ const candidates = [];
3044
+ for (const [ref, rankScore] of allRankScores) {
3045
+ if (alreadyInPool.has(ref))
3046
+ continue;
3047
+ const noOps = getConsecutiveNoOps(replayDb, ref);
3048
+ if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
3049
+ convergedSkipped++;
3050
+ continue;
3051
+ }
3052
+ candidates.push({ ref, rankScore });
3053
+ }
3054
+ candidates.sort((a, b) => b.rankScore !== a.rankScore ? b.rankScore - a.rankScore : a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0);
3055
+ const candidatePool = candidates.length;
3056
+ const selected = candidates.slice(0, replayBudget);
3057
+ const newReplayRefs = [];
3058
+ for (const { ref } of selected) {
3059
+ replayRefSet.add(ref);
3060
+ // Synthesise a stub (mirror the forgetting-safety stub). Resolve the
3061
+ // backing file from the planned-ref pool so the downstream existsSync
3062
+ // guard keeps the ref (a replay candidate from asset_salience whose file
3063
+ // is gone correctly drops out). Only refs present in the indexed pool can
3064
+ // be revisited — refs without a planned entry get no filePath and are
3065
+ // dropped by the disk check, which is the desired behavior.
3066
+ const planned = plannedRefs.find((p) => p.ref === ref);
3067
+ newReplayRefs.push({
3068
+ ref,
3069
+ reason: "scope-type",
3070
+ eligibilitySource: "replay",
3071
+ ...(planned?.filePath ? { filePath: planned.filePath } : {}),
3072
+ });
3073
+ // Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
3074
+ if (!salienceMap.has(ref)) {
3075
+ salienceMap.set(ref, {
3076
+ encoding: 0,
3077
+ outcome: 0,
3078
+ retrieval: 0,
3079
+ rankScore: allRankScores.get(ref) ?? 0,
3080
+ });
3081
+ }
3082
+ }
3083
+ if (newReplayRefs.length > 0) {
3084
+ mergedRefs = dedupeRefs([...mergedRefs, ...newReplayRefs]);
3085
+ // Replay is the WEAKEST lane: stamp 'replay' ONLY for refs not already
3086
+ // keyed by a stronger lane.
3087
+ for (const ref of replayRefSet) {
3088
+ if (!eligibilitySourceByRef.has(ref))
3089
+ eligibilitySourceByRef.set(ref, "replay");
3090
+ }
3091
+ for (const r of mergedRefs) {
3092
+ r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
3093
+ }
3094
+ }
3095
+ // Aggregated observability event (never per-ref).
3096
+ appendEvent({
3097
+ eventType: "improve_replay_selected",
3098
+ ref: undefined,
3099
+ metadata: {
3100
+ count: newReplayRefs.length,
3101
+ budget: replayBudget,
3102
+ convergedSkipped,
3103
+ candidatePool,
3104
+ },
3105
+ }, eventsCtx);
3106
+ }
3107
+ catch (err) {
3108
+ rethrowIfTestIsolationError(err);
3109
+ // best-effort: if DB unavailable, replayRefSet stays empty
3110
+ }
3111
+ finally {
3112
+ if (replayDb)
3113
+ replayDb.close();
3114
+ }
3115
+ }
3116
+ // Build no-op map for consolidation-selection dampener (plan §WS-1 step 8).
3117
+ // Reads consecutive_no_ops from the SAME pinned db handle used elsewhere in
3118
+ // this function. The effective score is used ONLY for processing/selection
3119
+ // order — the persisted rank_score in asset_salience is never mutated here.
3120
+ const noOpMap = new Map();
3121
+ try {
3122
+ const noOpDb = eventsCtx?.db ?? (eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
3123
+ if (noOpDb) {
3124
+ const ownsNoOpDb = !eventsCtx?.db;
3125
+ try {
3126
+ for (const r of mergedRefs) {
3127
+ noOpMap.set(r.ref, getConsecutiveNoOps(noOpDb, r.ref));
3128
+ }
3129
+ }
3130
+ finally {
3131
+ if (ownsNoOpDb)
3132
+ noOpDb.close();
3133
+ }
3134
+ }
3135
+ }
3136
+ catch {
3137
+ // best-effort: dampener failure never blocks selection
3138
+ }
3139
+ // Sort by effective selection score (desc), with explicit ref-string tie-break
3140
+ // for determinism. The effective score applies the consolidation-selection
3141
+ // dampener: assets that have been repeatedly skipped (consecutive_no_ops >=
3142
+ // THRESHOLD) are penalised by FACTOR so they sort after peers with similar
3143
+ // rankScore. The persisted rank_score is left unchanged — this is the whole
3144
+ // point of the dampener (stable assets stay fully retrievable).
3145
+ //
3146
+ // WIRING NOTE (plan §WS-1 step 8 / "consolidation-selection" disambiguation):
3147
+ // "consolidation-selection" in the plan refers to THIS reflect/distill
3148
+ // eligibility ordering — i.e. which assets are chosen for the reflect/distill
3149
+ // LLM pass — NOT to akmConsolidate (the cluster-merge phase at ~line 1994,
3150
+ // which runs earlier and never reads noOpMap). The no-op counter originates
3151
+ // from no-change reflect / quality-rejected distill outcomes; the dampener
3152
+ // suppresses repeated LLM attempts on those same assets without touching their
3153
+ // persisted rank_score (so they remain fully retrievable).
3154
+ //
3155
+ // This is the ONLY ranking path — negativeOnlyRatio and the legacy
3156
+ // symmetricValence branch are replaced. The three eligibilitySource lanes
3157
+ // (signal-delta / high-retrieval / proactive) survive as labels (set above).
3158
+ const effectiveScore = (ref) => {
3159
+ const rankScore = salienceMap.get(ref)?.rankScore ?? 0;
3160
+ const noOps = noOpMap.get(ref) ?? 0;
3161
+ return noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rankScore * SALIENCE_NO_OP_DAMPEN_FACTOR : rankScore;
3162
+ };
3163
+ const sorted = [...mergedRefs].sort((a, b) => {
3164
+ const scoreA = effectiveScore(a.ref);
3165
+ const scoreB = effectiveScore(b.ref);
3166
+ if (scoreB !== scoreA)
3167
+ return scoreB - scoreA;
3168
+ // Stable tie-break: deterministic regardless of input ordering.
3169
+ return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
3170
+ });
3171
+ // Phase 0: surface coverage gaps from zero-result search queries
3172
+ let coverageGaps = [];
3173
+ try {
3174
+ const dbForGaps = openExistingDatabase();
3175
+ try {
3176
+ coverageGaps = getZeroResultSearches(dbForGaps);
3177
+ }
3178
+ finally {
1952
3179
  closeDatabase(dbForGaps);
1953
3180
  }
1954
3181
  }
@@ -2015,8 +3242,22 @@ async function runImprovePreparationStage(args) {
2015
3242
  }
2016
3243
  }
2017
3244
  // ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
3245
+ //
3246
+ // #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
3247
+ // fresh slice. Without this split, a high-rankScore replay ref could sort above
3248
+ // a fresh ref in the single combined slice and STEAL its slot (violating AC2).
3249
+ // We partition into the replay lane vs the rest, apply --limit to the
3250
+ // non-replay (fresh) refs only, then APPEND up to `replayBudget` replay refs
3251
+ // after the fresh slice. Sort order within each partition is preserved.
3252
+ //
3253
+ // Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
3254
+ // replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
3255
+ // `replayLoop.slice(0, 0) === []` — byte-identical.
2018
3256
  const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
2019
- const loopRefs = options.limit ? allLoopRefs.slice(0, options.limit) : allLoopRefs;
3257
+ const replayLoop = allLoopRefs.filter((r) => r.eligibilitySource === "replay");
3258
+ const nonReplayLoop = allLoopRefs.filter((r) => r.eligibilitySource !== "replay");
3259
+ const baseLoop = options.limit ? nonReplayLoop.slice(0, options.limit) : nonReplayLoop;
3260
+ const loopRefs = [...baseLoop, ...replayLoop.slice(0, replayBudget)];
2020
3261
  // Update the returned distillOnlyRefs to the sorted order so callers see the
2021
3262
  // ranked view (loop stage uses it as a Set so order is irrelevant, but the
2022
3263
  // shape change keeps downstream consumers consistent).
@@ -2027,7 +3268,7 @@ async function runImprovePreparationStage(args) {
2027
3268
  `(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
2028
3269
  }
2029
3270
  if (signalAndRetrievalRefs.length > 0) {
2030
- info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval)`);
3271
+ info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
2031
3272
  }
2032
3273
  if (validationFailureRefs.size > 0) {
2033
3274
  info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
@@ -2038,6 +3279,17 @@ async function runImprovePreparationStage(args) {
2038
3279
  const deferredCount = actionableRefs.length - loopRefs.length;
2039
3280
  info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
2040
3281
  (options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
3282
+ // WS-4: Per-phase threshold auto-tune for the extract phase.
3283
+ // Persists result for the NEXT run's makeGateConfig to read.
3284
+ const extractTuneDbPath = eventsCtx?.dbPath;
3285
+ if (options.autoAccept !== undefined && extractTuneDbPath) {
3286
+ try {
3287
+ maybeAutoTuneThreshold(extractGateCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), extractTuneDbPath, undefined, "extract");
3288
+ }
3289
+ catch (err) {
3290
+ warn(`[improve] calibration auto-tune (extract) skipped: ${err instanceof Error ? err.message : String(err)}`);
3291
+ }
3292
+ }
2041
3293
  return {
2042
3294
  actions,
2043
3295
  cleanupWarnings,
@@ -2059,6 +3311,7 @@ async function runImprovePreparationStage(args) {
2059
3311
  gateAutoAcceptFailedCount,
2060
3312
  consolidation: consolidationPass.consolidation,
2061
3313
  consolidationRan: consolidationPass.consolidationRan,
3314
+ ...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
2062
3315
  };
2063
3316
  }
2064
3317
  async function runImproveLoopStage(args) {
@@ -2067,6 +3320,14 @@ async function runImproveLoopStage(args) {
2067
3320
  // receives only its fair share of the wall-clock budget.
2068
3321
  const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
2069
3322
  const RECENT_ERRORS_CAP = 3;
3323
+ // requirePlannedRefs guard: when the distill profile sets this flag, skip
3324
+ // distill for distill-only refs if the reflect phase produced no planned refs.
3325
+ // Prevents the distill loop from generating hundreds of distill-skipped events
3326
+ // on quiet passes (all refs on reflect cooldown, no new signal to distill).
3327
+ const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
3328
+ const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
3329
+ const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
3330
+ const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
2070
3331
  // R-2 / #389: Self-Consistency multi-sample voting helpers.
2071
3332
  // Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
2072
3333
  const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
@@ -2148,6 +3409,10 @@ async function runImproveLoopStage(args) {
2148
3409
  stashDir: primaryStashDir,
2149
3410
  config: options.config ?? loadConfig(),
2150
3411
  eventsCtx,
3412
+ stateDbPath: eventsCtx?.dbPath,
3413
+ // candidateCount drives the exploration budget. loopRefs is the per-phase
3414
+ // set for reflect/distill; pass it so exploration budget is proportional.
3415
+ candidateCount: loopRefs.length,
2151
3416
  });
2152
3417
  const distillGateCfg = makeGateConfig("distill", {
2153
3418
  globalThreshold: options.autoAccept,
@@ -2155,6 +3420,8 @@ async function runImproveLoopStage(args) {
2155
3420
  stashDir: primaryStashDir,
2156
3421
  config: options.config ?? loadConfig(),
2157
3422
  eventsCtx,
3423
+ stateDbPath: eventsCtx?.dbPath,
3424
+ candidateCount: loopRefs.length,
2158
3425
  });
2159
3426
  for (const planned of loopRefs) {
2160
3427
  if (Date.now() - startMs >= budgetMs) {
@@ -2227,6 +3494,9 @@ async function runImproveLoopStage(args) {
2227
3494
  eventSource: "improve",
2228
3495
  ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
2229
3496
  ...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
3497
+ // Attribution: carry the eligibility lane so reflect stamps it on
3498
+ // the reflect_invoked event and the persisted proposal.
3499
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2230
3500
  };
2231
3501
  // R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
2232
3502
  // Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
@@ -2251,6 +3521,9 @@ async function runImproveLoopStage(args) {
2251
3521
  source: "reflect",
2252
3522
  sourceRun: `reflect-sc-${Date.now()}`,
2253
3523
  payload: winner.proposal.payload,
3524
+ // Attribution: the self-consistency path persists the winner here
3525
+ // (draftMode skips reflect's own createProposal), so stamp the lane.
3526
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2254
3527
  });
2255
3528
  reflectResult = isProposalSkipped(persistResult)
2256
3529
  ? {
@@ -2326,6 +3599,28 @@ async function runImproveLoopStage(args) {
2326
3599
  reason: reflectResult.ok ? undefined : reflectResult.reason,
2327
3600
  },
2328
3601
  }, eventsCtx);
3602
+ // Plasticity counter (plan §WS-1 step 8): record no-ops so the
3603
+ // WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
3604
+ // repeatedly-silent assets during consolidation-selection.
3605
+ // A no_change reflect means the LLM was invoked but found nothing to
3606
+ // improve — the asset is stable. Track it. A successful reflect means
3607
+ // the asset changed; reset the counter so the dampener lifts.
3608
+ if (isNoChange && eventsCtx?.db) {
3609
+ try {
3610
+ recordNoOp(eventsCtx.db, planned.ref);
3611
+ }
3612
+ catch {
3613
+ // best-effort: plasticity counter failure never blocks the run
3614
+ }
3615
+ }
3616
+ else if (reflectResult.ok && eventsCtx?.db) {
3617
+ try {
3618
+ resetConsecutiveNoOps(eventsCtx.db, planned.ref);
3619
+ }
3620
+ catch {
3621
+ // best-effort
3622
+ }
3623
+ }
2329
3624
  if (reflectResult.ok) {
2330
3625
  const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
2331
3626
  gateAutoAcceptedCount += reflectGr.promoted.length;
@@ -2364,6 +3659,18 @@ async function runImproveLoopStage(args) {
2364
3659
  info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2365
3660
  continue;
2366
3661
  }
3662
+ // requirePlannedRefs guard: skip distill for distill-only refs when no
3663
+ // reflect-eligible refs were planned this run, preventing mass skip events.
3664
+ if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
3665
+ actions.push({
3666
+ ref: planned.ref,
3667
+ mode: "distill-skipped",
3668
+ result: { ok: true, reason: "require_planned_refs" },
3669
+ });
3670
+ completedCount++;
3671
+ info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
3672
+ continue;
3673
+ }
2367
3674
  // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
2368
3675
  // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
2369
3676
  // it will refuse.
@@ -2437,6 +3744,9 @@ async function runImproveLoopStage(args) {
2437
3744
  ref: planned.ref,
2438
3745
  ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
2439
3746
  ...(options.stashDir ? { stashDir: options.stashDir } : {}),
3747
+ // Attribution: carry the eligibility lane so distill stamps it on the
3748
+ // distill_invoked event and the persisted proposal.
3749
+ ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
2440
3750
  }));
2441
3751
  actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
2442
3752
  if (distillResult.outcome === "queued" && distillResult.proposal) {
@@ -2449,6 +3759,23 @@ async function runImproveLoopStage(args) {
2449
3759
  if (!promotedToKnowledge)
2450
3760
  memoryRefsForInference.add(planned.ref);
2451
3761
  }
3762
+ // Plasticity counter (plan §WS-1 step 8) for the distill path.
3763
+ // quality_rejected: the LLM ran but produced output that didn't pass the
3764
+ // quality gate — the asset is not yielding useful distill output.
3765
+ // queued: a proposal was produced; reset the no-op counter.
3766
+ if (eventsCtx?.db) {
3767
+ try {
3768
+ if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
3769
+ recordNoOp(eventsCtx.db, planned.ref);
3770
+ }
3771
+ else if (distillResult.outcome === "queued") {
3772
+ resetConsecutiveNoOps(eventsCtx.db, planned.ref);
3773
+ }
3774
+ }
3775
+ catch {
3776
+ // best-effort: plasticity counter failure never blocks the run
3777
+ }
3778
+ }
2452
3779
  if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
2453
3780
  const slug = planned.ref
2454
3781
  .replace(/[^a-z0-9]/gi, "-")
@@ -2515,6 +3842,26 @@ async function runImproveLoopStage(args) {
2515
3842
  completedCount++;
2516
3843
  info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
2517
3844
  }
3845
+ // WS-4: Per-phase threshold auto-tune — runs AFTER the loop so the gate
3846
+ // has processed all candidates for this run. Persists each phase's tuned
3847
+ // threshold to state.db for the NEXT run's makeGateConfig to read.
3848
+ // Best-effort: a tune failure must never fail the improve run.
3849
+ const stateDbPathForTune = eventsCtx?.dbPath;
3850
+ if (options.autoAccept !== undefined && stateDbPathForTune) {
3851
+ const phaseGateCfgMap = {
3852
+ reflect: reflectGateCfg,
3853
+ distill: distillGateCfg,
3854
+ };
3855
+ for (const phase of ["reflect", "distill"]) {
3856
+ const phaseCfg = phaseGateCfgMap[phase];
3857
+ try {
3858
+ maybeAutoTuneThreshold(phaseCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), stateDbPathForTune, undefined, phase);
3859
+ }
3860
+ catch (err) {
3861
+ warn(`[improve] calibration auto-tune (${phase}) skipped: ${err instanceof Error ? err.message : String(err)}`);
3862
+ }
3863
+ }
3864
+ }
2518
3865
  return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
2519
3866
  }
2520
3867
  async function runImprovePostLoopStage(args) {
@@ -2558,9 +3905,73 @@ async function runImprovePostLoopStage(args) {
2558
3905
  // best-effort
2559
3906
  }
2560
3907
  }
3908
+ // #609 — recombine / synthesize pass. Whole-corpus cross-episodic
3909
+ // generalization. Runs in the post-loop stage under consolidate.lock (it
3910
+ // reads the consolidated corpus and writes proposals). Opt-in: gated on the
3911
+ // `recombine` process being enabled, whole-stash / type scope (never `ref`),
3912
+ // and not a dry run. Mirrors the proactiveMaintenance opt-in wiring.
3913
+ let recombination;
3914
+ if (primaryStashDir &&
3915
+ improveProfile &&
3916
+ resolveProcessEnabled("recombine", improveProfile) &&
3917
+ scope.mode !== "ref" &&
3918
+ !options.dryRun) {
3919
+ const recombineFn = options.recombineFn ?? akmRecombine;
3920
+ try {
3921
+ recombination = await recombineFn({
3922
+ stashDir: primaryStashDir,
3923
+ config: options.config ?? loadConfig(),
3924
+ ...(options.runId ? { sourceRun: options.runId } : {}),
3925
+ ...(budgetSignal ? { signal: budgetSignal } : {}),
3926
+ ...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
3927
+ eligibilitySource: "recombine",
3928
+ ...(eventsCtx ? { ctx: eventsCtx } : {}),
3929
+ minClusterSize: improveProfile.processes?.recombine?.minClusterSize,
3930
+ maxClustersPerRun: improveProfile.processes?.recombine?.maxClustersPerRun,
3931
+ relatednessSource: improveProfile.processes?.recombine?.relatednessSource,
3932
+ confirmThreshold: improveProfile.processes?.recombine?.confirmThreshold,
3933
+ // #632 — clustering-tuning knobs. UNSET = pre-#632 behaviour.
3934
+ maxClusterSize: improveProfile.processes?.recombine?.maxClusterSize,
3935
+ excludeTags: improveProfile.processes?.recombine?.excludeTags,
3936
+ });
3937
+ }
3938
+ catch (e) {
3939
+ allWarnings.push(`recombine: ${String(e)}`);
3940
+ }
3941
+ }
3942
+ // #615 — procedural-compilation pass. Detects recurring successful ordered
3943
+ // action sequences and compiles them into workflow proposals. Opt-in: gated
3944
+ // on the `procedural` process being enabled, whole-stash / type scope (never
3945
+ // `ref`), and not a dry run. Mirrors the recombine opt-in wiring.
3946
+ let proceduralCompilation;
3947
+ if (primaryStashDir &&
3948
+ improveProfile &&
3949
+ resolveProcessEnabled("procedural", improveProfile) &&
3950
+ scope.mode !== "ref" &&
3951
+ !options.dryRun) {
3952
+ const proceduralFn = options.proceduralFn ?? akmProcedural;
3953
+ try {
3954
+ proceduralCompilation = await proceduralFn({
3955
+ stashDir: primaryStashDir,
3956
+ config: options.config ?? loadConfig(),
3957
+ ...(options.runId ? { sourceRun: options.runId } : {}),
3958
+ ...(budgetSignal ? { signal: budgetSignal } : {}),
3959
+ ...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
3960
+ eligibilitySource: "procedural",
3961
+ ...(eventsCtx ? { ctx: eventsCtx } : {}),
3962
+ minRecurrence: improveProfile.processes?.procedural?.minRecurrence,
3963
+ maxProposalsPerRun: improveProfile.processes?.procedural?.maxProposalsPerRun,
3964
+ });
3965
+ }
3966
+ catch (e) {
3967
+ allWarnings.push(`procedural: ${String(e)}`);
3968
+ }
3969
+ }
2561
3970
  return {
2562
3971
  allWarnings,
2563
3972
  deadUrls,
3973
+ ...(recombination ? { recombination } : {}),
3974
+ ...(proceduralCompilation ? { proceduralCompilation } : {}),
2564
3975
  ...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
2565
3976
  ...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
2566
3977
  ...(maintenanceResult.stalenessDetection ? { stalenessDetection: maintenanceResult.stalenessDetection } : {}),
@@ -2618,309 +4029,328 @@ export async function runImproveMaintenancePasses(args) {
2618
4029
  db = openIndexDb();
2619
4030
  }
2620
4031
  };
2621
- try {
2622
- db = openIndexDb();
2623
- // Memory inference candidate-discovery (post-Item 9 fix from
2624
- // memory:akm-improve-critical-review-2026-05-20). Previously this pass
2625
- // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
2626
- // candidateRefs filter. But memoryRefsForInference is populated from refs
2627
- // distilled THIS RUN by the time that happens, those parents are
2628
- // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
2629
- // them. The genuinely-pending parents in the stash never entered the
2630
- // filter. Result: 0/0/0 for 25 consecutive runs.
2631
- //
2632
- // Fix: always run the pass when the feature is enabled; let the pass's
2633
- // own `collectPendingMemories` + `isPendingMemory` predicate find
2634
- // candidates from the filesystem-of-truth. The this-run set is still
2635
- // logged as a hint but no longer used as a filter.
2636
- const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
2637
- if (memoryInferenceDisabledByProfile) {
2638
- info("[improve] memory inference skipped (disabled by improve profile)");
2639
- }
2640
- else {
2641
- const hintRefs = memoryRefsForInference.size;
2642
- info(hintRefs > 0
2643
- ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
2644
- : "[improve] memory inference starting (discovering pending parents)");
2645
- const inferenceStart = Date.now();
2646
- try {
2647
- // O-1 (#364): pass budget signal so a hung inference call is cancelled.
2648
- memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
2649
- config,
2650
- sources,
2651
- signal: budgetSignal,
2652
- db,
2653
- reEnrich: false,
2654
- onProgress: (event) => {
2655
- const current = event.currentRef ? ` ${event.currentRef}` : "";
2656
- info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
2657
- },
2658
- }));
2659
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2660
- actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
2661
- info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2662
- }
2663
- catch (err) {
2664
- memoryInferenceDurationMs = Date.now() - inferenceStart;
2665
- allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
4032
+ await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
4033
+ try {
4034
+ db = openIndexDb();
4035
+ // Memory inference candidate-discovery (post-Item 9 fix from
4036
+ // memory:akm-improve-critical-review-2026-05-20). Previously this pass
4037
+ // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
4038
+ // candidateRefs filter. But memoryRefsForInference is populated from refs
4039
+ // distilled THIS RUN by the time that happens, those parents are
4040
+ // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
4041
+ // them. The genuinely-pending parents in the stash never entered the
4042
+ // filter. Result: 0/0/0 for 25 consecutive runs.
4043
+ //
4044
+ // Fix: always run the pass when the feature is enabled; let the pass's
4045
+ // own `collectPendingMemories` + `isPendingMemory` predicate find
4046
+ // candidates from the filesystem-of-truth. The this-run set is still
4047
+ // logged as a hint but no longer used as a filter.
4048
+ const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
4049
+ const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
4050
+ const pendingBelowMinCount = (() => {
4051
+ if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
4052
+ return false;
4053
+ const pending = collectPendingMemories(primaryStashDir).length;
4054
+ if (pending < minPendingCount) {
4055
+ info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
4056
+ return true;
4057
+ }
4058
+ return false;
4059
+ })();
4060
+ if (memoryInferenceDisabledByProfile) {
4061
+ info("[improve] memory inference skipped (disabled by improve profile)");
2666
4062
  }
2667
- }
2668
- if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
2669
- info("[improve] reindexing after memory inference writes");
2670
- try {
2671
- await reindexWithIndexDbReleased(primaryStashDir);
2672
- reindexedAfterInference = true;
2673
- info("[improve] reindex after memory inference complete");
4063
+ else if (pendingBelowMinCount) {
4064
+ // skipped message already emitted above
2674
4065
  }
2675
- catch (err) {
2676
- allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2677
- }
2678
- }
2679
- const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
2680
- const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
2681
- const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
2682
- // Build the set of refs actually touched this run.
2683
- const touchedRefs = new Set();
2684
- for (const r of args.actionableRefs)
2685
- touchedRefs.add(r.ref);
2686
- for (const r of memoryRefsForInference)
2687
- touchedRefs.add(r);
2688
- // INVARIANT: graph extraction normally runs only on files touched by
2689
- // actionable refs (candidatePaths). Full-corpus scans are opt-in via
2690
- // profile.processes.graphExtraction.fullScan = true (used by the
2691
- // `graph-refresh` built-in profile and its weekly scheduled task).
2692
- // The empty-Set fallback is intentional when no refs were touched —
2693
- // the extractor's filter rejects every file and returns empty, keeping
2694
- // the pass invoked so the action is recorded and tests stay exercised.
2695
- if (graphExtractionDisabledByProfile) {
2696
- info("[improve] graph extraction skipped (disabled by improve profile)");
2697
- }
2698
- else if (sources.length > 0 && graphEnabled) {
2699
- info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
2700
- const extractionStart = Date.now();
2701
- try {
2702
- // D9: if consolidation ran but memory inference did not reindex, force a reindex
2703
- // so graph extraction sees current DB state after consolidation writes.
2704
- if (consolidationRan && !reindexedAfterInference) {
2705
- info("[improve] reindexing after consolidation (graph extraction needs current state)");
2706
- try {
2707
- await reindexWithIndexDbReleased(primaryStashDir);
2708
- reindexedAfterInference = true;
2709
- info("[improve] reindex after consolidation complete");
2710
- }
2711
- catch (err) {
2712
- allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
2713
- }
4066
+ else {
4067
+ const hintRefs = memoryRefsForInference.size;
4068
+ info(hintRefs > 0
4069
+ ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
4070
+ : "[improve] memory inference starting (discovering pending parents)");
4071
+ const inferenceStart = Date.now();
4072
+ try {
4073
+ // O-1 (#364): pass budget signal so a hung inference call is cancelled.
4074
+ memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
4075
+ config,
4076
+ sources,
4077
+ signal: budgetSignal,
4078
+ db,
4079
+ reEnrich: false,
4080
+ onProgress: (event) => {
4081
+ const current = event.currentRef ? ` ${event.currentRef}` : "";
4082
+ info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
4083
+ },
4084
+ }));
4085
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
4086
+ actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
4087
+ info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
2714
4088
  }
2715
- // #584: no close/reopen needed here — reindexWithIndexDbReleased
2716
- // already swapped in a fresh post-reindex handle.
2717
- // Resolve touched refs to absolute file paths. Skipped for fullScan
2718
- // (candidatePaths stays undefined → extractor processes all files).
2719
- let candidatePaths;
2720
- if (!graphExtractionFullScan) {
2721
- candidatePaths = new Set();
2722
- if (primaryStashDir && touchedRefs.size > 0) {
2723
- const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
2724
- const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
2725
- for (const p of resolved) {
2726
- if (typeof p === "string" && p.length > 0)
2727
- candidatePaths.add(p);
2728
- }
2729
- }
4089
+ catch (err) {
4090
+ memoryInferenceDurationMs = Date.now() - inferenceStart;
4091
+ allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2730
4092
  }
2731
- const progressHandler = (event) => {
2732
- const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
2733
- info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
2734
- };
2735
- // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
2736
- graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
2737
- config,
2738
- sources,
2739
- signal: budgetSignal,
2740
- db,
2741
- reEnrich: false,
2742
- onProgress: progressHandler,
2743
- options: { candidatePaths },
2744
- }));
2745
- graphExtractionDurationMs = Date.now() - extractionStart;
2746
- actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
2747
- info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
2748
- }
2749
- catch (err) {
2750
- graphExtractionDurationMs = Date.now() - extractionStart;
2751
- allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
2752
4093
  }
2753
- }
2754
- else if (sources.length > 0 && !graphEnabled) {
2755
- info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
2756
- }
2757
- // Orphan proposal purge — reject pending reflect proposals whose target
2758
- // asset no longer exists on disk. Runs after graph extraction so newly
2759
- // promoted assets from accept flows during this run are already present.
2760
- if (primaryStashDir) {
2761
- try {
2762
- const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
2763
- orphansPurged = purgeResult.rejected;
2764
- if (purgeResult.rejected > 0) {
2765
- info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
4094
+ if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
4095
+ info("[improve] reindexing after memory inference writes");
4096
+ try {
4097
+ await reindexWithIndexDbReleased(primaryStashDir);
4098
+ reindexedAfterInference = true;
4099
+ info("[improve] reindex after memory inference complete");
4100
+ }
4101
+ catch (err) {
4102
+ allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
2766
4103
  }
2767
- appendEvent({
2768
- eventType: "proposal_orphan_purge",
2769
- ref: "proposals:_orphan-purge",
2770
- metadata: {
2771
- checked: purgeResult.checked,
2772
- rejected: purgeResult.rejected,
2773
- durationMs: purgeResult.durationMs,
2774
- byType: purgeResult.byType,
2775
- orphans: purgeResult.orphans.map((o) => o.ref),
2776
- },
2777
- }, eventsCtx);
2778
4104
  }
2779
- catch (err) {
2780
- allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
4105
+ const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
4106
+ const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
4107
+ const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
4108
+ // #624 P2: optional incremental high-signal-first cap. Unset = process all
4109
+ // eligible (byte-identical to today; no ranking/slice).
4110
+ const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
4111
+ // Build the set of refs actually touched this run.
4112
+ const touchedRefs = new Set();
4113
+ for (const r of args.actionableRefs)
4114
+ touchedRefs.add(r.ref);
4115
+ for (const r of memoryRefsForInference)
4116
+ touchedRefs.add(r);
4117
+ // INVARIANT: graph extraction normally runs only on files touched by
4118
+ // actionable refs (candidatePaths). Full-corpus scans are opt-in via
4119
+ // profile.processes.graphExtraction.fullScan = true (used by the
4120
+ // `graph-refresh` built-in profile and its weekly scheduled task).
4121
+ // The empty-Set fallback is intentional when no refs were touched —
4122
+ // the extractor's filter rejects every file and returns empty, keeping
4123
+ // the pass invoked so the action is recorded and tests stay exercised.
4124
+ if (graphExtractionDisabledByProfile) {
4125
+ info("[improve] graph extraction skipped (disabled by improve profile)");
2781
4126
  }
2782
- // Phase 6B (Advantage D6b): expire pending proposals that have aged past
2783
- // the retention window. Runs AFTER orphan purge so we never double-archive
2784
- // a proposal that orphan-purge already moved. `expireStaleProposals` emits
2785
- // its own per-proposal `proposal_expired` events; we additionally emit a
2786
- // single roll-up event here for parity with the orphan-purge surface.
2787
- try {
2788
- const expireResult = expireStaleProposals(primaryStashDir, config);
2789
- proposalsExpired = expireResult.expired;
2790
- if (expireResult.expired > 0) {
2791
- info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
2792
- `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
4127
+ else if (sources.length > 0 && graphEnabled) {
4128
+ info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
4129
+ const extractionStart = Date.now();
4130
+ try {
4131
+ // D9: if consolidation ran but memory inference did not reindex, force a reindex
4132
+ // so graph extraction sees current DB state after consolidation writes.
4133
+ if (consolidationRan && !reindexedAfterInference) {
4134
+ info("[improve] reindexing after consolidation (graph extraction needs current state)");
4135
+ try {
4136
+ await reindexWithIndexDbReleased(primaryStashDir);
4137
+ reindexedAfterInference = true;
4138
+ info("[improve] reindex after consolidation complete");
4139
+ }
4140
+ catch (err) {
4141
+ allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
4142
+ }
4143
+ }
4144
+ // #584: no close/reopen needed here — reindexWithIndexDbReleased
4145
+ // already swapped in a fresh post-reindex handle.
4146
+ // Resolve touched refs to absolute file paths. Skipped for fullScan
4147
+ // (candidatePaths stays undefined → extractor processes all files).
4148
+ let candidatePaths;
4149
+ if (!graphExtractionFullScan) {
4150
+ candidatePaths = new Set();
4151
+ if (primaryStashDir && touchedRefs.size > 0) {
4152
+ const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
4153
+ const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
4154
+ for (const p of resolved) {
4155
+ if (typeof p === "string" && p.length > 0)
4156
+ candidatePaths.add(p);
4157
+ }
4158
+ }
4159
+ }
4160
+ const progressHandler = (event) => {
4161
+ const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
4162
+ info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
4163
+ };
4164
+ // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
4165
+ graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
4166
+ config,
4167
+ sources,
4168
+ signal: budgetSignal,
4169
+ db,
4170
+ reEnrich: false,
4171
+ onProgress: progressHandler,
4172
+ options: { candidatePaths, ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}) },
4173
+ }));
4174
+ graphExtractionDurationMs = Date.now() - extractionStart;
4175
+ actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
4176
+ info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
4177
+ }
4178
+ catch (err) {
4179
+ graphExtractionDurationMs = Date.now() - extractionStart;
4180
+ allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
2793
4181
  }
2794
- appendEvent({
2795
- eventType: "proposal_expiration_pass",
2796
- ref: "proposals:_expiration",
2797
- metadata: {
2798
- checked: expireResult.checked,
2799
- expired: expireResult.expired,
2800
- durationMs: expireResult.durationMs,
2801
- retentionDays: expireResult.retentionDays,
2802
- expiredProposals: expireResult.expiredProposals,
2803
- },
2804
- }, eventsCtx);
2805
4182
  }
2806
- catch (err) {
2807
- allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
4183
+ else if (sources.length > 0 && !graphEnabled) {
4184
+ info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
2808
4185
  }
2809
- }
2810
- // Fix #2 (observability 0.8.0): trim the events table in state.db so it
2811
- // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
2812
- // invocation, and every command surface emits at least one event besides —
2813
- // without this trim, state.db is a permanent append-only log. Config key
2814
- // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
2815
- // window. The purge runs against state.db (a different SQLite file from
2816
- // the index `db` above).
2817
- {
2818
- const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
2819
- if (retentionDays > 0) {
2820
- // #585: reuse the long-lived eventsCtx.db connection when akmImprove
2821
- // opened one — opening a second state.db write connection while
2822
- // eventsDb is still live made two simultaneous writers contend on the
2823
- // same WAL file ("database is locked"). Only the eventsCtx.dbPath
2824
- // fallback path (state.db failed to open up-front) opens — and then
2825
- // owns and closes — its own handle. C2 still holds: the fallback uses
2826
- // the boundary-pinned path, never a live `process.env` re-read.
2827
- const ownsStateDb = !eventsCtx?.db;
2828
- let stateDb;
4186
+ // Orphan proposal purge — reject pending reflect proposals whose target
4187
+ // asset no longer exists on disk. Runs after graph extraction so newly
4188
+ // promoted assets from accept flows during this run are already present.
4189
+ if (primaryStashDir) {
2829
4190
  try {
2830
- stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
2831
- const purgedCount = purgeOldEvents(stateDb, retentionDays);
2832
- if (purgedCount > 0) {
2833
- info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
2834
- }
2835
- appendEvent({
2836
- eventType: "events_purged",
2837
- ref: "events:_purge",
2838
- metadata: { purgedCount, retentionDays },
2839
- }, eventsCtx);
2840
- // improve_runs uses the same retention window as events — both are
2841
- // observability/audit data, both grow append-only, both have a
2842
- // dedicated purge helper. Mirroring the events purge here means a
2843
- // single retention knob (improve.eventRetentionDays) governs both.
2844
- const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
2845
- if (improveRunsPurged > 0) {
2846
- info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
4191
+ const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
4192
+ orphansPurged = purgeResult.rejected;
4193
+ if (purgeResult.rejected > 0) {
4194
+ info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
2847
4195
  }
2848
4196
  appendEvent({
2849
- eventType: "improve_runs_purged",
2850
- ref: "improve_runs:_purge",
2851
- metadata: { purgedCount: improveRunsPurged, retentionDays },
4197
+ eventType: "proposal_orphan_purge",
4198
+ ref: "proposals:_orphan-purge",
4199
+ metadata: {
4200
+ checked: purgeResult.checked,
4201
+ rejected: purgeResult.rejected,
4202
+ durationMs: purgeResult.durationMs,
4203
+ byType: purgeResult.byType,
4204
+ orphans: purgeResult.orphans.map((o) => o.ref),
4205
+ },
2852
4206
  }, eventsCtx);
2853
4207
  }
2854
4208
  catch (err) {
2855
- allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
2856
- }
2857
- finally {
2858
- if (ownsStateDb && stateDb) {
2859
- try {
2860
- stateDb.close();
2861
- }
2862
- catch {
2863
- // best-effort
2864
- }
2865
- }
4209
+ allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
2866
4210
  }
2867
- // task_logs in logs.db (#579) shares the same retention window as
2868
- // events/improve_runs all three are observability data governed by
2869
- // the single improve.eventRetentionDays knob. Separate try/finally
2870
- // because logs.db is a different file: a locked/missing logs.db must
2871
- // not block the state.db purges above.
2872
- let logsDb;
4211
+ // Phase 6B (Advantage D6b): expire pending proposals that have aged past
4212
+ // the retention window. Runs AFTER orphan purge so we never double-archive
4213
+ // a proposal that orphan-purge already moved. `expireStaleProposals` emits
4214
+ // its own per-proposal `proposal_expired` events; we additionally emit a
4215
+ // single roll-up event here for parity with the orphan-purge surface.
2873
4216
  try {
2874
- logsDb = openLogsDatabase();
2875
- const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
2876
- if (taskLogsPurged > 0) {
2877
- info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
4217
+ const expireResult = expireStaleProposals(primaryStashDir, config);
4218
+ proposalsExpired = expireResult.expired;
4219
+ if (expireResult.expired > 0) {
4220
+ info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
4221
+ `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
2878
4222
  }
2879
4223
  appendEvent({
2880
- eventType: "task_logs_purged",
2881
- ref: "task_logs:_purge",
2882
- metadata: { purgedCount: taskLogsPurged, retentionDays },
4224
+ eventType: "proposal_expiration_pass",
4225
+ ref: "proposals:_expiration",
4226
+ metadata: {
4227
+ checked: expireResult.checked,
4228
+ expired: expireResult.expired,
4229
+ durationMs: expireResult.durationMs,
4230
+ retentionDays: expireResult.retentionDays,
4231
+ expiredProposals: expireResult.expiredProposals,
4232
+ },
2883
4233
  }, eventsCtx);
2884
4234
  }
2885
4235
  catch (err) {
2886
- allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
4236
+ allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
2887
4237
  }
2888
- finally {
2889
- if (logsDb) {
2890
- try {
2891
- logsDb.close();
4238
+ }
4239
+ // Fix #2 (observability 0.8.0): trim the events table in state.db so it
4240
+ // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
4241
+ // invocation, and every command surface emits at least one event besides —
4242
+ // without this trim, state.db is a permanent append-only log. Config key
4243
+ // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
4244
+ // window. The purge runs against state.db (a different SQLite file from
4245
+ // the index `db` above).
4246
+ {
4247
+ const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
4248
+ if (retentionDays > 0) {
4249
+ // #585: reuse the long-lived eventsCtx.db connection when akmImprove
4250
+ // opened one — opening a second state.db write connection while
4251
+ // eventsDb is still live made two simultaneous writers contend on the
4252
+ // same WAL file ("database is locked"). Only the eventsCtx.dbPath
4253
+ // fallback path (state.db failed to open up-front) opens — and then
4254
+ // owns and closes — its own handle. C2 still holds: the fallback uses
4255
+ // the boundary-pinned path, never a live `process.env` re-read.
4256
+ const ownsStateDb = !eventsCtx?.db;
4257
+ let stateDb;
4258
+ try {
4259
+ stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
4260
+ const purgedCount = purgeOldEvents(stateDb, retentionDays);
4261
+ if (purgedCount > 0) {
4262
+ info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
2892
4263
  }
2893
- catch {
2894
- // best-effort
4264
+ appendEvent({
4265
+ eventType: "events_purged",
4266
+ ref: "events:_purge",
4267
+ metadata: { purgedCount, retentionDays },
4268
+ }, eventsCtx);
4269
+ // improve_runs uses the same retention window as events — both are
4270
+ // observability/audit data, both grow append-only, both have a
4271
+ // dedicated purge helper. Mirroring the events purge here means a
4272
+ // single retention knob (improve.eventRetentionDays) governs both.
4273
+ const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
4274
+ if (improveRunsPurged > 0) {
4275
+ info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
4276
+ }
4277
+ appendEvent({
4278
+ eventType: "improve_runs_purged",
4279
+ ref: "improve_runs:_purge",
4280
+ metadata: { purgedCount: improveRunsPurged, retentionDays },
4281
+ }, eventsCtx);
4282
+ }
4283
+ catch (err) {
4284
+ allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
4285
+ }
4286
+ finally {
4287
+ if (ownsStateDb && stateDb) {
4288
+ try {
4289
+ stateDb.close();
4290
+ }
4291
+ catch {
4292
+ // best-effort
4293
+ }
4294
+ }
4295
+ }
4296
+ // task_logs in logs.db (#579) shares the same retention window as
4297
+ // events/improve_runs — all three are observability data governed by
4298
+ // the single improve.eventRetentionDays knob. Separate try/finally
4299
+ // because logs.db is a different file: a locked/missing logs.db must
4300
+ // not block the state.db purges above.
4301
+ let logsDb;
4302
+ try {
4303
+ logsDb = openLogsDatabase();
4304
+ const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
4305
+ if (taskLogsPurged > 0) {
4306
+ info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
4307
+ }
4308
+ appendEvent({
4309
+ eventType: "task_logs_purged",
4310
+ ref: "task_logs:_purge",
4311
+ metadata: { purgedCount: taskLogsPurged, retentionDays },
4312
+ }, eventsCtx);
4313
+ }
4314
+ catch (err) {
4315
+ allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
4316
+ }
4317
+ finally {
4318
+ if (logsDb) {
4319
+ try {
4320
+ logsDb.close();
4321
+ }
4322
+ catch {
4323
+ // best-effort
4324
+ }
2895
4325
  }
2896
4326
  }
2897
4327
  }
2898
4328
  }
2899
- }
2900
- // Phase 4A (staleness detection). Activates the `deprecated` belief-state
2901
- // machinery shipped in Phase 1A. Default OFF gated by
2902
- // `features.index.staleness_detection.enabled`. Runs after orphan purge
2903
- // and before the URL check (which lives in the outer caller).
2904
- if (sources.length > 0) {
2905
- try {
2906
- stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
2907
- if (stalenessDetection.considered > 0) {
2908
- info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
2909
- `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
2910
- `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
4329
+ // Phase 4A (staleness detection). Activates the `deprecated` belief-state
4330
+ // machinery shipped in Phase 1A. Default OFF gated by
4331
+ // `features.index.staleness_detection.enabled`. Runs after orphan purge
4332
+ // and before the URL check (which lives in the outer caller).
4333
+ if (sources.length > 0) {
4334
+ try {
4335
+ stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
4336
+ if (stalenessDetection.considered > 0) {
4337
+ info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
4338
+ `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
4339
+ `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
4340
+ }
4341
+ for (const w of stalenessDetection.warnings)
4342
+ allWarnings.push(`[improve] staleness detection: ${w}`);
4343
+ }
4344
+ catch (err) {
4345
+ allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
2911
4346
  }
2912
- for (const w of stalenessDetection.warnings)
2913
- allWarnings.push(`[improve] staleness detection: ${w}`);
2914
- }
2915
- catch (err) {
2916
- allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
2917
4347
  }
2918
4348
  }
2919
- }
2920
- finally {
2921
- if (db)
2922
- closeDatabase(db);
2923
- }
4349
+ finally {
4350
+ if (db)
4351
+ closeDatabase(db);
4352
+ }
4353
+ });
2924
4354
  return {
2925
4355
  ...(memoryInference ? { memoryInference } : {}),
2926
4356
  ...(graphExtraction ? { graphExtraction } : {}),