akm-cli 0.9.0-beta.45 → 0.9.0-beta.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dist/cli/shared.js +28 -0
  2. package/dist/cli.js +1 -2
  3. package/dist/commands/env/env-cli.js +16 -24
  4. package/dist/commands/env/secret-cli.js +12 -20
  5. package/dist/commands/graph/graph-cli.js +5 -13
  6. package/dist/commands/graph/graph.js +3 -3
  7. package/dist/commands/improve/consolidate/chunking.js +141 -0
  8. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  9. package/dist/commands/improve/consolidate/merge.js +145 -0
  10. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  11. package/dist/commands/improve/consolidate/types.js +4 -0
  12. package/dist/commands/improve/consolidate.js +20 -571
  13. package/dist/commands/improve/distill.js +5 -9
  14. package/dist/commands/improve/eligibility.js +434 -0
  15. package/dist/commands/improve/extract-cli.js +9 -1
  16. package/dist/commands/improve/extract.js +5 -19
  17. package/dist/commands/improve/improve-auto-accept.js +4 -8
  18. package/dist/commands/improve/improve-cli.js +35 -60
  19. package/dist/commands/improve/improve-result-file.js +5 -23
  20. package/dist/commands/improve/improve-session.js +58 -0
  21. package/dist/commands/improve/improve.js +107 -3606
  22. package/dist/commands/improve/locks.js +154 -0
  23. package/dist/commands/improve/loop-stages.js +1079 -0
  24. package/dist/commands/improve/preparation.js +1963 -0
  25. package/dist/commands/improve/recombine.js +6 -12
  26. package/dist/commands/improve/reflect.js +29 -34
  27. package/dist/commands/proposal/drain.js +25 -48
  28. package/dist/commands/proposal/proposal-cli.js +21 -31
  29. package/dist/commands/proposal/validators/proposals.js +3 -7
  30. package/dist/commands/read/curate.js +70 -14
  31. package/dist/commands/read/knowledge.js +2 -2
  32. package/dist/commands/sources/self-update.js +2 -2
  33. package/dist/commands/sources/stash-cli.js +9 -37
  34. package/dist/commands/tasks/tasks-cli.js +19 -27
  35. package/dist/commands/wiki-cli.js +21 -35
  36. package/dist/core/config/config.js +18 -2
  37. package/dist/core/events.js +3 -7
  38. package/dist/core/logs-db.js +6 -63
  39. package/dist/core/state/migrations.js +714 -0
  40. package/dist/core/state-db.js +28 -779
  41. package/dist/indexer/db/db.js +82 -216
  42. package/dist/indexer/indexer.js +11 -112
  43. package/dist/indexer/passes/dir-staleness.js +114 -0
  44. package/dist/indexer/search/search-source.js +10 -24
  45. package/dist/integrations/agent/runner-dispatch.js +59 -0
  46. package/dist/llm/client.js +22 -11
  47. package/dist/llm/graph-extract.js +28 -39
  48. package/dist/llm/memory-infer.js +34 -22
  49. package/dist/llm/metadata-enhance.js +35 -30
  50. package/dist/llm/structured-call.js +49 -0
  51. package/dist/output/shapes/passthrough.js +0 -1
  52. package/dist/registry/providers/skills-sh.js +21 -147
  53. package/dist/registry/providers/static-index.js +15 -157
  54. package/dist/registry/resolve.js +22 -9
  55. package/dist/scripts/migrate-storage.js +892 -1186
  56. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
  57. package/dist/setup/setup.js +26 -5
  58. package/dist/sources/providers/filesystem.js +0 -1
  59. package/dist/sources/providers/git-install.js +206 -0
  60. package/dist/sources/providers/git-provider.js +234 -0
  61. package/dist/sources/providers/git-stash.js +248 -0
  62. package/dist/sources/providers/git.js +10 -671
  63. package/dist/sources/providers/npm.js +2 -6
  64. package/dist/sources/providers/sync-from-ref.js +9 -1
  65. package/dist/sources/providers/website.js +2 -3
  66. package/dist/sources/website-ingest.js +51 -9
  67. package/dist/sources/wiki-fetchers/registry.js +53 -0
  68. package/dist/sources/wiki-fetchers/youtube.js +181 -0
  69. package/dist/storage/database.js +45 -10
  70. package/dist/storage/managed-db.js +82 -0
  71. package/dist/storage/repositories/registry-cache.js +92 -0
  72. package/dist/tasks/runner.js +5 -13
  73. package/dist/workflows/runtime/runs.js +1 -117
  74. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  75. package/package.json +5 -5
  76. package/dist/commands/db-cli.js +0 -23
  77. package/dist/indexer/db/db-backup.js +0 -376
@@ -4,194 +4,45 @@
4
4
  import fs from "node:fs";
5
5
  import path from "node:path";
6
6
  import { assertNever } from "../../core/assert.js";
7
- import { makeAssetRef, parseAssetRef } from "../../core/asset/asset-ref.js";
8
- import { parseFrontmatter } from "../../core/asset/frontmatter.js";
9
- import { daysToMs, isAssetType } from "../../core/common.js";
10
- import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
11
- import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
7
+ import { daysToMs } from "../../core/common.js";
8
+ import { loadConfig } from "../../core/config/config.js";
9
+ import { rethrowIfTestIsolationError } from "../../core/errors.js";
12
10
  import { appendEvent, readEvents } from "../../core/events.js";
13
- import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../../core/file-lock.js";
14
11
  import { classifyImproveAction } from "../../core/improve-types.js";
15
- import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
16
12
  import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
17
- import { listProposalGateDecisions, listStateProposals, openStateDatabase, persistPhaseThreshold, purgeOldEvents, purgeOldImproveRuns, } from "../../core/state-db.js";
13
+ import { openStateDatabase } from "../../core/state-db.js";
18
14
  import { info, warn } from "../../core/warn.js";
19
- import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
15
+ import { closeDatabase, getEntryCount, openExistingDatabase } from "../../indexer/db/db.js";
20
16
  import { ensureIndex } from "../../indexer/ensure-index.js";
21
- import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
22
- import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
23
17
  import { akmIndex } from "../../indexer/indexer.js";
24
- import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
25
- import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
26
- import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
27
- import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
28
- import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
29
- import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
30
- import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
31
- import { isProcessEnabled } from "../../llm/feature-gate.js";
18
+ import { resolveSourceEntries } from "../../indexer/search/search-source.js";
19
+ import { resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
32
20
  import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
33
21
  import { withLlmStage } from "../../llm/usage-telemetry.js";
34
22
  import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
35
- import { akmLint } from "../lint/index.js";
36
23
  import { drainProposals } from "../proposal/drain.js";
37
24
  import { resolveDrainPolicy } from "../proposal/drain-policies.js";
38
- import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
39
- import { runSchemaRepairPass } from "../sources/schema-repair.js";
40
- import { checkDeadUrls } from "../url-checker.js";
41
- import { computeThresholdAutoTune, gateDecisionsToSamples, summarizeCalibration, } from "./calibration.js";
42
- import { akmConsolidate } from "./consolidate.js";
43
- import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
44
- import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
45
- import { countEvalCases, writeEvalCase } from "./eval-cases.js";
46
- import { akmExtract, countNewExtractCandidates } from "./extract.js";
47
- import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
48
- import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
49
- import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
25
+ import { akmDistill } from "./distill.js";
26
+ // Eligibility / candidate-selection predicates live in ./eligibility.
27
+ import { buildLatestProposalTsMap, collectEligibleRefs, memoryCleanupParentRef, resolveImproveScope, shouldAnalyzeMemoryCleanup, } from "./eligibility.js";
28
+ import { countEvalCases } from "./eval-cases.js";
29
+ import { resolveImproveProfile, resolveProcessEnabled } from "./improve-profiles.js";
30
+ // #607 per-process lock primitives live in ./locks. Imported for internal use;
31
+ // resetHeldProcessLocks is re-exported (the test seam imports it from here).
32
+ import { PROCESS_LOCK_DEFS, processLockPath, releaseAllProcessLocks, releaseHeldLocksIfOwned, releaseProcessLock, tryAcquireProcessLock, withOptionalProcessLock, } from "./locks.js";
33
+ // The cycle loop / post-loop / maintenance stages live in ./loop-stages.
34
+ import { runImproveLoopStage, runImprovePostLoopStage } from "./loop-stages.js";
50
35
  import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
51
- import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
52
- import { computeProxyAdequacy, getAllAssetOutcomes, getOutcomeScoresByRef, outcomeScoreToSalience, updateAssetOutcome, } from "./outcome-loop.js";
53
- import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, filterProactiveDue, selectProactiveMaintenanceRefs, } from "./proactive-maintenance.js";
54
- import { akmProcedural } from "./procedural.js";
55
- import { akmRecombine } from "./recombine.js";
36
+ import { analyzeMemoryCleanup } from "./memory/memory-improve.js";
37
+ // The pre-loop preparation pipeline lives in ./preparation.
38
+ import { runImprovePreparationStage } from "./preparation.js";
39
+ import { DEFAULT_DUE_DAYS, filterProactiveDue } from "./proactive-maintenance.js";
56
40
  import { akmReflect } from "./reflect.js";
57
- import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getConsecutiveNoOps, getLastUseMsByRef, isContentEncodingRow, recordNoOp, resetConsecutiveNoOps, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
58
- // #607 Lock Decomposition: fine-grained per-process locks replace the single
59
- // `improve.lock`. Three independent locks allow concurrent improve runs when
60
- // they touch different subsystems (e.g. quick-shredder consolidate can run
61
- // alongside daily reflect+distill).
62
- //
63
- // consolidate.lock — protects consolidate + memoryInference (both write index.db)
64
- // reflect-distill.lock — protects reflect + distill (both write state.db proposals)
65
- // triage.lock — protects triage (writes proposal promotions)
66
- //
67
- // Stale timeouts are per-lock, tuned to the expected runtime of the protected
68
- // processes: consolidate is disk-bound (1h), reflect+distill is GPU-bound (2h),
69
- // triage is fast (30min).
70
- const PROCESS_LOCK_DEFS = {
71
- consolidate: { fileName: "consolidate.lock", staleAfterMs: 60 * 60 * 1000 },
72
- reflectDistill: { fileName: "reflect-distill.lock", staleAfterMs: 2 * 60 * 60 * 1000 },
73
- triage: { fileName: "triage.lock", staleAfterMs: 30 * 60 * 1000 },
74
- };
75
- const heldProcessLocks = new Set();
76
- export function resetHeldProcessLocks() {
77
- heldProcessLocks.clear();
78
- }
79
- function processLockPath(lockBaseDir, lockName) {
80
- return path.join(lockBaseDir, PROCESS_LOCK_DEFS[lockName].fileName);
81
- }
82
- function tryAcquireProcessLock(lockPath, staleAfterMs, skipIfLocked, lockLabel) {
83
- fs.mkdirSync(path.dirname(lockPath), { recursive: true });
84
- const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
85
- if (tryAcquireLockSync(lockPath, lockPayload())) {
86
- heldProcessLocks.add(lockPath);
87
- return "acquired";
88
- }
89
- const probe = probeLock(lockPath, { staleAfterMs });
90
- const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
91
- const lock = rawContent
92
- ? (() => {
93
- try {
94
- return JSON.parse(rawContent);
95
- }
96
- catch {
97
- return null;
98
- }
99
- })()
100
- : null;
101
- if (probe.state === "stale") {
102
- try {
103
- appendEvent({
104
- eventType: "improve_lock_recovered",
105
- metadata: {
106
- lockName: lockLabel,
107
- stalePid: lock?.pid ?? null,
108
- lockedAt: lock?.startedAt ?? null,
109
- recoveredAt: new Date().toISOString(),
110
- lockAgeMs: probe.ageMs ?? null,
111
- reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
112
- },
113
- });
114
- }
115
- catch {
116
- /* event emission is best-effort; never block lock recovery */
117
- }
118
- releaseLock(lockPath);
119
- if (tryAcquireLockSync(lockPath, lockPayload())) {
120
- heldProcessLocks.add(lockPath);
121
- return "acquired";
122
- }
123
- if (skipIfLocked) {
124
- warn(`[improve] ${lockLabel} lock acquired by another run during stale recovery; skipping (--skip-if-locked)`);
125
- return "skipped";
126
- }
127
- throw new ConfigError(`akm improve ${lockLabel} is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
128
- }
129
- if (skipIfLocked) {
130
- warn(`[improve] ${lockLabel} lock held by another run (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
131
- return "skipped";
132
- }
133
- throw new ConfigError(`akm improve ${lockLabel} is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
134
- }
135
- function releaseProcessLock(lockPath) {
136
- try {
137
- fs.unlinkSync(lockPath);
138
- }
139
- catch {
140
- // ignore
141
- }
142
- heldProcessLocks.delete(lockPath);
143
- }
144
- function releaseAllProcessLocks() {
145
- for (const p of heldProcessLocks) {
146
- try {
147
- fs.unlinkSync(p);
148
- }
149
- catch {
150
- // ignore
151
- }
152
- }
153
- heldProcessLocks.clear();
154
- }
155
- function resolveImproveScope(scope) {
156
- const trimmed = scope?.trim();
157
- if (!trimmed)
158
- return { mode: "all" };
159
- try {
160
- parseAssetRef(trimmed);
161
- return { mode: "ref", value: trimmed };
162
- }
163
- catch {
164
- if (!isAssetType(trimmed)) {
165
- throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, secret, task.\n` +
166
- `If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
167
- }
168
- return { mode: "type", value: trimmed };
169
- }
170
- }
171
- /**
172
- * Render the end-of-run stash-sync commit message, expanding `{token}`
173
- * placeholders against this run's results. Unknown tokens are passed through
174
- * verbatim so adding new tokens later never breaks an existing template, and so
175
- * a literal brace in a message is harmless.
176
- *
177
- * Supported tokens (the "free" set — derived from data already on the result):
178
- * {timestamp} `YYYY-MM-DD HH:MM:SS` (UTC)
179
- * {date} `YYYY-MM-DD` (UTC)
180
- * {time} `HH:MM:SS` (UTC)
181
- * {scope} scope value (e.g. a ref/type) or the scope mode (`all`)
182
- * {refs} number of planned refs this run processed
183
- * {accepted} number of proposals auto-accepted by the confidence gate
184
- * {triage_promoted} proposals promoted by the triage pre-pass (0 if triage did not run)
185
- * {triage_rejected} proposals rejected by the triage pre-pass (0 if triage did not run)
186
- * {runId} this run's id (empty string when absent)
187
- *
188
- * The result is still passed through `sanitizeCommitMessage` downstream in
189
- * `saveGitStash`, so token values never widen the commit-message attack surface
190
- * (newlines/control chars are collapsed there).
191
- *
192
- * `nowMs` is injected (not read from `Date.now()`) so the function is pure and
193
- * deterministically testable.
194
- */
41
+ export { resetHeldProcessLocks } from "./locks.js";
42
+ // Re-exported from ./loop-stages for test importers (improve-db-locking).
43
+ export { runImproveMaintenancePasses } from "./loop-stages.js";
44
+ // Re-exported from ./preparation so existing importers (tests, callers) resolve.
45
+ export { maybeAutoTuneThreshold } from "./preparation.js";
195
46
  export function renderSyncCommitMessage(template, result, nowMs) {
196
47
  const iso = new Date(nowMs).toISOString();
197
48
  const tokens = {
@@ -207,350 +58,6 @@ export function renderSyncCommitMessage(template, result, nowMs) {
207
58
  };
208
59
  return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
209
60
  }
210
- /**
211
- * Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
212
- * merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
213
- * Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
214
- */
215
- function dedupeRefs(refs) {
216
- const seen = new Set();
217
- const out = [];
218
- for (const r of refs) {
219
- if (seen.has(r.ref))
220
- continue;
221
- seen.add(r.ref);
222
- out.push(r);
223
- }
224
- return out;
225
- }
226
- async function collectEligibleRefs(scope, stashDir, improveProfile) {
227
- if (scope.mode === "ref" && scope.value) {
228
- const parsed = parseAssetRef(scope.value);
229
- const writableDirs = new Set(getWritableStashDirs(stashDir).map((dir) => path.resolve(dir)));
230
- const filePath = await findAssetFilePath(scope.value, stashDir, writableDirs);
231
- if (!filePath) {
232
- return {
233
- plannedRefs: [],
234
- memorySummary: { eligible: 0, derived: 0 },
235
- profileFilteredRefs: [],
236
- };
237
- }
238
- return {
239
- plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
240
- memorySummary: {
241
- eligible: parsed.type === "memory" ? 1 : 0,
242
- derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
243
- },
244
- profileFilteredRefs: [],
245
- };
246
- }
247
- let sources;
248
- try {
249
- sources = resolveSourceEntries(stashDir);
250
- }
251
- catch {
252
- return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
253
- }
254
- if (sources.length === 0) {
255
- return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
256
- }
257
- // Only operate on writable sources — never mutate read-only registry caches
258
- // or remote stashes that the user did not mark writable.
259
- let writableDirs;
260
- try {
261
- writableDirs = getWritableStashDirs(stashDir);
262
- }
263
- catch {
264
- writableDirs = sources.slice(0, 1).map((s) => s.path); // fallback: primary only
265
- }
266
- const writableDirSet = new Set(writableDirs.map((d) => path.resolve(d)));
267
- let db;
268
- try {
269
- db = openExistingDatabase();
270
- const entries = getAllEntries(db, scope.mode === "type" ? scope.value : undefined).filter((indexed) => {
271
- // First apply the existing stashDir-scope filter (no-op when stashDir is unset).
272
- if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir))
273
- return false;
274
- // Then restrict to writable sources only.
275
- return isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirSet);
276
- });
277
- const planned = new Map();
278
- const profileFiltered = new Map();
279
- let memoryEligible = 0;
280
- let memoryDerived = 0;
281
- for (const indexed of entries) {
282
- const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
283
- const isDerived = indexed.entry.name.endsWith(".derived");
284
- // `.derived` memories are LLM-inferred and intentionally skip reflect
285
- // (see the synthetic `derived-memory-reflect-skipped` branch in the
286
- // improve loop). Enqueueing them here just produced one synthetic skip
287
- // per derived memory per hour with no real work — pure churn observed
288
- // 2026-05-21: 11 derived refs re-planned every hour during idle periods.
289
- // The cleanup phase (analyzeMemoryCleanup) inspects derived memories
290
- // independently of `plannedRefs`, so dropping them here loses nothing.
291
- if (!isDerived && !planned.has(ref) && !profileFiltered.has(ref)) {
292
- // 2026-05-27: extend the .derived precedent to profile-incompatible
293
- // refs. If every per-ref pass (reflect + distill) on the active
294
- // profile would refuse this ref, drop it from `plannedRefs`. The
295
- // caller emits `improve_skipped { reason: profile_filtered_all_passes }`
296
- // once `eventsCtx` is available so the audit trail is preserved in a
297
- // single event per ref instead of 2× synthetic actions per run.
298
- // Background: see /tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md
299
- if (improveProfile && isProfileFilteredForAllPasses(ref, improveProfile)) {
300
- profileFiltered.set(ref, {
301
- ref,
302
- reason: "profile_filtered_all_passes",
303
- filePath: indexed.filePath,
304
- });
305
- }
306
- else {
307
- planned.set(ref, {
308
- ref,
309
- reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
310
- filePath: indexed.filePath,
311
- });
312
- }
313
- }
314
- if (indexed.entry.type === "memory") {
315
- memoryEligible += 1;
316
- if (isDerived)
317
- memoryDerived += 1;
318
- }
319
- }
320
- return {
321
- plannedRefs: [...planned.values()],
322
- memorySummary: { eligible: memoryEligible, derived: memoryDerived },
323
- profileFilteredRefs: [...profileFiltered.values()],
324
- };
325
- }
326
- catch (error) {
327
- // The bun-test isolation guard must never be downgraded to "empty plan".
328
- rethrowIfTestIsolationError(error);
329
- if (error instanceof NotFoundError || error instanceof Error) {
330
- return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
331
- }
332
- throw error;
333
- }
334
- finally {
335
- if (db)
336
- closeDatabase(db);
337
- }
338
- }
339
- function isEntryInScope(entryStashDir, filePath, stashDir) {
340
- if (!stashDir)
341
- return true;
342
- const resolvedEntryStashDir = path.resolve(entryStashDir);
343
- const resolvedFilePath = path.resolve(filePath);
344
- const resolvedScopeStashDir = path.resolve(stashDir);
345
- return (resolvedEntryStashDir === resolvedScopeStashDir ||
346
- resolvedEntryStashDir.startsWith(`${resolvedScopeStashDir}${path.sep}`) ||
347
- resolvedFilePath.startsWith(`${resolvedScopeStashDir}${path.sep}`));
348
- }
349
- /**
350
- * Return true when the indexed entry belongs to one of the writable source
351
- * directories. Entries from read-only registry caches or remote stashes that
352
- * the user has not marked writable must never enter the improve/distill loop.
353
- */
354
- function isEntryInWritableSource(entryStashDir, filePath, writableDirSet) {
355
- const resolvedEntryStashDir = path.resolve(entryStashDir);
356
- const resolvedFilePath = path.resolve(filePath);
357
- for (const writableDir of writableDirSet) {
358
- if (resolvedEntryStashDir === writableDir ||
359
- resolvedEntryStashDir.startsWith(`${writableDir}${path.sep}`) ||
360
- resolvedFilePath.startsWith(`${writableDir}${path.sep}`)) {
361
- return true;
362
- }
363
- }
364
- return false;
365
- }
366
- function memoryCleanupParentRef(scope, stashDir) {
367
- if (scope.mode !== "ref" || !scope.value)
368
- return undefined;
369
- const parsed = parseAssetRef(scope.value);
370
- if (parsed.type !== "memory")
371
- return undefined;
372
- if (!parsed.name.endsWith(".derived"))
373
- return scope.value;
374
- const sources = resolveSourceEntries(stashDir);
375
- for (const source of sources) {
376
- const candidate = path.join(source.path, "memories", `${parsed.name}.md`);
377
- if (!fs.existsSync(candidate))
378
- continue;
379
- const raw = fs.readFileSync(candidate, "utf8");
380
- const fm = parseFrontmatter(raw).data;
381
- const sourceRef = typeof fm.source === "string" ? fm.source : undefined;
382
- if (sourceRef) {
383
- try {
384
- const parent = parseAssetRef(sourceRef.trim());
385
- if (parent.type === "memory")
386
- return makeAssetRef(parent.type, parent.name);
387
- }
388
- catch { }
389
- }
390
- }
391
- return makeAssetRef("memory", parsed.name.slice(0, -".derived".length));
392
- }
393
- function isLessonCandidate(ref) {
394
- // Only lesson assets need lesson-schema validation (description + when_to_use).
395
- // Memories have their own distill path via shouldDistillMemoryRef.
396
- // All other types go through reflect, not distill.
397
- return parseAssetRef(ref).type === "lesson";
398
- }
399
- /**
400
- * Planner-side check: should this ref enter the distill queue?
401
- *
402
- * Distill produces lessons from non-lesson sources. Two cases are eligible:
403
- *
404
- * 1. Memory refs that pass {@link shouldDistillMemoryRef} (the existing
405
- * memory→lesson/knowledge promotion path).
406
- *
407
- * Refs whose `type` is in {@link DISTILL_REFUSED_INPUT_TYPES} (currently
408
- * `lesson:*`) are explicitly excluded — distill refuses them at runtime and
409
- * queuing them just produces a no-op `skipped` outcome per ref per hour. That
410
- * planner waste was the bug fixed in commit
411
- * fix(improve): drop distill-refused types from planner.
412
- *
413
- * Note: prior to this fix the gate used `isLessonCandidate(ref)` directly,
414
- * which was true *only* for `lesson:*` refs — exactly the set distill refuses.
415
- * The result: every hourly run re-queued the same lesson refs, the same skip
416
- * message returned, and no work was ever done. See
417
- * `tests/commands/improve-distill-planner-skip-lessons.test.ts`.
418
- */
419
- function isDistillCandidateRef(ref, stashDir) {
420
- const parsed = parseAssetRef(ref);
421
- if (isDistillRefusedInputType(parsed.type))
422
- return false;
423
- return shouldDistillMemoryRef(ref, stashDir);
424
- }
425
- function shouldDistillMemoryRef(ref, stashDir) {
426
- const parsed = parseAssetRef(ref);
427
- if (parsed.type !== "memory")
428
- return false;
429
- const sources = resolveSourceEntries(stashDir);
430
- for (const source of sources) {
431
- const candidate = `${source.path}/memories/${parsed.name}.md`;
432
- if (!fs.existsSync(candidate))
433
- continue;
434
- const raw = fs.readFileSync(candidate, "utf8");
435
- const fm = parseFrontmatter(raw).data;
436
- const quality = typeof fm.quality === "string" ? fm.quality : undefined;
437
- if (quality === "proposed")
438
- return false;
439
- return !parsed.name.endsWith(".derived");
440
- }
441
- return !parsed.name.endsWith(".derived");
442
- }
443
- // ── Signal-delta eligibility helpers (0.8.0) ────────────────────────────────
444
- //
445
- // The 0.8.0 redesign replaced flat time-based cooldowns for reflect/distill
446
- // with a *signal-delta* gate: a ref is re-eligible iff new feedback has
447
- // landed since the last proposal was generated for it. These helpers build
448
- // the two timestamp maps the gate needs in bulk, so the planner avoids
449
- // N+1 queries across the full postCleanupRefs set.
450
- /**
451
- * Latest feedback event timestamp per ref in the active window. Reads all
452
- * `feedback` events newer than `sinceIso` in one query and indexes by ref,
453
- * keeping the maximum `ts` per ref.
454
- *
455
- * Only events with a meaningful payload count as "signal" — `metadata.signal`
456
- * (positive/negative) OR `metadata.note` (a free-form annotation). Empty
457
- * metadata events are ignored so a stray `akm feedback <ref>` invocation
458
- * without a flag doesn't trigger downstream re-processing.
459
- */
460
- function buildLatestFeedbackTsMap(refs, sinceIso) {
461
- const out = new Map();
462
- if (refs.length === 0)
463
- return out;
464
- const refSet = new Set(refs);
465
- const { events } = readEvents({ type: "feedback", since: sinceIso });
466
- for (const e of events) {
467
- const ref = e.ref;
468
- if (!ref || !refSet.has(ref))
469
- continue;
470
- const meta = e.metadata;
471
- const hasSignal = meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
472
- if (!hasSignal)
473
- continue;
474
- const ts = e.ts ?? "";
475
- if (ts > (out.get(ref) ?? ""))
476
- out.set(ref, ts);
477
- }
478
- return out;
479
- }
480
- /**
481
- * Latest proposal timestamp per input-ref, filtered by source ('reflect' or
482
- * 'distill'). Reads the corresponding `*_invoked` events from state.db —
483
- * these events are emitted at proposal creation time and carry the *input*
484
- * asset ref (memory:foo, skill:bar, etc.) directly. We use them rather than
485
- * `listProposals` because distill proposals are keyed by the derived
486
- * lesson/knowledge ref, not the source memory — joining back through the
487
- * payload would be fragile.
488
- */
489
- function buildLatestProposalTsMap(refs, source) {
490
- const out = new Map();
491
- if (refs.length === 0)
492
- return out;
493
- const refSet = new Set(refs);
494
- const eventType = source === "reflect" ? "reflect_invoked" : "distill_invoked";
495
- const { events } = readEvents({ type: eventType });
496
- for (const e of events) {
497
- const ref = e.ref;
498
- if (!ref || !refSet.has(ref))
499
- continue;
500
- // For distill_invoked we only count attempts that produced (or attempted
501
- // to produce) a real proposal — config_disabled / parse-error outcomes
502
- // should not move the signal-delta cursor forward.
503
- if (eventType === "distill_invoked") {
504
- const outcome = e.metadata?.outcome;
505
- if (outcome !== "queued" && outcome !== "skipped" && outcome !== "validation_failed")
506
- continue;
507
- }
508
- const ts = e.ts ?? "";
509
- if (ts > (out.get(ref) ?? ""))
510
- out.set(ref, ts);
511
- }
512
- return out;
513
- }
514
- /**
515
- * Signal-delta eligibility predicate.
516
- *
517
- * True iff `latestFeedback[ref]` is defined AND either no prior proposal
518
- * exists for this (ref, source) OR `latestFeedback[ref] > lastProposal[ref]`.
519
- *
520
- * Refs with no feedback signal at all are ineligible by definition — the
521
- * high-retrieval fallback path (see `noFeedbackCandidates` later in the
522
- * planner) handles never-touched-but-frequently-read assets separately.
523
- */
524
- function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
525
- const fb = latestFeedback.get(ref);
526
- if (!fb)
527
- return false;
528
- const lp = lastProposal.get(ref);
529
- if (!lp)
530
- return true;
531
- return fb > lp;
532
- }
533
- /**
534
- * H7 (#566): cooperative budget watchdog with a captured, RAII-cleared hard-kill.
535
- *
536
- * When the wall-clock budget expires, `onExhausted` (normally an
537
- * `AbortController.abort`) signals cooperative cancellation so the run can drain
538
- * its in-flight log/`state.db` flush and unwind naturally. A second hard-kill
539
- * timer is then armed as a watchdog: it only `exit(0)`s if the drain itself
540
- * overruns `hardKillGraceMs`, preventing the process from outliving the task
541
- * timeout window (lock-cascade fix).
542
- *
543
- * Both timers are captured; the returned dispose() clears whichever is still
544
- * pending. Callers invoke it from a `finally`, so a *clean* drain reaches the
545
- * `finally` and cancels the pending hard-kill before it can fire — the previous
546
- * detached `setTimeout(() => process.exit(0), 5000)` always fired, truncating a
547
- * clean flush. The hard-kill timer is `unref()`-ed so it never keeps the event
548
- * loop alive on its own: once the run drains it exits with its own code, not the
549
- * forced 0.
550
- *
551
- * Dependencies are injectable purely so the concurrency-sensitive timing
552
- * contract can be exercised deterministically in unit tests.
553
- */
554
61
  export function armBudgetWatchdog(budgetMs, controller, deps) {
555
62
  const setTimeoutFn = deps?.setTimeoutFn ?? setTimeout;
556
63
  const clearTimeoutFn = deps?.clearTimeoutFn ?? clearTimeout;
@@ -575,107 +82,6 @@ export function armBudgetWatchdog(budgetMs, controller, deps) {
575
82
  }
576
83
  };
577
84
  }
578
- /**
579
- * #612 / WS-4 — bounded, opt-in per-phase auto-accept threshold auto-tune.
580
- *
581
- * Reads `improve.calibration` from config. When `autoTune` is enabled, computes
582
- * the calibration of recent gate decisions, derives a bounded threshold
583
- * adjustment (clamped into the configured band, capped per step), logs it, and
584
- * records a `calibration_autotune` event. Returns the new threshold (integer
585
- * 0-100) when an adjustment was made, or `undefined` to leave the caller's
586
- * threshold unchanged.
587
- *
588
- * WS-4 change: accepts an optional `phase` parameter. When provided, the tuned
589
- * threshold is persisted to `improve_gate_thresholds` (state.db Migration 012)
590
- * keyed by phase so `makeGateConfig` can read it back on the next run and each
591
- * phase maintains its own calibrated threshold rather than a shared global.
592
- *
593
- * WS-4 ceiling: `maxThreshold` defaults to 85 (not 100) to prevent the gate
594
- * converging to pure exploitation and shutting down Gap-3/4 novelty throughput.
595
- *
596
- * DEFAULT OFF: with no `improve.calibration` block (or `autoTune: false`) this
597
- * returns `undefined` immediately, so the gate threshold is unchanged and
598
- * behaviour is byte-identical to today.
599
- */
600
- export function maybeAutoTuneThreshold(currentThreshold, config, stateDbPath, ctx, phase) {
601
- const cal = config.improve?.calibration;
602
- if (!cal?.autoTune)
603
- return undefined;
604
- // WS-4: default maxThreshold is now 85 (ceiling to prevent pure exploitation).
605
- // Callers that explicitly set maxThreshold in config override this default.
606
- const tuneConfig = {
607
- autoTune: true,
608
- minThreshold: cal.minThreshold ?? 0,
609
- maxThreshold: cal.maxThreshold ?? 85,
610
- maxStep: cal.maxStep ?? 5,
611
- minSamples: cal.minSamples ?? 20,
612
- targetAcceptRate: cal.targetAcceptRate ?? 0.9,
613
- };
614
- // Defensive: an inverted band disables tuning rather than clamping to nonsense.
615
- if (tuneConfig.minThreshold > tuneConfig.maxThreshold)
616
- return undefined;
617
- const db = openStateDatabase(stateDbPath);
618
- let summary;
619
- try {
620
- const allDecisions = listProposalGateDecisions(db);
621
- // WS-4 fix: when called with a phase label, restrict calibration to that
622
- // phase's decision pool so a reflect-dominated run cannot tighten the
623
- // consolidate gate (or vice-versa). The gate field is `improve:<phase>`,
624
- // matching what improve-auto-accept.ts stamps at line ~163.
625
- const gateLabel = phase ? `improve:${phase}` : undefined;
626
- const decisions = gateLabel ? allDecisions.filter((d) => d.gate === gateLabel) : allDecisions;
627
- summary = summarizeCalibration(gateDecisionsToSamples(decisions));
628
- }
629
- finally {
630
- db.close();
631
- }
632
- const result = computeThresholdAutoTune(currentThreshold, summary, tuneConfig);
633
- if (!result.adjusted)
634
- return undefined;
635
- const appendEventFn = ctx?.appendEventFn ?? appendEvent;
636
- const phaseLabel = phase ?? "global";
637
- info(`[improve] calibration auto-tune (${phaseLabel}): threshold ${result.previousThreshold} -> ${result.newThreshold} ` +
638
- `(${result.reason}; samples=${summary.samples}, acceptRate=${summary.overallAcceptRate}, ` +
639
- `gap=${summary.calibrationGap}, band=[${tuneConfig.minThreshold},${tuneConfig.maxThreshold}])`);
640
- try {
641
- appendEventFn({
642
- eventType: "calibration_autotune",
643
- ref: "improve:calibration",
644
- metadata: {
645
- phase: phaseLabel,
646
- previousThreshold: result.previousThreshold,
647
- newThreshold: result.newThreshold,
648
- delta: result.delta,
649
- reason: result.reason,
650
- samples: summary.samples,
651
- overallAcceptRate: summary.overallAcceptRate,
652
- calibrationGap: summary.calibrationGap,
653
- minThreshold: tuneConfig.minThreshold,
654
- maxThreshold: tuneConfig.maxThreshold,
655
- },
656
- });
657
- }
658
- catch (err) {
659
- warn(`[improve] calibration auto-tune event not recorded: ${err instanceof Error ? err.message : String(err)}`);
660
- }
661
- // WS-4: Persist the per-phase threshold so makeGateConfig reads it on the
662
- // next run. Best-effort — a write failure must not abort the improve run.
663
- if (phase) {
664
- try {
665
- const persistDb = openStateDatabase(stateDbPath);
666
- try {
667
- persistPhaseThreshold(persistDb, phase, result.newThreshold);
668
- }
669
- finally {
670
- persistDb.close();
671
- }
672
- }
673
- catch (err) {
674
- warn(`[improve] calibration auto-tune: failed to persist phase threshold for ${phase}: ${err instanceof Error ? err.message : String(err)}`);
675
- }
676
- }
677
- return result.newThreshold;
678
- }
679
85
  export async function akmImprove(options = {}) {
680
86
  const scope = resolveImproveScope(options.scope);
681
87
  const reflectFn = options.reflectFn ?? akmReflect;
@@ -846,6 +252,10 @@ export async function akmImprove(options = {}) {
846
252
  ? "Improve folds memory cleanup into the same proposal queue: speculative promotions still go through reflect/distill proposals, while high-confidence redundant derived memories are moved into a recoverable cleanup archive instead of being left active in the stash."
847
253
  : undefined;
848
254
  };
255
+ // Holds our own process.on("exit") backstop so the finally can remove EXACTLY
256
+ // that handler (not every exit listener in the process). Declared in the scope
257
+ // shared by the try and its finally; assigned when the backstop is registered.
258
+ let exitBackstop;
849
259
  try {
850
260
  // #607: Per-process lock acquisition. Each process acquires only the lock(s)
851
261
  // it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
@@ -853,11 +263,8 @@ export async function akmImprove(options = {}) {
853
263
  if (!options.dryRun) {
854
264
  // Backstop release on process.exit() (signal handler / budget watchdog),
855
265
  // which skips the finally below. Removed in that finally on the normal path.
856
- const releaseAllOnExit = () => {
857
- for (const p of heldProcessLocks) {
858
- releaseLockIfOwned(p, process.pid);
859
- }
860
- };
266
+ const releaseAllOnExit = () => releaseHeldLocksIfOwned(process.pid);
267
+ exitBackstop = releaseAllOnExit;
861
268
  process.on("exit", releaseAllOnExit);
862
269
  // #607 triage pre-pass: acquire triage.lock, drain the standing pending
863
270
  // backlog BEFORE ensureIndex so improve generates fresh proposals against
@@ -1137,8 +544,12 @@ export async function akmImprove(options = {}) {
1137
544
  // #607: acquire consolidate.lock for the preparation stage (consolidate,
1138
545
  // ensureIndex, extract all write index.db). Released immediately after.
1139
546
  const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
1140
- const consolidatePrepAcquired = tryAcquireProcessLock(consolidateLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
1141
- preparation = await runImprovePreparationStageImpl({
547
+ preparation = await withOptionalProcessLock({
548
+ lockPath: consolidateLPath,
549
+ staleAfterMs: PROCESS_LOCK_DEFS.consolidate.staleAfterMs,
550
+ skipIfLocked: options.skipIfLocked,
551
+ label: "consolidate",
552
+ }, () => runImprovePreparationStageImpl({
1142
553
  scope,
1143
554
  options,
1144
555
  plannedRefs,
@@ -1152,9 +563,7 @@ export async function akmImprove(options = {}) {
1152
563
  initialCleanupWarnings: preEnsureCleanupWarnings,
1153
564
  improveProfile,
1154
565
  budgetSignal: budgetAbortController.signal,
1155
- });
1156
- if (consolidatePrepAcquired)
1157
- releaseProcessLock(consolidateLPath);
566
+ }));
1158
567
  prepGateCount += preparation.gateAutoAcceptedCount;
1159
568
  prepGateFailedCount += preparation.gateAutoAcceptFailedCount;
1160
569
  // D6: pre-load all proposal_rejected events from the last 30 days once,
@@ -1171,48 +580,52 @@ export async function akmImprove(options = {}) {
1171
580
  // #607: acquire reflect-distill.lock for the loop stage (reflect + distill
1172
581
  // both write proposals to state.db). Released immediately after.
1173
582
  const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
1174
- const reflectDistillAcquired = tryAcquireProcessLock(reflectDistillLPath, PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs, options.skipIfLocked, "reflect-distill") === "acquired";
1175
- // Post-lock cooldown re-filter for proactive refs (#SELECT-TIME-LEAK).
1176
- // Planning built `lastReflectProposalTs` BEFORE acquiring this lock, so a
1177
- // concurrent run's `reflect_invoked` writes are invisible to it. Now that
1178
- // we hold the lock, re-read fresh timestamp maps for the proactive subset
1179
- // and drop any ref whose cooldown has been consumed by the concurrent run.
1180
- const proactiveLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource === "proactive");
1181
- let postLockLoopRefs = preparation.loopRefs;
1182
- if (proactiveLoopRefs.length > 0) {
1183
- const proactiveRefStrs = proactiveLoopRefs.map((r) => r.ref);
1184
- const freshReflectTs = buildLatestProposalTsMap(proactiveRefStrs, "reflect");
1185
- const freshDistillTs = buildLatestProposalTsMap(proactiveRefStrs, "distill");
1186
- const pmDueDays = improveProfile.processes?.proactiveMaintenance?.dueDays ?? DEFAULT_DUE_DAYS;
1187
- const stillDue = new Set(filterProactiveDue(proactiveLoopRefs, freshReflectTs, freshDistillTs, pmDueDays, Date.now()).map((r) => r.ref));
1188
- const dropped = proactiveLoopRefs.filter((r) => !stillDue.has(r.ref));
1189
- if (dropped.length > 0) {
1190
- info(`[improve] post-lock cooldown re-filter: dropped ${dropped.length} proactive ref(s) claimed by concurrent run (${dropped.map((r) => r.ref).join(", ")})`);
1191
- postLockLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource !== "proactive" || stillDue.has(r.ref));
583
+ const loopResult = await withOptionalProcessLock({
584
+ lockPath: reflectDistillLPath,
585
+ staleAfterMs: PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs,
586
+ skipIfLocked: options.skipIfLocked,
587
+ label: "reflect-distill",
588
+ }, () => {
589
+ // Post-lock cooldown re-filter for proactive refs (#SELECT-TIME-LEAK).
590
+ // Planning built `lastReflectProposalTs` BEFORE acquiring this lock, so a
591
+ // concurrent run's `reflect_invoked` writes are invisible to it. Now that
592
+ // we hold the lock, re-read fresh timestamp maps for the proactive subset
593
+ // and drop any ref whose cooldown has been consumed by the concurrent run.
594
+ const proactiveLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource === "proactive");
595
+ let postLockLoopRefs = preparation.loopRefs;
596
+ if (proactiveLoopRefs.length > 0) {
597
+ const proactiveRefStrs = proactiveLoopRefs.map((r) => r.ref);
598
+ const freshReflectTs = buildLatestProposalTsMap(proactiveRefStrs, "reflect");
599
+ const freshDistillTs = buildLatestProposalTsMap(proactiveRefStrs, "distill");
600
+ const pmDueDays = improveProfile.processes?.proactiveMaintenance?.dueDays ?? DEFAULT_DUE_DAYS;
601
+ const stillDue = new Set(filterProactiveDue(proactiveLoopRefs, freshReflectTs, freshDistillTs, pmDueDays, Date.now()).map((r) => r.ref));
602
+ const dropped = proactiveLoopRefs.filter((r) => !stillDue.has(r.ref));
603
+ if (dropped.length > 0) {
604
+ info(`[improve] post-lock cooldown re-filter: dropped ${dropped.length} proactive ref(s) claimed by concurrent run (${dropped.map((r) => r.ref).join(", ")})`);
605
+ postLockLoopRefs = preparation.loopRefs.filter((r) => r.eligibilitySource !== "proactive" || stillDue.has(r.ref));
606
+ }
1192
607
  }
1193
- }
1194
- const loopResult = await runImproveLoopStageImpl({
1195
- scope,
1196
- options,
1197
- primaryStashDir,
1198
- reflectFn,
1199
- distillFn,
1200
- loopRefs: postLockLoopRefs,
1201
- actions: preparation.actions,
1202
- signalBearingSet: preparation.signalBearingSet,
1203
- distillCooledRefs: preparation.distillCooledRefs,
1204
- distillOnlyRefs: preparation.distillOnlyRefs,
1205
- recentErrors: preparation.recentErrors,
1206
- rejectedProposalsByRef,
1207
- utilityMap: preparation.utilityMap,
1208
- startMs,
1209
- budgetMs,
1210
- eventsCtx,
1211
- improveProfile,
1212
- budgetSignal: budgetAbortController.signal,
608
+ return runImproveLoopStageImpl({
609
+ scope,
610
+ options,
611
+ primaryStashDir,
612
+ reflectFn,
613
+ distillFn,
614
+ loopRefs: postLockLoopRefs,
615
+ actions: preparation.actions,
616
+ signalBearingSet: preparation.signalBearingSet,
617
+ distillCooledRefs: preparation.distillCooledRefs,
618
+ distillOnlyRefs: preparation.distillOnlyRefs,
619
+ recentErrors: preparation.recentErrors,
620
+ rejectedProposalsByRef,
621
+ utilityMap: preparation.utilityMap,
622
+ startMs,
623
+ budgetMs,
624
+ eventsCtx,
625
+ improveProfile,
626
+ budgetSignal: budgetAbortController.signal,
627
+ });
1213
628
  });
1214
- if (reflectDistillAcquired)
1215
- releaseProcessLock(reflectDistillLPath);
1216
629
  const loopGateCountThisCycle = loopResult.gateAutoAcceptedCount;
1217
630
  reflectsWithErrorContext += loopResult.reflectsWithErrorContext;
1218
631
  loopGateCount += loopResult.gateAutoAcceptedCount;
@@ -1224,8 +637,12 @@ export async function akmImprove(options = {}) {
1224
637
  // #607: acquire consolidate.lock for the post-loop stage (memoryInference +
1225
638
  // graphExtraction both write index.db). Released immediately after.
1226
639
  const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
1227
- const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
1228
- const postLoopResult = await runImprovePostLoopStageImpl({
640
+ const postLoopResult = await withOptionalProcessLock({
641
+ lockPath: consolidatePostLPath,
642
+ staleAfterMs: PROCESS_LOCK_DEFS.consolidate.staleAfterMs,
643
+ skipIfLocked: options.skipIfLocked,
644
+ label: "consolidate",
645
+ }, () => runImprovePostLoopStageImpl({
1229
646
  scope,
1230
647
  options,
1231
648
  primaryStashDir,
@@ -1238,9 +655,7 @@ export async function akmImprove(options = {}) {
1238
655
  budgetSignal: budgetAbortController.signal,
1239
656
  improveProfile,
1240
657
  consolidationRan: preparation.consolidationRan,
1241
- });
1242
- if (consolidatePostAcquired)
1243
- releaseProcessLock(consolidatePostLPath);
658
+ }));
1244
659
  const postLoopGateCountThisCycle = postLoopResult.gateAutoAcceptedCount;
1245
660
  // Last-wins point-in-time objects.
1246
661
  memoryInference = postLoopResult.memoryInference;
@@ -1419,9 +834,15 @@ export async function akmImprove(options = {}) {
1419
834
  // #607: release any per-process locks still held (backstop for error paths;
1420
835
  // the normal path already released each lock after its stage completed).
1421
836
  releaseAllProcessLocks();
1422
- // Drop the process.exit backstop so it does not fire later (or accumulate
1423
- // across repeated in-process calls).
1424
- process.removeAllListeners("exit");
837
+ // Drop ONLY our own process.exit backstop so it does not fire later (or
838
+ // accumulate across repeated in-process calls). Must NOT use
839
+ // removeAllListeners("exit") here: in the in-process model (tests and
840
+ // programmatic callers import cli.ts) that would silently destroy exit
841
+ // handlers owned by the host or other commands.
842
+ if (exitBackstop) {
843
+ process.removeListener("exit", exitBackstop);
844
+ exitBackstop = undefined;
845
+ }
1425
846
  // I1: close the long-lived state.db connection opened at the top of the run.
1426
847
  try {
1427
848
  eventsDb?.close();
@@ -1553,2933 +974,13 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
1553
974
  },
1554
975
  }, eventsCtx);
1555
976
  }
1556
- /**
1557
- * Run (or gate-skip) the memory consolidation pass.
1558
- *
1559
- * #551 — two coordinated changes live here:
1560
- *
1561
- * 1. STRUCTURAL: this runs before extract in the improve pipeline (see
1562
- * `runImprovePreparationStage`). Consolidation therefore only ever judges
1563
- * PRIOR-run memories; current-run extract promotions are invisible to it.
1564
- *
1565
- * 2. SMARTER POOL-DELTA GATE: even among on-disk files, a memory whose only
1566
- * post-`lastConsolidateTs` mtime bump came from its OWN auto-accept
1567
- * promotion (i.e. it was just promoted by extract in the immediately
1568
- * preceding run and has not had a full improve cycle to settle) does NOT
1569
- * count as "work to do". We exclude those paths from the pool-delta check
1570
- * using the `promoted` events already emitted with each promotion's
1571
- * `assetPath`. A genuinely-settled prior memory — one edited by feedback,
1572
- * reflect, manual edit, or simply older than the last consolidate — still
1573
- * triggers the run. This is gate-option (a) from the issue (same-run /
1574
- * adjacent-run promotion exclusion), chosen over option (b) because there
1575
- * is no `extract_completed` event in the data model to gate against;
1576
- * `promoted` events with `assetPath` already carry exactly the signal we
1577
- * need, so the fix is non-invasive and provably correct.
1578
- */
1579
- async function runConsolidationPass(args) {
1580
- const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx, budgetSignal, runBudgetMs } = args;
1581
- const baseConfig = options.config ?? loadConfig();
1582
- const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
1583
- const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
1584
- const volumeTriggered = typeof memorySummary.eligible === "number" && memorySummary.eligible > MEMORY_VOLUME_THRESHOLD && hasLlm;
1585
- // When volume triggers a consolidation pass, force-enable the consolidate
1586
- // process on the default improve profile so the gate accepts the run even
1587
- // if the user's config disabled it. We synthesise a new profile override
1588
- // rather than mutating connection settings.
1589
- const consolidationConfig = volumeTriggered
1590
- ? {
1591
- ...baseConfig,
1592
- profiles: {
1593
- ...(baseConfig.profiles ?? {}),
1594
- improve: {
1595
- ...(baseConfig.profiles?.improve ?? {}),
1596
- default: {
1597
- ...(baseConfig.profiles?.improve?.default ?? {}),
1598
- processes: {
1599
- ...(baseConfig.profiles?.improve?.default?.processes ?? {}),
1600
- consolidate: {
1601
- ...(baseConfig.profiles?.improve?.default?.processes?.consolidate ?? {}),
1602
- enabled: true,
1603
- },
1604
- },
1605
- },
1606
- },
1607
- },
1608
- }
1609
- : baseConfig;
1610
- // 0.8.0 pool-delta gate for consolidate: re-eligible iff at least one
1611
- // memory file has been updated since the most recent successful
1612
- // consolidate_completed event. Time-based cooldowns produced the same
1613
- // synchronised-wave failure mode the reflect/distill cooldowns did; the
1614
- // pool-delta gate ties consolidation to actual work-to-do.
1615
- const recentConsolidations = readEvents({ type: "consolidate_completed" });
1616
- const lastConsolidation = recentConsolidations.events
1617
- .filter((e) => e.metadata?.processed && Number(e.metadata.processed) > 0)
1618
- .sort((a, b) => new Date(b.ts ?? 0).getTime() - new Date(a.ts ?? 0).getTime())[0];
1619
- const lastConsolidateTs = lastConsolidation?.ts;
1620
- // #551 smarter gate: build the set of memory asset paths whose only delta
1621
- // since the last consolidate is their OWN auto-accept promotion. Those files
1622
- // have not had a full improve cycle to settle, so they offer no merge /
1623
- // contradiction candidates yet — excluding them stops the gate firing on
1624
- // freshly-promoted single-source memories. We read `promoted` events emitted
1625
- // after the last consolidate; each carries the written `assetPath`.
1626
- const promotedSinceConsolidate = (() => {
1627
- const paths = new Set();
1628
- try {
1629
- const promoted = readEvents({
1630
- type: "promoted",
1631
- ...(lastConsolidateTs ? { since: lastConsolidateTs } : {}),
1632
- }).events;
1633
- for (const e of promoted) {
1634
- const ap = e.metadata?.assetPath;
1635
- if (typeof ap === "string" && ap.length > 0)
1636
- paths.add(path.resolve(ap));
1637
- }
1638
- }
1639
- catch {
1640
- // best-effort: if the events query fails, fall back to no exclusions
1641
- // (preserves pre-#551 behaviour rather than over-skipping).
1642
- }
1643
- return paths;
1644
- })();
1645
- // Pool-delta: any memory file with mtime > lastConsolidateTs flags work to do,
1646
- // EXCEPT files whose only post-consolidate change was their own promotion.
1647
- // Using file mtime keeps this query DB-free and matches what the indexer
1648
- // already uses as the canonical `memory.updated_at` proxy.
1649
- //
1650
- // Bootstrap: when no successful consolidate_completed event has ever been
1651
- // recorded, we cannot evaluate the pool-delta — treat as eligible so a
1652
- // fresh stash runs consolidate once before the steady-state gate kicks in.
1653
- const memoryUpdatedAfterLastConsolidate = (() => {
1654
- if (volumeTriggered)
1655
- return true; // volume override forces the run regardless.
1656
- if (!lastConsolidateTs)
1657
- return true; // bootstrap path: never consolidated.
1658
- if (!primaryStashDir)
1659
- return false;
1660
- const memoriesDir = path.join(primaryStashDir, "memories");
1661
- if (!fs.existsSync(memoriesDir))
1662
- return false;
1663
- try {
1664
- return fs.readdirSync(memoriesDir).some((f) => {
1665
- if (!f.endsWith(".md"))
1666
- return false;
1667
- const filePath = path.join(memoriesDir, f);
1668
- // #551: skip files that were only touched by their own promotion this
1669
- // cohort — they have no settled merge/contradiction candidates yet.
1670
- if (promotedSinceConsolidate.has(path.resolve(filePath)))
1671
- return false;
1672
- try {
1673
- return fs.statSync(filePath).mtime.toISOString() > lastConsolidateTs;
1674
- }
1675
- catch {
1676
- return false;
1677
- }
1678
- });
1679
- }
1680
- catch {
1681
- return false;
1682
- }
1683
- })();
1684
- const consolidationOnCooldown = !volumeTriggered && !memoryUpdatedAfterLastConsolidate;
1685
- // Profile gate: if profile explicitly disables consolidate, skip the entire pass.
1686
- const consolidateDisabledByProfile = improveProfile?.processes?.consolidate?.enabled === false;
1687
- // #553 minPoolSize guard: skip consolidation when the eligible memory pool is
1688
- // below a minimum size, rather than spending an LLM pass on a handful of
1689
- // memories. This is an INDEPENDENT skip condition from #551's mtime pool-delta
1690
- // gate — either can skip. Default 500; `minPoolSize: 0` disables the guard.
1691
- // Evaluated against the eligible-pool count BEFORE entering the LLM loop so a
1692
- // skip costs ZERO LLM calls.
1693
- const CONSOLIDATE_DEFAULT_MIN_POOL_SIZE = 500;
1694
- const configuredMinPoolSize = improveProfile?.processes?.consolidate?.minPoolSize;
1695
- const minPoolSize = typeof configuredMinPoolSize === "number" ? configuredMinPoolSize : CONSOLIDATE_DEFAULT_MIN_POOL_SIZE;
1696
- const eligiblePoolSize = typeof memorySummary.eligible === "number" ? memorySummary.eligible : 0;
1697
- // volumeTriggered means the pool already exceeds the volume threshold (100),
1698
- // so a force-triggered run never trips the pool-size guard. The guard only
1699
- // engages when minPoolSize > 0 and the eligible pool is strictly below it.
1700
- const poolBelowMinSize = !volumeTriggered && minPoolSize > 0 && eligiblePoolSize < minPoolSize;
1701
- let consolidation = {
1702
- schemaVersion: 1,
1703
- ok: true,
1704
- shape: "consolidate-result",
1705
- dryRun: false,
1706
- previewOnly: false,
1707
- target: "",
1708
- processed: 0,
1709
- merged: 0,
1710
- deleted: 0,
1711
- promoted: [],
1712
- contradicted: 0,
1713
- warnings: [],
1714
- durationMs: 0,
977
+ function shapeMemoryCleanup(plan) {
978
+ return {
979
+ analyzedDerived: plan.analyzedDerived,
980
+ pruneCandidates: plan.pruneCandidates,
981
+ contradictionCandidates: plan.contradictionCandidates,
982
+ beliefStateTransitions: plan.beliefStateTransitions,
983
+ consolidationCandidates: plan.consolidationCandidates,
984
+ ...(plan.relativeDateCandidates.length > 0 ? { relativeDateCandidates: plan.relativeDateCandidates } : {}),
1715
985
  };
1716
- let gateAutoAcceptedCount = 0;
1717
- let gateAutoAcceptFailedCount = 0;
1718
- const consolidateGateCfg = makeGateConfig("consolidate", {
1719
- globalThreshold: options.autoAccept,
1720
- dryRun: options.dryRun ?? false,
1721
- stashDir: primaryStashDir,
1722
- config: consolidationConfig,
1723
- eventsCtx,
1724
- stateDbPath: eventsCtx?.dbPath,
1725
- }, { minimumThreshold: 95 });
1726
- if (consolidateDisabledByProfile) {
1727
- info("[improve] consolidation skipped (disabled by improve profile)");
1728
- }
1729
- else if (poolBelowMinSize) {
1730
- // #553: eligible pool below the configured minimum — skip with zero LLM
1731
- // calls. Reuse the #551 `improve_skipped` emission path so health surfaces
1732
- // it via the dynamic skipReasons aggregation under `pool_below_min_size`.
1733
- appendEvent({
1734
- eventType: "improve_skipped",
1735
- ref: "memory:_consolidation",
1736
- metadata: {
1737
- reason: "pool_below_min_size",
1738
- poolSize: eligiblePoolSize,
1739
- minPoolSize,
1740
- },
1741
- }, eventsCtx);
1742
- info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
1743
- }
1744
- else if (!consolidationOnCooldown) {
1745
- consolidation = await withLlmStage("consolidate", () => akmConsolidate({
1746
- ...options.consolidateOptions,
1747
- config: consolidationConfig,
1748
- stashDir: options.stashDir,
1749
- autoTriggered: volumeTriggered,
1750
- // Tie consolidate proposals back to this improve invocation so
1751
- // accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
1752
- sourceRun: `consolidate-${Date.now()}`,
1753
- // Pass profile-configured options. incrementalSince narrows the pool to
1754
- // recently-changed memories + graph neighbours — use this for frequent
1755
- // passes (quick-shredder). Leave absent in the nightly default profile for
1756
- // a full-pool sweep that catches stale-but-unmerged duplicates.
1757
- incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
1758
- limit: improveProfile?.processes?.consolidate?.limit,
1759
- neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
1760
- maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
1761
- // #617 — deterministic near-duplicate dedup pre-pass. DEFAULT OFF; only
1762
- // runs when the profile explicitly sets `consolidate.dedup.enabled`.
1763
- dedup: improveProfile?.processes?.consolidate?.dedup,
1764
- // #581 — judged-state cache. DEFAULT OFF; only engages when the profile
1765
- // explicitly sets `consolidate.judgedCache.enabled`. Skips memories
1766
- // judged-unchanged since their last judge so one run sweeps the full
1767
- // corpus instead of narrowing to a time-window slice.
1768
- judgedCache: improveProfile?.processes?.consolidate?.judgedCache,
1769
- // Honor profile.autoAccept (already merged into options.autoAccept at the
1770
- // top of akmImprove). The CLI parser always supplies 90 when --auto-accept
1771
- // is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
1772
- // (which maps to undefined) from disabling consolidation auto-accept.
1773
- // options.consolidateOptions.autoAccept (if explicitly provided by caller)
1774
- // still wins because the spread above runs first.
1775
- autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
1776
- // WS-3a: forward budget signal for graceful abort on timeout, and pass
1777
- // the profile's p90 estimate for cold-start budget reduction.
1778
- signal: budgetSignal,
1779
- p90ChunkSecondsDefault: improveProfile?.processes?.consolidate?.p90ChunkSecondsDefault,
1780
- // WS-5: pass total run budget so perfTelemetry.estimatedBudgetFractionUsed
1781
- // can flag when consolidation alone exceeded the budget.
1782
- runBudgetMs,
1783
- }));
1784
- {
1785
- const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
1786
- try {
1787
- if (!primaryStashDir)
1788
- return { proposalId, confidence: undefined };
1789
- const proposal = getProposal(primaryStashDir, proposalId);
1790
- return { proposalId, confidence: proposal.confidence };
1791
- }
1792
- catch {
1793
- return { proposalId, confidence: undefined };
1794
- }
1795
- }), consolidateGateCfg);
1796
- gateAutoAcceptedCount += consolidateGr.promoted.length;
1797
- gateAutoAcceptFailedCount += consolidateGr.failed.length;
1798
- }
1799
- if (consolidation.processed > 0) {
1800
- appendEvent({
1801
- eventType: "consolidate_completed",
1802
- ref: "memory:_consolidation",
1803
- metadata: {
1804
- processed: consolidation.processed,
1805
- merged: consolidation.merged,
1806
- deleted: consolidation.deleted,
1807
- contradicted: consolidation.contradicted,
1808
- failedChunks: consolidation.failedChunks ?? 0,
1809
- durationMs: consolidation.durationMs,
1810
- },
1811
- }, eventsCtx);
1812
- }
1813
- }
1814
- else {
1815
- appendEvent({
1816
- eventType: "improve_skipped",
1817
- ref: "memory:_consolidation",
1818
- metadata: {
1819
- reason: "consolidation_no_memory_updates",
1820
- lastEventTs: lastConsolidation?.ts ?? null,
1821
- },
1822
- }, eventsCtx);
1823
- info("[improve] consolidation skipped (no memory updates since last run)");
1824
- }
1825
- // D9: track whether consolidation wrote any data so graph extraction can reindex if needed
1826
- const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
1827
- // WS-4: Per-phase threshold auto-tune for the consolidate phase.
1828
- // Persists result for the NEXT run's makeGateConfig to read.
1829
- const consolidateTuneDbPath = eventsCtx?.dbPath;
1830
- if (options.autoAccept !== undefined && consolidateTuneDbPath) {
1831
- try {
1832
- maybeAutoTuneThreshold(consolidateGateCfg.phaseThreshold ?? options.autoAccept, consolidationConfig, consolidateTuneDbPath, undefined, "consolidate");
1833
- }
1834
- catch (err) {
1835
- warn(`[improve] calibration auto-tune (consolidate) skipped: ${err instanceof Error ? err.message : String(err)}`);
1836
- }
1837
- }
1838
- return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
1839
- }
1840
- async function runImprovePreparationStage(args) {
1841
- const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, budgetSignal, } = args;
1842
- const actions = [];
1843
- const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
1844
- // Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
1845
- let memoryIndexHealth;
1846
- if (primaryStashDir) {
1847
- const memoryMdPath = path.join(primaryStashDir, "memories", "MEMORY.md");
1848
- if (fs.existsSync(memoryMdPath)) {
1849
- try {
1850
- const lines = fs.readFileSync(memoryMdPath, "utf8").split("\n").length;
1851
- const overBudget = lines >= 180;
1852
- memoryIndexHealth = { lineCount: lines, overBudget };
1853
- if (overBudget) {
1854
- cleanupWarnings.push(`MEMORY.md has ${lines} lines (budget: 200). Consolidation strongly recommended.`);
1855
- }
1856
- }
1857
- catch {
1858
- // best-effort
1859
- }
1860
- }
1861
- }
1862
- // Phase 0.3 — memory consolidation pass (#551).
1863
- //
1864
- // Consolidation runs BEFORE the session-extract pass. This is the structural
1865
- // half of the #551 fix: extract auto-accept writes brand-new memory .md files
1866
- // on every run, which previously made the consolidation pool-delta gate fire
1867
- // unconditionally (any new file => "memory updated since last consolidate").
1868
- // By running consolidation first, the gate and akmConsolidate only ever see
1869
- // memories that existed at the start of the run — current-run extract
1870
- // promotions are not on disk yet. The complementary smarter-gate logic
1871
- // (excluding adjacent-run promotions) lives in `runConsolidationPass`.
1872
- const consolidationPass = await runConsolidationPass({
1873
- options,
1874
- primaryStashDir,
1875
- memorySummary,
1876
- improveProfile,
1877
- eventsCtx,
1878
- budgetSignal,
1879
- runBudgetMs: budgetMs,
1880
- });
1881
- // Phase 0.4 — session-extract pass.
1882
- //
1883
- // Reads native session files (claude-code JSONL, opencode storage tree)
1884
- // through the SessionLogHarness registry, pre-filters noise, and asks a
1885
- // bounded in-tree LLM to produce candidate memory/lesson/knowledge
1886
- // proposals for content the agent did NOT preserve via inline `akm remember`
1887
- // / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
1888
- // hook with an on-demand pull pipeline.
1889
- //
1890
- // Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
1891
- // (#593: the gate respects the resolved improve profile, not just the
1892
- // hardcoded `default` profile path the legacy feature flag reads).
1893
- // Each available harness gets one call with the default --since window;
1894
- // already-seen sessions (tracked in state.db.extract_sessions_seen) are
1895
- // skipped automatically so re-runs don't burn LLM calls on unchanged data.
1896
- //
1897
- // Failures are non-fatal — one harness throwing doesn't abort improve.
1898
- // The extract envelope's own `warnings` field surfaces what went wrong.
1899
- let extractResults;
1900
- // Seed the preparation-stage gate counters with consolidation's auto-accept
1901
- // gate results (#551: consolidation now runs in this stage), then accumulate
1902
- // extract's gate results on top.
1903
- let gateAutoAcceptedCount = consolidationPass.gateAutoAcceptedCount;
1904
- let gateAutoAcceptFailedCount = consolidationPass.gateAutoAcceptFailedCount;
1905
- const extractConfig = options.config ?? loadConfig();
1906
- const extractGateCfg = makeGateConfig("extract", {
1907
- globalThreshold: options.autoAccept,
1908
- dryRun: options.dryRun ?? false,
1909
- stashDir: primaryStashDir,
1910
- config: extractConfig,
1911
- eventsCtx,
1912
- stateDbPath: eventsCtx?.dbPath,
1913
- });
1914
- // #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
1915
- // already done upstream; here we elide every akmExtract/processSession call)
1916
- // when the NEW (unseen, in-window) candidate-session pool is below a minimum.
1917
- // 22% of improve runs produce zero memory-inference writes because extract
1918
- // finds no new sessions, yet still burns the full extract pipeline. Default 0
1919
- // (disabled) preserves existing always-run behaviour; only opted-in profiles
1920
- // (e.g. `frequent`) set it. Evaluated BEFORE any LLM call so a skip costs zero
1921
- // LLM work AND writes nothing — which also means no extract auto-accept bumps
1922
- // memory mtimes, so a skipped extract never flags work for the NEXT run's
1923
- // consolidation mtime-gate (the downstream trigger #554 asks us to suppress).
1924
- const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
1925
- // Read from the ACTIVE resolved profile (not always `default`), matching how
1926
- // `extract.enabled` resolves — otherwise a non-default profile (e.g.
1927
- // `frequent`) setting `minNewSessions` was silently ignored.
1928
- const configuredMinNewSessions = improveProfile.processes?.extract?.minNewSessions;
1929
- const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
1930
- // #593/#594: the ACTIVE resolved improve profile is the single source of
1931
- // truth for whether extract runs. (Previously this also ANDed in the legacy
1932
- // `session_extraction` feature flag, which only reads
1933
- // `profiles.improve.default.processes.extract.enabled`; that made the default
1934
- // profile a global kill switch, so a non-default profile enabling extract was
1935
- // silently overridden. The default profile is now just another profile.)
1936
- // `akmExtract` re-checks the same active profile internally via `improveProfile`.
1937
- if (resolveProcessEnabled("extract", improveProfile)) {
1938
- const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
1939
- // The guard engages only when minNewSessions > 0; 0 disables it entirely.
1940
- let belowMinNewSessions = false;
1941
- if (minNewSessions > 0 && availableHarnesses.length > 0) {
1942
- const countFn = options.extractCandidateCountFn ?? countNewExtractCandidates;
1943
- const newCandidateCount = countFn(extractConfig, {
1944
- ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1945
- // Use the ACTIVE profile's discovery window so the gate counts over the
1946
- // same window akmExtract will scan (not always `default`).
1947
- ...(improveProfile.processes?.extract?.defaultSince
1948
- ? { since: improveProfile.processes.extract.defaultSince }
1949
- : {}),
1950
- // C2: pin the candidate-count state.db open to the boundary-resolved path.
1951
- ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
1952
- });
1953
- if (newCandidateCount < minNewSessions) {
1954
- belowMinNewSessions = true;
1955
- // Reuse the #551/#553 `improve_skipped` emission path so health's dynamic
1956
- // skipReasons aggregation surfaces this under `below_min_new_sessions`.
1957
- appendEvent({
1958
- eventType: "improve_skipped",
1959
- ref: "memory:_extract",
1960
- metadata: {
1961
- reason: "below_min_new_sessions",
1962
- newSessions: newCandidateCount,
1963
- minNewSessions,
1964
- },
1965
- }, eventsCtx);
1966
- info(`[improve] extract skipped (new sessions ${newCandidateCount} < minNewSessions ${minNewSessions})`);
1967
- }
1968
- }
1969
- if (!belowMinNewSessions && availableHarnesses.length > 0) {
1970
- extractResults = [];
1971
- for (const h of availableHarnesses) {
1972
- try {
1973
- const result = await withLlmStage("session-extraction", () => akmExtract({
1974
- type: h.name,
1975
- ...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
1976
- config: extractConfig,
1977
- // Thread the ACTIVE profile so extract's internal gate + per-process
1978
- // config read the running profile, not always `default`.
1979
- improveProfile,
1980
- dryRun: options.dryRun ?? false,
1981
- ...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
1982
- // C2: pin extract's skip-tracking state.db open to the boundary path.
1983
- ...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
1984
- }));
1985
- extractResults.push(result);
1986
- {
1987
- const gr = await runAutoAcceptGate(primaryStashDir
1988
- ? result.proposals.map((proposalId) => {
1989
- const proposal = getProposal(primaryStashDir, proposalId);
1990
- return { proposalId, confidence: resolveExtractConfidence(proposal) };
1991
- })
1992
- : [], extractGateCfg);
1993
- gateAutoAcceptedCount += gr.promoted.length;
1994
- gateAutoAcceptFailedCount += gr.failed.length;
1995
- }
1996
- }
1997
- catch (err) {
1998
- const msg = err instanceof Error ? err.message : String(err);
1999
- cleanupWarnings.push(`extract(${h.name}) failed: ${msg}`);
2000
- }
2001
- }
2002
- if (extractResults.length === 0) {
2003
- // All harnesses threw — clear so the envelope's `extract` field is
2004
- // absent rather than misleadingly empty.
2005
- extractResults = undefined;
2006
- }
2007
- }
2008
- }
2009
- // Backlog drain: gate any pending extract proposals that weren't created in
2010
- // this run (i.e. pre-date the gate or were produced by a run that timed out
2011
- // before the gate fired). Without this, eligible proposals accumulate
2012
- // indefinitely — the fresh-gate only covers the current run's output.
2013
- if (primaryStashDir && !options.dryRun && options.autoAccept !== undefined) {
2014
- const freshIds = new Set((extractResults ?? []).flatMap((r) => r.proposals));
2015
- const backlog = listProposals(primaryStashDir, { status: "pending" }).filter((p) => p.source === "extract" && !freshIds.has(p.id));
2016
- if (backlog.length > 0) {
2017
- const backlogCandidates = backlog.map((p) => ({
2018
- proposalId: p.id,
2019
- confidence: resolveExtractConfidence(p),
2020
- }));
2021
- const backlogGr = await runAutoAcceptGate(backlogCandidates, extractGateCfg);
2022
- gateAutoAcceptedCount += backlogGr.promoted.length;
2023
- gateAutoAcceptFailedCount += backlogGr.failed.length;
2024
- }
2025
- }
2026
- // eligibleCount = raw pre-filter count (before cooldown/signal/cleanup filters).
2027
- // improve_completed.plannedRefs = post-filter count of refs that actually entered the loop.
2028
- appendEvent({
2029
- eventType: "improve_invoked",
2030
- ref: scope.mode === "ref" ? scope.value : `improve:${scope.mode}:${scope.value ?? "all"}`,
2031
- metadata: { scope, dryRun: options.dryRun ?? false, eligibleCount: plannedRefs.length },
2032
- }, eventsCtx);
2033
- // ensureIndex now runs in akmImprove() BEFORE collectEligibleRefs so the
2034
- // eligible-ref query sees a populated `entries` table on the very first
2035
- // pass after a DB version upgrade (#339). Any failure messages from that
2036
- // earlier call were threaded in via args.initialCleanupWarnings.
2037
- let appliedCleanup;
2038
- try {
2039
- appliedCleanup =
2040
- primaryStashDir && memoryCleanupPlan ? applyMemoryCleanup(primaryStashDir, memoryCleanupPlan) : undefined;
2041
- }
2042
- catch (err) {
2043
- cleanupWarnings.push(`applyMemoryCleanup failed: ${err instanceof Error ? err.message : String(err)}`);
2044
- }
2045
- const archivedRefs = appliedCleanup?.archived.map((record) => record.ref) ?? [];
2046
- const removed = new Set(archivedRefs);
2047
- const postCleanupRefs = archivedRefs.length === 0 ? plannedRefs : plannedRefs.filter((r) => !removed.has(r.ref));
2048
- // ── Phase 1: validation pass + schema repair (run on full postCleanupRefs) ──
2049
- // Identifies refs whose on-disk asset has structural problems. Validation
2050
- // failures are excluded from every downstream bucket. Run early so the
2051
- // cooldown partition operates on a clean set.
2052
- if (appliedCleanup) {
2053
- for (const candidate of memoryCleanupPlan?.pruneCandidates ?? []) {
2054
- const archived = appliedCleanup.archived.find((record) => record.ref === candidate.ref);
2055
- if (!archived)
2056
- continue;
2057
- actions.push({
2058
- ref: candidate.ref,
2059
- mode: "memory-prune",
2060
- result: { ok: true, pruned: true, reason: candidate.reason },
2061
- });
2062
- }
2063
- if ((appliedCleanup.archived.length > 0 || appliedCleanup.beliefStateTransitions.length > 0) && primaryStashDir) {
2064
- try {
2065
- await reindexFn({ stashDir: primaryStashDir });
2066
- }
2067
- catch (err) {
2068
- cleanupWarnings.push(`reindex after cleanup failed: ${err instanceof Error ? err.message : String(err)}`);
2069
- }
2070
- }
2071
- }
2072
- const validationFailures = [];
2073
- for (const candidate of postCleanupRefs) {
2074
- try {
2075
- // #591: use the path pre-resolved at planning time when it is still on
2076
- // disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
2077
- // stash. Fall back to findAssetFilePath only for refs that bypassed
2078
- // collectEligibleRefs' index scan or whose file moved since planning.
2079
- const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
2080
- ? candidate.filePath
2081
- : await findAssetFilePath(candidate.ref, options.stashDir);
2082
- if (!filePath) {
2083
- validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
2084
- continue;
2085
- }
2086
- if (path.extname(filePath).toLowerCase() !== ".md") {
2087
- continue;
2088
- }
2089
- if (isLessonCandidate(candidate.ref)) {
2090
- const raw = fs.readFileSync(filePath, "utf8");
2091
- const fm = parseFrontmatter(raw).data;
2092
- if (!fm.description)
2093
- validationFailures.push({ ref: candidate.ref, reason: "missing description" });
2094
- }
2095
- }
2096
- catch (e) {
2097
- validationFailures.push({ ref: candidate.ref, reason: String(e) });
2098
- }
2099
- }
2100
- if (validationFailures.length > 0) {
2101
- info(`[improve] ${validationFailures.length} assets have validation issues (will attempt schema repair):`);
2102
- for (const f of validationFailures)
2103
- info(` ${f.ref}: ${f.reason}`);
2104
- }
2105
- let schemaRepairs = [];
2106
- let repairedRefs = new Set();
2107
- // Schema repair pass: attempt to fix validation failures via LLM before skipping.
2108
- if (validationFailures.length > 0 && options.repairValidationFailures !== false) {
2109
- const baseConfigForRepair = options.config ?? loadConfig();
2110
- const llmCfg = getDefaultLlmConfig(baseConfigForRepair);
2111
- if (llmCfg) {
2112
- const result = await runSchemaRepairPass(validationFailures, {
2113
- startMs,
2114
- budgetMs,
2115
- llmConfig: llmCfg,
2116
- stashDir: options.stashDir,
2117
- findFilePath: findAssetFilePath,
2118
- isLessonCandidateFn: isLessonCandidate,
2119
- });
2120
- schemaRepairs = result.repairs;
2121
- repairedRefs = result.repairedRefs;
2122
- }
2123
- }
2124
- const validationFailureRefs = new Set(validationFailures.filter((f) => !repairedRefs.has(f.ref)).map((f) => f.ref));
2125
- if (repairedRefs.size > 0) {
2126
- info(`[improve] schema repair fixed ${repairedRefs.size}/${validationFailures.length} validation failures; ${validationFailureRefs.size} remain`);
2127
- }
2128
- // Phase 0.5 — structural hygiene pass
2129
- let lintSummary;
2130
- if (primaryStashDir) {
2131
- try {
2132
- const lintResult = akmLint({ fix: true, dir: primaryStashDir });
2133
- lintSummary = { fixed: lintResult.summary.fixed, flagged: lintResult.summary.flagged };
2134
- }
2135
- catch {
2136
- // lint is best-effort; never block improve
2137
- }
2138
- }
2139
- // O-5 / #378: Per-originator rolling error windows.
2140
- // Reflexion (arXiv:2303.11366) warns that cross-task verbal critique
2141
- // contamination degrades below single-shot baseline. Each originator key
2142
- // ("schema-repair", "reflect") maintains its own rolling window so that
2143
- // schema-repair failures are not injected as avoidPatterns into reflect calls.
2144
- const recentErrors = {};
2145
- const RECENT_ERRORS_CAP = 3;
2146
- // Helper: push an error onto an originator's rolling window.
2147
- function pushRecentError(originator, msg) {
2148
- if (!recentErrors[originator])
2149
- recentErrors[originator] = [];
2150
- recentErrors[originator].push(msg);
2151
- if (recentErrors[originator].length > RECENT_ERRORS_CAP)
2152
- recentErrors[originator].shift();
2153
- }
2154
- // Seed schema-repair originator window from any schema-repair errors.
2155
- for (const repair of schemaRepairs) {
2156
- if (repair.outcome === "error") {
2157
- const errMsg = repair.error ?? `schema repair error: ${repair.reason}`;
2158
- pushRecentError("schema-repair", errMsg);
2159
- }
2160
- }
2161
- // ── Phase 2: signal-delta eligibility sets built EARLY ────────────────────
2162
- // 0.8.0 replaces the flat time-based cooldowns (which produced synchronised
2163
- // waves whenever many refs cooled at the same instant — see the 2026-05-26
2164
- // 54-ref simultaneous-reflect incident) with a *signal-delta* gate:
2165
- //
2166
- // reflectEligible(ref) ≡ latestFeedbackTs(ref) > lastReflectProposalTs(ref)
2167
- // distillEligible(ref) ≡ latestFeedbackTs(ref) > lastDistillProposalTs(ref)
2168
- //
2169
- // i.e. a ref is re-eligible iff new feedback has landed since the last
2170
- // proposal was generated for it. Stable content with no new signal stays
2171
- // out of the queue regardless of clock time; a sudden burst of feedback
2172
- // surfaces only the refs that the burst actually touches.
2173
- //
2174
- // The 30-day FEEDBACK_SIGNAL_WINDOW_DAYS bound still applies — only feedback
2175
- // events newer than that count as "current signal". Ancient one-off
2176
- // negatives don't permanently lock a ref into every run.
2177
- //
2178
- // High-retrieval refs (P0-A path) use a simpler "eligible once" rule: a
2179
- // ref with no feedback signal but retrievalCount ≥ threshold is eligible
2180
- // exactly once (no prior reflect proposal). Subsequent re-eligibility for
2181
- // those refs requires either a new feedback event (then the normal
2182
- // signal-delta gate applies) or human action. Documented limitation: this
2183
- // path does not re-fire on retrieval-count growth alone in 0.8.0; storing
2184
- // the retrieval count in proposal metadata for proper delta-tracking is
2185
- // captured as future work.
2186
- const FEEDBACK_SIGNAL_WINDOW_DAYS = 30;
2187
- const feedbackSinceCutoff = new Date(Date.now() - daysToMs(FEEDBACK_SIGNAL_WINDOW_DAYS)).toISOString();
2188
- // Build the three timestamp maps once across the entire postCleanupRefs set.
2189
- // Per-ref queries would be N+1 and the planner is already the hottest path
2190
- // in `akm improve`.
2191
- const candidateRefs = postCleanupRefs.filter((r) => !validationFailureRefs.has(r.ref)).map((r) => r.ref);
2192
- const latestFeedbackTs = buildLatestFeedbackTsMap(candidateRefs, feedbackSinceCutoff);
2193
- const lastReflectProposalTs = buildLatestProposalTsMap(candidateRefs, "reflect");
2194
- const lastDistillProposalTs = buildLatestProposalTsMap(candidateRefs, "distill");
2195
- // Refs the distill signal-delta gate rejected at planning time. The main
2196
- // loop reads this to skip distill for these refs without re-checking
2197
- // eligibility per iteration.
2198
- const distillCooledRefs = new Set();
2199
- const preCooldownCount = postCleanupRefs.length;
2200
- // ── Phase 3: partition postCleanupRefs by signal-delta eligibility ────────
2201
- // Three buckets (validation failures are excluded entirely):
2202
- // eligibleRefs — reflect signal-delta passes (full reflect+distill
2203
- // loop path; distill guard remains in the loop for
2204
- // refs that fail the distill signal-delta gate).
2205
- // distillOnlyRefs — reflect blocked but distill signal-delta passes
2206
- // AND ref is a distill candidate.
2207
- // noFeedbackPool — neither signal-delta gate passes *and* the ref has
2208
- // no recent feedback signal at all. These are NOT
2209
- // skipped here: they are handed to the high-retrieval
2210
- // fallback (P0-A) below so frequently-retrieved but
2211
- // never-rated assets can still be improved. Only refs
2212
- // that P0-A declines are ultimately fully skipped.
2213
- // fullySkippedCount — has stale feedback but no signal delta → genuine
2214
- // skip (counted, aggregated event emitted post-loop),
2215
- // excluded from sort.
2216
- const eligibleRefs = [];
2217
- const distillOnlyRefs = [];
2218
- // Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
2219
- const noFeedbackPool = [];
2220
- let fullySkippedCount = 0;
2221
- // O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
2222
- const scopeRefBypass = scope.mode === "ref";
2223
- for (const r of postCleanupRefs) {
2224
- if (validationFailureRefs.has(r.ref))
2225
- continue;
2226
- if (scopeRefBypass) {
2227
- eligibleRefs.push(r);
2228
- continue;
2229
- }
2230
- const reflectOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastReflectProposalTs);
2231
- const distillOk = isSignalDeltaEligible(r.ref, latestFeedbackTs, lastDistillProposalTs);
2232
- const isDistillCandidate = isDistillCandidateRef(r.ref, options.stashDir);
2233
- if (reflectOk) {
2234
- if (!distillOk && isDistillCandidate) {
2235
- // Reflect passes the gate, distill does not — emit the synthetic
2236
- // distill-skipped action and event up-front so the in-loop guard
2237
- // does not have to re-derive eligibility.
2238
- distillCooledRefs.add(r.ref);
2239
- actions.push({ ref: r.ref, mode: "distill-skipped", result: { ok: true, reason: "distill signal-delta" } });
2240
- appendEvent({
2241
- eventType: "improve_skipped",
2242
- ref: r.ref,
2243
- metadata: { reason: "distill_no_new_signal" },
2244
- }, eventsCtx);
2245
- }
2246
- else if (!distillOk) {
2247
- // Not a distill candidate AND distill gate doesn't pass — just mark
2248
- // distillCooled so the loop's distill section is a no-op.
2249
- distillCooledRefs.add(r.ref);
2250
- }
2251
- eligibleRefs.push(r);
2252
- }
2253
- else if (distillOk && isDistillCandidate) {
2254
- // Reflect blocked but distill passes → distill-only bucket.
2255
- distillOnlyRefs.push(r);
2256
- }
2257
- else if (!latestFeedbackTs.has(r.ref)) {
2258
- // Neither signal-delta gate passes AND there is no recent feedback signal
2259
- // at all. Rather than skip outright, defer to the high-retrieval fallback
2260
- // (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
2261
- // what that path is meant to rescue. Refs P0-A declines are skipped there.
2262
- noFeedbackPool.push(r);
2263
- }
2264
- else {
2265
- // Has feedback on record but no signal delta since the last proposal —
2266
- // genuinely fully skipped. Counted here; a single aggregated
2267
- // improve_skipped event is emitted after the loop (mirrors
2268
- // profile_filtered_all_passes) instead of one event per ref.
2269
- fullySkippedCount++;
2270
- actions.push({
2271
- ref: r.ref,
2272
- mode: "distill-skipped",
2273
- result: { ok: true, reason: "no new signal since last proposal" },
2274
- });
2275
- }
2276
- }
2277
- // Emit ONE aggregated skip event for the fully-skipped bucket rather than one
2278
- // improve_skipped event per ref (#592 pattern, mirrors
2279
- // profile_filtered_all_passes above). The per-ref loop previously produced
2280
- // ~11K state.db writes per run on a large stash, the dominant contributor to
2281
- // 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
2282
- // run summary; no downstream consumer needs a per-ref DB audit trail (health's
2283
- // skip histogram reads the `no_new_signal` counter from the count field).
2284
- if (fullySkippedCount > 0) {
2285
- appendEvent({
2286
- eventType: "improve_skipped",
2287
- ref: undefined,
2288
- metadata: {
2289
- reason: "no_new_signal",
2290
- count: fullySkippedCount,
2291
- },
2292
- }, eventsCtx);
2293
- }
2294
- // ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
2295
- // Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
2296
- // deferred noFeedbackPool that may be rescued by the high-retrieval fallback
2297
- // (P0-A). The fully-skipped bucket has already been routed and its aggregated
2298
- // event emitted; we deliberately avoid spending DB/CPU on refs that the
2299
- // signal-delta gate rejected with feedback already on record.
2300
- const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
2301
- // Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
2302
- // partition above could not place these in a reflect/distill bucket, but they
2303
- // may still qualify if they have been retrieved often enough. Two disjoint
2304
- // sources feed this set:
2305
- // 1. noFeedbackPool — refs with no recent feedback that the partition loop
2306
- // deliberately deferred here (otherwise they would never reach P0-A).
2307
- // 2. processableRefs entries that turn out to carry no recent feedback
2308
- // *signal* once feedbackSummary is computed below.
2309
- // (1) is added here; (2) is folded in after feedbackSummary is built.
2310
- // Gap 6: only surface feedback signals from the last 30 days so that
2311
- // ancient one-off feedback events don't permanently lock an asset into
2312
- // every improve run. Assets with only stale signals fall through to the
2313
- // high-retrieval path (P0-A) or are skipped until new signals arrive.
2314
- // (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
2315
- // Phase 2 above for the signal-delta gate; we reuse them here.)
2316
- // Pre-compute feedback summary per ref in a SINGLE bulk read so we don't
2317
- // open state.db once per asset (which caused 5000+ accumulated FDs and a
2318
- // 2-hour runaway on a 13K-asset stash). Pattern mirrors buildLatestFeedbackTsMap
2319
- // above: one readEvents() call fetches ALL feedback events, then we aggregate
2320
- // in-memory by ref — O(1) DB opens regardless of candidate set size.
2321
- // Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
2322
- // ratios are available for any noFeedbackPool ref that P0-A rescues below.
2323
- //
2324
- // Behavioral note: positive/negative COUNTS are all-time (same as the old
2325
- // per-ref readEvents call which had no `since` filter); hasSignal is bounded
2326
- // to feedbackSinceCutoff (same as the old inline `(e.ts ?? "") >= cutoff` guard).
2327
- const feedbackSummary = new Map();
2328
- {
2329
- const feedbackCandidateSet = new Set([...processableRefs, ...noFeedbackPool].map((r) => r.ref));
2330
- if (feedbackCandidateSet.size > 0) {
2331
- // Fetch ALL feedback events in one query (no ref filter, no since filter =
2332
- // single full table scan). Filtering per-ref in memory avoids N sequential
2333
- // state.db opens — the dominant FD-leak path on large stashes.
2334
- const { events: allFeedbackEvents } = readEvents({ type: "feedback" }, eventsCtx);
2335
- for (const e of allFeedbackEvents) {
2336
- const ref = e.ref;
2337
- if (!ref || !feedbackCandidateSet.has(ref))
2338
- continue;
2339
- const entry = feedbackSummary.get(ref) ?? { hasSignal: false, positive: 0, negative: 0 };
2340
- const meta = e.metadata;
2341
- // hasSignal: only count feedback events within the 30-day window.
2342
- if (!entry.hasSignal &&
2343
- (e.ts ?? "") >= feedbackSinceCutoff &&
2344
- meta !== undefined &&
2345
- (typeof meta.signal === "string" || typeof meta.note === "string")) {
2346
- entry.hasSignal = true;
2347
- }
2348
- // positive/negative: all-time counts (no since filter, matching prior behaviour).
2349
- if (meta?.signal === "positive")
2350
- entry.positive++;
2351
- else if (meta?.signal === "negative")
2352
- entry.negative++;
2353
- feedbackSummary.set(ref, entry);
2354
- }
2355
- // Ensure every candidate has an entry (even refs with zero feedback events).
2356
- for (const ref of feedbackCandidateSet) {
2357
- if (!feedbackSummary.has(ref)) {
2358
- feedbackSummary.set(ref, { hasSignal: false, positive: 0, negative: 0 });
2359
- }
2360
- }
2361
- }
2362
- }
2363
- const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
2364
- // P0-A: also surface zero-feedback assets that have been retrieved many times.
2365
- const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
2366
- const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
2367
- // Zero-feedback candidates for P0-A: processableRefs without a recent signal,
2368
- // plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
2369
- // disjoint by construction, but guard against overlap defensively).
2370
- const noFeedbackSeen = new Set();
2371
- const noFeedbackCandidates = [];
2372
- for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
2373
- if (noFeedbackSeen.has(r.ref))
2374
- continue;
2375
- noFeedbackSeen.add(r.ref);
2376
- noFeedbackCandidates.push(r);
2377
- }
2378
- let highRetrievalRefs = [];
2379
- // Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
2380
- // proactive-maintenance selector below can reuse them without a second DB pass.
2381
- // Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
2382
- // step 2: recency is MANDATORY — never pinned to floor).
2383
- let retrievalCounts = new Map();
2384
- let lastUseMsForProactive = new Map();
2385
- let dbForRetrieval;
2386
- try {
2387
- dbForRetrieval = openExistingDatabase();
2388
- const showEventCount = countUsageEventsByType(dbForRetrieval, "show");
2389
- if (showEventCount === 0) {
2390
- warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
2391
- }
2392
- // Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
2393
- // Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
2394
- // had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
2395
- // to 0 regardless of actual use. Two assets of the same type — one
2396
- // heavily-retrieved, one never-touched — would receive identical rankScores.
2397
- // Fix (WS-1 blocker 3): union the feedback pool into the lookup.
2398
- const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
2399
- retrievalCounts = getRetrievalCounts(dbForRetrieval, allCandidateRefs);
2400
- lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
2401
- // High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
2402
- // ref qualifies exactly once — when it has actually been retrieved
2403
- // (retrievalCount ≥ 1) AND retrievalCount ≥ threshold AND no prior reflect
2404
- // proposal exists for it. Once a reflect proposal is on record, subsequent
2405
- // re-eligibility requires explicit feedback (which flows through the normal
2406
- // signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
2407
- // from rescuing genuinely never-retrieved assets — the fallback is for
2408
- // *retrieved* assets, not silent ones. Tracking growth in retrieval count
2409
- // would require persisting the count in proposal metadata; deferred to a
2410
- // follow-up.
2411
- highRetrievalRefs = noFeedbackCandidates.filter((r) => {
2412
- const count = retrievalCounts.get(r.ref) ?? 0;
2413
- return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
2414
- });
2415
- }
2416
- catch (err) {
2417
- rethrowIfTestIsolationError(err);
2418
- // best-effort: if DB unavailable, highRetrievalRefs stays empty
2419
- }
2420
- finally {
2421
- if (dbForRetrieval)
2422
- closeDatabase(dbForRetrieval);
2423
- }
2424
- // ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
2425
- // The signal-delta gate and P0-A only surface assets with fresh feedback or a
2426
- // raw-retrieval spike. Neither revisits a stable, high-value asset on a
2427
- // schedule, so on a quiet stash useful assets drift stale and are never
2428
- // refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
2429
- // and the run is whole-stash / type scope, this selector ranks the eligible
2430
- // population by a composite maintenance priority, gates on staleness ("due"),
2431
- // bounds to top-N, and folds the winners into the SAME candidate set the other
2432
- // two sources feed — so they flow through the existing #580 empty-diff /
2433
- // cosmetic suppression and additive-distill gates. It adds no new mutation
2434
- // logic of its own. The due gate doubles as the rotation cooldown: a freshly
2435
- // reflected asset is excluded until it ages back past `dueDays`, so successive
2436
- // runs rotate through the due pool rather than re-selecting the same heads.
2437
- let proactiveRefs = [];
2438
- let proactiveMaintenanceSummary;
2439
- const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
2440
- if (proactiveEnabled) {
2441
- const pmCfg = improveProfile.processes?.proactiveMaintenance;
2442
- const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
2443
- const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
2444
- // Candidate population: the zero-feedback / non-signal pool — exactly the
2445
- // assets the other two sources would NOT pick this run. Exclude any P0-A
2446
- // rescued this run so we never double-select the same ref.
2447
- const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
2448
- const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
2449
- const selection = selectProactiveMaintenanceRefs({
2450
- candidates: pmCandidates,
2451
- lastReflectTs: lastReflectProposalTs,
2452
- lastDistillTs: lastDistillProposalTs,
2453
- retrievalCounts,
2454
- // WS-1: wire lastUseMs so the recency decay term is genuine (plan §step 2).
2455
- lastUseMs: lastUseMsForProactive,
2456
- sizeBytesOf: (r) => {
2457
- const fp = r.filePath;
2458
- if (!fp)
2459
- return undefined;
2460
- try {
2461
- return fs.statSync(fp).size;
2462
- }
2463
- catch {
2464
- return undefined;
2465
- }
2466
- },
2467
- dueDays,
2468
- maxPerRun,
2469
- });
2470
- proactiveRefs = selection.selected;
2471
- proactiveMaintenanceSummary = {
2472
- selected: selection.selected.length,
2473
- dueTotal: selection.dueTotal,
2474
- neverReflected: selection.neverReflected,
2475
- };
2476
- // Aggregated observability event (never per-ref — avoids the event flood the
2477
- // Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
2478
- appendEvent({
2479
- eventType: "proactive_selected",
2480
- ref: undefined,
2481
- metadata: {
2482
- count: selection.selected.length,
2483
- dueTotal: selection.dueTotal,
2484
- neverReflected: selection.neverReflected,
2485
- },
2486
- }, eventsCtx);
2487
- if (selection.selected.length > 0) {
2488
- info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
2489
- `(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
2490
- }
2491
- }
2492
- // ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
2493
- // Zero-feedback refs whose encoding_salience (set at distill time by
2494
- // scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
2495
- // into the improve run even without retrieval or feedback signal. This rescues
2496
- // newly distilled assets that the stash has not yet surfaced to users.
2497
- //
2498
- // Cap: at most 10% of the effective run limit so the lane cannot crowd out
2499
- // reactive feedback. Requires state.db to have an asset_salience row — refs
2500
- // without a row (pre-#608 assets still on the type-weight stub) are skipped.
2501
- //
2502
- // Cooldown: a ref qualifies at most once — when no prior reflect proposal
2503
- // exists for it (`!lastReflectProposalTs.has`). Without this guard the lane
2504
- // re-selects the same high-salience refs on EVERY run (auto-accept emits a
2505
- // `promoted` event, not `feedback`, so the ref never leaves
2506
- // noFeedbackCandidates), burning LLM calls and churning the asset. This
2507
- // mirrors the P0-A high-retrieval gate's `!lastReflectProposalTs.has(r.ref)`
2508
- // guard above so all "rescue" lanes share the same once-per-asset semantics.
2509
- //
2510
- // Content-provenance gate (#644 follow-up): the row must ALSO carry a genuine
2511
- // content-derived encoding score (`isContentEncodingRow`). Otherwise the lane
2512
- // admits the per-type WEIGHT STUB (skill/agent 0.9, command/workflow 0.8,
2513
- // lesson 0.75 from DEFAULT_TYPE_ENCODING_WEIGHTS) for every distill-unscored
2514
- // asset — i.e. "high-salience" degenerates into "is a skill/agent/command/
2515
- // lesson", which selected the lore-writer type-stub agent on every run. Only
2516
- // content-scored assets earn the high-salience rescue; type-stub rows must
2517
- // earn retrieval/feedback signal via the other lanes. This PRESERVES #608's
2518
- // intent — distilled assets (the lane's real targets) keep their real content
2519
- // score and still qualify — while cutting the type-stub waste. See §5 F1 of
2520
- // docs/design/improve-salience-working-reference.md and #608/#644.
2521
- const highSalienceRefs = [];
2522
- const salienceCfg = (options.config ?? loadConfig()).improve?.salience;
2523
- const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
2524
- const proactiveAndRetrievalSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
2525
- {
2526
- let dbForHighSalience;
2527
- try {
2528
- dbForHighSalience = openStateDatabase(eventsCtx?.dbPath);
2529
- const effectiveLimit = options.limit ?? 10;
2530
- const highSalienceCap = Math.max(1, Math.floor(effectiveLimit * 0.1));
2531
- const candidates = noFeedbackCandidates.filter((r) => !proactiveAndRetrievalSet.has(r.ref));
2532
- for (const r of candidates) {
2533
- if (highSalienceRefs.length >= highSalienceCap)
2534
- break;
2535
- const row = getAssetSalience(dbForHighSalience, r.ref);
2536
- if (row &&
2537
- isContentEncodingRow(row, parseAssetRef(r.ref).type) &&
2538
- row.encoding_salience >= salienceThreshold &&
2539
- !lastReflectProposalTs.has(r.ref)) {
2540
- highSalienceRefs.push(r);
2541
- }
2542
- }
2543
- }
2544
- catch (err) {
2545
- rethrowIfTestIsolationError(err);
2546
- // best-effort: if DB unavailable, highSalienceRefs stays empty
2547
- }
2548
- finally {
2549
- if (dbForHighSalience)
2550
- dbForHighSalience.close();
2551
- }
2552
- if (highSalienceRefs.length > 0) {
2553
- info(`[improve] high-salience lane admitted ${highSalienceRefs.length} content-scored ref(s) ` +
2554
- `(threshold=${salienceThreshold}, requires content-derived encoding_source)`);
2555
- }
2556
- }
2557
- // Record an in-memory skip action for every zero-feedback ref that the
2558
- // partition loop deferred to P0-A but P0-A then declined (retrievalCount below
2559
- // threshold, or a prior reflect proposal already on record). These never make
2560
- // it into mergedRefs, so without this they would silently vanish from the run
2561
- // summary. No DB event is written here — these refs carry no signal at all, so
2562
- // there is nothing for the skip histogram to aggregate; the action log alone
2563
- // preserves the per-ref audit trail (mirrors the fully-skipped action above).
2564
- const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs, ...highSalienceRefs].map((r) => r.ref));
2565
- for (const r of noFeedbackPool) {
2566
- if (rescuedSet.has(r.ref))
2567
- continue;
2568
- actions.push({
2569
- ref: r.ref,
2570
- mode: "distill-skipped",
2571
- result: { ok: true, reason: "no new signal since last proposal" },
2572
- });
2573
- }
2574
- // If the user explicitly scoped to a single ref, always act on it —
2575
- // skip the signal/retrieval filter entirely. The filter exists to avoid
2576
- // noisy "improve everything" runs; it should not gate an intentional
2577
- // per-ref invocation where the user's explicit choice is the signal.
2578
- //
2579
- // For type/all scope: only process refs with usage signals (recent feedback
2580
- // or sufficient retrievals). A stash with no signals has 0 eligible refs —
2581
- // usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
2582
- // to bring them into the eligible pool.
2583
- // Layer-2 proactive refs join the eligible set alongside feedback-signal and
2584
- // high-retrieval (P0-A) refs. The four sources are disjoint by construction
2585
- // (proactive draws from noFeedbackCandidates with the P0-A picks removed, and
2586
- // high-salience draws from the remainder), but dedupe defensively so a ref can
2587
- // never enter the loop twice. `requireFeedbackSignal` still suppresses all
2588
- // fallback sources for callers that want feedback-only runs.
2589
- const signalAndRetrievalRefs = dedupeRefs([
2590
- ...signalFiltered,
2591
- ...highRetrievalRefs,
2592
- ...proactiveRefs,
2593
- ...highSalienceRefs,
2594
- ]);
2595
- let mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
2596
- // ── Attribution tagging: stamp each ref with the eligibility lane that
2597
- // selected it ──────────────────────────────────────────────────────────────
2598
- // Every reflect/distill proposal must record WHICH lane chose its source asset
2599
- // so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
2600
- // (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
2601
- // lane map here — the one place all four lanes are known — and stamp it onto
2602
- // each ImproveEligibleRef object. Because the ref objects are shared by
2603
- // reference across buckets, the stamp travels with the ref through the sort,
2604
- // disk-check, and loop stages down to the reflect/distill event emit sites and
2605
- // createProposal calls. See EligibilitySource for the lane vocabulary.
2606
- //
2607
- // Precedence (prefer the most specific reactive signal):
2608
- // scope > signal-delta > high-retrieval > proactive > high-salience
2609
- // A ref with real feedback is attributed to feedback even if it was also due
2610
- // for proactive maintenance or had high encoding salience. We apply lanes
2611
- // weakest-first so the strongest overwrites; the explicit --scope <ref> bypass
2612
- // wins outright (user intent).
2613
- const eligibilitySourceByRef = new Map();
2614
- for (const r of highSalienceRefs)
2615
- eligibilitySourceByRef.set(r.ref, "high-salience");
2616
- for (const r of proactiveRefs)
2617
- eligibilitySourceByRef.set(r.ref, "proactive");
2618
- for (const r of highRetrievalRefs)
2619
- eligibilitySourceByRef.set(r.ref, "high-retrieval");
2620
- for (const r of signalFiltered)
2621
- eligibilitySourceByRef.set(r.ref, "signal-delta");
2622
- if (scope.mode === "ref") {
2623
- // O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
2624
- // arrived via the scopeRefBypass branch, so attribute the whole set to scope.
2625
- for (const r of processableRefs)
2626
- eligibilitySourceByRef.set(r.ref, "scope");
2627
- }
2628
- for (const r of mergedRefs) {
2629
- // "unknown" is a genuine fallback, never a silent alias for signal-delta:
2630
- // only refs we truly cannot attribute land here (none in practice, since
2631
- // mergedRefs is always a subset of the four lanes above).
2632
- r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
2633
- }
2634
- // WS-1 — Unified salience vector (S1 seam).
2635
- //
2636
- // The legacy sort combined three independent formulas (utility EMA, negative-only
2637
- // ratio / symmetric-valence magnitude, and the proactive-maintenance priority
2638
- // formula). WS-1 converges them into one `computeSalience()` call per ref, with
2639
- // three independently-stored sub-scores and one documented rankScore projection.
2640
- //
2641
- // Migration note: if a profile still has `symmetricValence` set, emit a one-time
2642
- // warning — its behaviour (symmetric |valence| attention) is now always-on as
2643
- // part of the salience vector, so the knob is a no-op and will be removed in 0.10.
2644
- if (improveProfile.symmetricValence === true) {
2645
- warn("[improve] Profile option 'symmetricValence' is deprecated (WS-1 salience vector). " +
2646
- "Symmetric valence is now always active; remove the option from your improve profile.");
2647
- }
2648
- // Fetch last-use timestamps from the index DB for the full merged set so the
2649
- // recency term in retrievalSalience is genuinely decayable (plan §WS-1 step 2).
2650
- // This reuses the index DB opened earlier for retrieval counts; a separate
2651
- // lightweight open is used here to avoid holding the connection longer than needed.
2652
- let lastUseMsByRef = new Map();
2653
- // utilityMap is kept for backward-compatible observability (health report reads it).
2654
- const utilityMap = buildUtilityMap(mergedRefs);
2655
- let dbForSalience;
2656
- try {
2657
- dbForSalience = openExistingDatabase();
2658
- lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs.map((r) => r.ref));
2659
- }
2660
- catch (err) {
2661
- rethrowIfTestIsolationError(err);
2662
- // best-effort: if DB unavailable, recency term stays at floor (lastUseMs=0)
2663
- }
2664
- finally {
2665
- if (dbForSalience)
2666
- closeDatabase(dbForSalience);
2667
- }
2668
- // ── WS-2 Outcome loop ─────────────────────────────────────────────────────
2669
- //
2670
- // Update asset_outcome for every ref in the merged set BEFORE computing the
2671
- // salience vector so the updated outcome_score feeds outcomeSalience this run.
2672
- //
2673
- // Inputs per ref:
2674
- // - currentRetrievalCount: from retrievalCounts (index DB)
2675
- // - lastRetrievedAt: from lastUseMsByRef (utility_scores.last_used_at)
2676
- // - negativeFeedbackCount: cumulative negatives from feedbackSummary
2677
- // - acceptedChangeCount: accepted proposals for this ref (state.db)
2678
- // - valence: net valence from computeValenceScore(feedbackSummary.get(ref))
2679
- // - utilityScore: from utilityMap (for warm-start seed on new rows)
2680
- //
2681
- // Best-effort: outcome failures never block the salience or ranking pass.
2682
- const outcomeSalienceByRef = new Map();
2683
- try {
2684
- const outcomeDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
2685
- const ownsOutcomeDb = !eventsCtx?.db;
2686
- try {
2687
- // Count accepted proposals per ref in one pass (avoid N separate queries).
2688
- // Scoped to primaryStashDir when available so multi-stash installs don't
2689
- // inflate counts with proposals from other stashes.
2690
- const acceptedCountByRef = new Map();
2691
- try {
2692
- const acceptedProposals = listStateProposals(outcomeDb, {
2693
- status: "accepted",
2694
- ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
2695
- });
2696
- for (const p of acceptedProposals) {
2697
- acceptedCountByRef.set(p.ref, (acceptedCountByRef.get(p.ref) ?? 0) + 1);
2698
- }
2699
- }
2700
- catch {
2701
- // best-effort: if proposals query fails, accepted counts stay at 0
2702
- }
2703
- // Update each ref's outcome row and collect the resulting outcome scores.
2704
- const rawOutcomeScores = new Map();
2705
- const nowForOutcome = Date.now();
2706
- for (const r of mergedRefs) {
2707
- const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
2708
- const valenceResult = computeValenceScore(fb);
2709
- try {
2710
- const result = updateAssetOutcome(outcomeDb, {
2711
- ref: r.ref,
2712
- currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
2713
- lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
2714
- acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
2715
- negativeFeedbackCount: fb.negative,
2716
- valence: valenceResult.valence,
2717
- utilityScore: utilityMap.get(r.ref),
2718
- now: nowForOutcome,
2719
- });
2720
- rawOutcomeScores.set(r.ref, result.outcomeScore);
2721
- }
2722
- catch {
2723
- // best-effort per-ref: skip this ref's outcome update on failure
2724
- }
2725
- }
2726
- // Compute stash-wide max outcome_score for normalisation (diversity floor).
2727
- // Read ALL rows (not just this run's batch) so the normalisation is
2728
- // stash-relative, not pool-relative.
2729
- let maxOutcomeScore = 0;
2730
- try {
2731
- const allOutcomes = getAllAssetOutcomes(outcomeDb);
2732
- for (const row of allOutcomes) {
2733
- if (row.outcome_score > maxOutcomeScore)
2734
- maxOutcomeScore = row.outcome_score;
2735
- }
2736
- // Proxy-adequacy tripwire: emit a health event if outcome_score is
2737
- // negatively correlated with accepted_change_rate (inverted proxy).
2738
- const adequacy = computeProxyAdequacy(allOutcomes);
2739
- if (adequacy.isInverted) {
2740
- appendEvent({
2741
- eventType: "outcome_proxy_inverted",
2742
- ref: undefined,
2743
- metadata: {
2744
- correlation: adequacy.correlation,
2745
- n: adequacy.n,
2746
- note: "corr(outcome_score, accepted_change_rate) < −0.3: high-outcome_score assets have LOW accepted-change rates — the proxy's 'doing well' signal is inverted, so the coarse retrieval-delta signal is no longer trustworthy and the 0.10+ rich in-session signal is no longer deferrable. See plan §WS-2 proxy-adequacy tripwire.",
2747
- },
2748
- }, eventsCtx);
2749
- }
2750
- }
2751
- catch {
2752
- // best-effort: tripwire failure never blocks ranking
2753
- }
2754
- // Convert raw outcome scores → normalised outcomeSalience values in [0,1].
2755
- for (const [ref, score] of rawOutcomeScores) {
2756
- const normalised = outcomeScoreToSalience(score, maxOutcomeScore);
2757
- outcomeSalienceByRef.set(ref, normalised);
2758
- }
2759
- // Also fetch outcome scores for refs NOT updated this run (stale or absent)
2760
- // so the outcomeSalience read path works for all refs in the batch.
2761
- const missingRefs = mergedRefs.map((r) => r.ref).filter((ref) => !rawOutcomeScores.has(ref));
2762
- if (missingRefs.length > 0) {
2763
- const storedScores = getOutcomeScoresByRef(outcomeDb, missingRefs);
2764
- for (const [ref, score] of storedScores) {
2765
- outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
2766
- }
2767
- }
2768
- }
2769
- finally {
2770
- if (ownsOutcomeDb)
2771
- outcomeDb.close();
2772
- }
2773
- }
2774
- catch (err) {
2775
- rethrowIfTestIsolationError(err);
2776
- // best-effort: outcome failures never block salience computation
2777
- }
2778
- // Compute the salience vector for every ref in the merged set.
2779
- // retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
2780
- // so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
2781
- // outcomeSalienceByRef is populated by WS-2 above (or empty on first run).
2782
- //
2783
- // Part-V gate: read the operator opt-in flag from config. Default false
2784
- // (WS-1 parity weights) until the maintainer runs scripts/akm-eval and sets
2785
- // improve.salience.outcomeWeightEnabled: true in the config.
2786
- const salienceConfig = (options.config ?? loadConfig()).improve?.salience;
2787
- const outcomeWeightEnabled = salienceConfig?.outcomeWeightEnabled === true;
2788
- const salienceMap = new Map();
2789
- const nowForSalience = Date.now();
2790
- // #644 — preserve content-derived encoding scores across runs.
2791
- //
2792
- // Before computing the salience vector, load each ref's stored encoding score
2793
- // and its provenance. When the stored row carries a genuine content-derived
2794
- // score (written by the distill path via `scoreEncodingSalience`), pass that
2795
- // value back in as `inputs.encodingSalience` so `computeSalience` does NOT fall
2796
- // back to the type-weight stub — keeping both the persisted `encoding_salience`
2797
- // AND the derived `rank_score` keyed on real novelty/magnitude/prediction-error.
2798
- // Refs that have never been content-scored keep the type-weight stub fallback.
2799
- const storedEncodingByRef = new Map();
2800
- {
2801
- let dbForStoredEncoding;
2802
- try {
2803
- dbForStoredEncoding = openStateDatabase(eventsCtx?.dbPath);
2804
- for (const r of mergedRefs) {
2805
- const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
2806
- const row = getAssetSalience(dbForStoredEncoding, r.ref);
2807
- if (row && isContentEncodingRow(row, type)) {
2808
- storedEncodingByRef.set(r.ref, row.encoding_salience);
2809
- }
2810
- }
2811
- }
2812
- catch (err) {
2813
- rethrowIfTestIsolationError(err);
2814
- // best-effort: if DB unavailable, fall back to type-weight stub (prior behaviour)
2815
- }
2816
- finally {
2817
- if (dbForStoredEncoding)
2818
- dbForStoredEncoding.close();
2819
- }
2820
- }
2821
- for (const r of mergedRefs) {
2822
- const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
2823
- const sizeBytes = (() => {
2824
- const fp = r.filePath;
2825
- if (!fp)
2826
- return undefined;
2827
- try {
2828
- return fs.statSync(fp).size;
2829
- }
2830
- catch {
2831
- return undefined;
2832
- }
2833
- })();
2834
- const storedEncoding = storedEncodingByRef.get(r.ref);
2835
- const vector = computeSalience({
2836
- ref: r.ref,
2837
- type,
2838
- // #644: pass the stored content-derived score (if any) so the type-weight
2839
- // stub is NOT re-asserted over a real distill-written encoding score.
2840
- ...(storedEncoding !== undefined ? { encodingSalience: storedEncoding } : {}),
2841
- retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
2842
- lastUseMs: lastUseMsByRef.get(r.ref),
2843
- utilityScore: utilityMap.get(r.ref),
2844
- outcomeSalience: outcomeSalienceByRef.get(r.ref),
2845
- sizeBytes,
2846
- now: nowForSalience,
2847
- outcomeWeightEnabled,
2848
- });
2849
- salienceMap.set(r.ref, vector);
2850
- }
2851
- // Persist salience vectors to state.db (best-effort, non-blocking).
2852
- // The canonical store enables WS-3 homeostatic demotion and WS-2 outcome reads.
2853
- //
2854
- // Forgetting-safety report (plan §WS-1 step 7) — stash-wide rank comparison:
2855
- //
2856
- // BEFORE persisting the new rankScores, read ALL existing rows from state.db
2857
- // (not just the per-run candidate pool). This gives stash-wide rank positions so
2858
- // the top-200/below-500 thresholds are meaningful.
2859
- //
2860
- // Two distinct scenarios:
2861
- //
2862
- // A. First WS-1 run (table empty): the old stash-wide combinedEligibilityScore
2863
- // ordering was never persisted in state.db (asset_salience is a new WS-1 table).
2864
- // However, the old formula's inputs are available in-scope for every candidate
2865
- // in the current pool: utility comes from utilityMap and the attention term
2866
- // from feedbackSummary (positive/negative counts). We reconstruct the old
2867
- // combinedEligibilityScore = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT
2868
- // for every ref in salienceMap and rank them, giving a candidate-pool-scoped
2869
- // old ordering. This is a partial reconstruction (only current-pool refs, not
2870
- // stash-wide), but it is the most faithful comparison possible at cutover and
2871
- // allows the top-200→below-500 forgetting guard to fire if the formula change
2872
- // dramatically reorders the candidate pool.
2873
- // See docs/design/improve-reconciliation-plan.md §WS-1 step 7 — the stash-wide
2874
- // ordering was unreconstructable (no prior state.db snapshot), so this candidate-
2875
- // pool partial reconstruction is the documented resolution for the first-run case.
2876
- // Emit `improve_salience_first_run` to mark the cutover moment and include the
2877
- // reconstructed comparison result in the metadata.
2878
- //
2879
- // B. Subsequent runs (table has rows): use ALL existing rows as old ranks, merge
2880
- // them with the current run's salienceMap updates for new ranks, and call
2881
- // buildRankChangeReport with stash-wide positions. This detects real rank drift
2882
- // — e.g. a retrieval-pattern shift causing a previously top-200 asset to slip
2883
- // below position 500.
2884
- //
2885
- // Measurement-protocol deferral (plan §269, Part-V):
2886
- // The Part-V T0 baseline (scripts/akm-eval + health report) and the throughput/
2887
- // quality gate are deferred pending owner sign-off. Full measurement requires a
2888
- // before/after `akm health` report. Owner-acknowledged deferral: WS-2 landing
2889
- // will re-introduce outcome salience and trigger the full re-tuning pass at that
2890
- // time. salience.ts already accepts outcomeSalience directly as an input
2891
- // (see SalienceInputs.outcomeSalience); no separate hook is needed.
2892
- //
2893
- // Forgetting-safety collection: populated inside scenario B below, consumed
2894
- // after the try/catch to union candidates into mergedRefs before the sort.
2895
- // Only refs from a real pre-existing ordering (scenario B) are collected;
2896
- // empty on scenario A or when no candidates dropped below the threshold.
2897
- let pendingForgettingRefs = [];
2898
- try {
2899
- const stateDb = openStateDatabase(eventsCtx?.dbPath);
2900
- try {
2901
- // Step 7: stash-wide rank-change report BEFORE overwriting the table.
2902
- //
2903
- // Load ALL existing rows so rank positions are stash-relative, not pool-relative.
2904
- const existingAllScores = getAllRankScores(stateDb);
2905
- if (existingAllScores.size === 0) {
2906
- // Scenario A: first WS-1 run — table empty.
2907
- //
2908
- // Reconstruct the old combinedEligibilityScore ordering for the current
2909
- // candidate pool using inputs that are already in-scope: utility from
2910
- // utilityMap and the attention term from feedbackSummary (positive/negative
2911
- // counts). Old formula: score = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT.
2912
- //
2913
- // Limitation: this covers only the current-run candidate pool, not the full
2914
- // stash. The stash-wide ordering was never persisted (asset_salience is a new
2915
- // WS-1 table), so this is the most faithful comparison possible at cutover.
2916
- // See docs/design/improve-reconciliation-plan.md §WS-1 step 7.
2917
- const reconstructedOldScores = new Map();
2918
- for (const ref of salienceMap.keys()) {
2919
- const utility = utilityMap.get(ref) ?? 0;
2920
- const fb = feedbackSummary.get(ref) ?? { positive: 0, negative: 0 };
2921
- const attention = computeValenceScore(fb).attention;
2922
- reconstructedOldScores.set(ref, utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT);
2923
- }
2924
- // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2925
- const toRanks = (scores) => {
2926
- const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
2927
- return new Map(sorted.map(([ref], i) => [ref, i + 1]));
2928
- };
2929
- const oldRanks = toRanks(reconstructedOldScores);
2930
- const newRanks = toRanks(new Map([...salienceMap.entries()].map(([ref, v]) => [ref, v.rankScore])));
2931
- const firstRunReport = buildRankChangeReport(oldRanks, newRanks);
2932
- if (firstRunReport.forgettingCandidates.length > 0) {
2933
- warn(`[improve/salience] WS-1 first-run rank-change report: ${firstRunReport.forgettingCandidates.length} asset(s) fell from top-200 to below position 500 (cutover formula change). ` +
2934
- `Top drops: ${firstRunReport.forgettingCandidates
2935
- .slice(0, 5)
2936
- .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2937
- .join(", ")}`);
2938
- pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
2939
- }
2940
- appendEvent({
2941
- eventType: "improve_salience_first_run",
2942
- ref: undefined,
2943
- metadata: {
2944
- candidateCount: salienceMap.size,
2945
- note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); see improve-reconciliation-plan.md §WS-1 step 7",
2946
- forgettingCandidates: firstRunReport.forgettingCandidates.length,
2947
- topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
2948
- ref: e.ref,
2949
- oldRank: e.oldRank,
2950
- newRank: e.newRank,
2951
- })),
2952
- },
2953
- }, eventsCtx);
2954
- }
2955
- else {
2956
- // Scenario B: subsequent run — compare stash-wide old vs. new ranks.
2957
- //
2958
- // Build new scores by merging the full table with this run's updates.
2959
- // Refs in salienceMap override their stored value; refs not in this run
2960
- // retain their stored value unchanged. This gives a complete stash-wide
2961
- // picture of what the new ordering looks like after this run.
2962
- const mergedNewScores = new Map(existingAllScores);
2963
- for (const [ref, vector] of salienceMap) {
2964
- mergedNewScores.set(ref, vector.rankScore);
2965
- }
2966
- // Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
2967
- const toRanks = (scores) => {
2968
- const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
2969
- return new Map(sorted.map(([ref], i) => [ref, i + 1]));
2970
- };
2971
- const oldRanks = toRanks(existingAllScores);
2972
- const newRanks = toRanks(mergedNewScores);
2973
- const report = buildRankChangeReport(oldRanks, newRanks);
2974
- if (report.forgettingCandidates.length > 0) {
2975
- warn(`[improve/salience] WS-1 rank-change report: ${report.forgettingCandidates.length} asset(s) fell from top-200 to below position 500. ` +
2976
- `Top drops: ${report.forgettingCandidates
2977
- .slice(0, 5)
2978
- .map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
2979
- .join(", ")}`);
2980
- // Collect refs for protective consolidation pass (plan §WS-1 step 7).
2981
- // These are force-included in the candidate pool (mergedRefs) after
2982
- // this try block, bypassing cooldown/signal-delta gating.
2983
- pendingForgettingRefs = report.forgettingCandidates.map((e) => e.ref);
2984
- }
2985
- appendEvent({
2986
- eventType: "improve_salience_rank_change",
2987
- ref: undefined,
2988
- metadata: {
2989
- stashSize: existingAllScores.size,
2990
- totalChanged: report.allChanges.length,
2991
- forgettingCandidates: report.forgettingCandidates.length,
2992
- topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
2993
- ref: e.ref,
2994
- oldRank: e.oldRank,
2995
- newRank: e.newRank,
2996
- })),
2997
- },
2998
- }, eventsCtx);
2999
- }
3000
- for (const [ref, vector] of salienceMap) {
3001
- upsertAssetSalience(stateDb, ref, vector, nowForSalience);
3002
- }
3003
- }
3004
- finally {
3005
- stateDb.close();
3006
- }
3007
- }
3008
- catch (err) {
3009
- rethrowIfTestIsolationError(err);
3010
- // best-effort: salience persistence failure never blocks ranking
3011
- }
3012
- // ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
3013
- // Forgetting candidates detected in scenario B are force-injected into
3014
- // mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
3015
- // signal-delta gating. Any ref already present in mergedRefs keeps its
3016
- // existing eligibilitySource (stronger reactive signals win); refs not yet in
3017
- // the pool are synthesised as minimal ImproveEligibleRef stubs and labelled
3018
- // 'forgetting-safety' so S5/WS-5 can slice by lane. The dedupeRefs call
3019
- // ensures no ref can enter the loop twice.
3020
- if (pendingForgettingRefs.length > 0 && scope.mode !== "ref") {
3021
- const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
3022
- const newForgettingRefs = [];
3023
- for (const ref of pendingForgettingRefs) {
3024
- if (!existingRefSet.has(ref)) {
3025
- // Ref not already in the candidate pool — synthesise a stub so it
3026
- // participates in the reflect/distill loop with proper attribution.
3027
- newForgettingRefs.push({ ref, reason: "scope-type", eligibilitySource: "forgetting-safety" });
3028
- }
3029
- // Always stamp the lane in the attribution map (overwrites weaker lanes;
3030
- // stronger reactive signals — scope/signal-delta/high-retrieval/proactive
3031
- // — are written after this block so they take precedence).
3032
- eligibilitySourceByRef.set(ref, "forgetting-safety");
3033
- }
3034
- if (newForgettingRefs.length > 0) {
3035
- mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
3036
- }
3037
- // Re-stamp attribution for any refs whose lane needs updating.
3038
- // Precedence (weakest → strongest, each overwrites the previous):
3039
- // proactive < high-retrieval < forgetting-safety < signal-delta
3040
- // Scope mode is already excluded by the outer guard (`scope.mode !== "ref"`).
3041
- // forgetting-safety sits above proactive and high-retrieval so that a ref
3042
- // flagged as a forgetting candidate is always visible to S5/WS-5 as such,
3043
- // even when it was also due for a proactive maintenance run. signal-delta
3044
- // overrides forgetting-safety because a ref with fresh feedback is reactive
3045
- // and doesn't need the protective pass label for measurement purposes.
3046
- for (const r of highSalienceRefs)
3047
- eligibilitySourceByRef.set(r.ref, "high-salience");
3048
- for (const r of proactiveRefs)
3049
- eligibilitySourceByRef.set(r.ref, "proactive");
3050
- for (const r of highRetrievalRefs)
3051
- eligibilitySourceByRef.set(r.ref, "high-retrieval");
3052
- // Apply forgetting-safety OVER proactive, high-retrieval, and high-salience
3053
- // (already stamped in the loop above via
3054
- // `eligibilitySourceByRef.set(ref, "forgetting-safety")`). No-op here: the
3055
- // set() calls above for proactive/high-retrieval/high-salience overwrite the
3056
- // earlier forgetting-safety stamp — so we re-apply forgetting-safety now for
3057
- // those refs that are both forgetting candidates AND in another fallback lane.
3058
- for (const ref of pendingForgettingRefs) {
3059
- eligibilitySourceByRef.set(ref, "forgetting-safety");
3060
- }
3061
- // signal-delta is the strongest reactive signal and overrides forgetting-safety.
3062
- for (const r of signalFiltered)
3063
- eligibilitySourceByRef.set(r.ref, "signal-delta");
3064
- // Update eligibilitySource on the ref objects themselves for any refs whose
3065
- // lane changed (covers both new stubs and pre-existing refs).
3066
- for (const r of mergedRefs) {
3067
- r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
3068
- }
3069
- }
3070
- // ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
3071
- // Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
3072
- // revisited even with zero reactive signal (no feedback, no retrieval) and
3073
- // regardless of cooldown — exactly like the forgetting-safety lane, replay is
3074
- // injected AFTER cooldown/signal-delta partitioning so it bypasses those gates.
3075
- //
3076
- // Strictly additive: the replay slice is appended AFTER the --limit fresh slice
3077
- // (see the loopRefs partition below), so it can never shrink the fresh-ref set.
3078
- // Replay is the WEAKEST lane — it only stamps refs no other lane already claimed,
3079
- // and budget is spent only on refs not already in mergedRefs (so a stronger lane
3080
- // never has its budget wasted or its label overwritten).
3081
- //
3082
- // Default replayBudget=0 ⇒ this whole block is a no-op (no DB open, no event,
3083
- // no mergedRefs mutation), preserving byte-identical pre-#610 selection behavior.
3084
- const replayBudget = (options.config ?? loadConfig()).improve?.salience?.replayBudget ?? 0;
3085
- const replayRefSet = new Set();
3086
- if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
3087
- let replayDb;
3088
- try {
3089
- replayDb = openStateDatabase(eventsCtx?.dbPath);
3090
- const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
3091
- const allRankScores = getAllRankScores(replayDb);
3092
- // Candidate universe = every salience row NOT already in the pool, ordered by
3093
- // rank_score desc with a deterministic ref-string tie-break (mirrors the main
3094
- // sort). Converged refs (consecutive_no_ops >= dampener threshold) are fully
3095
- // EXCLUDED — a stronger skip than the dampener (which only halves order).
3096
- let convergedSkipped = 0;
3097
- const candidates = [];
3098
- for (const [ref, rankScore] of allRankScores) {
3099
- if (alreadyInPool.has(ref))
3100
- continue;
3101
- const noOps = getConsecutiveNoOps(replayDb, ref);
3102
- if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
3103
- convergedSkipped++;
3104
- continue;
3105
- }
3106
- candidates.push({ ref, rankScore });
3107
- }
3108
- candidates.sort((a, b) => b.rankScore !== a.rankScore ? b.rankScore - a.rankScore : a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0);
3109
- const candidatePool = candidates.length;
3110
- const selected = candidates.slice(0, replayBudget);
3111
- const newReplayRefs = [];
3112
- for (const { ref } of selected) {
3113
- replayRefSet.add(ref);
3114
- // Synthesise a stub (mirror the forgetting-safety stub). Resolve the
3115
- // backing file from the planned-ref pool so the downstream existsSync
3116
- // guard keeps the ref (a replay candidate from asset_salience whose file
3117
- // is gone correctly drops out). Only refs present in the indexed pool can
3118
- // be revisited — refs without a planned entry get no filePath and are
3119
- // dropped by the disk check, which is the desired behavior.
3120
- const planned = plannedRefs.find((p) => p.ref === ref);
3121
- newReplayRefs.push({
3122
- ref,
3123
- reason: "scope-type",
3124
- eligibilitySource: "replay",
3125
- ...(planned?.filePath ? { filePath: planned.filePath } : {}),
3126
- });
3127
- // Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
3128
- if (!salienceMap.has(ref)) {
3129
- salienceMap.set(ref, {
3130
- encoding: 0,
3131
- outcome: 0,
3132
- retrieval: 0,
3133
- rankScore: allRankScores.get(ref) ?? 0,
3134
- });
3135
- }
3136
- }
3137
- if (newReplayRefs.length > 0) {
3138
- mergedRefs = dedupeRefs([...mergedRefs, ...newReplayRefs]);
3139
- // Replay is the WEAKEST lane: stamp 'replay' ONLY for refs not already
3140
- // keyed by a stronger lane.
3141
- for (const ref of replayRefSet) {
3142
- if (!eligibilitySourceByRef.has(ref))
3143
- eligibilitySourceByRef.set(ref, "replay");
3144
- }
3145
- for (const r of mergedRefs) {
3146
- r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
3147
- }
3148
- }
3149
- // Aggregated observability event (never per-ref).
3150
- appendEvent({
3151
- eventType: "improve_replay_selected",
3152
- ref: undefined,
3153
- metadata: {
3154
- count: newReplayRefs.length,
3155
- budget: replayBudget,
3156
- convergedSkipped,
3157
- candidatePool,
3158
- },
3159
- }, eventsCtx);
3160
- }
3161
- catch (err) {
3162
- rethrowIfTestIsolationError(err);
3163
- // best-effort: if DB unavailable, replayRefSet stays empty
3164
- }
3165
- finally {
3166
- if (replayDb)
3167
- replayDb.close();
3168
- }
3169
- }
3170
- // Build no-op map for consolidation-selection dampener (plan §WS-1 step 8).
3171
- // Reads consecutive_no_ops from the SAME pinned db handle used elsewhere in
3172
- // this function. The effective score is used ONLY for processing/selection
3173
- // order — the persisted rank_score in asset_salience is never mutated here.
3174
- const noOpMap = new Map();
3175
- try {
3176
- const noOpDb = eventsCtx?.db ?? (eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
3177
- if (noOpDb) {
3178
- const ownsNoOpDb = !eventsCtx?.db;
3179
- try {
3180
- for (const r of mergedRefs) {
3181
- noOpMap.set(r.ref, getConsecutiveNoOps(noOpDb, r.ref));
3182
- }
3183
- }
3184
- finally {
3185
- if (ownsNoOpDb)
3186
- noOpDb.close();
3187
- }
3188
- }
3189
- }
3190
- catch {
3191
- // best-effort: dampener failure never blocks selection
3192
- }
3193
- // Sort by effective selection score (desc), with explicit ref-string tie-break
3194
- // for determinism. The effective score applies the consolidation-selection
3195
- // dampener: assets that have been repeatedly skipped (consecutive_no_ops >=
3196
- // THRESHOLD) are penalised by FACTOR so they sort after peers with similar
3197
- // rankScore. The persisted rank_score is left unchanged — this is the whole
3198
- // point of the dampener (stable assets stay fully retrievable).
3199
- //
3200
- // WIRING NOTE (plan §WS-1 step 8 / "consolidation-selection" disambiguation):
3201
- // "consolidation-selection" in the plan refers to THIS reflect/distill
3202
- // eligibility ordering — i.e. which assets are chosen for the reflect/distill
3203
- // LLM pass — NOT to akmConsolidate (the cluster-merge phase at ~line 1994,
3204
- // which runs earlier and never reads noOpMap). The no-op counter originates
3205
- // from no-change reflect / quality-rejected distill outcomes; the dampener
3206
- // suppresses repeated LLM attempts on those same assets without touching their
3207
- // persisted rank_score (so they remain fully retrievable).
3208
- //
3209
- // This is the ONLY ranking path — negativeOnlyRatio and the legacy
3210
- // symmetricValence branch are replaced. The three eligibilitySource lanes
3211
- // (signal-delta / high-retrieval / proactive) survive as labels (set above).
3212
- const effectiveScore = (ref) => {
3213
- const rankScore = salienceMap.get(ref)?.rankScore ?? 0;
3214
- const noOps = noOpMap.get(ref) ?? 0;
3215
- return noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rankScore * SALIENCE_NO_OP_DAMPEN_FACTOR : rankScore;
3216
- };
3217
- const sorted = [...mergedRefs].sort((a, b) => {
3218
- const scoreA = effectiveScore(a.ref);
3219
- const scoreB = effectiveScore(b.ref);
3220
- if (scoreB !== scoreA)
3221
- return scoreB - scoreA;
3222
- // Stable tie-break: deterministic regardless of input ordering.
3223
- return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
3224
- });
3225
- // Phase 0: surface coverage gaps from zero-result search queries
3226
- let coverageGaps = [];
3227
- try {
3228
- const dbForGaps = openExistingDatabase();
3229
- try {
3230
- coverageGaps = getZeroResultSearches(dbForGaps);
3231
- }
3232
- finally {
3233
- closeDatabase(dbForGaps);
3234
- }
3235
- }
3236
- catch (err) {
3237
- rethrowIfTestIsolationError(err);
3238
- // best-effort
3239
- }
3240
- // actionableRefs is the post-cooldown, post-validation, post-signal, post-sort
3241
- // set — i.e. the genuinely processable refs in priority order. Note: this is
3242
- // a semantic shift from earlier code where actionableRefs was the pre-cooldown
3243
- // sorted set; the new meaning matches reality and is documented on
3244
- // ImprovePreparationResult.actionableRefs.
3245
- //
3246
- // Final guard: drop any candidate whose backing file is no longer on disk.
3247
- // Phase 1 validation captures missing files at the start of preparation, but
3248
- // the gap between that check and dispatch can be minutes on large stashes —
3249
- // long enough for a checkpoint / git checkout / external cleanup to delete
3250
- // the asset. Empirically (improve-critical-review 2026-05-20) the single
3251
- // biggest reject category was "Asset no longer exists on disk" (604/1407 =
3252
- // 43%), meaning reflect/distill was producing proposals against deleted refs.
3253
- // A cheap existsSync per surviving candidate eliminates that wasted work.
3254
- const assetMissingOnDisk = [];
3255
- const existsCheckedActionable = [];
3256
- for (const candidate of sorted) {
3257
- // #591: prefer the path pre-resolved at planning time (synchronous
3258
- // existsSync) over a serial async DB lookup per ref.
3259
- const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
3260
- ? candidate.filePath
3261
- : await findAssetFilePath(candidate.ref, options.stashDir);
3262
- if (filePath && fs.existsSync(filePath)) {
3263
- existsCheckedActionable.push(candidate);
3264
- }
3265
- else {
3266
- assetMissingOnDisk.push(candidate.ref);
3267
- }
3268
- }
3269
- // #592 audit: one summary event instead of one per missing ref. Normally
3270
- // tiny, but a stash deletion racing the run could make this O(n) sequential
3271
- // state.db writes. `refs` is capped so the metadata row stays bounded.
3272
- if (assetMissingOnDisk.length > 0) {
3273
- appendEvent({
3274
- eventType: "improve_skipped",
3275
- ref: undefined,
3276
- metadata: {
3277
- reason: "asset_missing_on_disk",
3278
- count: assetMissingOnDisk.length,
3279
- refs: assetMissingOnDisk.slice(0, 50),
3280
- },
3281
- }, eventsCtx);
3282
- }
3283
- const actionableRefs = existsCheckedActionable;
3284
- // Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
3285
- // preserving sort order. distillOnlyRefs participate in the sort so --limit
3286
- // picks them by score, not by arbitrary position.
3287
- const distillOnlyRefSetForSort = new Set(distillOnlyRefs.map((r) => r.ref));
3288
- const reflectAndDistillRefsAfterSort = [];
3289
- const distillOnlyRefsAfterSort = [];
3290
- for (const r of actionableRefs) {
3291
- if (distillOnlyRefSetForSort.has(r.ref)) {
3292
- distillOnlyRefsAfterSort.push(r);
3293
- }
3294
- else {
3295
- reflectAndDistillRefsAfterSort.push(r);
3296
- }
3297
- }
3298
- // ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
3299
- //
3300
- // #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
3301
- // fresh slice. Without this split, a high-rankScore replay ref could sort above
3302
- // a fresh ref in the single combined slice and STEAL its slot (violating AC2).
3303
- // We partition into the replay lane vs the rest, apply --limit to the
3304
- // non-replay (fresh) refs only, then APPEND up to `replayBudget` replay refs
3305
- // after the fresh slice. Sort order within each partition is preserved.
3306
- //
3307
- // Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
3308
- // replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
3309
- // `replayLoop.slice(0, 0) === []` — byte-identical.
3310
- const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
3311
- const replayLoop = allLoopRefs.filter((r) => r.eligibilitySource === "replay");
3312
- const nonReplayLoop = allLoopRefs.filter((r) => r.eligibilitySource !== "replay");
3313
- const baseLoop = options.limit ? nonReplayLoop.slice(0, options.limit) : nonReplayLoop;
3314
- const loopRefs = [...baseLoop, ...replayLoop.slice(0, replayBudget)];
3315
- // Update the returned distillOnlyRefs to the sorted order so callers see the
3316
- // ranked view (loop stage uses it as a Set so order is irrelevant, but the
3317
- // shape change keeps downstream consumers consistent).
3318
- const distillOnlyRefsResult = distillOnlyRefsAfterSort;
3319
- const totalReflectBlocked = fullySkippedCount + distillOnlyRefs.length;
3320
- if (totalReflectBlocked > 0) {
3321
- info(`[improve] ${totalReflectBlocked} of ${preCooldownCount} indexed refs blocked by reflect signal-delta ` +
3322
- `(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
3323
- }
3324
- if (signalAndRetrievalRefs.length > 0) {
3325
- info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
3326
- }
3327
- if (validationFailureRefs.size > 0) {
3328
- info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
3329
- }
3330
- if (assetMissingOnDisk.length > 0) {
3331
- info(`[improve] ${assetMissingOnDisk.length} candidates dropped — file not on disk`);
3332
- }
3333
- const deferredCount = actionableRefs.length - loopRefs.length;
3334
- info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
3335
- (options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
3336
- // WS-4: Per-phase threshold auto-tune for the extract phase.
3337
- // Persists result for the NEXT run's makeGateConfig to read.
3338
- const extractTuneDbPath = eventsCtx?.dbPath;
3339
- if (options.autoAccept !== undefined && extractTuneDbPath) {
3340
- try {
3341
- maybeAutoTuneThreshold(extractGateCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), extractTuneDbPath, undefined, "extract");
3342
- }
3343
- catch (err) {
3344
- warn(`[improve] calibration auto-tune (extract) skipped: ${err instanceof Error ? err.message : String(err)}`);
3345
- }
3346
- }
3347
- return {
3348
- actions,
3349
- cleanupWarnings,
3350
- appliedCleanup,
3351
- memoryIndexHealth,
3352
- extract: extractResults,
3353
- actionableRefs,
3354
- signalBearingSet,
3355
- validationFailures,
3356
- schemaRepairs,
3357
- lintSummary,
3358
- loopRefs,
3359
- distillCooledRefs,
3360
- distillOnlyRefs: distillOnlyRefsResult,
3361
- coverageGaps,
3362
- recentErrors,
3363
- utilityMap,
3364
- gateAutoAcceptedCount,
3365
- gateAutoAcceptFailedCount,
3366
- consolidation: consolidationPass.consolidation,
3367
- consolidationRan: consolidationPass.consolidationRan,
3368
- ...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
3369
- };
3370
- }
3371
- async function runImproveLoopStage(args) {
3372
- const { scope, options, primaryStashDir, reflectFn, distillFn, loopRefs, actions, signalBearingSet, distillCooledRefs, distillOnlyRefs, recentErrors, rejectedProposalsByRef, utilityMap, startMs, budgetMs, eventsCtx, improveProfile, } = args;
3373
- // O-1 (#364): compute remaining budget at call time so each sub-call
3374
- // receives only its fair share of the wall-clock budget.
3375
- const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
3376
- const RECENT_ERRORS_CAP = 3;
3377
- // requirePlannedRefs guard: when the distill profile sets this flag, skip
3378
- // distill for distill-only refs if the reflect phase produced no planned refs.
3379
- // Prevents the distill loop from generating hundreds of distill-skipped events
3380
- // on quiet passes (all refs on reflect cooldown, no new signal to distill).
3381
- const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
3382
- const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
3383
- const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
3384
- const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
3385
- // R-2 / #389: Self-Consistency multi-sample voting helpers.
3386
- // Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
3387
- const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
3388
- const SC_N = Math.min(Math.max(2, options.selfConsistencyN ?? 3), 5);
3389
- /**
3390
- * Compute Jaccard token overlap between two strings.
3391
- * Tokenizes by whitespace; returns 0 when both are empty.
3392
- */
3393
- function jaccardSimilarity(a, b) {
3394
- const tokensA = new Set(a.split(/\s+/).filter(Boolean));
3395
- const tokensB = new Set(b.split(/\s+/).filter(Boolean));
3396
- if (tokensA.size === 0 && tokensB.size === 0)
3397
- return 1;
3398
- let intersection = 0;
3399
- for (const t of tokensA) {
3400
- if (tokensB.has(t))
3401
- intersection++;
3402
- }
3403
- const union = tokensA.size + tokensB.size - intersection;
3404
- return union > 0 ? intersection / union : 0;
3405
- }
3406
- /**
3407
- * Given N reflect results, return the one with the highest average Jaccard
3408
- * similarity to all other successful results (majority-vote winner).
3409
- * Falls back to the first successful result when N < 2.
3410
- */
3411
- function pickMajorityVote(results) {
3412
- const successful = results.filter((r) => r.ok);
3413
- if (successful.length === 0)
3414
- return (results[0] ?? {
3415
- schemaVersion: 1,
3416
- ok: false,
3417
- reason: "non_zero_exit",
3418
- error: "all samples failed",
3419
- exitCode: null,
3420
- });
3421
- if (successful.length === 1)
3422
- return successful[0];
3423
- let bestIdx = 0;
3424
- let bestScore = -1;
3425
- for (let i = 0; i < successful.length; i++) {
3426
- let totalSim = 0;
3427
- for (let j = 0; j < successful.length; j++) {
3428
- if (i === j)
3429
- continue;
3430
- totalSim += jaccardSimilarity(successful[i].proposal.payload.content ?? "", successful[j].proposal.payload.content ?? "");
3431
- }
3432
- const avgSim = totalSim / (successful.length - 1);
3433
- if (avgSim > bestScore) {
3434
- bestScore = avgSim;
3435
- bestIdx = i;
3436
- }
3437
- }
3438
- return successful[bestIdx] ?? successful[0];
3439
- }
3440
- // O-5 / #378: helper to push per-originator errors into the rolling window.
3441
- function pushRecentError(originator, msg) {
3442
- if (!recentErrors[originator])
3443
- recentErrors[originator] = [];
3444
- recentErrors[originator].push(msg);
3445
- if (recentErrors[originator].length > RECENT_ERRORS_CAP)
3446
- recentErrors[originator].shift();
3447
- }
3448
- // Build a Set for O(1) membership test — these refs skip the reflect call (Bug D2).
3449
- const distillOnlyRefSet = new Set(distillOnlyRefs.map((r) => r.ref));
3450
- let completedCount = 0;
3451
- let reflectsWithErrorContext = 0;
3452
- const memoryRefsForInference = new Set();
3453
- // Pre-load all pending proposals once instead of querying per asset in the loop.
3454
- const dedupeStashDirForProposals = primaryStashDir ?? options.stashDir;
3455
- const pendingProposalRefSet = new Set(dedupeStashDirForProposals
3456
- ? listProposals(dedupeStashDirForProposals, { status: "pending" }).map((p) => p.ref)
3457
- : []);
3458
- let gateAutoAcceptedCount = 0;
3459
- let gateAutoAcceptFailedCount = 0;
3460
- const reflectGateCfg = makeGateConfig("reflect", {
3461
- globalThreshold: options.autoAccept,
3462
- dryRun: options.dryRun ?? false,
3463
- stashDir: primaryStashDir,
3464
- config: options.config ?? loadConfig(),
3465
- eventsCtx,
3466
- stateDbPath: eventsCtx?.dbPath,
3467
- // candidateCount drives the exploration budget. loopRefs is the per-phase
3468
- // set for reflect/distill; pass it so exploration budget is proportional.
3469
- candidateCount: loopRefs.length,
3470
- });
3471
- const distillGateCfg = makeGateConfig("distill", {
3472
- globalThreshold: options.autoAccept,
3473
- dryRun: options.dryRun ?? false,
3474
- stashDir: primaryStashDir,
3475
- config: options.config ?? loadConfig(),
3476
- eventsCtx,
3477
- stateDbPath: eventsCtx?.dbPath,
3478
- candidateCount: loopRefs.length,
3479
- });
3480
- for (const planned of loopRefs) {
3481
- if (Date.now() - startMs >= budgetMs) {
3482
- const remaining = loopRefs.length - completedCount;
3483
- info(`[improve] budget exhausted after ${Math.round((Date.now() - startMs) / 60000)}min — ${remaining} assets skipped`);
3484
- appendEvent({
3485
- eventType: "improve_skipped",
3486
- ref: planned.ref,
3487
- metadata: {
3488
- reason: "budget_exhausted",
3489
- remaining,
3490
- },
3491
- }, eventsCtx);
3492
- // B11: Emit improve_skipped for all remaining assets that will not be processed.
3493
- for (const remainingRef of loopRefs.slice(completedCount + 1)) {
3494
- appendEvent({
3495
- eventType: "improve_skipped",
3496
- ref: remainingRef.ref,
3497
- metadata: { reason: "budget_exhausted_batch", remaining: loopRefs.length - completedCount - 1 },
3498
- }, eventsCtx);
3499
- }
3500
- actions.push({
3501
- ref: planned.ref,
3502
- mode: "error",
3503
- result: { ok: false, error: "timeout: improve wall-clock budget exhausted" },
3504
- });
3505
- break;
3506
- }
3507
- try {
3508
- // Bug D2: distillOnlyRefs skip the reflect call but still run the distill path.
3509
- // Bug D1: in-loop distill-cooldown check removed — distill-cooled candidates
3510
- // have their synthetic actions emitted in runImprovePreparationStage.
3511
- const isDistillOnly = distillOnlyRefSet.has(planned.ref);
3512
- const parsedPlannedRef = parseAssetRef(planned.ref);
3513
- // B6: derived memories are machine-generated; skip reflect to avoid noisy proposals.
3514
- // shouldDistillMemoryRef already returns false for .derived refs, so the distill
3515
- // path is also a no-op for them — we just avoid unnecessary agent spawns.
3516
- // D2: distillOnlyRefs also skip the reflect call (reflect-cooled, distill path only).
3517
- if (!isDistillOnly && !planned.ref.endsWith(".derived")) {
3518
- // Type guard: skip reflect for unsupported types (script, env, task, etc.)
3519
- // and raw wiki directories, driven by the active improve profile.
3520
- const reflectSkip = shouldSkipRef(planned.ref, "reflect", improveProfile);
3521
- if (reflectSkip.skip) {
3522
- actions.push({
3523
- ref: planned.ref,
3524
- mode: "reflect-skipped",
3525
- result: { ok: true, reason: reflectSkip.reason },
3526
- });
3527
- }
3528
- else {
3529
- // O-5 / #378: only inject reflect-originator errors into the reflect call.
3530
- // Cross-task errors (e.g. schema-repair) must NOT contaminate reflect prompts.
3531
- const reflectErrors = recentErrors.reflect ?? [];
3532
- if (reflectErrors.length > 0)
3533
- reflectsWithErrorContext++;
3534
- // O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
3535
- // bounded by the wall-clock deadline rather than the default per-profile timeout.
3536
- const reflectBudgetMs = remainingBudgetMs();
3537
- // Wire profile.processes.reflect.{mode, profile, timeoutMs} into the reflect
3538
- // dispatch when present. Falls back to akmReflect's own config-based resolution
3539
- // (profiles.improve.<name>.processes.reflect → defaults.llm) when the profile
3540
- // does not specify.
3541
- const reflectProfileRunner = resolveImproveProcessRunnerFromProfile(improveProfile.processes?.reflect, options.config ?? loadConfig());
3542
- const reflectCallArgs = {
3543
- ref: planned.ref,
3544
- task: options.task,
3545
- ...(options.stashDir ? { stashDir: options.stashDir } : {}),
3546
- ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
3547
- agentProcess: options.agentProcess ?? "reflect",
3548
- eventSource: "improve",
3549
- ...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
3550
- ...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
3551
- // Attribution: carry the eligibility lane so reflect stamps it on
3552
- // the reflect_invoked event and the persisted proposal.
3553
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
3554
- };
3555
- // R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
3556
- // Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
3557
- const refUtility = utilityMap.get(planned.ref) ?? 0;
3558
- const useConsistency = refUtility >= SC_THRESHOLD && SC_N >= 2;
3559
- let reflectResult;
3560
- if (useConsistency) {
3561
- const samples = [];
3562
- for (let s = 0; s < SC_N; s++) {
3563
- if (remainingBudgetMs() <= 0)
3564
- break;
3565
- // draftMode: skip DB write so each sample doesn't create a proposal.
3566
- samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
3567
- }
3568
- const winner = pickMajorityVote(samples.length > 0
3569
- ? samples
3570
- : [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
3571
- // Persist only the majority-vote winner as a single real proposal.
3572
- if (winner.ok && primaryStashDir) {
3573
- const persistResult = createProposal(primaryStashDir, {
3574
- ref: winner.proposal.ref,
3575
- source: "reflect",
3576
- sourceRun: `reflect-sc-${Date.now()}`,
3577
- payload: winner.proposal.payload,
3578
- // Attribution: the self-consistency path persists the winner here
3579
- // (draftMode skips reflect's own createProposal), so stamp the lane.
3580
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
3581
- });
3582
- reflectResult = isProposalSkipped(persistResult)
3583
- ? {
3584
- schemaVersion: 1,
3585
- ok: false,
3586
- reason: "cooldown",
3587
- error: `SC proposal skipped: ${persistResult.message}`,
3588
- ref: winner.ref,
3589
- exitCode: null,
3590
- }
3591
- : { ...winner, proposal: persistResult };
3592
- }
3593
- else {
3594
- reflectResult = winner;
3595
- }
3596
- }
3597
- else {
3598
- reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
3599
- }
3600
- const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
3601
- // Content-policy guard hits (reflect size-rail rejections) are NOT
3602
- // LLM faults — the agent responded fine, the downstream guard
3603
- // blocked the output. Route them to a distinct `reflect-guard-rejected`
3604
- // mode so health metrics can split deterministic guard hits out of
3605
- // true LLM failures. See
3606
- // `/tmp/akm-health-investigations/metrics-taxonomy-review.md` §1a.
3607
- const isGuardReject = !reflectResult.ok && reflectResult.reason === "content_policy_reject";
3608
- // Type-guard rejection (reflect refused a script/env/task ref) is
3609
- // also NOT an LLM failure — the LLM is never invoked. Route to the
3610
- // existing `reflect-skipped` bucket so it does not inflate the
3611
- // failure-rate numerator. ~9% of `reflect-failed` events in the
3612
- // user's stack were this case; see review §1a row "Reflect refused
3613
- // asset type".
3614
- const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
3615
- // Noise-gate suppression (#580): the candidate edit was an empty
3616
- // diff or a cosmetic-only reformat of the current asset. Like
3617
- // `unsupported_type`, this is a deterministic skip — not an LLM
3618
- // fault — so it routes to the `reflect-skipped` bucket and stays
3619
- // out of recentErrors/avoidPatterns.
3620
- const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
3621
- actions.push({
3622
- ref: planned.ref,
3623
- mode: reflectResult.ok
3624
- ? "reflect"
3625
- : isCooldown
3626
- ? "reflect-cooldown"
3627
- : isGuardReject
3628
- ? "reflect-guard-rejected"
3629
- : isTypeRefused || isNoChange
3630
- ? "reflect-skipped"
3631
- : "reflect-failed",
3632
- result: reflectResult,
3633
- });
3634
- // Cooldown skips, guard rejects, type-refused skips, and noise-gate
3635
- // skips are not failures — do not pollute recentErrors with them
3636
- // (those get injected as `avoidPatterns` into the next reflect
3637
- // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
3638
- // so the next iteration sees "your last expansion was too large";
3639
- // type-refused and no-change are deterministic and add no learning
3640
- // signal.
3641
- if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
3642
- const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
3643
- pushRecentError("reflect", errMsg);
3644
- }
3645
- // improve_reflect_outcome — per-asset metric for tuning the reflect path.
3646
- appendEvent({
3647
- eventType: "improve_reflect_outcome",
3648
- ref: planned.ref,
3649
- metadata: {
3650
- ok: reflectResult.ok,
3651
- durationMs: reflectResult.ok ? reflectResult.durationMs : undefined,
3652
- agentProfile: reflectResult.ok ? reflectResult.agentProfile : undefined,
3653
- reason: reflectResult.ok ? undefined : reflectResult.reason,
3654
- },
3655
- }, eventsCtx);
3656
- // Plasticity counter (plan §WS-1 step 8): record no-ops so the
3657
- // WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
3658
- // repeatedly-silent assets during consolidation-selection.
3659
- // A no_change reflect means the LLM was invoked but found nothing to
3660
- // improve — the asset is stable. Track it. A successful reflect means
3661
- // the asset changed; reset the counter so the dampener lifts.
3662
- if (isNoChange && eventsCtx?.db) {
3663
- try {
3664
- recordNoOp(eventsCtx.db, planned.ref);
3665
- }
3666
- catch {
3667
- // best-effort: plasticity counter failure never blocks the run
3668
- }
3669
- }
3670
- else if (reflectResult.ok && eventsCtx?.db) {
3671
- try {
3672
- resetConsecutiveNoOps(eventsCtx.db, planned.ref);
3673
- }
3674
- catch {
3675
- // best-effort
3676
- }
3677
- }
3678
- if (reflectResult.ok) {
3679
- const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
3680
- gateAutoAcceptedCount += reflectGr.promoted.length;
3681
- gateAutoAcceptFailedCount += reflectGr.failed.length;
3682
- }
3683
- } // end else (reflect type/profile check)
3684
- }
3685
- else if (!isDistillOnly && planned.ref.endsWith(".derived")) {
3686
- // B6: .derived refs skip reflect; record synthetic skip action.
3687
- actions.push({
3688
- ref: planned.ref,
3689
- mode: "distill-skipped",
3690
- result: { ok: true, reason: "derived-memory-reflect-skipped" },
3691
- });
3692
- appendEvent({
3693
- eventType: "improve_skipped",
3694
- ref: planned.ref,
3695
- metadata: { reason: "derived_memory_reflect_skipped" },
3696
- }, eventsCtx);
3697
- }
3698
- // isDistillOnly refs: no reflect action emitted — proceed directly to distill path below.
3699
- const hasRecentFeedbackSignal = signalBearingSet.has(planned.ref);
3700
- const explicitRefScope = scope.mode === "ref";
3701
- // Profile gate: apply the full type-filter / raw-wiki / disabled rules to
3702
- // distill so callers who configure `profile.processes.distill.allowedTypes`
3703
- // or land on raw-wiki refs get a recorded skip action instead of silently
3704
- // proceeding.
3705
- const distillSkip = shouldSkipRef(planned.ref, "distill", improveProfile);
3706
- if (distillSkip.skip) {
3707
- actions.push({
3708
- ref: planned.ref,
3709
- mode: "distill-skipped",
3710
- result: { ok: true, reason: distillSkip.reason },
3711
- });
3712
- completedCount++;
3713
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
3714
- continue;
3715
- }
3716
- // requirePlannedRefs guard: skip distill for distill-only refs when no
3717
- // reflect-eligible refs were planned this run, preventing mass skip events.
3718
- if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
3719
- actions.push({
3720
- ref: planned.ref,
3721
- mode: "distill-skipped",
3722
- result: { ok: true, reason: "require_planned_refs" },
3723
- });
3724
- completedCount++;
3725
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
3726
- continue;
3727
- }
3728
- // See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
3729
- // DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
3730
- // it will refuse.
3731
- const shouldAttemptDistill = isDistillCandidateRef(planned.ref, options.stashDir);
3732
- const skipMemoryDistillForWeakSignal = !isDistillOnly && parsedPlannedRef.type === "memory" && !hasRecentFeedbackSignal && !explicitRefScope;
3733
- // distillCooledRefs guard: pre-filter emitted synthetic actions for distill-candidate
3734
- // refs; non-candidate refs in the set are blocked here.
3735
- // O-2 (#365): bypass the distill cooldown when the user explicitly targeted
3736
- // this ref via --scope — their intent overrides unattended-run policies.
3737
- if (shouldAttemptDistill &&
3738
- !skipMemoryDistillForWeakSignal &&
3739
- (!distillCooledRefs.has(planned.ref) || explicitRefScope)) {
3740
- // TODO(refactor): single call site needs both lesson+knowledge refs for proposal dedup. If a third target ref type is added, extract deriveAllTargetRefs(inputRef): string[].
3741
- const lessonRef = deriveLessonRef(planned.ref);
3742
- const knowledgeRef = deriveKnowledgeRef(planned.ref);
3743
- const dedupeStashDir = primaryStashDir ?? options.stashDir;
3744
- if (dedupeStashDir) {
3745
- // B2: check both lesson ref and knowledge ref since auto-promoted memories
3746
- // create knowledge: proposals, not lesson: proposals.
3747
- const hasExistingPending = pendingProposalRefSet.has(lessonRef) || pendingProposalRefSet.has(knowledgeRef);
3748
- if (hasExistingPending) {
3749
- actions.push({
3750
- ref: planned.ref,
3751
- mode: "distill-skipped",
3752
- result: { ok: true, reason: "pending proposal exists" },
3753
- });
3754
- appendEvent({
3755
- eventType: "improve_skipped",
3756
- ref: planned.ref,
3757
- metadata: { reason: "pending_proposal_exists" },
3758
- }, eventsCtx);
3759
- completedCount++;
3760
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
3761
- continue;
3762
- }
3763
- // D-2 (#370): reject-aware cooldown for distill. When the reviewer
3764
- // recently rejected a distilled lesson or knowledge proposal for this
3765
- // asset, skip re-distillation for a 1-day grace window. Prevents the
3766
- // same rejected proposal from being regenerated immediately. The
3767
- // window is fixed (the 0.8.0 redesign moved per-ref cooldowns to
3768
- // signal-delta gates and dropped --distill-cooldown-days; a short
3769
- // reject grace is preserved here so a fresh rejection isn't
3770
- // overridden by the same run).
3771
- // References: ExpeL arXiv:2308.10144, STaR arXiv:2203.14465.
3772
- const DISTILL_REJECT_COOLDOWN_MS = daysToMs(1);
3773
- const recentlyRejectedLesson = !explicitRefScope && // O-2: bypass when --scope <ref> is explicit
3774
- (rejectedProposalsByRef.has(lessonRef) || rejectedProposalsByRef.has(knowledgeRef));
3775
- if (recentlyRejectedLesson) {
3776
- const rejectedEntry = rejectedProposalsByRef.get(lessonRef) ?? rejectedProposalsByRef.get(knowledgeRef);
3777
- const rejectedAgeMs = rejectedEntry ? Date.now() - new Date(rejectedEntry.ts).getTime() : 0;
3778
- if (rejectedAgeMs < DISTILL_REJECT_COOLDOWN_MS) {
3779
- actions.push({
3780
- ref: planned.ref,
3781
- mode: "distill-skipped",
3782
- result: { ok: true, reason: "distill reject grace window" },
3783
- });
3784
- appendEvent({
3785
- eventType: "improve_skipped",
3786
- ref: planned.ref,
3787
- metadata: {
3788
- reason: "distill_reject_grace_window",
3789
- },
3790
- }, eventsCtx);
3791
- completedCount++;
3792
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
3793
- continue;
3794
- }
3795
- }
3796
- }
3797
- const distillResult = await withLlmStage("distill", () => distillFn({
3798
- ref: planned.ref,
3799
- ...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
3800
- ...(options.stashDir ? { stashDir: options.stashDir } : {}),
3801
- // Attribution: carry the eligibility lane so distill stamps it on the
3802
- // distill_invoked event and the persisted proposal.
3803
- ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
3804
- }));
3805
- actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
3806
- if (distillResult.outcome === "queued" && distillResult.proposal) {
3807
- const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
3808
- gateAutoAcceptedCount += distillGr.promoted.length;
3809
- gateAutoAcceptFailedCount += distillGr.failed.length;
3810
- }
3811
- if (parsedPlannedRef.type === "memory") {
3812
- const promotedToKnowledge = distillResult.outcome === "queued" && distillResult.proposalKind === "knowledge";
3813
- if (!promotedToKnowledge)
3814
- memoryRefsForInference.add(planned.ref);
3815
- }
3816
- // Plasticity counter (plan §WS-1 step 8) for the distill path.
3817
- // quality_rejected: the LLM ran but produced output that didn't pass the
3818
- // quality gate — the asset is not yielding useful distill output.
3819
- // queued: a proposal was produced; reset the no-op counter.
3820
- if (eventsCtx?.db) {
3821
- try {
3822
- if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
3823
- recordNoOp(eventsCtx.db, planned.ref);
3824
- }
3825
- else if (distillResult.outcome === "queued") {
3826
- resetConsecutiveNoOps(eventsCtx.db, planned.ref);
3827
- }
3828
- }
3829
- catch {
3830
- // best-effort: plasticity counter failure never blocks the run
3831
- }
3832
- }
3833
- if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
3834
- const slug = planned.ref
3835
- .replace(/[^a-z0-9]/gi, "-")
3836
- .toLowerCase()
3837
- .slice(0, 60);
3838
- writeEvalCase(primaryStashDir, {
3839
- ref: planned.ref,
3840
- failureReason: distillResult.reason ?? "quality gate rejected",
3841
- assetType: parseAssetRef(planned.ref).type ?? "unknown",
3842
- rejectedAt: Date.now(),
3843
- source: "distill_quality_rejected",
3844
- slug: `${slug}-${Date.now()}`,
3845
- });
3846
- }
3847
- // D6: use pre-loaded map instead of per-iteration DB query
3848
- const rejectedProposalEvent = rejectedProposalsByRef.get(planned.ref);
3849
- if (rejectedProposalEvent && primaryStashDir) {
3850
- const slug = planned.ref
3851
- .replace(/[^a-z0-9]/gi, "-")
3852
- .toLowerCase()
3853
- .slice(0, 60);
3854
- writeEvalCase(primaryStashDir, {
3855
- ref: planned.ref,
3856
- failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
3857
- assetType: parseAssetRef(planned.ref).type ?? "unknown",
3858
- rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
3859
- source: "proposal_rejected",
3860
- slug: `${slug}-rejected`,
3861
- });
3862
- }
3863
- }
3864
- else if (skipMemoryDistillForWeakSignal) {
3865
- actions.push({
3866
- ref: planned.ref,
3867
- mode: "distill-skipped",
3868
- result: { ok: true, reason: "memory requires recent feedback signal" },
3869
- });
3870
- appendEvent({
3871
- eventType: "improve_skipped",
3872
- ref: planned.ref,
3873
- metadata: { reason: "memory_distill_requires_feedback" },
3874
- }, eventsCtx);
3875
- }
3876
- }
3877
- catch (err) {
3878
- // B7: UsageError thrown by akmDistill on validation_failed should be recorded
3879
- // as mode:"distill" with outcome:"validation_failed", NOT as a generic error.
3880
- // The distill_invoked event was already emitted inside akmDistill before the throw.
3881
- if (err instanceof UsageError) {
3882
- actions.push({
3883
- ref: planned.ref,
3884
- mode: "distill",
3885
- result: { ok: false, outcome: "validation_failed", error: err.message },
3886
- });
3887
- }
3888
- else {
3889
- actions.push({
3890
- ref: planned.ref,
3891
- mode: "error",
3892
- result: { ok: false, error: err instanceof Error ? err.message : String(err) },
3893
- });
3894
- }
3895
- }
3896
- completedCount++;
3897
- info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
3898
- }
3899
- // WS-4: Per-phase threshold auto-tune — runs AFTER the loop so the gate
3900
- // has processed all candidates for this run. Persists each phase's tuned
3901
- // threshold to state.db for the NEXT run's makeGateConfig to read.
3902
- // Best-effort: a tune failure must never fail the improve run.
3903
- const stateDbPathForTune = eventsCtx?.dbPath;
3904
- if (options.autoAccept !== undefined && stateDbPathForTune) {
3905
- const phaseGateCfgMap = {
3906
- reflect: reflectGateCfg,
3907
- distill: distillGateCfg,
3908
- };
3909
- for (const phase of ["reflect", "distill"]) {
3910
- const phaseCfg = phaseGateCfgMap[phase];
3911
- try {
3912
- maybeAutoTuneThreshold(phaseCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), stateDbPathForTune, undefined, phase);
3913
- }
3914
- catch (err) {
3915
- warn(`[improve] calibration auto-tune (${phase}) skipped: ${err instanceof Error ? err.message : String(err)}`);
3916
- }
3917
- }
3918
- }
3919
- return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
3920
- }
3921
- async function runImprovePostLoopStage(args) {
3922
- const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, consolidationRan, } = args;
3923
- const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
3924
- info("[improve] post-loop maintenance starting");
3925
- const maintenanceResult = await runImproveMaintenancePasses({
3926
- options,
3927
- primaryStashDir,
3928
- actionableRefs,
3929
- memoryRefsForInference,
3930
- allWarnings,
3931
- reindexFn,
3932
- consolidationRan,
3933
- // O-1 (#364): forward the budget signal to memory inference + graph extraction.
3934
- budgetSignal,
3935
- eventsCtx,
3936
- improveProfile,
3937
- });
3938
- let deadUrls;
3939
- if (scope.mode === "all" && primaryStashDir && actionableRefs.length > 0) {
3940
- try {
3941
- const knowledgeEntries = actionableRefs
3942
- .filter((r) => {
3943
- try {
3944
- return parseAssetRef(r.ref).type === "knowledge";
3945
- }
3946
- catch {
3947
- return false;
3948
- }
3949
- })
3950
- .slice(0, 10)
3951
- .map((r) => ({ ref: r.ref, body: "" }));
3952
- if (knowledgeEntries.length > 0) {
3953
- info(`[improve] checking URLs in ${knowledgeEntries.length} knowledge refs`);
3954
- deadUrls = await checkDeadUrls(primaryStashDir, knowledgeEntries);
3955
- info(`[improve] URL check complete (${deadUrls.length} dead/timeout URLs)`);
3956
- }
3957
- }
3958
- catch {
3959
- // best-effort
3960
- }
3961
- }
3962
- // #609 — recombine / synthesize pass. Whole-corpus cross-episodic
3963
- // generalization. Runs in the post-loop stage under consolidate.lock (it
3964
- // reads the consolidated corpus and writes proposals). Opt-in: gated on the
3965
- // `recombine` process being enabled, whole-stash / type scope (never `ref`),
3966
- // and not a dry run. Mirrors the proactiveMaintenance opt-in wiring.
3967
- let recombination;
3968
- if (primaryStashDir &&
3969
- improveProfile &&
3970
- resolveProcessEnabled("recombine", improveProfile) &&
3971
- scope.mode !== "ref" &&
3972
- !options.dryRun) {
3973
- const recombineFn = options.recombineFn ?? akmRecombine;
3974
- try {
3975
- recombination = await recombineFn({
3976
- stashDir: primaryStashDir,
3977
- config: options.config ?? loadConfig(),
3978
- ...(options.runId ? { sourceRun: options.runId } : {}),
3979
- ...(budgetSignal ? { signal: budgetSignal } : {}),
3980
- ...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
3981
- eligibilitySource: "recombine",
3982
- ...(eventsCtx ? { ctx: eventsCtx } : {}),
3983
- minClusterSize: improveProfile.processes?.recombine?.minClusterSize,
3984
- maxClustersPerRun: improveProfile.processes?.recombine?.maxClustersPerRun,
3985
- relatednessSource: improveProfile.processes?.recombine?.relatednessSource,
3986
- confirmThreshold: improveProfile.processes?.recombine?.confirmThreshold,
3987
- // #632 — clustering-tuning knobs. UNSET = pre-#632 behaviour.
3988
- maxClusterSize: improveProfile.processes?.recombine?.maxClusterSize,
3989
- excludeTags: improveProfile.processes?.recombine?.excludeTags,
3990
- excludeEntities: improveProfile.processes?.recombine?.excludeEntities,
3991
- });
3992
- }
3993
- catch (e) {
3994
- allWarnings.push(`recombine: ${String(e)}`);
3995
- }
3996
- }
3997
- // #615 — procedural-compilation pass. Detects recurring successful ordered
3998
- // action sequences and compiles them into workflow proposals. Opt-in: gated
3999
- // on the `procedural` process being enabled, whole-stash / type scope (never
4000
- // `ref`), and not a dry run. Mirrors the recombine opt-in wiring.
4001
- let proceduralCompilation;
4002
- if (primaryStashDir &&
4003
- improveProfile &&
4004
- resolveProcessEnabled("procedural", improveProfile) &&
4005
- scope.mode !== "ref" &&
4006
- !options.dryRun) {
4007
- const proceduralFn = options.proceduralFn ?? akmProcedural;
4008
- try {
4009
- proceduralCompilation = await proceduralFn({
4010
- stashDir: primaryStashDir,
4011
- config: options.config ?? loadConfig(),
4012
- ...(options.runId ? { sourceRun: options.runId } : {}),
4013
- ...(budgetSignal ? { signal: budgetSignal } : {}),
4014
- ...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
4015
- eligibilitySource: "procedural",
4016
- ...(eventsCtx ? { ctx: eventsCtx } : {}),
4017
- minRecurrence: improveProfile.processes?.procedural?.minRecurrence,
4018
- maxProposalsPerRun: improveProfile.processes?.procedural?.maxProposalsPerRun,
4019
- });
4020
- }
4021
- catch (e) {
4022
- allWarnings.push(`procedural: ${String(e)}`);
4023
- }
4024
- }
4025
- return {
4026
- allWarnings,
4027
- deadUrls,
4028
- ...(recombination ? { recombination } : {}),
4029
- ...(proceduralCompilation ? { proceduralCompilation } : {}),
4030
- ...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
4031
- ...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
4032
- ...(maintenanceResult.stalenessDetection ? { stalenessDetection: maintenanceResult.stalenessDetection } : {}),
4033
- ...(maintenanceResult.actions && maintenanceResult.actions.length > 0
4034
- ? { maintenanceActions: maintenanceResult.actions }
4035
- : {}),
4036
- memoryInferenceDurationMs: maintenanceResult.memoryInferenceDurationMs,
4037
- graphExtractionDurationMs: maintenanceResult.graphExtractionDurationMs,
4038
- orphansPurged: maintenanceResult.orphansPurged,
4039
- proposalsExpired: maintenanceResult.proposalsExpired,
4040
- // Consolidation's auto-accept gate counts now accrue in the preparation
4041
- // stage (#551); post-loop no longer runs an auto-accept gate of its own.
4042
- gateAutoAcceptedCount: 0,
4043
- gateAutoAcceptFailedCount: 0,
4044
- };
4045
- }
4046
- // TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
4047
- // Exported for tests (#584/#585 DB-locking regression coverage); production
4048
- // callers reach it only through akmImprove → runImprovePostLoopStage.
4049
- export async function runImproveMaintenancePasses(args) {
4050
- const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
4051
- if (!primaryStashDir)
4052
- return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
4053
- const config = options.config ?? loadConfig();
4054
- const sources = resolveSourceEntries(options.stashDir, config);
4055
- const memoryInferenceFn = options.memoryInferenceFn ?? runMemoryInferencePass;
4056
- const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
4057
- const stalenessDetectionFn = options.stalenessDetectionFn ?? runStalenessDetectionPass;
4058
- let db;
4059
- let memoryInference;
4060
- let graphExtraction;
4061
- let stalenessDetection;
4062
- let reindexedAfterInference = false;
4063
- const actions = [];
4064
- let memoryInferenceDurationMs = 0;
4065
- let graphExtractionDurationMs = 0;
4066
- let orphansPurged = 0;
4067
- let proposalsExpired = 0;
4068
- const openIndexDb = () => openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
4069
- // #584: reindexFn opens its own write handle on the same index.db WAL file.
4070
- // Holding our handle across that call produced SQLITE_BUSY / "database is
4071
- // locked" failures in production, so the handle is closed BEFORE every
4072
- // reindex and reopened after — the fresh handle also sees the post-reindex
4073
- // state that graph extraction and staleness detection below rely on. The
4074
- // reopen runs in `finally` so a failed reindex still leaves a usable handle.
4075
- const reindexWithIndexDbReleased = async (stashDir) => {
4076
- if (db) {
4077
- closeDatabase(db);
4078
- db = undefined;
4079
- }
4080
- try {
4081
- await reindexFn({ stashDir });
4082
- }
4083
- finally {
4084
- db = openIndexDb();
4085
- }
4086
- };
4087
- await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
4088
- try {
4089
- db = openIndexDb();
4090
- // Memory inference candidate-discovery (post-Item 9 fix from
4091
- // memory:akm-improve-critical-review-2026-05-20). Previously this pass
4092
- // was gated on memoryRefsForInference.size > 0 AND passed those refs as a
4093
- // candidateRefs filter. But memoryRefsForInference is populated from refs
4094
- // distilled THIS RUN — by the time that happens, those parents are
4095
- // already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
4096
- // them. The genuinely-pending parents in the stash never entered the
4097
- // filter. Result: 0/0/0 for 25 consecutive runs.
4098
- //
4099
- // Fix: always run the pass when the feature is enabled; let the pass's
4100
- // own `collectPendingMemories` + `isPendingMemory` predicate find
4101
- // candidates from the filesystem-of-truth. The this-run set is still
4102
- // logged as a hint but no longer used as a filter.
4103
- const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
4104
- const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
4105
- const pendingBelowMinCount = (() => {
4106
- if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
4107
- return false;
4108
- const pending = collectPendingMemories(primaryStashDir).length;
4109
- if (pending < minPendingCount) {
4110
- info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
4111
- return true;
4112
- }
4113
- return false;
4114
- })();
4115
- if (memoryInferenceDisabledByProfile) {
4116
- info("[improve] memory inference skipped (disabled by improve profile)");
4117
- }
4118
- else if (pendingBelowMinCount) {
4119
- // skipped — message already emitted above
4120
- }
4121
- else {
4122
- const hintRefs = memoryRefsForInference.size;
4123
- info(hintRefs > 0
4124
- ? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
4125
- : "[improve] memory inference starting (discovering pending parents)");
4126
- const inferenceStart = Date.now();
4127
- try {
4128
- // O-1 (#364): pass budget signal so a hung inference call is cancelled.
4129
- memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
4130
- config,
4131
- sources,
4132
- signal: budgetSignal,
4133
- db,
4134
- reEnrich: false,
4135
- onProgress: (event) => {
4136
- const current = event.currentRef ? ` ${event.currentRef}` : "";
4137
- info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
4138
- },
4139
- }));
4140
- memoryInferenceDurationMs = Date.now() - inferenceStart;
4141
- actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
4142
- info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
4143
- }
4144
- catch (err) {
4145
- memoryInferenceDurationMs = Date.now() - inferenceStart;
4146
- allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
4147
- }
4148
- }
4149
- if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
4150
- info("[improve] reindexing after memory inference writes");
4151
- try {
4152
- await reindexWithIndexDbReleased(primaryStashDir);
4153
- reindexedAfterInference = true;
4154
- info("[improve] reindex after memory inference complete");
4155
- }
4156
- catch (err) {
4157
- allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
4158
- }
4159
- }
4160
- const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
4161
- const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
4162
- const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
4163
- // #624 P2: optional incremental high-signal-first cap. Unset = process all
4164
- // eligible (byte-identical to today; no ranking/slice).
4165
- const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
4166
- // Build the set of refs actually touched this run.
4167
- const touchedRefs = new Set();
4168
- for (const r of args.actionableRefs)
4169
- touchedRefs.add(r.ref);
4170
- for (const r of memoryRefsForInference)
4171
- touchedRefs.add(r);
4172
- // INVARIANT: graph extraction normally runs only on files touched by
4173
- // actionable refs (candidatePaths). Full-corpus scans are opt-in via
4174
- // profile.processes.graphExtraction.fullScan = true (used by the
4175
- // `graph-refresh` built-in profile and its weekly scheduled task).
4176
- // The empty-Set fallback is intentional when no refs were touched —
4177
- // the extractor's filter rejects every file and returns empty, keeping
4178
- // the pass invoked so the action is recorded and tests stay exercised.
4179
- if (graphExtractionDisabledByProfile) {
4180
- info("[improve] graph extraction skipped (disabled by improve profile)");
4181
- }
4182
- else if (sources.length > 0 && graphEnabled) {
4183
- info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
4184
- const extractionStart = Date.now();
4185
- try {
4186
- // D9: if consolidation ran but memory inference did not reindex, force a reindex
4187
- // so graph extraction sees current DB state after consolidation writes.
4188
- if (consolidationRan && !reindexedAfterInference) {
4189
- info("[improve] reindexing after consolidation (graph extraction needs current state)");
4190
- try {
4191
- await reindexWithIndexDbReleased(primaryStashDir);
4192
- reindexedAfterInference = true;
4193
- info("[improve] reindex after consolidation complete");
4194
- }
4195
- catch (err) {
4196
- allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
4197
- }
4198
- }
4199
- // #584: no close/reopen needed here — reindexWithIndexDbReleased
4200
- // already swapped in a fresh post-reindex handle.
4201
- // Resolve touched refs to absolute file paths. Skipped for fullScan
4202
- // (candidatePaths stays undefined → extractor processes all files).
4203
- let candidatePaths;
4204
- if (!graphExtractionFullScan) {
4205
- candidatePaths = new Set();
4206
- if (primaryStashDir && touchedRefs.size > 0) {
4207
- const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
4208
- const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
4209
- for (const p of resolved) {
4210
- if (typeof p === "string" && p.length > 0)
4211
- candidatePaths.add(p);
4212
- }
4213
- }
4214
- }
4215
- const progressHandler = (event) => {
4216
- const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
4217
- info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
4218
- };
4219
- // O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
4220
- graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
4221
- config,
4222
- sources,
4223
- signal: budgetSignal,
4224
- db,
4225
- reEnrich: false,
4226
- onProgress: progressHandler,
4227
- options: { candidatePaths, ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}) },
4228
- }));
4229
- graphExtractionDurationMs = Date.now() - extractionStart;
4230
- actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
4231
- info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
4232
- }
4233
- catch (err) {
4234
- graphExtractionDurationMs = Date.now() - extractionStart;
4235
- allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
4236
- }
4237
- }
4238
- else if (sources.length > 0 && !graphEnabled) {
4239
- info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
4240
- }
4241
- // Orphan proposal purge — reject pending reflect proposals whose target
4242
- // asset no longer exists on disk. Runs after graph extraction so newly
4243
- // promoted assets from accept flows during this run are already present.
4244
- if (primaryStashDir) {
4245
- try {
4246
- const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
4247
- orphansPurged = purgeResult.rejected;
4248
- if (purgeResult.rejected > 0) {
4249
- info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
4250
- }
4251
- appendEvent({
4252
- eventType: "proposal_orphan_purge",
4253
- ref: "proposals:_orphan-purge",
4254
- metadata: {
4255
- checked: purgeResult.checked,
4256
- rejected: purgeResult.rejected,
4257
- durationMs: purgeResult.durationMs,
4258
- byType: purgeResult.byType,
4259
- orphans: purgeResult.orphans.map((o) => o.ref),
4260
- },
4261
- }, eventsCtx);
4262
- }
4263
- catch (err) {
4264
- allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
4265
- }
4266
- // Phase 6B (Advantage D6b): expire pending proposals that have aged past
4267
- // the retention window. Runs AFTER orphan purge so we never double-archive
4268
- // a proposal that orphan-purge already moved. `expireStaleProposals` emits
4269
- // its own per-proposal `proposal_expired` events; we additionally emit a
4270
- // single roll-up event here for parity with the orphan-purge surface.
4271
- try {
4272
- const expireResult = expireStaleProposals(primaryStashDir, config);
4273
- proposalsExpired = expireResult.expired;
4274
- if (expireResult.expired > 0) {
4275
- info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
4276
- `(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
4277
- }
4278
- appendEvent({
4279
- eventType: "proposal_expiration_pass",
4280
- ref: "proposals:_expiration",
4281
- metadata: {
4282
- checked: expireResult.checked,
4283
- expired: expireResult.expired,
4284
- durationMs: expireResult.durationMs,
4285
- retentionDays: expireResult.retentionDays,
4286
- expiredProposals: expireResult.expiredProposals,
4287
- },
4288
- }, eventsCtx);
4289
- }
4290
- catch (err) {
4291
- allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
4292
- }
4293
- }
4294
- // Fix #2 (observability 0.8.0): trim the events table in state.db so it
4295
- // doesn't grow unbounded. `akm health` writes a `health_probe` row on every
4296
- // invocation, and every command surface emits at least one event besides —
4297
- // without this trim, state.db is a permanent append-only log. Config key
4298
- // `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
4299
- // window. The purge runs against state.db (a different SQLite file from
4300
- // the index `db` above).
4301
- {
4302
- const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
4303
- if (retentionDays > 0) {
4304
- // #585: reuse the long-lived eventsCtx.db connection when akmImprove
4305
- // opened one — opening a second state.db write connection while
4306
- // eventsDb is still live made two simultaneous writers contend on the
4307
- // same WAL file ("database is locked"). Only the eventsCtx.dbPath
4308
- // fallback path (state.db failed to open up-front) opens — and then
4309
- // owns and closes — its own handle. C2 still holds: the fallback uses
4310
- // the boundary-pinned path, never a live `process.env` re-read.
4311
- const ownsStateDb = !eventsCtx?.db;
4312
- let stateDb;
4313
- try {
4314
- stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
4315
- const purgedCount = purgeOldEvents(stateDb, retentionDays);
4316
- if (purgedCount > 0) {
4317
- info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
4318
- }
4319
- appendEvent({
4320
- eventType: "events_purged",
4321
- ref: "events:_purge",
4322
- metadata: { purgedCount, retentionDays },
4323
- }, eventsCtx);
4324
- // improve_runs uses the same retention window as events — both are
4325
- // observability/audit data, both grow append-only, both have a
4326
- // dedicated purge helper. Mirroring the events purge here means a
4327
- // single retention knob (improve.eventRetentionDays) governs both.
4328
- const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
4329
- if (improveRunsPurged > 0) {
4330
- info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
4331
- }
4332
- appendEvent({
4333
- eventType: "improve_runs_purged",
4334
- ref: "improve_runs:_purge",
4335
- metadata: { purgedCount: improveRunsPurged, retentionDays },
4336
- }, eventsCtx);
4337
- }
4338
- catch (err) {
4339
- allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
4340
- }
4341
- finally {
4342
- if (ownsStateDb && stateDb) {
4343
- try {
4344
- stateDb.close();
4345
- }
4346
- catch {
4347
- // best-effort
4348
- }
4349
- }
4350
- }
4351
- // task_logs in logs.db (#579) shares the same retention window as
4352
- // events/improve_runs — all three are observability data governed by
4353
- // the single improve.eventRetentionDays knob. Separate try/finally
4354
- // because logs.db is a different file: a locked/missing logs.db must
4355
- // not block the state.db purges above.
4356
- let logsDb;
4357
- try {
4358
- logsDb = openLogsDatabase();
4359
- const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
4360
- if (taskLogsPurged > 0) {
4361
- info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
4362
- }
4363
- appendEvent({
4364
- eventType: "task_logs_purged",
4365
- ref: "task_logs:_purge",
4366
- metadata: { purgedCount: taskLogsPurged, retentionDays },
4367
- }, eventsCtx);
4368
- }
4369
- catch (err) {
4370
- allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
4371
- }
4372
- finally {
4373
- if (logsDb) {
4374
- try {
4375
- logsDb.close();
4376
- }
4377
- catch {
4378
- // best-effort
4379
- }
4380
- }
4381
- }
4382
- }
4383
- }
4384
- // Phase 4A (staleness detection). Activates the `deprecated` belief-state
4385
- // machinery shipped in Phase 1A. Default OFF — gated by
4386
- // `features.index.staleness_detection.enabled`. Runs after orphan purge
4387
- // and before the URL check (which lives in the outer caller).
4388
- if (sources.length > 0) {
4389
- try {
4390
- stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
4391
- if (stalenessDetection.considered > 0) {
4392
- info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
4393
- `deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
4394
- `skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
4395
- }
4396
- for (const w of stalenessDetection.warnings)
4397
- allWarnings.push(`[improve] staleness detection: ${w}`);
4398
- }
4399
- catch (err) {
4400
- allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
4401
- }
4402
- }
4403
- }
4404
- finally {
4405
- if (db)
4406
- closeDatabase(db);
4407
- }
4408
- });
4409
- return {
4410
- ...(memoryInference ? { memoryInference } : {}),
4411
- ...(graphExtraction ? { graphExtraction } : {}),
4412
- ...(stalenessDetection ? { stalenessDetection } : {}),
4413
- ...(actions.length > 0 ? { actions } : {}),
4414
- memoryInferenceDurationMs,
4415
- graphExtractionDurationMs,
4416
- orphansPurged,
4417
- proposalsExpired,
4418
- };
4419
- }
4420
- function shouldAnalyzeMemoryCleanup(scope, eligibleMemories, primaryStashDir) {
4421
- if (!primaryStashDir || eligibleMemories === 0)
4422
- return false;
4423
- if (scope.mode === "all")
4424
- return true;
4425
- if (scope.mode === "type")
4426
- return scope.value === "memory";
4427
- if (!scope.value)
4428
- return false;
4429
- return parseAssetRef(scope.value).type === "memory";
4430
- }
4431
- function shapeMemoryCleanup(plan) {
4432
- return {
4433
- analyzedDerived: plan.analyzedDerived,
4434
- pruneCandidates: plan.pruneCandidates,
4435
- contradictionCandidates: plan.contradictionCandidates,
4436
- beliefStateTransitions: plan.beliefStateTransitions,
4437
- consolidationCandidates: plan.consolidationCandidates,
4438
- ...(plan.relativeDateCandidates.length > 0 ? { relativeDateCandidates: plan.relativeDateCandidates } : {}),
4439
- };
4440
- }
4441
- function buildUtilityMap(refs) {
4442
- const map = new Map();
4443
- if (refs.length === 0)
4444
- return map;
4445
- const refSet = new Set(refs.map((r) => r.ref));
4446
- let db;
4447
- try {
4448
- db = openExistingDatabase();
4449
- const allDbEntries = getAllEntries(db);
4450
- const idToRef = new Map();
4451
- for (const indexed of allDbEntries) {
4452
- const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
4453
- if (refSet.has(ref))
4454
- idToRef.set(indexed.id, ref);
4455
- }
4456
- const ids = [...idToRef.keys()];
4457
- if (ids.length > 0) {
4458
- const { global: scores } = getUtilityScoresByIds(db, ids);
4459
- for (const [id, score] of scores) {
4460
- const ref = idToRef.get(id);
4461
- if (ref)
4462
- map.set(ref, score.utility);
4463
- }
4464
- }
4465
- }
4466
- catch (err) {
4467
- rethrowIfTestIsolationError(err);
4468
- // best-effort: if DB unavailable, all utilities default to 0
4469
- }
4470
- finally {
4471
- if (db)
4472
- closeDatabase(db);
4473
- }
4474
- return map;
4475
- }
4476
- async function findAssetFilePath(ref, stashDir, writableDirSet) {
4477
- return resolveAssetPath(ref, {
4478
- stashDir,
4479
- mode: "disk-only",
4480
- writableDirSet,
4481
- directoryIndexNames: ["SKILL.md"],
4482
- preserveDirectNameFallback: true,
4483
- honorOrigin: false,
4484
- });
4485
986
  }