akm-cli 0.9.0-beta.0 → 0.9.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +511 -0
- package/dist/assets/profiles/quick.json +2 -1
- package/dist/assets/templates/html/default.html +78 -0
- package/dist/assets/templates/html/health.html +732 -0
- package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
- package/dist/cli/shared.js +21 -5
- package/dist/cli.js +47 -5
- package/dist/commands/config-cli.js +0 -10
- package/dist/commands/feedback-cli.js +42 -37
- package/dist/commands/graph/graph.js +75 -71
- package/dist/commands/health/checks.js +48 -0
- package/dist/commands/health/html-report.js +666 -0
- package/dist/commands/health.js +186 -13
- package/dist/commands/improve/consolidate.js +39 -6
- package/dist/commands/improve/distill.js +26 -5
- package/dist/commands/improve/extract-prompt.js +1 -1
- package/dist/commands/improve/extract.js +52 -8
- package/dist/commands/improve/improve-auto-accept.js +33 -1
- package/dist/commands/improve/improve-cli.js +7 -0
- package/dist/commands/improve/improve-profiles.js +4 -0
- package/dist/commands/improve/improve.js +877 -433
- package/dist/commands/improve/proactive-maintenance.js +113 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +31 -0
- package/dist/commands/proposal/drain.js +73 -6
- package/dist/commands/proposal/proposal-cli.js +22 -10
- package/dist/commands/proposal/proposal.js +17 -1
- package/dist/commands/proposal/validators/proposals.js +365 -329
- package/dist/commands/read/curate.js +17 -0
- package/dist/commands/remember.js +6 -2
- package/dist/commands/sources/stash-cli.js +10 -2
- package/dist/commands/tasks/tasks.js +32 -8
- package/dist/core/config/config-schema.js +30 -0
- package/dist/core/file-lock.js +22 -0
- package/dist/core/logs-db.js +304 -0
- package/dist/core/paths.js +3 -0
- package/dist/core/state-db.js +152 -14
- package/dist/indexer/db/db.js +99 -13
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +114 -111
- package/dist/indexer/passes/memory-inference.js +61 -22
- package/dist/integrations/harnesses/claude/session-log.js +17 -5
- package/dist/llm/client.js +38 -4
- package/dist/llm/usage-persist.js +77 -0
- package/dist/llm/usage-telemetry.js +103 -0
- package/dist/output/context.js +3 -2
- package/dist/output/html-render.js +73 -0
- package/dist/output/shapes/helpers.js +17 -1
- package/dist/output/text/helpers.js +69 -1
- package/dist/scripts/migrate-storage.js +153 -25
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +21 -2
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/tasks/backends/cron.js +46 -9
- package/dist/tasks/runner.js +99 -16
- package/dist/workflows/db.js +4 -0
- package/package.json +3 -2
- package/dist/commands/config-edit.js +0 -344
|
@@ -10,23 +10,27 @@ import { daysToMs, isAssetType } from "../../core/common.js";
|
|
|
10
10
|
import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
|
|
11
11
|
import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
12
12
|
import { appendEvent, readEvents } from "../../core/events.js";
|
|
13
|
-
import { probeLock, releaseLock, tryAcquireLockSync } from "../../core/file-lock.js";
|
|
13
|
+
import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../../core/file-lock.js";
|
|
14
14
|
import { classifyImproveAction } from "../../core/improve-types.js";
|
|
15
|
+
import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
|
|
15
16
|
import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
|
|
16
17
|
import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
|
|
17
18
|
import { info, warn } from "../../core/warn.js";
|
|
18
19
|
import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
|
|
19
20
|
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
20
21
|
import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
|
|
22
|
+
import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
|
|
21
23
|
import { akmIndex } from "../../indexer/indexer.js";
|
|
22
|
-
import { runMemoryInferencePass } from "../../indexer/passes/memory-inference.js";
|
|
24
|
+
import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
|
|
23
25
|
import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
|
|
24
26
|
import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
25
27
|
import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
|
|
26
28
|
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
27
29
|
import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
|
|
28
30
|
import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
|
|
29
|
-
import {
|
|
31
|
+
import { isProcessEnabled } from "../../llm/feature-gate.js";
|
|
32
|
+
import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
|
|
33
|
+
import { withLlmStage } from "../../llm/usage-telemetry.js";
|
|
30
34
|
import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
|
|
31
35
|
import { akmLint } from "../lint/index.js";
|
|
32
36
|
import { drainProposals } from "../proposal/drain.js";
|
|
@@ -43,7 +47,105 @@ import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./i
|
|
|
43
47
|
import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
|
|
44
48
|
import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
|
|
45
49
|
import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
|
|
50
|
+
import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
|
|
46
51
|
import { akmReflect } from "./reflect.js";
|
|
52
|
+
// #607 Lock Decomposition: fine-grained per-process locks replace the single
|
|
53
|
+
// `improve.lock`. Three independent locks allow concurrent improve runs when
|
|
54
|
+
// they touch different subsystems (e.g. quick-shredder consolidate can run
|
|
55
|
+
// alongside daily reflect+distill).
|
|
56
|
+
//
|
|
57
|
+
// consolidate.lock — protects consolidate + memoryInference (both write index.db)
|
|
58
|
+
// reflect-distill.lock — protects reflect + distill (both write state.db proposals)
|
|
59
|
+
// triage.lock — protects triage (writes proposal promotions)
|
|
60
|
+
//
|
|
61
|
+
// Stale timeouts are per-lock, tuned to the expected runtime of the protected
|
|
62
|
+
// processes: consolidate is disk-bound (1h), reflect+distill is GPU-bound (2h),
|
|
63
|
+
// triage is fast (30min).
|
|
64
|
+
const PROCESS_LOCK_DEFS = {
|
|
65
|
+
consolidate: { fileName: "consolidate.lock", staleAfterMs: 60 * 60 * 1000 },
|
|
66
|
+
reflectDistill: { fileName: "reflect-distill.lock", staleAfterMs: 2 * 60 * 60 * 1000 },
|
|
67
|
+
triage: { fileName: "triage.lock", staleAfterMs: 30 * 60 * 1000 },
|
|
68
|
+
};
|
|
69
|
+
const heldProcessLocks = new Set();
|
|
70
|
+
export function resetHeldProcessLocks() {
|
|
71
|
+
heldProcessLocks.clear();
|
|
72
|
+
}
|
|
73
|
+
function processLockPath(lockBaseDir, lockName) {
|
|
74
|
+
return path.join(lockBaseDir, PROCESS_LOCK_DEFS[lockName].fileName);
|
|
75
|
+
}
|
|
76
|
+
function tryAcquireProcessLock(lockPath, staleAfterMs, skipIfLocked, lockLabel) {
|
|
77
|
+
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
78
|
+
const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
|
|
79
|
+
if (tryAcquireLockSync(lockPath, lockPayload())) {
|
|
80
|
+
heldProcessLocks.add(lockPath);
|
|
81
|
+
return "acquired";
|
|
82
|
+
}
|
|
83
|
+
const probe = probeLock(lockPath, { staleAfterMs });
|
|
84
|
+
const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
|
|
85
|
+
const lock = rawContent
|
|
86
|
+
? (() => {
|
|
87
|
+
try {
|
|
88
|
+
return JSON.parse(rawContent);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
})()
|
|
94
|
+
: null;
|
|
95
|
+
if (probe.state === "stale") {
|
|
96
|
+
try {
|
|
97
|
+
appendEvent({
|
|
98
|
+
eventType: "improve_lock_recovered",
|
|
99
|
+
metadata: {
|
|
100
|
+
lockName: lockLabel,
|
|
101
|
+
stalePid: lock?.pid ?? null,
|
|
102
|
+
lockedAt: lock?.startedAt ?? null,
|
|
103
|
+
recoveredAt: new Date().toISOString(),
|
|
104
|
+
lockAgeMs: probe.ageMs ?? null,
|
|
105
|
+
reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
/* event emission is best-effort; never block lock recovery */
|
|
111
|
+
}
|
|
112
|
+
releaseLock(lockPath);
|
|
113
|
+
if (tryAcquireLockSync(lockPath, lockPayload())) {
|
|
114
|
+
heldProcessLocks.add(lockPath);
|
|
115
|
+
return "acquired";
|
|
116
|
+
}
|
|
117
|
+
if (skipIfLocked) {
|
|
118
|
+
warn(`[improve] ${lockLabel} lock acquired by another run during stale recovery; skipping (--skip-if-locked)`);
|
|
119
|
+
return "skipped";
|
|
120
|
+
}
|
|
121
|
+
throw new ConfigError(`akm improve ${lockLabel} is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
122
|
+
}
|
|
123
|
+
if (skipIfLocked) {
|
|
124
|
+
warn(`[improve] ${lockLabel} lock held by another run (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
|
|
125
|
+
return "skipped";
|
|
126
|
+
}
|
|
127
|
+
throw new ConfigError(`akm improve ${lockLabel} is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
128
|
+
}
|
|
129
|
+
function releaseProcessLock(lockPath) {
|
|
130
|
+
try {
|
|
131
|
+
fs.unlinkSync(lockPath);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// ignore
|
|
135
|
+
}
|
|
136
|
+
heldProcessLocks.delete(lockPath);
|
|
137
|
+
}
|
|
138
|
+
function releaseAllProcessLocks() {
|
|
139
|
+
for (const p of heldProcessLocks) {
|
|
140
|
+
try {
|
|
141
|
+
fs.unlinkSync(p);
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
// ignore
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
heldProcessLocks.clear();
|
|
148
|
+
}
|
|
47
149
|
function resolveImproveScope(scope) {
|
|
48
150
|
const trimmed = scope?.trim();
|
|
49
151
|
if (!trimmed)
|
|
@@ -99,6 +201,22 @@ export function renderSyncCommitMessage(template, result, nowMs) {
|
|
|
99
201
|
};
|
|
100
202
|
return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
|
|
101
203
|
}
|
|
204
|
+
/**
|
|
205
|
+
* Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
|
|
206
|
+
* merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
|
|
207
|
+
* Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
|
|
208
|
+
*/
|
|
209
|
+
function dedupeRefs(refs) {
|
|
210
|
+
const seen = new Set();
|
|
211
|
+
const out = [];
|
|
212
|
+
for (const r of refs) {
|
|
213
|
+
if (seen.has(r.ref))
|
|
214
|
+
continue;
|
|
215
|
+
seen.add(r.ref);
|
|
216
|
+
out.push(r);
|
|
217
|
+
}
|
|
218
|
+
return out;
|
|
219
|
+
}
|
|
102
220
|
async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
103
221
|
if (scope.mode === "ref" && scope.value) {
|
|
104
222
|
const parsed = parseAssetRef(scope.value);
|
|
@@ -112,7 +230,7 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
|
112
230
|
};
|
|
113
231
|
}
|
|
114
232
|
return {
|
|
115
|
-
plannedRefs: [{ ref: scope.value, reason: "scope-ref" }],
|
|
233
|
+
plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
|
|
116
234
|
memorySummary: {
|
|
117
235
|
eligible: parsed.type === "memory" ? 1 : 0,
|
|
118
236
|
derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
|
|
@@ -176,12 +294,14 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
|
176
294
|
profileFiltered.set(ref, {
|
|
177
295
|
ref,
|
|
178
296
|
reason: "profile_filtered_all_passes",
|
|
297
|
+
filePath: indexed.filePath,
|
|
179
298
|
});
|
|
180
299
|
}
|
|
181
300
|
else {
|
|
182
301
|
planned.set(ref, {
|
|
183
302
|
ref,
|
|
184
303
|
reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
|
|
304
|
+
filePath: indexed.filePath,
|
|
185
305
|
});
|
|
186
306
|
}
|
|
187
307
|
}
|
|
@@ -466,7 +586,9 @@ export async function akmImprove(options = {}) {
|
|
|
466
586
|
options = {
|
|
467
587
|
...options,
|
|
468
588
|
autoAccept: options.autoAccept ?? improveProfile.autoAccept,
|
|
469
|
-
limit
|
|
589
|
+
// Profile-level limit, then process-level reflect.limit as fallback.
|
|
590
|
+
// CLI --limit takes precedence over both.
|
|
591
|
+
limit: options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit,
|
|
470
592
|
};
|
|
471
593
|
let primaryStashDir;
|
|
472
594
|
try {
|
|
@@ -484,82 +606,16 @@ export async function akmImprove(options = {}) {
|
|
|
484
606
|
// timeout root cause). Because beforeEach runs synchronously, env is still the
|
|
485
607
|
// calling test's own at this point; we capture it before yielding the loop.
|
|
486
608
|
const resolvedStateDbPath = getStateDbPathInDataDir();
|
|
487
|
-
//
|
|
488
|
-
//
|
|
489
|
-
//
|
|
490
|
-
//
|
|
491
|
-
//
|
|
492
|
-
//
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
const
|
|
497
|
-
const acquireLock = () => {
|
|
498
|
-
fs.mkdirSync(path.dirname(resolvedLockPath), { recursive: true });
|
|
499
|
-
const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
|
|
500
|
-
if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
|
|
501
|
-
return;
|
|
502
|
-
// Lock file already exists — probe to determine whether it's still held
|
|
503
|
-
// or whether the prior run died without cleaning up.
|
|
504
|
-
const probe = probeLock(resolvedLockPath, { staleAfterMs: MAX_LOCK_AGE_MS });
|
|
505
|
-
const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
|
|
506
|
-
const lock = rawContent
|
|
507
|
-
? (() => {
|
|
508
|
-
try {
|
|
509
|
-
return JSON.parse(rawContent);
|
|
510
|
-
}
|
|
511
|
-
catch {
|
|
512
|
-
return null;
|
|
513
|
-
}
|
|
514
|
-
})()
|
|
515
|
-
: null;
|
|
516
|
-
if (probe.state === "stale") {
|
|
517
|
-
// O-7 / #394: Emit improve_lock_recovered event before recovery so the
|
|
518
|
-
// audit trail records the abnormal prior-run exit (Temporal/Airflow pattern).
|
|
519
|
-
try {
|
|
520
|
-
appendEvent({
|
|
521
|
-
eventType: "improve_lock_recovered",
|
|
522
|
-
metadata: {
|
|
523
|
-
stalePid: lock?.pid ?? null,
|
|
524
|
-
lockedAt: lock?.startedAt ?? null,
|
|
525
|
-
recoveredAt: new Date().toISOString(),
|
|
526
|
-
lockAgeMs: probe.ageMs ?? null,
|
|
527
|
-
reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
|
|
528
|
-
},
|
|
529
|
-
});
|
|
530
|
-
}
|
|
531
|
-
catch {
|
|
532
|
-
/* event emission is best-effort; never block lock recovery */
|
|
533
|
-
}
|
|
534
|
-
releaseLock(resolvedLockPath);
|
|
535
|
-
if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
|
|
536
|
-
return;
|
|
537
|
-
throw new ConfigError(`akm improve is already running. Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
538
|
-
}
|
|
539
|
-
throw new ConfigError(`akm improve is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
540
|
-
};
|
|
541
|
-
// Phase 4 lock-leak guard (§7 ordering hazard): hoisting `improve.lock` above
|
|
542
|
-
// the pre-index region (so the triage pre-pass runs under it) means the lock is
|
|
543
|
-
// held while ensureIndex / collectEligibleRefs / contradiction-detection /
|
|
544
|
-
// memory-cleanup analysis run — but the main protecting `try { … } finally {
|
|
545
|
-
// unlinkSync(resolvedLockPath) }` does not begin until after them. A throw in
|
|
546
|
-
// any of those steps would leak the lock. We close that window by wrapping the
|
|
547
|
-
// whole region in a try whose catch releases the lock (when held) and
|
|
548
|
-
// re-throws. The values this region computes are declared in the outer scope so
|
|
549
|
-
// they remain visible to the main run below. The dry-run path never sets
|
|
550
|
-
// `lockAcquired`, so its early return releases nothing.
|
|
551
|
-
let lockAcquired = false;
|
|
552
|
-
const releaseLockOnError = () => {
|
|
553
|
-
if (!lockAcquired)
|
|
554
|
-
return;
|
|
555
|
-
try {
|
|
556
|
-
fs.unlinkSync(resolvedLockPath);
|
|
557
|
-
}
|
|
558
|
-
catch {
|
|
559
|
-
// best-effort release on the error path
|
|
560
|
-
}
|
|
561
|
-
lockAcquired = false;
|
|
562
|
-
};
|
|
609
|
+
// #607 Lock decomposition: three per-process locks replace the single
|
|
610
|
+
// `improve.lock`. Each process acquires only the lock(s) it needs, so
|
|
611
|
+
// quick-shredder consolidate can run alongside daily reflect+distill.
|
|
612
|
+
//
|
|
613
|
+
// consolidate.lock — protects consolidate + memoryInference + graphExtraction (index.db writers)
|
|
614
|
+
// reflect-distill.lock — protects reflect + distill (state.db proposal writers)
|
|
615
|
+
// triage.lock — protects triage pre-pass (state.db proposal promotions)
|
|
616
|
+
//
|
|
617
|
+
// Lock base directory — same `.akm/` under the primary stash dir.
|
|
618
|
+
const lockBaseDir = primaryStashDir ? path.join(primaryStashDir, ".akm") : path.join(options.stashDir ?? ".", ".akm");
|
|
563
619
|
const preEnsureCleanupWarnings = [];
|
|
564
620
|
let plannedRefs;
|
|
565
621
|
let memorySummary;
|
|
@@ -568,48 +624,59 @@ export async function akmImprove(options = {}) {
|
|
|
568
624
|
let guidance;
|
|
569
625
|
let triageDrain;
|
|
570
626
|
try {
|
|
571
|
-
//
|
|
572
|
-
// The dry-run branch
|
|
573
|
-
//
|
|
627
|
+
// #607: Per-process lock acquisition. Each process acquires only the lock(s)
|
|
628
|
+
// it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
|
|
629
|
+
// locks (decision: dry-run never mutates the queue).
|
|
574
630
|
if (!options.dryRun) {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
// triage
|
|
584
|
-
//
|
|
631
|
+
// Backstop release on process.exit() (signal handler / budget watchdog),
|
|
632
|
+
// which skips the finally below. Removed in that finally on the normal path.
|
|
633
|
+
const releaseAllOnExit = () => {
|
|
634
|
+
for (const p of heldProcessLocks) {
|
|
635
|
+
releaseLockIfOwned(p, process.pid);
|
|
636
|
+
}
|
|
637
|
+
};
|
|
638
|
+
process.on("exit", releaseAllOnExit);
|
|
639
|
+
// #607 triage pre-pass: acquire triage.lock, drain the standing pending
|
|
640
|
+
// backlog BEFORE ensureIndex so improve generates fresh proposals against
|
|
641
|
+
// a cleared queue (no `duplicate_pending` collisions) and ensureIndex
|
|
642
|
+
// absorbs triage's promotions for free. Release immediately after —
|
|
643
|
+
// triage.lock is not needed again until the next improve run.
|
|
585
644
|
if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
|
|
586
645
|
if (scope.mode === "ref") {
|
|
587
646
|
warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
|
|
588
647
|
}
|
|
589
648
|
else {
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
|
|
595
|
-
const judgment = triageConfig?.judgment
|
|
596
|
-
? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
|
|
597
|
-
: null;
|
|
598
|
-
triageDrain = await drainProposalsFn({
|
|
599
|
-
stashDir: primaryStashDir,
|
|
600
|
-
policy,
|
|
601
|
-
applyMode,
|
|
602
|
-
maxAccepts,
|
|
603
|
-
dryRun: false,
|
|
604
|
-
// No fresh ids exist yet — triage runs before improve generates any.
|
|
605
|
-
excludeIds: new Set(),
|
|
606
|
-
...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
|
|
607
|
-
judgment,
|
|
608
|
-
});
|
|
649
|
+
const triageLPath = processLockPath(lockBaseDir, "triage");
|
|
650
|
+
const triageResult = tryAcquireProcessLock(triageLPath, PROCESS_LOCK_DEFS.triage.staleAfterMs, options.skipIfLocked, "triage");
|
|
651
|
+
if (triageResult === "skipped") {
|
|
652
|
+
triageDrain = undefined;
|
|
609
653
|
}
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
654
|
+
else {
|
|
655
|
+
try {
|
|
656
|
+
const triageConfig = improveProfile.processes?.triage;
|
|
657
|
+
const policy = resolveDrainPolicy(triageConfig?.policy);
|
|
658
|
+
const applyMode = triageConfig?.applyMode ?? "queue";
|
|
659
|
+
const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
|
|
660
|
+
const judgment = triageConfig?.judgment
|
|
661
|
+
? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
|
|
662
|
+
: null;
|
|
663
|
+
triageDrain = await drainProposalsFn({
|
|
664
|
+
stashDir: primaryStashDir,
|
|
665
|
+
policy,
|
|
666
|
+
applyMode,
|
|
667
|
+
maxAccepts,
|
|
668
|
+
dryRun: false,
|
|
669
|
+
excludeIds: new Set(),
|
|
670
|
+
...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
|
|
671
|
+
judgment,
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
catch (err) {
|
|
675
|
+
warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
676
|
+
}
|
|
677
|
+
finally {
|
|
678
|
+
releaseProcessLock(triageLPath);
|
|
679
|
+
}
|
|
613
680
|
}
|
|
614
681
|
}
|
|
615
682
|
}
|
|
@@ -641,7 +708,7 @@ export async function akmImprove(options = {}) {
|
|
|
641
708
|
// best-effort; leave preEnsureEntryCount undefined
|
|
642
709
|
}
|
|
643
710
|
try {
|
|
644
|
-
await ensureIndexFn(primaryStashDir);
|
|
711
|
+
await ensureIndexFn(primaryStashDir, { mode: "blocking" });
|
|
645
712
|
}
|
|
646
713
|
catch (err) {
|
|
647
714
|
preEnsureCleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -679,7 +746,7 @@ export async function akmImprove(options = {}) {
|
|
|
679
746
|
if (primaryStashDir && shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)) {
|
|
680
747
|
try {
|
|
681
748
|
// Reuse the config resolved at the top of the run instead of a second load.
|
|
682
|
-
await detectAndWriteContradictions(primaryStashDir, _earlyConfig);
|
|
749
|
+
await withLlmStage("memory-contradiction", () => detectAndWriteContradictions(primaryStashDir, _earlyConfig));
|
|
683
750
|
}
|
|
684
751
|
catch (err) {
|
|
685
752
|
// Non-fatal: contradiction detection is a best-effort pass.
|
|
@@ -709,17 +776,14 @@ export async function akmImprove(options = {}) {
|
|
|
709
776
|
}
|
|
710
777
|
}
|
|
711
778
|
catch (err) {
|
|
712
|
-
|
|
779
|
+
releaseAllProcessLocks();
|
|
713
780
|
throw err;
|
|
714
781
|
}
|
|
715
|
-
//
|
|
716
|
-
//
|
|
717
|
-
//
|
|
718
|
-
//
|
|
719
|
-
//
|
|
720
|
-
// any of them used to leak the lock (blocking the next improve up to 4h);
|
|
721
|
-
// now the finally releases it exactly once. The dry-run path already returned
|
|
722
|
-
// above without acquiring the lock, so it never reaches this finally; the
|
|
782
|
+
// #607: per-process locks are acquired/released around each stage below.
|
|
783
|
+
// The triage pre-pass already ran under triage.lock (released). The
|
|
784
|
+
// preparation stage runs under consolidate.lock, the loop stage under
|
|
785
|
+
// reflect-distill.lock, and the post-loop stage under consolidate.lock again.
|
|
786
|
+
// Each stage acquires its lock just before starting and releases in finally.
|
|
723
787
|
// best-effort `unlinkSync` is a no-op when no lock file exists.
|
|
724
788
|
const startMs = Date.now();
|
|
725
789
|
const budgetMs = options.timeoutMs ?? 2 * 60 * 60 * 1000; // default 2 hours
|
|
@@ -739,6 +803,9 @@ export async function akmImprove(options = {}) {
|
|
|
739
803
|
// Pinned to the boundary snapshot so the fallback per-call `appendEvent`
|
|
740
804
|
// opens (when the long-lived handle below fails to open) never re-read env.
|
|
741
805
|
let eventsCtx = { dbPath: resolvedStateDbPath };
|
|
806
|
+
// #576: clears the per-run LLM usage sink. Defaults to a no-op until the sink
|
|
807
|
+
// is installed inside the try; the `finally` always calls it.
|
|
808
|
+
let disposeLlmUsageSink = () => { };
|
|
742
809
|
try {
|
|
743
810
|
// H7 (#566): arm the budget watchdog. `armBudgetWatchdog` captures both the
|
|
744
811
|
// budget timer and the hard-kill timer it schedules on exhaustion, returning
|
|
@@ -758,19 +825,32 @@ export async function akmImprove(options = {}) {
|
|
|
758
825
|
// still pinned to the boundary-resolved path, never a live env re-read.
|
|
759
826
|
eventsCtx = { dbPath: resolvedStateDbPath };
|
|
760
827
|
}
|
|
761
|
-
//
|
|
828
|
+
// #576: persist per-call LLM usage telemetry for this run as `llm_usage`
|
|
829
|
+
// events, reusing the same boundary-pinned events context (and long-lived
|
|
830
|
+
// handle when available). Disposed in `finally` so the sink never leaks
|
|
831
|
+
// across runs. Wrapping is best-effort end to end — see usage-telemetry.ts.
|
|
832
|
+
disposeLlmUsageSink = installLlmUsagePersistence(eventsCtx);
|
|
833
|
+
// 2026-05-27: emit an `improve_skipped` audit event for refs the planner
|
|
762
834
|
// pre-filtered (reflect AND distill both refuse them under the active
|
|
763
|
-
// profile).
|
|
764
|
-
//
|
|
765
|
-
//
|
|
766
|
-
//
|
|
767
|
-
|
|
835
|
+
// profile). Emitted as a single summary event (count only) rather than one
|
|
836
|
+
// event per ref (#592) — the per-ref loop caused O(n) sequential state.db
|
|
837
|
+
// writes that consumed ~500 s on a 9 000-ref stash. No downstream consumer
|
|
838
|
+
// needs the per-ref audit trail: health's skip histogram reads the
|
|
839
|
+
// `profile_filtered_all_passes` counters from `improve_completed` metadata.
|
|
840
|
+
if (profileFilteredRefs.length > 0) {
|
|
768
841
|
appendEvent({
|
|
769
842
|
eventType: "improve_skipped",
|
|
770
|
-
ref:
|
|
771
|
-
metadata: {
|
|
843
|
+
ref: undefined,
|
|
844
|
+
metadata: {
|
|
845
|
+
reason: "profile_filtered_all_passes",
|
|
846
|
+
count: profileFilteredRefs.length,
|
|
847
|
+
},
|
|
772
848
|
}, eventsCtx);
|
|
773
849
|
}
|
|
850
|
+
// #607: acquire consolidate.lock for the preparation stage (consolidate,
|
|
851
|
+
// ensureIndex, extract all write index.db). Released immediately after.
|
|
852
|
+
const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
|
|
853
|
+
const consolidatePrepAcquired = tryAcquireProcessLock(consolidateLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
|
|
774
854
|
const preparation = await runImprovePreparationStage({
|
|
775
855
|
scope,
|
|
776
856
|
options,
|
|
@@ -785,6 +865,8 @@ export async function akmImprove(options = {}) {
|
|
|
785
865
|
initialCleanupWarnings: preEnsureCleanupWarnings,
|
|
786
866
|
improveProfile,
|
|
787
867
|
});
|
|
868
|
+
if (consolidatePrepAcquired)
|
|
869
|
+
releaseProcessLock(consolidateLPath);
|
|
788
870
|
// D6: pre-load all proposal_rejected events from the last 30 days once,
|
|
789
871
|
// so the per-asset loop can use a Map lookup instead of N DB round trips.
|
|
790
872
|
const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
|
|
@@ -796,6 +878,10 @@ export async function akmImprove(options = {}) {
|
|
|
796
878
|
rejectedProposalsByRef.set(e.ref, e);
|
|
797
879
|
}
|
|
798
880
|
}
|
|
881
|
+
// #607: acquire reflect-distill.lock for the loop stage (reflect + distill
|
|
882
|
+
// both write proposals to state.db). Released immediately after.
|
|
883
|
+
const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
|
|
884
|
+
const reflectDistillAcquired = tryAcquireProcessLock(reflectDistillLPath, PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs, options.skipIfLocked, "reflect-distill") === "acquired";
|
|
799
885
|
const { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount: loopGateCount, gateAutoAcceptFailedCount: loopGateFailedCount, } = await runImproveLoopStage({
|
|
800
886
|
scope,
|
|
801
887
|
options,
|
|
@@ -815,9 +901,15 @@ export async function akmImprove(options = {}) {
|
|
|
815
901
|
eventsCtx,
|
|
816
902
|
improveProfile,
|
|
817
903
|
});
|
|
904
|
+
if (reflectDistillAcquired)
|
|
905
|
+
releaseProcessLock(reflectDistillLPath);
|
|
818
906
|
// #551: consolidation now runs in the preparation stage (before extract);
|
|
819
907
|
// its result and run-flag are read from `preparation`, not the post-loop.
|
|
820
908
|
const consolidation = preparation.consolidation;
|
|
909
|
+
// #607: acquire consolidate.lock for the post-loop stage (memoryInference +
|
|
910
|
+
// graphExtraction both write index.db). Released immediately after.
|
|
911
|
+
const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
|
|
912
|
+
const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
|
|
821
913
|
const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
|
|
822
914
|
scope,
|
|
823
915
|
options,
|
|
@@ -828,11 +920,12 @@ export async function akmImprove(options = {}) {
|
|
|
828
920
|
memoryRefsForInference,
|
|
829
921
|
reindexFn,
|
|
830
922
|
eventsCtx,
|
|
831
|
-
// O-1 (#364): propagate wall-clock budget signal to post-loop maintenance.
|
|
832
923
|
budgetSignal: budgetAbortController.signal,
|
|
833
924
|
improveProfile,
|
|
834
925
|
consolidationRan: preparation.consolidationRan,
|
|
835
926
|
});
|
|
927
|
+
if (consolidatePostAcquired)
|
|
928
|
+
releaseProcessLock(consolidatePostLPath);
|
|
836
929
|
const finalActions = maintenanceActions && maintenanceActions.length > 0
|
|
837
930
|
? [...preparation.actions, ...maintenanceActions]
|
|
838
931
|
: preparation.actions;
|
|
@@ -917,6 +1010,7 @@ export async function akmImprove(options = {}) {
|
|
|
917
1010
|
},
|
|
918
1011
|
}
|
|
919
1012
|
: {}),
|
|
1013
|
+
...(preparation.proactiveMaintenance ? { proactiveMaintenance: preparation.proactiveMaintenance } : {}),
|
|
920
1014
|
...(options.runId !== undefined ? { runId: options.runId } : {}),
|
|
921
1015
|
};
|
|
922
1016
|
if (!result.dryRun)
|
|
@@ -993,15 +1087,18 @@ export async function akmImprove(options = {}) {
|
|
|
993
1087
|
throw err;
|
|
994
1088
|
}
|
|
995
1089
|
finally {
|
|
1090
|
+
// #576: clear the per-run LLM usage sink BEFORE closing `eventsDb` below, so
|
|
1091
|
+
// no late sink invocation can write through a closed handle.
|
|
1092
|
+
disposeLlmUsageSink();
|
|
996
1093
|
// O-1 (#364): Clear the budget abort timer so it does not keep the event
|
|
997
1094
|
// loop alive after the run completes.
|
|
998
1095
|
clearBudgetTimer();
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1096
|
+
// #607: release any per-process locks still held (backstop for error paths;
|
|
1097
|
+
// the normal path already released each lock after its stage completed).
|
|
1098
|
+
releaseAllProcessLocks();
|
|
1099
|
+
// Drop the process.exit backstop so it does not fire later (or accumulate
|
|
1100
|
+
// across repeated in-process calls).
|
|
1101
|
+
process.removeAllListeners("exit");
|
|
1005
1102
|
// I1: close the long-lived state.db connection opened at the top of the run.
|
|
1006
1103
|
try {
|
|
1007
1104
|
eventsDb?.close();
|
|
@@ -1114,6 +1211,11 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1114
1211
|
memoryInferenceDurationMs: durations.memoryInferenceDurationMs,
|
|
1115
1212
|
graphExtractionExtractedFiles: result.graphExtraction?.quality.extractedFiles ?? 0,
|
|
1116
1213
|
graphExtractionDurationMs: durations.graphExtractionDurationMs,
|
|
1214
|
+
// Layer-2 proactive-maintenance coverage (0 when the process is disabled
|
|
1215
|
+
// or the run was ref-scoped) so a scheduled sweep's reach is trackable.
|
|
1216
|
+
proactiveSelected: result.proactiveMaintenance?.selected ?? 0,
|
|
1217
|
+
proactiveDueTotal: result.proactiveMaintenance?.dueTotal ?? 0,
|
|
1218
|
+
proactiveNeverReflected: result.proactiveMaintenance?.neverReflected ?? 0,
|
|
1117
1219
|
// New metrics for tuning the improve loop.
|
|
1118
1220
|
...(durations.totalDurationMs !== undefined ? { durationMs: durations.totalDurationMs } : {}),
|
|
1119
1221
|
...(durations.warningCount !== undefined ? { warningCount: durations.warningCount } : {}),
|
|
@@ -1316,7 +1418,7 @@ async function runConsolidationPass(args) {
|
|
|
1316
1418
|
info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
|
|
1317
1419
|
}
|
|
1318
1420
|
else if (!consolidationOnCooldown) {
|
|
1319
|
-
consolidation = await akmConsolidate({
|
|
1421
|
+
consolidation = await withLlmStage("consolidate", () => akmConsolidate({
|
|
1320
1422
|
...options.consolidateOptions,
|
|
1321
1423
|
config: consolidationConfig,
|
|
1322
1424
|
stashDir: options.stashDir,
|
|
@@ -1324,16 +1426,13 @@ async function runConsolidationPass(args) {
|
|
|
1324
1426
|
// Tie consolidate proposals back to this improve invocation so
|
|
1325
1427
|
// accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
|
|
1326
1428
|
sourceRun: `consolidate-${Date.now()}`,
|
|
1327
|
-
//
|
|
1328
|
-
//
|
|
1329
|
-
//
|
|
1330
|
-
//
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
// volumeTriggered=true on every run, permanently forcing full 12-chunk
|
|
1335
|
-
// scans (~264s) instead of the intended 1-2 chunk incremental path (~44s).
|
|
1336
|
-
incrementalSince: lastConsolidateTs,
|
|
1429
|
+
// Pass profile-configured options. incrementalSince narrows the pool to
|
|
1430
|
+
// recently-changed memories + graph neighbours — use this for frequent
|
|
1431
|
+
// passes (quick-shredder). Leave absent in the nightly default profile for
|
|
1432
|
+
// a full-pool sweep that catches stale-but-unmerged duplicates.
|
|
1433
|
+
incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
|
|
1434
|
+
limit: improveProfile?.processes?.consolidate?.limit,
|
|
1435
|
+
neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
|
|
1337
1436
|
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
1338
1437
|
// Honor profile.autoAccept (already merged into options.autoAccept at the
|
|
1339
1438
|
// top of akmImprove). The CLI parser always supplies 90 when --auto-accept
|
|
@@ -1342,7 +1441,7 @@ async function runConsolidationPass(args) {
|
|
|
1342
1441
|
// options.consolidateOptions.autoAccept (if explicitly provided by caller)
|
|
1343
1442
|
// still wins because the spread above runs first.
|
|
1344
1443
|
autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
|
|
1345
|
-
});
|
|
1444
|
+
}));
|
|
1346
1445
|
{
|
|
1347
1446
|
const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
|
|
1348
1447
|
try {
|
|
@@ -1362,7 +1461,14 @@ async function runConsolidationPass(args) {
|
|
|
1362
1461
|
appendEvent({
|
|
1363
1462
|
eventType: "consolidate_completed",
|
|
1364
1463
|
ref: "memory:_consolidation",
|
|
1365
|
-
metadata: {
|
|
1464
|
+
metadata: {
|
|
1465
|
+
processed: consolidation.processed,
|
|
1466
|
+
merged: consolidation.merged,
|
|
1467
|
+
deleted: consolidation.deleted,
|
|
1468
|
+
contradicted: consolidation.contradicted,
|
|
1469
|
+
failedChunks: consolidation.failedChunks ?? 0,
|
|
1470
|
+
durationMs: consolidation.durationMs,
|
|
1471
|
+
},
|
|
1366
1472
|
}, eventsCtx);
|
|
1367
1473
|
}
|
|
1368
1474
|
}
|
|
@@ -1429,7 +1535,9 @@ async function runImprovePreparationStage(args) {
|
|
|
1429
1535
|
// / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
|
|
1430
1536
|
// hook with an on-demand pull pipeline.
|
|
1431
1537
|
//
|
|
1432
|
-
// Default-on; opt out via `
|
|
1538
|
+
// Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
|
|
1539
|
+
// (#593: the gate respects the resolved improve profile, not just the
|
|
1540
|
+
// hardcoded `default` profile path the legacy feature flag reads).
|
|
1433
1541
|
// Each available harness gets one call with the default --since window;
|
|
1434
1542
|
// already-seen sessions (tracked in state.db.extract_sessions_seen) are
|
|
1435
1543
|
// skipped automatically so re-runs don't burn LLM calls on unchanged data.
|
|
@@ -1463,7 +1571,14 @@ async function runImprovePreparationStage(args) {
|
|
|
1463
1571
|
const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
|
|
1464
1572
|
const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
|
|
1465
1573
|
const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
|
|
1466
|
-
|
|
1574
|
+
// #593/#594: the ACTIVE resolved improve profile is the single source of
|
|
1575
|
+
// truth for whether extract runs. (Previously this also ANDed in the legacy
|
|
1576
|
+
// `session_extraction` feature flag, which only reads
|
|
1577
|
+
// `profiles.improve.default.processes.extract.enabled`; that made the default
|
|
1578
|
+
// profile a global kill switch, so a non-default profile enabling extract was
|
|
1579
|
+
// silently overridden. The default profile is now just another profile.)
|
|
1580
|
+
// `akmExtract` re-checks the same active profile internally via `improveProfile`.
|
|
1581
|
+
if (resolveProcessEnabled("extract", improveProfile)) {
|
|
1467
1582
|
const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
|
|
1468
1583
|
// The guard engages only when minNewSessions > 0; 0 disables it entirely.
|
|
1469
1584
|
let belowMinNewSessions = false;
|
|
@@ -1494,15 +1609,18 @@ async function runImprovePreparationStage(args) {
|
|
|
1494
1609
|
extractResults = [];
|
|
1495
1610
|
for (const h of availableHarnesses) {
|
|
1496
1611
|
try {
|
|
1497
|
-
const result = await akmExtract({
|
|
1612
|
+
const result = await withLlmStage("session-extraction", () => akmExtract({
|
|
1498
1613
|
type: h.name,
|
|
1499
1614
|
...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
|
|
1500
1615
|
config: extractConfig,
|
|
1616
|
+
// Thread the ACTIVE profile so extract's internal gate + per-process
|
|
1617
|
+
// config read the running profile, not always `default`.
|
|
1618
|
+
improveProfile,
|
|
1501
1619
|
dryRun: options.dryRun ?? false,
|
|
1502
1620
|
...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
|
|
1503
1621
|
// C2: pin extract's skip-tracking state.db open to the boundary path.
|
|
1504
1622
|
...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
1505
|
-
});
|
|
1623
|
+
}));
|
|
1506
1624
|
extractResults.push(result);
|
|
1507
1625
|
{
|
|
1508
1626
|
const gr = await runAutoAcceptGate(primaryStashDir
|
|
@@ -1593,7 +1711,13 @@ async function runImprovePreparationStage(args) {
|
|
|
1593
1711
|
const validationFailures = [];
|
|
1594
1712
|
for (const candidate of postCleanupRefs) {
|
|
1595
1713
|
try {
|
|
1596
|
-
|
|
1714
|
+
// #591: use the path pre-resolved at planning time when it is still on
|
|
1715
|
+
// disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
|
|
1716
|
+
// stash. Fall back to findAssetFilePath only for refs that bypassed
|
|
1717
|
+
// collectEligibleRefs' index scan or whose file moved since planning.
|
|
1718
|
+
const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
|
|
1719
|
+
? candidate.filePath
|
|
1720
|
+
: await findAssetFilePath(candidate.ref, options.stashDir);
|
|
1597
1721
|
if (!filePath) {
|
|
1598
1722
|
validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
|
|
1599
1723
|
continue;
|
|
@@ -1719,10 +1843,19 @@ async function runImprovePreparationStage(args) {
|
|
|
1719
1843
|
// refs that fail the distill signal-delta gate).
|
|
1720
1844
|
// distillOnlyRefs — reflect blocked but distill signal-delta passes
|
|
1721
1845
|
// AND ref is a distill candidate.
|
|
1722
|
-
//
|
|
1723
|
-
//
|
|
1846
|
+
// noFeedbackPool — neither signal-delta gate passes *and* the ref has
|
|
1847
|
+
// no recent feedback signal at all. These are NOT
|
|
1848
|
+
// skipped here: they are handed to the high-retrieval
|
|
1849
|
+
// fallback (P0-A) below so frequently-retrieved but
|
|
1850
|
+
// never-rated assets can still be improved. Only refs
|
|
1851
|
+
// that P0-A declines are ultimately fully skipped.
|
|
1852
|
+
// fullySkippedCount — has stale feedback but no signal delta → genuine
|
|
1853
|
+
// skip (counted, aggregated event emitted post-loop),
|
|
1854
|
+
// excluded from sort.
|
|
1724
1855
|
const eligibleRefs = [];
|
|
1725
1856
|
const distillOnlyRefs = [];
|
|
1857
|
+
// Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
|
|
1858
|
+
const noFeedbackPool = [];
|
|
1726
1859
|
let fullySkippedCount = 0;
|
|
1727
1860
|
// O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
|
|
1728
1861
|
const scopeRefBypass = scope.mode === "ref";
|
|
@@ -1760,22 +1893,59 @@ async function runImprovePreparationStage(args) {
|
|
|
1760
1893
|
// Reflect blocked but distill passes → distill-only bucket.
|
|
1761
1894
|
distillOnlyRefs.push(r);
|
|
1762
1895
|
}
|
|
1896
|
+
else if (!latestFeedbackTs.has(r.ref)) {
|
|
1897
|
+
// Neither signal-delta gate passes AND there is no recent feedback signal
|
|
1898
|
+
// at all. Rather than skip outright, defer to the high-retrieval fallback
|
|
1899
|
+
// (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
|
|
1900
|
+
// what that path is meant to rescue. Refs P0-A declines are skipped there.
|
|
1901
|
+
noFeedbackPool.push(r);
|
|
1902
|
+
}
|
|
1763
1903
|
else {
|
|
1764
|
-
//
|
|
1904
|
+
// Has feedback on record but no signal delta since the last proposal —
|
|
1905
|
+
// genuinely fully skipped. Counted here; a single aggregated
|
|
1906
|
+
// improve_skipped event is emitted after the loop (mirrors
|
|
1907
|
+
// profile_filtered_all_passes) instead of one event per ref.
|
|
1765
1908
|
fullySkippedCount++;
|
|
1766
1909
|
actions.push({
|
|
1767
1910
|
ref: r.ref,
|
|
1768
1911
|
mode: "distill-skipped",
|
|
1769
1912
|
result: { ok: true, reason: "no new signal since last proposal" },
|
|
1770
1913
|
});
|
|
1771
|
-
appendEvent({ eventType: "improve_skipped", ref: r.ref, metadata: { reason: "no_new_signal" } }, eventsCtx);
|
|
1772
1914
|
}
|
|
1773
1915
|
}
|
|
1916
|
+
// Emit ONE aggregated skip event for the fully-skipped bucket rather than one
|
|
1917
|
+
// improve_skipped event per ref (#592 pattern, mirrors
|
|
1918
|
+
// profile_filtered_all_passes above). The per-ref loop previously produced
|
|
1919
|
+
// ~11K state.db writes per run on a large stash, the dominant contributor to
|
|
1920
|
+
// 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
|
|
1921
|
+
// run summary; no downstream consumer needs a per-ref DB audit trail (health's
|
|
1922
|
+
// skip histogram reads the `no_new_signal` counter from the count field).
|
|
1923
|
+
if (fullySkippedCount > 0) {
|
|
1924
|
+
appendEvent({
|
|
1925
|
+
eventType: "improve_skipped",
|
|
1926
|
+
ref: undefined,
|
|
1927
|
+
metadata: {
|
|
1928
|
+
reason: "no_new_signal",
|
|
1929
|
+
count: fullySkippedCount,
|
|
1930
|
+
},
|
|
1931
|
+
}, eventsCtx);
|
|
1932
|
+
}
|
|
1774
1933
|
// ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
|
|
1775
|
-
// Everything from here works
|
|
1776
|
-
//
|
|
1777
|
-
//
|
|
1934
|
+
// Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
|
|
1935
|
+
// deferred noFeedbackPool that may be rescued by the high-retrieval fallback
|
|
1936
|
+
// (P0-A). The fully-skipped bucket has already been routed and its aggregated
|
|
1937
|
+
// event emitted; we deliberately avoid spending DB/CPU on refs that the
|
|
1938
|
+
// signal-delta gate rejected with feedback already on record.
|
|
1778
1939
|
const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
|
|
1940
|
+
// Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
|
|
1941
|
+
// partition above could not place these in a reflect/distill bucket, but they
|
|
1942
|
+
// may still qualify if they have been retrieved often enough. Two disjoint
|
|
1943
|
+
// sources feed this set:
|
|
1944
|
+
// 1. noFeedbackPool — refs with no recent feedback that the partition loop
|
|
1945
|
+
// deliberately deferred here (otherwise they would never reach P0-A).
|
|
1946
|
+
// 2. processableRefs entries that turn out to carry no recent feedback
|
|
1947
|
+
// *signal* once feedbackSummary is computed below.
|
|
1948
|
+
// (1) is added here; (2) is folded in after feedbackSummary is built.
|
|
1779
1949
|
// Gap 6: only surface feedback signals from the last 30 days so that
|
|
1780
1950
|
// ancient one-off feedback events don't permanently lock an asset into
|
|
1781
1951
|
// every improve run. Assets with only stale signals fall through to the
|
|
@@ -1785,8 +1955,12 @@ async function runImprovePreparationStage(args) {
|
|
|
1785
1955
|
// Pre-compute feedback summary per ref in a single pass so we don't issue
|
|
1786
1956
|
// two readEvents({type:"feedback", ref}) per asset (one for signal filtering,
|
|
1787
1957
|
// one for ratio computation).
|
|
1958
|
+
// Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
|
|
1959
|
+
// ratios are available for any noFeedbackPool ref that P0-A rescues below.
|
|
1788
1960
|
const feedbackSummary = new Map();
|
|
1789
|
-
for (const candidate of processableRefs) {
|
|
1961
|
+
for (const candidate of [...processableRefs, ...noFeedbackPool]) {
|
|
1962
|
+
if (feedbackSummary.has(candidate.ref))
|
|
1963
|
+
continue;
|
|
1790
1964
|
const { events } = readEvents({ type: "feedback", ref: candidate.ref });
|
|
1791
1965
|
let hasSignal = false;
|
|
1792
1966
|
let positive = 0;
|
|
@@ -1809,8 +1983,21 @@ async function runImprovePreparationStage(args) {
|
|
|
1809
1983
|
// P0-A: also surface zero-feedback assets that have been retrieved many times.
|
|
1810
1984
|
const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
|
|
1811
1985
|
const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
|
|
1812
|
-
|
|
1986
|
+
// Zero-feedback candidates for P0-A: processableRefs without a recent signal,
|
|
1987
|
+
// plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
|
|
1988
|
+
// disjoint by construction, but guard against overlap defensively).
|
|
1989
|
+
const noFeedbackSeen = new Set();
|
|
1990
|
+
const noFeedbackCandidates = [];
|
|
1991
|
+
for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
|
|
1992
|
+
if (noFeedbackSeen.has(r.ref))
|
|
1993
|
+
continue;
|
|
1994
|
+
noFeedbackSeen.add(r.ref);
|
|
1995
|
+
noFeedbackCandidates.push(r);
|
|
1996
|
+
}
|
|
1813
1997
|
let highRetrievalRefs = [];
|
|
1998
|
+
// Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
|
|
1999
|
+
// proactive-maintenance selector below can reuse them without a second DB pass.
|
|
2000
|
+
let retrievalCounts = new Map();
|
|
1814
2001
|
let dbForRetrieval;
|
|
1815
2002
|
try {
|
|
1816
2003
|
dbForRetrieval = openExistingDatabase();
|
|
@@ -1818,15 +2005,21 @@ async function runImprovePreparationStage(args) {
|
|
|
1818
2005
|
if (showEventCount === 0) {
|
|
1819
2006
|
warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
|
|
1820
2007
|
}
|
|
1821
|
-
|
|
2008
|
+
retrievalCounts = getRetrievalCounts(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
|
|
1822
2009
|
// High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
|
|
1823
|
-
// ref qualifies exactly once — when
|
|
1824
|
-
//
|
|
1825
|
-
//
|
|
1826
|
-
//
|
|
1827
|
-
//
|
|
1828
|
-
//
|
|
1829
|
-
|
|
2010
|
+
// ref qualifies exactly once — when it has actually been retrieved
|
|
2011
|
+
// (retrievalCount ≥ 1) AND retrievalCount ≥ threshold AND no prior reflect
|
|
2012
|
+
// proposal exists for it. Once a reflect proposal is on record, subsequent
|
|
2013
|
+
// re-eligibility requires explicit feedback (which flows through the normal
|
|
2014
|
+
// signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
|
|
2015
|
+
// from rescuing genuinely never-retrieved assets — the fallback is for
|
|
2016
|
+
// *retrieved* assets, not silent ones. Tracking growth in retrieval count
|
|
2017
|
+
// would require persisting the count in proposal metadata; deferred to a
|
|
2018
|
+
// follow-up.
|
|
2019
|
+
highRetrievalRefs = noFeedbackCandidates.filter((r) => {
|
|
2020
|
+
const count = retrievalCounts.get(r.ref) ?? 0;
|
|
2021
|
+
return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
|
|
2022
|
+
});
|
|
1830
2023
|
}
|
|
1831
2024
|
catch (err) {
|
|
1832
2025
|
rethrowIfTestIsolationError(err);
|
|
@@ -1836,6 +2029,91 @@ async function runImprovePreparationStage(args) {
|
|
|
1836
2029
|
if (dbForRetrieval)
|
|
1837
2030
|
closeDatabase(dbForRetrieval);
|
|
1838
2031
|
}
|
|
2032
|
+
// ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
|
|
2033
|
+
// The signal-delta gate and P0-A only surface assets with fresh feedback or a
|
|
2034
|
+
// raw-retrieval spike. Neither revisits a stable, high-value asset on a
|
|
2035
|
+
// schedule, so on a quiet stash useful assets drift stale and are never
|
|
2036
|
+
// refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
|
|
2037
|
+
// and the run is whole-stash / type scope, this selector ranks the eligible
|
|
2038
|
+
// population by a composite maintenance priority, gates on staleness ("due"),
|
|
2039
|
+
// bounds to top-N, and folds the winners into the SAME candidate set the other
|
|
2040
|
+
// two sources feed — so they flow through the existing #580 empty-diff /
|
|
2041
|
+
// cosmetic suppression and additive-distill gates. It adds no new mutation
|
|
2042
|
+
// logic of its own. The due gate doubles as the rotation cooldown: a freshly
|
|
2043
|
+
// reflected asset is excluded until it ages back past `dueDays`, so successive
|
|
2044
|
+
// runs rotate through the due pool rather than re-selecting the same heads.
|
|
2045
|
+
let proactiveRefs = [];
|
|
2046
|
+
let proactiveMaintenanceSummary;
|
|
2047
|
+
const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
|
|
2048
|
+
if (proactiveEnabled) {
|
|
2049
|
+
const pmCfg = improveProfile.processes?.proactiveMaintenance;
|
|
2050
|
+
const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
|
|
2051
|
+
const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
|
|
2052
|
+
const importanceWeights = pmCfg?.importanceWeights;
|
|
2053
|
+
// Candidate population: the zero-feedback / non-signal pool — exactly the
|
|
2054
|
+
// assets the other two sources would NOT pick this run. Exclude any P0-A
|
|
2055
|
+
// rescued this run so we never double-select the same ref.
|
|
2056
|
+
const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
|
|
2057
|
+
const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
|
|
2058
|
+
const selection = selectProactiveMaintenanceRefs({
|
|
2059
|
+
candidates: pmCandidates,
|
|
2060
|
+
lastReflectTs: lastReflectProposalTs,
|
|
2061
|
+
lastDistillTs: lastDistillProposalTs,
|
|
2062
|
+
retrievalCounts,
|
|
2063
|
+
sizeBytesOf: (r) => {
|
|
2064
|
+
const fp = r.filePath;
|
|
2065
|
+
if (!fp)
|
|
2066
|
+
return undefined;
|
|
2067
|
+
try {
|
|
2068
|
+
return fs.statSync(fp).size;
|
|
2069
|
+
}
|
|
2070
|
+
catch {
|
|
2071
|
+
return undefined;
|
|
2072
|
+
}
|
|
2073
|
+
},
|
|
2074
|
+
dueDays,
|
|
2075
|
+
maxPerRun,
|
|
2076
|
+
importanceWeights,
|
|
2077
|
+
});
|
|
2078
|
+
proactiveRefs = selection.selected;
|
|
2079
|
+
proactiveMaintenanceSummary = {
|
|
2080
|
+
selected: selection.selected.length,
|
|
2081
|
+
dueTotal: selection.dueTotal,
|
|
2082
|
+
neverReflected: selection.neverReflected,
|
|
2083
|
+
};
|
|
2084
|
+
// Aggregated observability event (never per-ref — avoids the event flood the
|
|
2085
|
+
// Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
|
|
2086
|
+
appendEvent({
|
|
2087
|
+
eventType: "proactive_selected",
|
|
2088
|
+
ref: undefined,
|
|
2089
|
+
metadata: {
|
|
2090
|
+
count: selection.selected.length,
|
|
2091
|
+
dueTotal: selection.dueTotal,
|
|
2092
|
+
neverReflected: selection.neverReflected,
|
|
2093
|
+
},
|
|
2094
|
+
}, eventsCtx);
|
|
2095
|
+
if (selection.selected.length > 0) {
|
|
2096
|
+
info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
|
|
2097
|
+
`(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
// Record an in-memory skip action for every zero-feedback ref that the
|
|
2101
|
+
// partition loop deferred to P0-A but P0-A then declined (retrievalCount below
|
|
2102
|
+
// threshold, or a prior reflect proposal already on record). These never make
|
|
2103
|
+
// it into mergedRefs, so without this they would silently vanish from the run
|
|
2104
|
+
// summary. No DB event is written here — these refs carry no signal at all, so
|
|
2105
|
+
// there is nothing for the skip histogram to aggregate; the action log alone
|
|
2106
|
+
// preserves the per-ref audit trail (mirrors the fully-skipped action above).
|
|
2107
|
+
const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
|
|
2108
|
+
for (const r of noFeedbackPool) {
|
|
2109
|
+
if (rescuedSet.has(r.ref))
|
|
2110
|
+
continue;
|
|
2111
|
+
actions.push({
|
|
2112
|
+
ref: r.ref,
|
|
2113
|
+
mode: "distill-skipped",
|
|
2114
|
+
result: { ok: true, reason: "no new signal since last proposal" },
|
|
2115
|
+
});
|
|
2116
|
+
}
|
|
1839
2117
|
// If the user explicitly scoped to a single ref, always act on it —
|
|
1840
2118
|
// skip the signal/retrieval filter entirely. The filter exists to avoid
|
|
1841
2119
|
// noisy "improve everything" runs; it should not gate an intentional
|
|
@@ -1845,8 +2123,48 @@ async function runImprovePreparationStage(args) {
|
|
|
1845
2123
|
// or sufficient retrievals). A stash with no signals has 0 eligible refs —
|
|
1846
2124
|
// usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
|
|
1847
2125
|
// to bring them into the eligible pool.
|
|
1848
|
-
|
|
2126
|
+
// Layer-2 proactive refs join the eligible set alongside feedback-signal and
|
|
2127
|
+
// high-retrieval (P0-A) refs. The three sources are disjoint by construction
|
|
2128
|
+
// (proactive draws from noFeedbackCandidates with the P0-A picks removed), but
|
|
2129
|
+
// dedupe defensively so a ref can never enter the loop twice. `requireFeedbackSignal`
|
|
2130
|
+
// still suppresses both fallback sources for callers that want feedback-only runs.
|
|
2131
|
+
const signalAndRetrievalRefs = dedupeRefs([...signalFiltered, ...highRetrievalRefs, ...proactiveRefs]);
|
|
1849
2132
|
const mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
|
|
2133
|
+
// ── Attribution tagging: stamp each ref with the eligibility lane that
|
|
2134
|
+
// selected it ──────────────────────────────────────────────────────────────
|
|
2135
|
+
// Every reflect/distill proposal must record WHICH lane chose its source asset
|
|
2136
|
+
// so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
|
|
2137
|
+
// (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
|
|
2138
|
+
// lane map here — the one place all four lanes are known — and stamp it onto
|
|
2139
|
+
// each ImproveEligibleRef object. Because the ref objects are shared by
|
|
2140
|
+
// reference across buckets, the stamp travels with the ref through the sort,
|
|
2141
|
+
// disk-check, and loop stages down to the reflect/distill event emit sites and
|
|
2142
|
+
// createProposal calls. See EligibilitySource for the lane vocabulary.
|
|
2143
|
+
//
|
|
2144
|
+
// Precedence (prefer the most specific reactive signal):
|
|
2145
|
+
// scope > signal-delta > high-retrieval > proactive
|
|
2146
|
+
// A ref with real feedback is attributed to feedback even if it was also due
|
|
2147
|
+
// for proactive maintenance. We apply lanes weakest-first so the strongest
|
|
2148
|
+
// overwrites; the explicit --scope <ref> bypass wins outright (user intent).
|
|
2149
|
+
const eligibilitySourceByRef = new Map();
|
|
2150
|
+
for (const r of proactiveRefs)
|
|
2151
|
+
eligibilitySourceByRef.set(r.ref, "proactive");
|
|
2152
|
+
for (const r of highRetrievalRefs)
|
|
2153
|
+
eligibilitySourceByRef.set(r.ref, "high-retrieval");
|
|
2154
|
+
for (const r of signalFiltered)
|
|
2155
|
+
eligibilitySourceByRef.set(r.ref, "signal-delta");
|
|
2156
|
+
if (scope.mode === "ref") {
|
|
2157
|
+
// O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
|
|
2158
|
+
// arrived via the scopeRefBypass branch, so attribute the whole set to scope.
|
|
2159
|
+
for (const r of processableRefs)
|
|
2160
|
+
eligibilitySourceByRef.set(r.ref, "scope");
|
|
2161
|
+
}
|
|
2162
|
+
for (const r of mergedRefs) {
|
|
2163
|
+
// "unknown" is a genuine fallback, never a silent alias for signal-delta:
|
|
2164
|
+
// only refs we truly cannot attribute land here (none in practice, since
|
|
2165
|
+
// mergedRefs is always a subset of the four lanes above).
|
|
2166
|
+
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
2167
|
+
}
|
|
1850
2168
|
const utilityMap = buildUtilityMap(mergedRefs);
|
|
1851
2169
|
// Load feedback ratio per ref from the pre-computed summary (no extra DB pass).
|
|
1852
2170
|
const feedbackRatios = new Map();
|
|
@@ -1901,15 +2219,32 @@ async function runImprovePreparationStage(args) {
|
|
|
1901
2219
|
const assetMissingOnDisk = [];
|
|
1902
2220
|
const existsCheckedActionable = [];
|
|
1903
2221
|
for (const candidate of sorted) {
|
|
1904
|
-
|
|
2222
|
+
// #591: prefer the path pre-resolved at planning time (synchronous
|
|
2223
|
+
// existsSync) over a serial async DB lookup per ref.
|
|
2224
|
+
const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
|
|
2225
|
+
? candidate.filePath
|
|
2226
|
+
: await findAssetFilePath(candidate.ref, options.stashDir);
|
|
1905
2227
|
if (filePath && fs.existsSync(filePath)) {
|
|
1906
2228
|
existsCheckedActionable.push(candidate);
|
|
1907
2229
|
}
|
|
1908
2230
|
else {
|
|
1909
2231
|
assetMissingOnDisk.push(candidate.ref);
|
|
1910
|
-
appendEvent({ eventType: "improve_skipped", ref: candidate.ref, metadata: { reason: "asset_missing_on_disk" } }, eventsCtx);
|
|
1911
2232
|
}
|
|
1912
2233
|
}
|
|
2234
|
+
// #592 audit: one summary event instead of one per missing ref. Normally
|
|
2235
|
+
// tiny, but a stash deletion racing the run could make this O(n) sequential
|
|
2236
|
+
// state.db writes. `refs` is capped so the metadata row stays bounded.
|
|
2237
|
+
if (assetMissingOnDisk.length > 0) {
|
|
2238
|
+
appendEvent({
|
|
2239
|
+
eventType: "improve_skipped",
|
|
2240
|
+
ref: undefined,
|
|
2241
|
+
metadata: {
|
|
2242
|
+
reason: "asset_missing_on_disk",
|
|
2243
|
+
count: assetMissingOnDisk.length,
|
|
2244
|
+
refs: assetMissingOnDisk.slice(0, 50),
|
|
2245
|
+
},
|
|
2246
|
+
}, eventsCtx);
|
|
2247
|
+
}
|
|
1913
2248
|
const actionableRefs = existsCheckedActionable;
|
|
1914
2249
|
// Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
|
|
1915
2250
|
// preserving sort order. distillOnlyRefs participate in the sort so --limit
|
|
@@ -1970,6 +2305,7 @@ async function runImprovePreparationStage(args) {
|
|
|
1970
2305
|
gateAutoAcceptFailedCount,
|
|
1971
2306
|
consolidation: consolidationPass.consolidation,
|
|
1972
2307
|
consolidationRan: consolidationPass.consolidationRan,
|
|
2308
|
+
...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
|
|
1973
2309
|
};
|
|
1974
2310
|
}
|
|
1975
2311
|
async function runImproveLoopStage(args) {
|
|
@@ -1978,6 +2314,14 @@ async function runImproveLoopStage(args) {
|
|
|
1978
2314
|
// receives only its fair share of the wall-clock budget.
|
|
1979
2315
|
const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
|
|
1980
2316
|
const RECENT_ERRORS_CAP = 3;
|
|
2317
|
+
// requirePlannedRefs guard: when the distill profile sets this flag, skip
|
|
2318
|
+
// distill for distill-only refs if the reflect phase produced no planned refs.
|
|
2319
|
+
// Prevents the distill loop from generating hundreds of distill-skipped events
|
|
2320
|
+
// on quiet passes (all refs on reflect cooldown, no new signal to distill).
|
|
2321
|
+
const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
|
|
2322
|
+
const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
|
|
2323
|
+
const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
|
|
2324
|
+
const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
|
|
1981
2325
|
// R-2 / #389: Self-Consistency multi-sample voting helpers.
|
|
1982
2326
|
// Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
|
|
1983
2327
|
const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
|
|
@@ -2138,6 +2482,9 @@ async function runImproveLoopStage(args) {
|
|
|
2138
2482
|
eventSource: "improve",
|
|
2139
2483
|
...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
|
|
2140
2484
|
...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
|
|
2485
|
+
// Attribution: carry the eligibility lane so reflect stamps it on
|
|
2486
|
+
// the reflect_invoked event and the persisted proposal.
|
|
2487
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
2141
2488
|
};
|
|
2142
2489
|
// R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
|
|
2143
2490
|
// Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
|
|
@@ -2150,9 +2497,11 @@ async function runImproveLoopStage(args) {
|
|
|
2150
2497
|
if (remainingBudgetMs() <= 0)
|
|
2151
2498
|
break;
|
|
2152
2499
|
// draftMode: skip DB write so each sample doesn't create a proposal.
|
|
2153
|
-
samples.push(await reflectFn({ ...reflectCallArgs, draftMode: true }));
|
|
2500
|
+
samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
|
|
2154
2501
|
}
|
|
2155
|
-
const winner = pickMajorityVote(samples.length > 0
|
|
2502
|
+
const winner = pickMajorityVote(samples.length > 0
|
|
2503
|
+
? samples
|
|
2504
|
+
: [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
|
|
2156
2505
|
// Persist only the majority-vote winner as a single real proposal.
|
|
2157
2506
|
if (winner.ok && primaryStashDir) {
|
|
2158
2507
|
const persistResult = createProposal(primaryStashDir, {
|
|
@@ -2160,6 +2509,9 @@ async function runImproveLoopStage(args) {
|
|
|
2160
2509
|
source: "reflect",
|
|
2161
2510
|
sourceRun: `reflect-sc-${Date.now()}`,
|
|
2162
2511
|
payload: winner.proposal.payload,
|
|
2512
|
+
// Attribution: the self-consistency path persists the winner here
|
|
2513
|
+
// (draftMode skips reflect's own createProposal), so stamp the lane.
|
|
2514
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
2163
2515
|
});
|
|
2164
2516
|
reflectResult = isProposalSkipped(persistResult)
|
|
2165
2517
|
? {
|
|
@@ -2177,7 +2529,7 @@ async function runImproveLoopStage(args) {
|
|
|
2177
2529
|
}
|
|
2178
2530
|
}
|
|
2179
2531
|
else {
|
|
2180
|
-
reflectResult = await reflectFn(reflectCallArgs);
|
|
2532
|
+
reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
|
|
2181
2533
|
}
|
|
2182
2534
|
const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
|
|
2183
2535
|
// Content-policy guard hits (reflect size-rail rejections) are NOT
|
|
@@ -2194,6 +2546,12 @@ async function runImproveLoopStage(args) {
|
|
|
2194
2546
|
// user's stack were this case; see review §1a row "Reflect refused
|
|
2195
2547
|
// asset type".
|
|
2196
2548
|
const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
|
|
2549
|
+
// Noise-gate suppression (#580): the candidate edit was an empty
|
|
2550
|
+
// diff or a cosmetic-only reformat of the current asset. Like
|
|
2551
|
+
// `unsupported_type`, this is a deterministic skip — not an LLM
|
|
2552
|
+
// fault — so it routes to the `reflect-skipped` bucket and stays
|
|
2553
|
+
// out of recentErrors/avoidPatterns.
|
|
2554
|
+
const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
|
|
2197
2555
|
actions.push({
|
|
2198
2556
|
ref: planned.ref,
|
|
2199
2557
|
mode: reflectResult.ok
|
|
@@ -2202,18 +2560,19 @@ async function runImproveLoopStage(args) {
|
|
|
2202
2560
|
? "reflect-cooldown"
|
|
2203
2561
|
: isGuardReject
|
|
2204
2562
|
? "reflect-guard-rejected"
|
|
2205
|
-
: isTypeRefused
|
|
2563
|
+
: isTypeRefused || isNoChange
|
|
2206
2564
|
? "reflect-skipped"
|
|
2207
2565
|
: "reflect-failed",
|
|
2208
2566
|
result: reflectResult,
|
|
2209
2567
|
});
|
|
2210
|
-
// Cooldown skips, guard rejects,
|
|
2211
|
-
// failures — do not pollute recentErrors with them
|
|
2212
|
-
// injected as `avoidPatterns` into the next reflect
|
|
2213
|
-
// rejects ARE worth showing the LLM as a learn-signal
|
|
2214
|
-
// iteration sees "your last expansion was too large";
|
|
2215
|
-
//
|
|
2216
|
-
|
|
2568
|
+
// Cooldown skips, guard rejects, type-refused skips, and noise-gate
|
|
2569
|
+
// skips are not failures — do not pollute recentErrors with them
|
|
2570
|
+
// (those get injected as `avoidPatterns` into the next reflect
|
|
2571
|
+
// prompt). Guard rejects ARE worth showing the LLM as a learn-signal
|
|
2572
|
+
// so the next iteration sees "your last expansion was too large";
|
|
2573
|
+
// type-refused and no-change are deterministic and add no learning
|
|
2574
|
+
// signal.
|
|
2575
|
+
if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
|
|
2217
2576
|
const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
|
|
2218
2577
|
pushRecentError("reflect", errMsg);
|
|
2219
2578
|
}
|
|
@@ -2266,6 +2625,18 @@ async function runImproveLoopStage(args) {
|
|
|
2266
2625
|
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
2267
2626
|
continue;
|
|
2268
2627
|
}
|
|
2628
|
+
// requirePlannedRefs guard: skip distill for distill-only refs when no
|
|
2629
|
+
// reflect-eligible refs were planned this run, preventing mass skip events.
|
|
2630
|
+
if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
|
|
2631
|
+
actions.push({
|
|
2632
|
+
ref: planned.ref,
|
|
2633
|
+
mode: "distill-skipped",
|
|
2634
|
+
result: { ok: true, reason: "require_planned_refs" },
|
|
2635
|
+
});
|
|
2636
|
+
completedCount++;
|
|
2637
|
+
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
2638
|
+
continue;
|
|
2639
|
+
}
|
|
2269
2640
|
// See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
|
|
2270
2641
|
// DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
|
|
2271
2642
|
// it will refuse.
|
|
@@ -2335,11 +2706,14 @@ async function runImproveLoopStage(args) {
|
|
|
2335
2706
|
}
|
|
2336
2707
|
}
|
|
2337
2708
|
}
|
|
2338
|
-
const distillResult = await distillFn({
|
|
2709
|
+
const distillResult = await withLlmStage("distill", () => distillFn({
|
|
2339
2710
|
ref: planned.ref,
|
|
2340
2711
|
...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
|
|
2341
2712
|
...(options.stashDir ? { stashDir: options.stashDir } : {}),
|
|
2342
|
-
|
|
2713
|
+
// Attribution: carry the eligibility lane so distill stamps it on the
|
|
2714
|
+
// distill_invoked event and the persisted proposal.
|
|
2715
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
2716
|
+
}));
|
|
2343
2717
|
actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
|
|
2344
2718
|
if (distillResult.outcome === "queued" && distillResult.proposal) {
|
|
2345
2719
|
const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
|
|
@@ -2480,7 +2854,9 @@ async function runImprovePostLoopStage(args) {
|
|
|
2480
2854
|
};
|
|
2481
2855
|
}
|
|
2482
2856
|
// TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
|
|
2483
|
-
|
|
2857
|
+
// Exported for tests (#584/#585 DB-locking regression coverage); production
|
|
2858
|
+
// callers reach it only through akmImprove → runImprovePostLoopStage.
|
|
2859
|
+
export async function runImproveMaintenancePasses(args) {
|
|
2484
2860
|
const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
|
|
2485
2861
|
if (!primaryStashDir)
|
|
2486
2862
|
return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
|
|
@@ -2499,276 +2875,344 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2499
2875
|
let graphExtractionDurationMs = 0;
|
|
2500
2876
|
let orphansPurged = 0;
|
|
2501
2877
|
let proposalsExpired = 0;
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
// Fix: always run the pass when the feature is enabled; let the pass's
|
|
2514
|
-
// own `collectPendingMemories` + `isPendingMemory` predicate find
|
|
2515
|
-
// candidates from the filesystem-of-truth. The this-run set is still
|
|
2516
|
-
// logged as a hint but no longer used as a filter.
|
|
2517
|
-
const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
|
|
2518
|
-
if (memoryInferenceDisabledByProfile) {
|
|
2519
|
-
info("[improve] memory inference skipped (disabled by improve profile)");
|
|
2878
|
+
const openIndexDb = () => openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
2879
|
+
// #584: reindexFn opens its own write handle on the same index.db WAL file.
|
|
2880
|
+
// Holding our handle across that call produced SQLITE_BUSY / "database is
|
|
2881
|
+
// locked" failures in production, so the handle is closed BEFORE every
|
|
2882
|
+
// reindex and reopened after — the fresh handle also sees the post-reindex
|
|
2883
|
+
// state that graph extraction and staleness detection below rely on. The
|
|
2884
|
+
// reopen runs in `finally` so a failed reindex still leaves a usable handle.
|
|
2885
|
+
const reindexWithIndexDbReleased = async (stashDir) => {
|
|
2886
|
+
if (db) {
|
|
2887
|
+
closeDatabase(db);
|
|
2888
|
+
db = undefined;
|
|
2520
2889
|
}
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
info(hintRefs > 0
|
|
2524
|
-
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
2525
|
-
: "[improve] memory inference starting (discovering pending parents)");
|
|
2526
|
-
const inferenceStart = Date.now();
|
|
2527
|
-
try {
|
|
2528
|
-
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
2529
|
-
memoryInference = await memoryInferenceFn({
|
|
2530
|
-
config,
|
|
2531
|
-
sources,
|
|
2532
|
-
signal: budgetSignal,
|
|
2533
|
-
db,
|
|
2534
|
-
reEnrich: false,
|
|
2535
|
-
onProgress: (event) => {
|
|
2536
|
-
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
2537
|
-
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
2538
|
-
},
|
|
2539
|
-
});
|
|
2540
|
-
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2541
|
-
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
2542
|
-
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
2543
|
-
}
|
|
2544
|
-
catch (err) {
|
|
2545
|
-
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2546
|
-
allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2547
|
-
}
|
|
2890
|
+
try {
|
|
2891
|
+
await reindexFn({ stashDir });
|
|
2548
2892
|
}
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
try {
|
|
2552
|
-
await reindexFn({ stashDir: primaryStashDir });
|
|
2553
|
-
reindexedAfterInference = true;
|
|
2554
|
-
info("[improve] reindex after memory inference complete");
|
|
2555
|
-
}
|
|
2556
|
-
catch (err) {
|
|
2557
|
-
allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2558
|
-
}
|
|
2893
|
+
finally {
|
|
2894
|
+
db = openIndexDb();
|
|
2559
2895
|
}
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
2587
|
-
try {
|
|
2588
|
-
await reindexFn({ stashDir: primaryStashDir });
|
|
2589
|
-
reindexedAfterInference = true;
|
|
2590
|
-
info("[improve] reindex after consolidation complete");
|
|
2591
|
-
}
|
|
2592
|
-
catch (err) {
|
|
2593
|
-
allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2594
|
-
}
|
|
2896
|
+
};
|
|
2897
|
+
await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
|
|
2898
|
+
try {
|
|
2899
|
+
db = openIndexDb();
|
|
2900
|
+
// Memory inference candidate-discovery (post-Item 9 fix from
|
|
2901
|
+
// memory:akm-improve-critical-review-2026-05-20). Previously this pass
|
|
2902
|
+
// was gated on memoryRefsForInference.size > 0 AND passed those refs as a
|
|
2903
|
+
// candidateRefs filter. But memoryRefsForInference is populated from refs
|
|
2904
|
+
// distilled THIS RUN — by the time that happens, those parents are
|
|
2905
|
+
// already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
|
|
2906
|
+
// them. The genuinely-pending parents in the stash never entered the
|
|
2907
|
+
// filter. Result: 0/0/0 for 25 consecutive runs.
|
|
2908
|
+
//
|
|
2909
|
+
// Fix: always run the pass when the feature is enabled; let the pass's
|
|
2910
|
+
// own `collectPendingMemories` + `isPendingMemory` predicate find
|
|
2911
|
+
// candidates from the filesystem-of-truth. The this-run set is still
|
|
2912
|
+
// logged as a hint but no longer used as a filter.
|
|
2913
|
+
const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
|
|
2914
|
+
const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
|
|
2915
|
+
const pendingBelowMinCount = (() => {
|
|
2916
|
+
if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
|
|
2917
|
+
return false;
|
|
2918
|
+
const pending = collectPendingMemories(primaryStashDir).length;
|
|
2919
|
+
if (pending < minPendingCount) {
|
|
2920
|
+
info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
|
|
2921
|
+
return true;
|
|
2595
2922
|
}
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2923
|
+
return false;
|
|
2924
|
+
})();
|
|
2925
|
+
if (memoryInferenceDisabledByProfile) {
|
|
2926
|
+
info("[improve] memory inference skipped (disabled by improve profile)");
|
|
2927
|
+
}
|
|
2928
|
+
else if (pendingBelowMinCount) {
|
|
2929
|
+
// skipped — message already emitted above
|
|
2930
|
+
}
|
|
2931
|
+
else {
|
|
2932
|
+
const hintRefs = memoryRefsForInference.size;
|
|
2933
|
+
info(hintRefs > 0
|
|
2934
|
+
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
2935
|
+
: "[improve] memory inference starting (discovering pending parents)");
|
|
2936
|
+
const inferenceStart = Date.now();
|
|
2937
|
+
try {
|
|
2938
|
+
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
2939
|
+
memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
|
|
2940
|
+
config,
|
|
2941
|
+
sources,
|
|
2942
|
+
signal: budgetSignal,
|
|
2943
|
+
db,
|
|
2944
|
+
reEnrich: false,
|
|
2945
|
+
onProgress: (event) => {
|
|
2946
|
+
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
2947
|
+
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
2948
|
+
},
|
|
2949
|
+
}));
|
|
2950
|
+
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2951
|
+
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
2952
|
+
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
2599
2953
|
}
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
if (!graphExtractionFullScan) {
|
|
2604
|
-
candidatePaths = new Set();
|
|
2605
|
-
if (primaryStashDir && touchedRefs.size > 0) {
|
|
2606
|
-
const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
|
|
2607
|
-
const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
|
|
2608
|
-
for (const p of resolved) {
|
|
2609
|
-
if (typeof p === "string" && p.length > 0)
|
|
2610
|
-
candidatePaths.add(p);
|
|
2611
|
-
}
|
|
2612
|
-
}
|
|
2954
|
+
catch (err) {
|
|
2955
|
+
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2956
|
+
allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2613
2957
|
}
|
|
2614
|
-
const progressHandler = (event) => {
|
|
2615
|
-
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
2616
|
-
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
2617
|
-
};
|
|
2618
|
-
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
2619
|
-
graphExtraction = await graphExtractionFn({
|
|
2620
|
-
config,
|
|
2621
|
-
sources,
|
|
2622
|
-
signal: budgetSignal,
|
|
2623
|
-
db,
|
|
2624
|
-
reEnrich: false,
|
|
2625
|
-
onProgress: progressHandler,
|
|
2626
|
-
options: { candidatePaths },
|
|
2627
|
-
});
|
|
2628
|
-
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
2629
|
-
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
2630
|
-
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
2631
2958
|
}
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
// asset no longer exists on disk. Runs after graph extraction so newly
|
|
2642
|
-
// promoted assets from accept flows during this run are already present.
|
|
2643
|
-
if (primaryStashDir) {
|
|
2644
|
-
try {
|
|
2645
|
-
const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
|
|
2646
|
-
orphansPurged = purgeResult.rejected;
|
|
2647
|
-
if (purgeResult.rejected > 0) {
|
|
2648
|
-
info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
|
|
2959
|
+
if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
|
|
2960
|
+
info("[improve] reindexing after memory inference writes");
|
|
2961
|
+
try {
|
|
2962
|
+
await reindexWithIndexDbReleased(primaryStashDir);
|
|
2963
|
+
reindexedAfterInference = true;
|
|
2964
|
+
info("[improve] reindex after memory inference complete");
|
|
2965
|
+
}
|
|
2966
|
+
catch (err) {
|
|
2967
|
+
allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2649
2968
|
}
|
|
2650
|
-
appendEvent({
|
|
2651
|
-
eventType: "proposal_orphan_purge",
|
|
2652
|
-
ref: "proposals:_orphan-purge",
|
|
2653
|
-
metadata: {
|
|
2654
|
-
checked: purgeResult.checked,
|
|
2655
|
-
rejected: purgeResult.rejected,
|
|
2656
|
-
durationMs: purgeResult.durationMs,
|
|
2657
|
-
byType: purgeResult.byType,
|
|
2658
|
-
orphans: purgeResult.orphans.map((o) => o.ref),
|
|
2659
|
-
},
|
|
2660
|
-
}, eventsCtx);
|
|
2661
2969
|
}
|
|
2662
|
-
|
|
2663
|
-
|
|
2970
|
+
const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
|
|
2971
|
+
const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
|
|
2972
|
+
const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
|
|
2973
|
+
// Build the set of refs actually touched this run.
|
|
2974
|
+
const touchedRefs = new Set();
|
|
2975
|
+
for (const r of args.actionableRefs)
|
|
2976
|
+
touchedRefs.add(r.ref);
|
|
2977
|
+
for (const r of memoryRefsForInference)
|
|
2978
|
+
touchedRefs.add(r);
|
|
2979
|
+
// INVARIANT: graph extraction normally runs only on files touched by
|
|
2980
|
+
// actionable refs (candidatePaths). Full-corpus scans are opt-in via
|
|
2981
|
+
// profile.processes.graphExtraction.fullScan = true (used by the
|
|
2982
|
+
// `graph-refresh` built-in profile and its weekly scheduled task).
|
|
2983
|
+
// The empty-Set fallback is intentional when no refs were touched —
|
|
2984
|
+
// the extractor's filter rejects every file and returns empty, keeping
|
|
2985
|
+
// the pass invoked so the action is recorded and tests stay exercised.
|
|
2986
|
+
if (graphExtractionDisabledByProfile) {
|
|
2987
|
+
info("[improve] graph extraction skipped (disabled by improve profile)");
|
|
2664
2988
|
}
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2989
|
+
else if (sources.length > 0 && graphEnabled) {
|
|
2990
|
+
info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
|
|
2991
|
+
const extractionStart = Date.now();
|
|
2992
|
+
try {
|
|
2993
|
+
// D9: if consolidation ran but memory inference did not reindex, force a reindex
|
|
2994
|
+
// so graph extraction sees current DB state after consolidation writes.
|
|
2995
|
+
if (consolidationRan && !reindexedAfterInference) {
|
|
2996
|
+
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
2997
|
+
try {
|
|
2998
|
+
await reindexWithIndexDbReleased(primaryStashDir);
|
|
2999
|
+
reindexedAfterInference = true;
|
|
3000
|
+
info("[improve] reindex after consolidation complete");
|
|
3001
|
+
}
|
|
3002
|
+
catch (err) {
|
|
3003
|
+
allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
// #584: no close/reopen needed here — reindexWithIndexDbReleased
|
|
3007
|
+
// already swapped in a fresh post-reindex handle.
|
|
3008
|
+
// Resolve touched refs to absolute file paths. Skipped for fullScan
|
|
3009
|
+
// (candidatePaths stays undefined → extractor processes all files).
|
|
3010
|
+
let candidatePaths;
|
|
3011
|
+
if (!graphExtractionFullScan) {
|
|
3012
|
+
candidatePaths = new Set();
|
|
3013
|
+
if (primaryStashDir && touchedRefs.size > 0) {
|
|
3014
|
+
const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
|
|
3015
|
+
const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
|
|
3016
|
+
for (const p of resolved) {
|
|
3017
|
+
if (typeof p === "string" && p.length > 0)
|
|
3018
|
+
candidatePaths.add(p);
|
|
3019
|
+
}
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
3022
|
+
const progressHandler = (event) => {
|
|
3023
|
+
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
3024
|
+
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
3025
|
+
};
|
|
3026
|
+
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
3027
|
+
graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
|
|
3028
|
+
config,
|
|
3029
|
+
sources,
|
|
3030
|
+
signal: budgetSignal,
|
|
3031
|
+
db,
|
|
3032
|
+
reEnrich: false,
|
|
3033
|
+
onProgress: progressHandler,
|
|
3034
|
+
options: { candidatePaths },
|
|
3035
|
+
}));
|
|
3036
|
+
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
3037
|
+
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
3038
|
+
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
3039
|
+
}
|
|
3040
|
+
catch (err) {
|
|
3041
|
+
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
3042
|
+
allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2676
3043
|
}
|
|
2677
|
-
appendEvent({
|
|
2678
|
-
eventType: "proposal_expiration_pass",
|
|
2679
|
-
ref: "proposals:_expiration",
|
|
2680
|
-
metadata: {
|
|
2681
|
-
checked: expireResult.checked,
|
|
2682
|
-
expired: expireResult.expired,
|
|
2683
|
-
durationMs: expireResult.durationMs,
|
|
2684
|
-
retentionDays: expireResult.retentionDays,
|
|
2685
|
-
expiredProposals: expireResult.expiredProposals,
|
|
2686
|
-
},
|
|
2687
|
-
}, eventsCtx);
|
|
2688
3044
|
}
|
|
2689
|
-
|
|
2690
|
-
|
|
3045
|
+
else if (sources.length > 0 && !graphEnabled) {
|
|
3046
|
+
info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
|
|
2691
3047
|
}
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
// without this trim, state.db is a permanent append-only log. Config key
|
|
2697
|
-
// `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
|
|
2698
|
-
// window. `purgeOldEvents()` opens its own state.db handle separate from
|
|
2699
|
-
// the index `db` above (different SQLite file).
|
|
2700
|
-
{
|
|
2701
|
-
const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
|
|
2702
|
-
if (retentionDays > 0) {
|
|
2703
|
-
let stateDb;
|
|
3048
|
+
// Orphan proposal purge — reject pending reflect proposals whose target
|
|
3049
|
+
// asset no longer exists on disk. Runs after graph extraction so newly
|
|
3050
|
+
// promoted assets from accept flows during this run are already present.
|
|
3051
|
+
if (primaryStashDir) {
|
|
2704
3052
|
try {
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
stateDb = openStateDatabase(eventsCtx?.dbPath);
|
|
2710
|
-
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
2711
|
-
if (purgedCount > 0) {
|
|
2712
|
-
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
3053
|
+
const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
|
|
3054
|
+
orphansPurged = purgeResult.rejected;
|
|
3055
|
+
if (purgeResult.rejected > 0) {
|
|
3056
|
+
info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
|
|
2713
3057
|
}
|
|
2714
3058
|
appendEvent({
|
|
2715
|
-
eventType: "
|
|
2716
|
-
ref: "
|
|
2717
|
-
metadata: {
|
|
3059
|
+
eventType: "proposal_orphan_purge",
|
|
3060
|
+
ref: "proposals:_orphan-purge",
|
|
3061
|
+
metadata: {
|
|
3062
|
+
checked: purgeResult.checked,
|
|
3063
|
+
rejected: purgeResult.rejected,
|
|
3064
|
+
durationMs: purgeResult.durationMs,
|
|
3065
|
+
byType: purgeResult.byType,
|
|
3066
|
+
orphans: purgeResult.orphans.map((o) => o.ref),
|
|
3067
|
+
},
|
|
2718
3068
|
}, eventsCtx);
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
3069
|
+
}
|
|
3070
|
+
catch (err) {
|
|
3071
|
+
allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
3072
|
+
}
|
|
3073
|
+
// Phase 6B (Advantage D6b): expire pending proposals that have aged past
|
|
3074
|
+
// the retention window. Runs AFTER orphan purge so we never double-archive
|
|
3075
|
+
// a proposal that orphan-purge already moved. `expireStaleProposals` emits
|
|
3076
|
+
// its own per-proposal `proposal_expired` events; we additionally emit a
|
|
3077
|
+
// single roll-up event here for parity with the orphan-purge surface.
|
|
3078
|
+
try {
|
|
3079
|
+
const expireResult = expireStaleProposals(primaryStashDir, config);
|
|
3080
|
+
proposalsExpired = expireResult.expired;
|
|
3081
|
+
if (expireResult.expired > 0) {
|
|
3082
|
+
info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
|
|
3083
|
+
`(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
|
|
2726
3084
|
}
|
|
2727
3085
|
appendEvent({
|
|
2728
|
-
eventType: "
|
|
2729
|
-
ref: "
|
|
2730
|
-
metadata: {
|
|
3086
|
+
eventType: "proposal_expiration_pass",
|
|
3087
|
+
ref: "proposals:_expiration",
|
|
3088
|
+
metadata: {
|
|
3089
|
+
checked: expireResult.checked,
|
|
3090
|
+
expired: expireResult.expired,
|
|
3091
|
+
durationMs: expireResult.durationMs,
|
|
3092
|
+
retentionDays: expireResult.retentionDays,
|
|
3093
|
+
expiredProposals: expireResult.expiredProposals,
|
|
3094
|
+
},
|
|
2731
3095
|
}, eventsCtx);
|
|
2732
3096
|
}
|
|
2733
3097
|
catch (err) {
|
|
2734
|
-
allWarnings.push(`
|
|
3098
|
+
allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2735
3099
|
}
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
3100
|
+
}
|
|
3101
|
+
// Fix #2 (observability 0.8.0): trim the events table in state.db so it
|
|
3102
|
+
// doesn't grow unbounded. `akm health` writes a `health_probe` row on every
|
|
3103
|
+
// invocation, and every command surface emits at least one event besides —
|
|
3104
|
+
// without this trim, state.db is a permanent append-only log. Config key
|
|
3105
|
+
// `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
|
|
3106
|
+
// window. The purge runs against state.db (a different SQLite file from
|
|
3107
|
+
// the index `db` above).
|
|
3108
|
+
{
|
|
3109
|
+
const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
|
|
3110
|
+
if (retentionDays > 0) {
|
|
3111
|
+
// #585: reuse the long-lived eventsCtx.db connection when akmImprove
|
|
3112
|
+
// opened one — opening a second state.db write connection while
|
|
3113
|
+
// eventsDb is still live made two simultaneous writers contend on the
|
|
3114
|
+
// same WAL file ("database is locked"). Only the eventsCtx.dbPath
|
|
3115
|
+
// fallback path (state.db failed to open up-front) opens — and then
|
|
3116
|
+
// owns and closes — its own handle. C2 still holds: the fallback uses
|
|
3117
|
+
// the boundary-pinned path, never a live `process.env` re-read.
|
|
3118
|
+
const ownsStateDb = !eventsCtx?.db;
|
|
3119
|
+
let stateDb;
|
|
3120
|
+
try {
|
|
3121
|
+
stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
|
|
3122
|
+
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
3123
|
+
if (purgedCount > 0) {
|
|
3124
|
+
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
3125
|
+
}
|
|
3126
|
+
appendEvent({
|
|
3127
|
+
eventType: "events_purged",
|
|
3128
|
+
ref: "events:_purge",
|
|
3129
|
+
metadata: { purgedCount, retentionDays },
|
|
3130
|
+
}, eventsCtx);
|
|
3131
|
+
// improve_runs uses the same retention window as events — both are
|
|
3132
|
+
// observability/audit data, both grow append-only, both have a
|
|
3133
|
+
// dedicated purge helper. Mirroring the events purge here means a
|
|
3134
|
+
// single retention knob (improve.eventRetentionDays) governs both.
|
|
3135
|
+
const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
|
|
3136
|
+
if (improveRunsPurged > 0) {
|
|
3137
|
+
info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
|
|
3138
|
+
}
|
|
3139
|
+
appendEvent({
|
|
3140
|
+
eventType: "improve_runs_purged",
|
|
3141
|
+
ref: "improve_runs:_purge",
|
|
3142
|
+
metadata: { purgedCount: improveRunsPurged, retentionDays },
|
|
3143
|
+
}, eventsCtx);
|
|
3144
|
+
}
|
|
3145
|
+
catch (err) {
|
|
3146
|
+
allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
3147
|
+
}
|
|
3148
|
+
finally {
|
|
3149
|
+
if (ownsStateDb && stateDb) {
|
|
3150
|
+
try {
|
|
3151
|
+
stateDb.close();
|
|
3152
|
+
}
|
|
3153
|
+
catch {
|
|
3154
|
+
// best-effort
|
|
3155
|
+
}
|
|
2740
3156
|
}
|
|
2741
|
-
|
|
2742
|
-
|
|
3157
|
+
}
|
|
3158
|
+
// task_logs in logs.db (#579) shares the same retention window as
|
|
3159
|
+
// events/improve_runs — all three are observability data governed by
|
|
3160
|
+
// the single improve.eventRetentionDays knob. Separate try/finally
|
|
3161
|
+
// because logs.db is a different file: a locked/missing logs.db must
|
|
3162
|
+
// not block the state.db purges above.
|
|
3163
|
+
let logsDb;
|
|
3164
|
+
try {
|
|
3165
|
+
logsDb = openLogsDatabase();
|
|
3166
|
+
const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
|
|
3167
|
+
if (taskLogsPurged > 0) {
|
|
3168
|
+
info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
|
|
3169
|
+
}
|
|
3170
|
+
appendEvent({
|
|
3171
|
+
eventType: "task_logs_purged",
|
|
3172
|
+
ref: "task_logs:_purge",
|
|
3173
|
+
metadata: { purgedCount: taskLogsPurged, retentionDays },
|
|
3174
|
+
}, eventsCtx);
|
|
3175
|
+
}
|
|
3176
|
+
catch (err) {
|
|
3177
|
+
allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
3178
|
+
}
|
|
3179
|
+
finally {
|
|
3180
|
+
if (logsDb) {
|
|
3181
|
+
try {
|
|
3182
|
+
logsDb.close();
|
|
3183
|
+
}
|
|
3184
|
+
catch {
|
|
3185
|
+
// best-effort
|
|
3186
|
+
}
|
|
2743
3187
|
}
|
|
2744
3188
|
}
|
|
2745
3189
|
}
|
|
2746
3190
|
}
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
3191
|
+
// Phase 4A (staleness detection). Activates the `deprecated` belief-state
|
|
3192
|
+
// machinery shipped in Phase 1A. Default OFF — gated by
|
|
3193
|
+
// `features.index.staleness_detection.enabled`. Runs after orphan purge
|
|
3194
|
+
// and before the URL check (which lives in the outer caller).
|
|
3195
|
+
if (sources.length > 0) {
|
|
3196
|
+
try {
|
|
3197
|
+
stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
|
|
3198
|
+
if (stalenessDetection.considered > 0) {
|
|
3199
|
+
info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
|
|
3200
|
+
`deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
|
|
3201
|
+
`skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
|
|
3202
|
+
}
|
|
3203
|
+
for (const w of stalenessDetection.warnings)
|
|
3204
|
+
allWarnings.push(`[improve] staleness detection: ${w}`);
|
|
3205
|
+
}
|
|
3206
|
+
catch (err) {
|
|
3207
|
+
allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2759
3208
|
}
|
|
2760
|
-
for (const w of stalenessDetection.warnings)
|
|
2761
|
-
allWarnings.push(`[improve] staleness detection: ${w}`);
|
|
2762
|
-
}
|
|
2763
|
-
catch (err) {
|
|
2764
|
-
allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2765
3209
|
}
|
|
2766
3210
|
}
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
}
|
|
3211
|
+
finally {
|
|
3212
|
+
if (db)
|
|
3213
|
+
closeDatabase(db);
|
|
3214
|
+
}
|
|
3215
|
+
});
|
|
2772
3216
|
return {
|
|
2773
3217
|
...(memoryInference ? { memoryInference } : {}),
|
|
2774
3218
|
...(graphExtraction ? { graphExtraction } : {}),
|