akm-cli 0.9.0-beta.2 → 0.9.0-beta.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +660 -0
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +5 -1
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/templates/html/default.html +78 -0
- package/dist/assets/templates/html/health.html +730 -0
- package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
- package/dist/cli/shared.js +21 -5
- package/dist/cli.js +47 -5
- package/dist/commands/agent/contribute-cli.js +16 -3
- package/dist/commands/feedback-cli.js +15 -6
- package/dist/commands/graph/graph.js +75 -71
- package/dist/commands/health/checks.js +48 -0
- package/dist/commands/health/html-report.js +790 -0
- package/dist/commands/health.js +478 -15
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate.js +634 -111
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +145 -69
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +115 -1
- package/dist/commands/improve/extract-prompt.js +33 -2
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +280 -35
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +139 -6
- package/dist/commands/improve/improve-profiles.js +12 -0
- package/dist/commands/improve/improve.js +2079 -608
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/proactive-maintenance.js +87 -0
- package/dist/commands/improve/procedural.js +409 -0
- package/dist/commands/improve/recombine.js +488 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +51 -1
- package/dist/commands/improve/related-sessions.js +120 -0
- package/dist/commands/improve/salience.js +386 -0
- package/dist/commands/improve/triage.js +95 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +34 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/drain.js +73 -6
- package/dist/commands/proposal/proposal-cli.js +22 -10
- package/dist/commands/proposal/proposal.js +17 -1
- package/dist/commands/proposal/validators/proposals.js +369 -329
- package/dist/commands/read/curate.js +344 -80
- package/dist/commands/read/search-cli.js +7 -0
- package/dist/commands/read/search.js +1 -0
- package/dist/commands/read/show.js +67 -2
- package/dist/commands/remember.js +6 -2
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/commands/sources/stash-cli.js +10 -2
- package/dist/core/asset/asset-registry.js +2 -0
- package/dist/core/asset/asset-spec.js +14 -0
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/config/config-schema.js +255 -2
- package/dist/core/config/config.js +2 -2
- package/dist/core/logs-db.js +305 -0
- package/dist/core/paths.js +3 -0
- package/dist/core/state-db.js +706 -42
- package/dist/indexer/db/db.js +364 -38
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +203 -3
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +114 -111
- package/dist/indexer/passes/memory-inference.js +71 -25
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +15 -4
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/prompts.js +1 -0
- package/dist/integrations/harnesses/claude/session-log.js +27 -5
- package/dist/integrations/harnesses/opencode/session-log.js +9 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/client.js +38 -4
- package/dist/llm/embedder.js +27 -3
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/graph-extract.js +2 -1
- package/dist/llm/memory-infer.js +4 -8
- package/dist/llm/metadata-enhance.js +9 -1
- package/dist/llm/usage-persist.js +77 -0
- package/dist/llm/usage-telemetry.js +103 -0
- package/dist/output/context.js +3 -2
- package/dist/output/html-render.js +73 -0
- package/dist/output/renderers.js +73 -1
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/helpers.js +17 -1
- package/dist/output/text/helpers.js +78 -1
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +1262 -591
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +485 -270
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/runner.js +99 -16
- package/dist/workflows/db.js +5 -2
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +1 -0
- package/package.json +9 -6
|
@@ -12,21 +12,25 @@ import { ConfigError, NotFoundError, rethrowIfTestIsolationError, UsageError } f
|
|
|
12
12
|
import { appendEvent, readEvents } from "../../core/events.js";
|
|
13
13
|
import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../../core/file-lock.js";
|
|
14
14
|
import { classifyImproveAction } from "../../core/improve-types.js";
|
|
15
|
+
import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
|
|
15
16
|
import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
|
|
16
|
-
import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
|
|
17
|
+
import { listProposalGateDecisions, listStateProposals, openStateDatabase, persistPhaseThreshold, purgeOldEvents, purgeOldImproveRuns, } from "../../core/state-db.js";
|
|
17
18
|
import { info, warn } from "../../core/warn.js";
|
|
18
19
|
import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
|
|
19
20
|
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
20
21
|
import { runGraphExtractionPass } from "../../indexer/graph/graph-extraction.js";
|
|
22
|
+
import { withIndexWriterLease } from "../../indexer/index-writer-lock.js";
|
|
21
23
|
import { akmIndex } from "../../indexer/indexer.js";
|
|
22
|
-
import { runMemoryInferencePass } from "../../indexer/passes/memory-inference.js";
|
|
24
|
+
import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
|
|
23
25
|
import { runStalenessDetectionPass } from "../../indexer/passes/staleness-detect.js";
|
|
24
26
|
import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
25
27
|
import { countUsageEventsByType } from "../../indexer/usage/usage-events.js";
|
|
26
28
|
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
27
29
|
import { resolveImproveProcessRunnerFromProfile, resolveTriageJudgmentRunner } from "../../integrations/agent/runner.js";
|
|
28
30
|
import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
|
|
29
|
-
import {
|
|
31
|
+
import { isProcessEnabled } from "../../llm/feature-gate.js";
|
|
32
|
+
import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
|
|
33
|
+
import { withLlmStage } from "../../llm/usage-telemetry.js";
|
|
30
34
|
import { isGitBackedStash, resolveWritableOverride, saveGitStash } from "../../sources/providers/git.js";
|
|
31
35
|
import { akmLint } from "../lint/index.js";
|
|
32
36
|
import { drainProposals } from "../proposal/drain.js";
|
|
@@ -34,16 +38,120 @@ import { resolveDrainPolicy } from "../proposal/drain-policies.js";
|
|
|
34
38
|
import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
|
|
35
39
|
import { runSchemaRepairPass } from "../sources/schema-repair.js";
|
|
36
40
|
import { checkDeadUrls } from "../url-checker.js";
|
|
41
|
+
import { computeThresholdAutoTune, gateDecisionsToSamples, summarizeCalibration, } from "./calibration.js";
|
|
37
42
|
import { akmConsolidate } from "./consolidate.js";
|
|
38
43
|
import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
|
|
39
44
|
import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
|
|
40
45
|
import { countEvalCases, writeEvalCase } from "./eval-cases.js";
|
|
41
46
|
import { akmExtract, countNewExtractCandidates } from "./extract.js";
|
|
47
|
+
import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
|
|
42
48
|
import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
|
|
43
49
|
import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
|
|
44
50
|
import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
|
|
45
51
|
import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
|
|
52
|
+
import { computeProxyAdequacy, getAllAssetOutcomes, getOutcomeScoresByRef, outcomeScoreToSalience, updateAssetOutcome, } from "./outcome-loop.js";
|
|
53
|
+
import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
|
|
54
|
+
import { akmProcedural } from "./procedural.js";
|
|
55
|
+
import { akmRecombine } from "./recombine.js";
|
|
46
56
|
import { akmReflect } from "./reflect.js";
|
|
57
|
+
import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getConsecutiveNoOps, getLastUseMsByRef, recordNoOp, resetConsecutiveNoOps, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
|
|
58
|
+
// #607 Lock Decomposition: fine-grained per-process locks replace the single
|
|
59
|
+
// `improve.lock`. Three independent locks allow concurrent improve runs when
|
|
60
|
+
// they touch different subsystems (e.g. quick-shredder consolidate can run
|
|
61
|
+
// alongside daily reflect+distill).
|
|
62
|
+
//
|
|
63
|
+
// consolidate.lock — protects consolidate + memoryInference (both write index.db)
|
|
64
|
+
// reflect-distill.lock — protects reflect + distill (both write state.db proposals)
|
|
65
|
+
// triage.lock — protects triage (writes proposal promotions)
|
|
66
|
+
//
|
|
67
|
+
// Stale timeouts are per-lock, tuned to the expected runtime of the protected
|
|
68
|
+
// processes: consolidate is disk-bound (1h), reflect+distill is GPU-bound (2h),
|
|
69
|
+
// triage is fast (30min).
|
|
70
|
+
const PROCESS_LOCK_DEFS = {
|
|
71
|
+
consolidate: { fileName: "consolidate.lock", staleAfterMs: 60 * 60 * 1000 },
|
|
72
|
+
reflectDistill: { fileName: "reflect-distill.lock", staleAfterMs: 2 * 60 * 60 * 1000 },
|
|
73
|
+
triage: { fileName: "triage.lock", staleAfterMs: 30 * 60 * 1000 },
|
|
74
|
+
};
|
|
75
|
+
const heldProcessLocks = new Set();
|
|
76
|
+
export function resetHeldProcessLocks() {
|
|
77
|
+
heldProcessLocks.clear();
|
|
78
|
+
}
|
|
79
|
+
function processLockPath(lockBaseDir, lockName) {
|
|
80
|
+
return path.join(lockBaseDir, PROCESS_LOCK_DEFS[lockName].fileName);
|
|
81
|
+
}
|
|
82
|
+
function tryAcquireProcessLock(lockPath, staleAfterMs, skipIfLocked, lockLabel) {
|
|
83
|
+
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
84
|
+
const lockPayload = () => JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() });
|
|
85
|
+
if (tryAcquireLockSync(lockPath, lockPayload())) {
|
|
86
|
+
heldProcessLocks.add(lockPath);
|
|
87
|
+
return "acquired";
|
|
88
|
+
}
|
|
89
|
+
const probe = probeLock(lockPath, { staleAfterMs });
|
|
90
|
+
const rawContent = probe.state === "absent" ? undefined : probe.rawContent;
|
|
91
|
+
const lock = rawContent
|
|
92
|
+
? (() => {
|
|
93
|
+
try {
|
|
94
|
+
return JSON.parse(rawContent);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
})()
|
|
100
|
+
: null;
|
|
101
|
+
if (probe.state === "stale") {
|
|
102
|
+
try {
|
|
103
|
+
appendEvent({
|
|
104
|
+
eventType: "improve_lock_recovered",
|
|
105
|
+
metadata: {
|
|
106
|
+
lockName: lockLabel,
|
|
107
|
+
stalePid: lock?.pid ?? null,
|
|
108
|
+
lockedAt: lock?.startedAt ?? null,
|
|
109
|
+
recoveredAt: new Date().toISOString(),
|
|
110
|
+
lockAgeMs: probe.ageMs ?? null,
|
|
111
|
+
reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
/* event emission is best-effort; never block lock recovery */
|
|
117
|
+
}
|
|
118
|
+
releaseLock(lockPath);
|
|
119
|
+
if (tryAcquireLockSync(lockPath, lockPayload())) {
|
|
120
|
+
heldProcessLocks.add(lockPath);
|
|
121
|
+
return "acquired";
|
|
122
|
+
}
|
|
123
|
+
if (skipIfLocked) {
|
|
124
|
+
warn(`[improve] ${lockLabel} lock acquired by another run during stale recovery; skipping (--skip-if-locked)`);
|
|
125
|
+
return "skipped";
|
|
126
|
+
}
|
|
127
|
+
throw new ConfigError(`akm improve ${lockLabel} is already running. Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
128
|
+
}
|
|
129
|
+
if (skipIfLocked) {
|
|
130
|
+
warn(`[improve] ${lockLabel} lock held by another run (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
|
|
131
|
+
return "skipped";
|
|
132
|
+
}
|
|
133
|
+
throw new ConfigError(`akm improve ${lockLabel} is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${lockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
134
|
+
}
|
|
135
|
+
function releaseProcessLock(lockPath) {
|
|
136
|
+
try {
|
|
137
|
+
fs.unlinkSync(lockPath);
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
// ignore
|
|
141
|
+
}
|
|
142
|
+
heldProcessLocks.delete(lockPath);
|
|
143
|
+
}
|
|
144
|
+
function releaseAllProcessLocks() {
|
|
145
|
+
for (const p of heldProcessLocks) {
|
|
146
|
+
try {
|
|
147
|
+
fs.unlinkSync(p);
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
// ignore
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
heldProcessLocks.clear();
|
|
154
|
+
}
|
|
47
155
|
function resolveImproveScope(scope) {
|
|
48
156
|
const trimmed = scope?.trim();
|
|
49
157
|
if (!trimmed)
|
|
@@ -99,6 +207,22 @@ export function renderSyncCommitMessage(template, result, nowMs) {
|
|
|
99
207
|
};
|
|
100
208
|
return template.replace(/\{(\w+)\}/g, (match, key) => (Object.hasOwn(tokens, key) ? tokens[key] : match));
|
|
101
209
|
}
|
|
210
|
+
/**
|
|
211
|
+
* Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
|
|
212
|
+
* merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
|
|
213
|
+
* Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
|
|
214
|
+
*/
|
|
215
|
+
function dedupeRefs(refs) {
|
|
216
|
+
const seen = new Set();
|
|
217
|
+
const out = [];
|
|
218
|
+
for (const r of refs) {
|
|
219
|
+
if (seen.has(r.ref))
|
|
220
|
+
continue;
|
|
221
|
+
seen.add(r.ref);
|
|
222
|
+
out.push(r);
|
|
223
|
+
}
|
|
224
|
+
return out;
|
|
225
|
+
}
|
|
102
226
|
async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
103
227
|
if (scope.mode === "ref" && scope.value) {
|
|
104
228
|
const parsed = parseAssetRef(scope.value);
|
|
@@ -112,7 +236,7 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
|
112
236
|
};
|
|
113
237
|
}
|
|
114
238
|
return {
|
|
115
|
-
plannedRefs: [{ ref: scope.value, reason: "scope-ref" }],
|
|
239
|
+
plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
|
|
116
240
|
memorySummary: {
|
|
117
241
|
eligible: parsed.type === "memory" ? 1 : 0,
|
|
118
242
|
derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
|
|
@@ -176,12 +300,14 @@ async function collectEligibleRefs(scope, stashDir, improveProfile) {
|
|
|
176
300
|
profileFiltered.set(ref, {
|
|
177
301
|
ref,
|
|
178
302
|
reason: "profile_filtered_all_passes",
|
|
303
|
+
filePath: indexed.filePath,
|
|
179
304
|
});
|
|
180
305
|
}
|
|
181
306
|
else {
|
|
182
307
|
planned.set(ref, {
|
|
183
308
|
ref,
|
|
184
309
|
reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
|
|
310
|
+
filePath: indexed.filePath,
|
|
185
311
|
});
|
|
186
312
|
}
|
|
187
313
|
}
|
|
@@ -449,6 +575,107 @@ export function armBudgetWatchdog(budgetMs, controller, deps) {
|
|
|
449
575
|
}
|
|
450
576
|
};
|
|
451
577
|
}
|
|
578
|
+
/**
|
|
579
|
+
* #612 / WS-4 — bounded, opt-in per-phase auto-accept threshold auto-tune.
|
|
580
|
+
*
|
|
581
|
+
* Reads `improve.calibration` from config. When `autoTune` is enabled, computes
|
|
582
|
+
* the calibration of recent gate decisions, derives a bounded threshold
|
|
583
|
+
* adjustment (clamped into the configured band, capped per step), logs it, and
|
|
584
|
+
* records a `calibration_autotune` event. Returns the new threshold (integer
|
|
585
|
+
* 0-100) when an adjustment was made, or `undefined` to leave the caller's
|
|
586
|
+
* threshold unchanged.
|
|
587
|
+
*
|
|
588
|
+
* WS-4 change: accepts an optional `phase` parameter. When provided, the tuned
|
|
589
|
+
* threshold is persisted to `improve_gate_thresholds` (state.db Migration 012)
|
|
590
|
+
* keyed by phase so `makeGateConfig` can read it back on the next run and each
|
|
591
|
+
* phase maintains its own calibrated threshold rather than a shared global.
|
|
592
|
+
*
|
|
593
|
+
* WS-4 ceiling: `maxThreshold` defaults to 85 (not 100) to prevent the gate
|
|
594
|
+
* converging to pure exploitation and shutting down Gap-3/4 novelty throughput.
|
|
595
|
+
*
|
|
596
|
+
* DEFAULT OFF: with no `improve.calibration` block (or `autoTune: false`) this
|
|
597
|
+
* returns `undefined` immediately, so the gate threshold is unchanged and
|
|
598
|
+
* behaviour is byte-identical to today.
|
|
599
|
+
*/
|
|
600
|
+
export function maybeAutoTuneThreshold(currentThreshold, config, stateDbPath, ctx, phase) {
|
|
601
|
+
const cal = config.improve?.calibration;
|
|
602
|
+
if (!cal?.autoTune)
|
|
603
|
+
return undefined;
|
|
604
|
+
// WS-4: default maxThreshold is now 85 (ceiling to prevent pure exploitation).
|
|
605
|
+
// Callers that explicitly set maxThreshold in config override this default.
|
|
606
|
+
const tuneConfig = {
|
|
607
|
+
autoTune: true,
|
|
608
|
+
minThreshold: cal.minThreshold ?? 0,
|
|
609
|
+
maxThreshold: cal.maxThreshold ?? 85,
|
|
610
|
+
maxStep: cal.maxStep ?? 5,
|
|
611
|
+
minSamples: cal.minSamples ?? 20,
|
|
612
|
+
targetAcceptRate: cal.targetAcceptRate ?? 0.9,
|
|
613
|
+
};
|
|
614
|
+
// Defensive: an inverted band disables tuning rather than clamping to nonsense.
|
|
615
|
+
if (tuneConfig.minThreshold > tuneConfig.maxThreshold)
|
|
616
|
+
return undefined;
|
|
617
|
+
const db = openStateDatabase(stateDbPath);
|
|
618
|
+
let summary;
|
|
619
|
+
try {
|
|
620
|
+
const allDecisions = listProposalGateDecisions(db);
|
|
621
|
+
// WS-4 fix: when called with a phase label, restrict calibration to that
|
|
622
|
+
// phase's decision pool so a reflect-dominated run cannot tighten the
|
|
623
|
+
// consolidate gate (or vice-versa). The gate field is `improve:<phase>`,
|
|
624
|
+
// matching what improve-auto-accept.ts stamps at line ~163.
|
|
625
|
+
const gateLabel = phase ? `improve:${phase}` : undefined;
|
|
626
|
+
const decisions = gateLabel ? allDecisions.filter((d) => d.gate === gateLabel) : allDecisions;
|
|
627
|
+
summary = summarizeCalibration(gateDecisionsToSamples(decisions));
|
|
628
|
+
}
|
|
629
|
+
finally {
|
|
630
|
+
db.close();
|
|
631
|
+
}
|
|
632
|
+
const result = computeThresholdAutoTune(currentThreshold, summary, tuneConfig);
|
|
633
|
+
if (!result.adjusted)
|
|
634
|
+
return undefined;
|
|
635
|
+
const appendEventFn = ctx?.appendEventFn ?? appendEvent;
|
|
636
|
+
const phaseLabel = phase ?? "global";
|
|
637
|
+
info(`[improve] calibration auto-tune (${phaseLabel}): threshold ${result.previousThreshold} -> ${result.newThreshold} ` +
|
|
638
|
+
`(${result.reason}; samples=${summary.samples}, acceptRate=${summary.overallAcceptRate}, ` +
|
|
639
|
+
`gap=${summary.calibrationGap}, band=[${tuneConfig.minThreshold},${tuneConfig.maxThreshold}])`);
|
|
640
|
+
try {
|
|
641
|
+
appendEventFn({
|
|
642
|
+
eventType: "calibration_autotune",
|
|
643
|
+
ref: "improve:calibration",
|
|
644
|
+
metadata: {
|
|
645
|
+
phase: phaseLabel,
|
|
646
|
+
previousThreshold: result.previousThreshold,
|
|
647
|
+
newThreshold: result.newThreshold,
|
|
648
|
+
delta: result.delta,
|
|
649
|
+
reason: result.reason,
|
|
650
|
+
samples: summary.samples,
|
|
651
|
+
overallAcceptRate: summary.overallAcceptRate,
|
|
652
|
+
calibrationGap: summary.calibrationGap,
|
|
653
|
+
minThreshold: tuneConfig.minThreshold,
|
|
654
|
+
maxThreshold: tuneConfig.maxThreshold,
|
|
655
|
+
},
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
catch (err) {
|
|
659
|
+
warn(`[improve] calibration auto-tune event not recorded: ${err instanceof Error ? err.message : String(err)}`);
|
|
660
|
+
}
|
|
661
|
+
// WS-4: Persist the per-phase threshold so makeGateConfig reads it on the
|
|
662
|
+
// next run. Best-effort — a write failure must not abort the improve run.
|
|
663
|
+
if (phase) {
|
|
664
|
+
try {
|
|
665
|
+
const persistDb = openStateDatabase(stateDbPath);
|
|
666
|
+
try {
|
|
667
|
+
persistPhaseThreshold(persistDb, phase, result.newThreshold);
|
|
668
|
+
}
|
|
669
|
+
finally {
|
|
670
|
+
persistDb.close();
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
catch (err) {
|
|
674
|
+
warn(`[improve] calibration auto-tune: failed to persist phase threshold for ${phase}: ${err instanceof Error ? err.message : String(err)}`);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
return result.newThreshold;
|
|
678
|
+
}
|
|
452
679
|
export async function akmImprove(options = {}) {
|
|
453
680
|
const scope = resolveImproveScope(options.scope);
|
|
454
681
|
const reflectFn = options.reflectFn ?? akmReflect;
|
|
@@ -456,6 +683,11 @@ export async function akmImprove(options = {}) {
|
|
|
456
683
|
const ensureIndexFn = options.ensureIndexFn ?? ensureIndex;
|
|
457
684
|
const reindexFn = options.reindexFn ?? akmIndex;
|
|
458
685
|
const drainProposalsFn = options.drainProposalsFn ?? drainProposals;
|
|
686
|
+
// #616 multi-cycle test seams. Default to the real module-local fns.
|
|
687
|
+
const collectEligibleRefsImpl = options.collectEligibleRefsFn ?? collectEligibleRefs;
|
|
688
|
+
const runImprovePreparationStageImpl = options.runImprovePreparationStageFn ?? runImprovePreparationStage;
|
|
689
|
+
const runImproveLoopStageImpl = options.runImproveLoopStageFn ?? runImproveLoopStage;
|
|
690
|
+
const runImprovePostLoopStageImpl = options.runImprovePostLoopStageFn ?? runImprovePostLoopStage;
|
|
459
691
|
// Resolve the improve profile for this run. Profile drives type filtering,
|
|
460
692
|
// process gating, and default autoAccept/limit values.
|
|
461
693
|
const _earlyConfig = options.config ?? loadConfig();
|
|
@@ -466,8 +698,13 @@ export async function akmImprove(options = {}) {
|
|
|
466
698
|
options = {
|
|
467
699
|
...options,
|
|
468
700
|
autoAccept: options.autoAccept ?? improveProfile.autoAccept,
|
|
469
|
-
limit
|
|
701
|
+
// Profile-level limit, then process-level reflect.limit as fallback.
|
|
702
|
+
// CLI --limit takes precedence over both.
|
|
703
|
+
limit: options.limit ?? improveProfile?.processes?.reflect?.limit ?? improveProfile.limit,
|
|
470
704
|
};
|
|
705
|
+
// #616 — bounded multi-cycle phasing. CLI/programmatic override wins over
|
|
706
|
+
// profile.maxCycles; default 1 => single pass (byte-identical to pre-#616).
|
|
707
|
+
const maxCycles = Math.max(1, Math.trunc(options.maxCycles ?? improveProfile.maxCycles ?? 1));
|
|
471
708
|
let primaryStashDir;
|
|
472
709
|
try {
|
|
473
710
|
primaryStashDir = resolveSourceEntries(options.stashDir)[0]?.path;
|
|
@@ -484,174 +721,51 @@ export async function akmImprove(options = {}) {
|
|
|
484
721
|
// timeout root cause). Because beforeEach runs synchronously, env is still the
|
|
485
722
|
// calling test's own at this point; we capture it before yielding the loop.
|
|
486
723
|
const resolvedStateDbPath = getStateDbPathInDataDir();
|
|
487
|
-
//
|
|
488
|
-
//
|
|
489
|
-
//
|
|
490
|
-
//
|
|
491
|
-
//
|
|
492
|
-
//
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
})()
|
|
515
|
-
: null;
|
|
516
|
-
if (probe.state === "stale") {
|
|
517
|
-
// O-7 / #394: Emit improve_lock_recovered event before recovery so the
|
|
518
|
-
// audit trail records the abnormal prior-run exit (Temporal/Airflow pattern).
|
|
519
|
-
try {
|
|
520
|
-
appendEvent({
|
|
521
|
-
eventType: "improve_lock_recovered",
|
|
522
|
-
metadata: {
|
|
523
|
-
stalePid: lock?.pid ?? null,
|
|
524
|
-
lockedAt: lock?.startedAt ?? null,
|
|
525
|
-
recoveredAt: new Date().toISOString(),
|
|
526
|
-
lockAgeMs: probe.ageMs ?? null,
|
|
527
|
-
reason: probe.reason === "pid_dead" ? "pid_not_alive" : probe.reason,
|
|
528
|
-
},
|
|
529
|
-
});
|
|
530
|
-
}
|
|
531
|
-
catch {
|
|
532
|
-
/* event emission is best-effort; never block lock recovery */
|
|
533
|
-
}
|
|
534
|
-
releaseLock(resolvedLockPath);
|
|
535
|
-
if (tryAcquireLockSync(resolvedLockPath, lockPayload()))
|
|
536
|
-
return "acquired";
|
|
537
|
-
// Lost the race to another run that grabbed the freed stale lock.
|
|
538
|
-
if (options.skipIfLocked) {
|
|
539
|
-
warn("[improve] another run acquired the lock during stale recovery; skipping (--skip-if-locked)");
|
|
540
|
-
return "skipped";
|
|
541
|
-
}
|
|
542
|
-
throw new ConfigError(`akm improve is already running. Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
543
|
-
}
|
|
544
|
-
// Lock is held by a live run within the staleness window.
|
|
545
|
-
if (options.skipIfLocked) {
|
|
546
|
-
warn(`[improve] another improve run holds the lock (PID ${lock?.pid}, started ${lock?.startedAt}); skipping (--skip-if-locked)`);
|
|
547
|
-
return "skipped";
|
|
548
|
-
}
|
|
549
|
-
throw new ConfigError(`akm improve is already running (PID ${lock?.pid}, started ${lock?.startedAt}). Delete ${resolvedLockPath} to force.`, "INVALID_CONFIG_FILE");
|
|
550
|
-
};
|
|
551
|
-
// Phase 4 lock-leak guard (§7 ordering hazard): hoisting `improve.lock` above
|
|
552
|
-
// the pre-index region (so the triage pre-pass runs under it) means the lock is
|
|
553
|
-
// held while ensureIndex / collectEligibleRefs / contradiction-detection /
|
|
554
|
-
// memory-cleanup analysis run — but the main protecting `try { … } finally {
|
|
555
|
-
// unlinkSync(resolvedLockPath) }` does not begin until after them. A throw in
|
|
556
|
-
// any of those steps would leak the lock. We close that window by wrapping the
|
|
557
|
-
// whole region in a try whose catch releases the lock (when held) and
|
|
558
|
-
// re-throws. The values this region computes are declared in the outer scope so
|
|
559
|
-
// they remain visible to the main run below. The dry-run path never sets
|
|
560
|
-
// `lockAcquired`, so its early return releases nothing.
|
|
561
|
-
let lockAcquired = false;
|
|
562
|
-
const releaseLockOnError = () => {
|
|
563
|
-
if (!lockAcquired)
|
|
564
|
-
return;
|
|
565
|
-
try {
|
|
566
|
-
fs.unlinkSync(resolvedLockPath);
|
|
567
|
-
}
|
|
568
|
-
catch {
|
|
569
|
-
// best-effort release on the error path
|
|
570
|
-
}
|
|
571
|
-
lockAcquired = false;
|
|
572
|
-
};
|
|
573
|
-
// Signal-safe lock release. The SIGTERM/SIGINT/SIGHUP handler in improve-cli.ts
|
|
574
|
-
// calls `process.exit()`, which does NOT run the `finally` below that owns lock
|
|
575
|
-
// release — so a cron-timeout SIGTERM leaked `improve.lock` every run.
|
|
576
|
-
// `process.exit()` DOES fire `'exit'` listeners, so we release the lock from
|
|
577
|
-
// one. `releaseLockIfOwned` only unlinks a lock still owned by this PID, so it
|
|
578
|
-
// is safe even if a later run re-acquired it. The listener is removed in the
|
|
579
|
-
// `finally` so the normal path stays single-release and repeated in-process
|
|
580
|
-
// `akmImprove` calls (tests) do not accumulate listeners.
|
|
581
|
-
const releaseLockOnExit = () => {
|
|
582
|
-
releaseLockIfOwned(resolvedLockPath, process.pid);
|
|
583
|
-
};
|
|
724
|
+
// #612 / WS-4 — bounded, OPT-IN per-phase auto-accept threshold auto-tune.
|
|
725
|
+
// DEFAULT OFF: `autoTune: false` (or absent) is a complete no-op.
|
|
726
|
+
//
|
|
727
|
+
// WS-4 change: thresholds are now PER PHASE. The old single global mutation
|
|
728
|
+
// of `options.autoAccept` is retired — it caused every phase to share one
|
|
729
|
+
// calibration signal, so a reflect-dominated run could tighten the consolidate
|
|
730
|
+
// gate (or vice-versa). Instead:
|
|
731
|
+
// - Each `makeGateConfig` call reads the phase's stored threshold from
|
|
732
|
+
// state.db (Migration 012) and uses it as `phaseThreshold`, overriding
|
|
733
|
+
// the `globalThreshold` (= options.autoAccept) for that phase.
|
|
734
|
+
// - Per-phase `maybeAutoTuneThreshold` calls fire AFTER each phase's gate
|
|
735
|
+
// has run and persist the new threshold to state.db for the NEXT run.
|
|
736
|
+
// - `options.autoAccept` stays unchanged (it is the operator-supplied
|
|
737
|
+
// baseline, not a mutable run-time state).
|
|
738
|
+
//
|
|
739
|
+
// The global tune call is intentionally removed here. See per-phase calls
|
|
740
|
+
// below (near each makeGateConfig / runAutoAcceptGate block).
|
|
741
|
+
// #607 Lock decomposition: three per-process locks replace the single
|
|
742
|
+
// `improve.lock`. Each process acquires only the lock(s) it needs, so
|
|
743
|
+
// quick-shredder consolidate can run alongside daily reflect+distill.
|
|
744
|
+
//
|
|
745
|
+
// consolidate.lock — protects consolidate + memoryInference + graphExtraction (index.db writers)
|
|
746
|
+
// reflect-distill.lock — protects reflect + distill (state.db proposal writers)
|
|
747
|
+
// triage.lock — protects triage pre-pass (state.db proposal promotions)
|
|
748
|
+
//
|
|
749
|
+
// Lock base directory — same `.akm/` under the primary stash dir.
|
|
750
|
+
const lockBaseDir = primaryStashDir ? path.join(primaryStashDir, ".akm") : path.join(options.stashDir ?? ".", ".akm");
|
|
584
751
|
const preEnsureCleanupWarnings = [];
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
752
|
+
// #616: assigned by runIndexAndCollect() (closure) so TS cannot prove definite
|
|
753
|
+
// assignment — seed with empty values; the first runIndexAndCollect() call
|
|
754
|
+
// (cycle 1, in the first try) always overwrites them before any read.
|
|
755
|
+
let plannedRefs = [];
|
|
756
|
+
let memorySummary = { eligible: 0, derived: 0 };
|
|
757
|
+
let profileFilteredRefs = [];
|
|
588
758
|
let memoryCleanupPlan;
|
|
589
759
|
let guidance;
|
|
590
760
|
let triageDrain;
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
// work — never registered the exit listener, never set lockAcquired,
|
|
600
|
-
// so we release nothing belonging to the run that owns the lock.
|
|
601
|
-
return {
|
|
602
|
-
schemaVersion: 1,
|
|
603
|
-
ok: true,
|
|
604
|
-
scope,
|
|
605
|
-
dryRun: false,
|
|
606
|
-
skipped: { reason: "lock-held" },
|
|
607
|
-
memorySummary: { eligible: 0, derived: 0 },
|
|
608
|
-
plannedRefs: [],
|
|
609
|
-
};
|
|
610
|
-
}
|
|
611
|
-
lockAcquired = true;
|
|
612
|
-
// Backstop release on process.exit() (signal handler / budget watchdog),
|
|
613
|
-
// which skips the finally below. Removed in that finally on the normal path.
|
|
614
|
-
process.on("exit", releaseLockOnExit);
|
|
615
|
-
// Phase 4 triage pre-pass (§7, §13): drain the standing pending backlog
|
|
616
|
-
// BEFORE ensureIndex so improve generates fresh proposals against a cleared
|
|
617
|
-
// queue (no `duplicate_pending` collisions) and ensureIndex absorbs triage's
|
|
618
|
-
// promotions for free. Gated on the triage process being enabled (opt-in,
|
|
619
|
-
// defaults off) and on a whole-stash / type-scoped run — a single-ref
|
|
620
|
-
// `akm improve skill:x` must never drain the whole queue. Best-effort: a
|
|
621
|
-
// triage failure is a non-fatal warning, never an abort (mirrors the
|
|
622
|
-
// contradiction-detection pass below).
|
|
623
|
-
if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
|
|
624
|
-
if (scope.mode === "ref") {
|
|
625
|
-
warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
|
|
626
|
-
}
|
|
627
|
-
else {
|
|
628
|
-
try {
|
|
629
|
-
const triageConfig = improveProfile.processes?.triage;
|
|
630
|
-
const policy = resolveDrainPolicy(triageConfig?.policy);
|
|
631
|
-
const applyMode = triageConfig?.applyMode ?? "queue";
|
|
632
|
-
const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
|
|
633
|
-
const judgment = triageConfig?.judgment
|
|
634
|
-
? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
|
|
635
|
-
: null;
|
|
636
|
-
triageDrain = await drainProposalsFn({
|
|
637
|
-
stashDir: primaryStashDir,
|
|
638
|
-
policy,
|
|
639
|
-
applyMode,
|
|
640
|
-
maxAccepts,
|
|
641
|
-
dryRun: false,
|
|
642
|
-
// No fresh ids exist yet — triage runs before improve generates any.
|
|
643
|
-
excludeIds: new Set(),
|
|
644
|
-
...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
|
|
645
|
-
judgment,
|
|
646
|
-
});
|
|
647
|
-
}
|
|
648
|
-
catch (err) {
|
|
649
|
-
// Non-fatal: triage is a best-effort pre-pass and must never abort improve.
|
|
650
|
-
warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
}
|
|
654
|
-
}
|
|
761
|
+
// #616 — ensureIndex + collectEligibleRefs + memory-cleanup recompute, lifted
|
|
762
|
+
// into a helper so the SAME sequence runs once for cycle 1 (below, in the
|
|
763
|
+
// first try) and is re-run at the top of each subsequent multi-cycle cycle.
|
|
764
|
+
// Re-running ensureIndex between cycles makes cycle N's gate-promoted
|
|
765
|
+
// proposals visible to cycle N+1's collectEligibleRefs. Mutates the
|
|
766
|
+
// outer-scope plannedRefs/memorySummary/profileFilteredRefs/memoryCleanupPlan/
|
|
767
|
+
// guidance so for maxCycles:1 the body is byte-identical to pre-#616.
|
|
768
|
+
const runIndexAndCollect = async () => {
|
|
655
769
|
// #339 fix: ensureIndex MUST run BEFORE collectEligibleRefs. The eligible-ref
|
|
656
770
|
// query reads the `entries` table; if a DB version upgrade just dropped that
|
|
657
771
|
// table (or the index is otherwise empty), the prior run order silently
|
|
@@ -679,7 +793,7 @@ export async function akmImprove(options = {}) {
|
|
|
679
793
|
// best-effort; leave preEnsureEntryCount undefined
|
|
680
794
|
}
|
|
681
795
|
try {
|
|
682
|
-
await ensureIndexFn(primaryStashDir);
|
|
796
|
+
await ensureIndexFn(primaryStashDir, { mode: "blocking" });
|
|
683
797
|
}
|
|
684
798
|
catch (err) {
|
|
685
799
|
preEnsureCleanupWarnings.push(`ensureIndex failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -709,7 +823,7 @@ export async function akmImprove(options = {}) {
|
|
|
709
823
|
}
|
|
710
824
|
}
|
|
711
825
|
}
|
|
712
|
-
({ plannedRefs, memorySummary, profileFilteredRefs } = await
|
|
826
|
+
({ plannedRefs, memorySummary, profileFilteredRefs } = await collectEligibleRefsImpl(scope, options.stashDir, improveProfile));
|
|
713
827
|
const cleanupParentRef = memoryCleanupParentRef(scope, options.stashDir);
|
|
714
828
|
// M-1 (#367): Run contradiction-detection BEFORE analyzeMemoryCleanup so
|
|
715
829
|
// the SCC resolver in resolveFamilyContradictions has edges to work on.
|
|
@@ -717,7 +831,7 @@ export async function akmImprove(options = {}) {
|
|
|
717
831
|
if (primaryStashDir && shouldAnalyzeMemoryCleanup(scope, memorySummary.eligible, primaryStashDir)) {
|
|
718
832
|
try {
|
|
719
833
|
// Reuse the config resolved at the top of the run instead of a second load.
|
|
720
|
-
await detectAndWriteContradictions(primaryStashDir, _earlyConfig);
|
|
834
|
+
await withLlmStage("memory-contradiction", () => detectAndWriteContradictions(primaryStashDir, _earlyConfig));
|
|
721
835
|
}
|
|
722
836
|
catch (err) {
|
|
723
837
|
// Non-fatal: contradiction detection is a best-effort pass.
|
|
@@ -731,6 +845,69 @@ export async function akmImprove(options = {}) {
|
|
|
731
845
|
memorySummary.eligible > 0
|
|
732
846
|
? "Improve folds memory cleanup into the same proposal queue: speculative promotions still go through reflect/distill proposals, while high-confidence redundant derived memories are moved into a recoverable cleanup archive instead of being left active in the stash."
|
|
733
847
|
: undefined;
|
|
848
|
+
};
|
|
849
|
+
try {
|
|
850
|
+
// #607: Per-process lock acquisition. Each process acquires only the lock(s)
|
|
851
|
+
// it needs. The dry-run branch produces plannedRefs/memorySummary WITHOUT any
|
|
852
|
+
// locks (decision: dry-run never mutates the queue).
|
|
853
|
+
if (!options.dryRun) {
|
|
854
|
+
// Backstop release on process.exit() (signal handler / budget watchdog),
|
|
855
|
+
// which skips the finally below. Removed in that finally on the normal path.
|
|
856
|
+
const releaseAllOnExit = () => {
|
|
857
|
+
for (const p of heldProcessLocks) {
|
|
858
|
+
releaseLockIfOwned(p, process.pid);
|
|
859
|
+
}
|
|
860
|
+
};
|
|
861
|
+
process.on("exit", releaseAllOnExit);
|
|
862
|
+
// #607 triage pre-pass: acquire triage.lock, drain the standing pending
|
|
863
|
+
// backlog BEFORE ensureIndex so improve generates fresh proposals against
|
|
864
|
+
// a cleared queue (no `duplicate_pending` collisions) and ensureIndex
|
|
865
|
+
// absorbs triage's promotions for free. Release immediately after —
|
|
866
|
+
// triage.lock is not needed again until the next improve run.
|
|
867
|
+
if (primaryStashDir && resolveProcessEnabled("triage", improveProfile)) {
|
|
868
|
+
if (scope.mode === "ref") {
|
|
869
|
+
warn("[improve] triage pre-pass skipped (single-ref scope never drains the whole queue)");
|
|
870
|
+
}
|
|
871
|
+
else {
|
|
872
|
+
const triageLPath = processLockPath(lockBaseDir, "triage");
|
|
873
|
+
const triageResult = tryAcquireProcessLock(triageLPath, PROCESS_LOCK_DEFS.triage.staleAfterMs, options.skipIfLocked, "triage");
|
|
874
|
+
if (triageResult === "skipped") {
|
|
875
|
+
triageDrain = undefined;
|
|
876
|
+
}
|
|
877
|
+
else {
|
|
878
|
+
try {
|
|
879
|
+
const triageConfig = improveProfile.processes?.triage;
|
|
880
|
+
const policy = resolveDrainPolicy(triageConfig?.policy);
|
|
881
|
+
const applyMode = triageConfig?.applyMode ?? "queue";
|
|
882
|
+
const maxAccepts = triageConfig?.maxAcceptsPerRun ?? 25;
|
|
883
|
+
const judgment = triageConfig?.judgment
|
|
884
|
+
? resolveTriageJudgmentRunner(triageConfig.judgment, _earlyConfig)
|
|
885
|
+
: null;
|
|
886
|
+
triageDrain = await drainProposalsFn({
|
|
887
|
+
stashDir: primaryStashDir,
|
|
888
|
+
policy,
|
|
889
|
+
applyMode,
|
|
890
|
+
maxAccepts,
|
|
891
|
+
dryRun: false,
|
|
892
|
+
excludeIds: new Set(),
|
|
893
|
+
...(triageConfig?.maxDiffLines !== undefined ? { maxDiffLines: triageConfig.maxDiffLines } : {}),
|
|
894
|
+
judgment,
|
|
895
|
+
});
|
|
896
|
+
}
|
|
897
|
+
catch (err) {
|
|
898
|
+
warn(`[improve] triage pre-pass failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
899
|
+
}
|
|
900
|
+
finally {
|
|
901
|
+
releaseProcessLock(triageLPath);
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
// #339 fix: ensureIndex MUST run BEFORE collectEligibleRefs (now inside the
|
|
908
|
+
// helper). Cycle 1 runs it here; subsequent multi-cycle cycles re-run it via
|
|
909
|
+
// the same helper at the top of each cycle below.
|
|
910
|
+
await runIndexAndCollect();
|
|
734
911
|
if (options.dryRun) {
|
|
735
912
|
const result = {
|
|
736
913
|
schemaVersion: 1,
|
|
@@ -747,17 +924,14 @@ export async function akmImprove(options = {}) {
|
|
|
747
924
|
}
|
|
748
925
|
}
|
|
749
926
|
catch (err) {
|
|
750
|
-
|
|
927
|
+
releaseAllProcessLocks();
|
|
751
928
|
throw err;
|
|
752
929
|
}
|
|
753
|
-
//
|
|
754
|
-
//
|
|
755
|
-
//
|
|
756
|
-
//
|
|
757
|
-
//
|
|
758
|
-
// any of them used to leak the lock (blocking the next improve up to 4h);
|
|
759
|
-
// now the finally releases it exactly once. The dry-run path already returned
|
|
760
|
-
// above without acquiring the lock, so it never reaches this finally; the
|
|
930
|
+
// #607: per-process locks are acquired/released around each stage below.
|
|
931
|
+
// The triage pre-pass already ran under triage.lock (released). The
|
|
932
|
+
// preparation stage runs under consolidate.lock, the loop stage under
|
|
933
|
+
// reflect-distill.lock, and the post-loop stage under consolidate.lock again.
|
|
934
|
+
// Each stage acquires its lock just before starting and releases in finally.
|
|
761
935
|
// best-effort `unlinkSync` is a no-op when no lock file exists.
|
|
762
936
|
const startMs = Date.now();
|
|
763
937
|
const budgetMs = options.timeoutMs ?? 2 * 60 * 60 * 1000; // default 2 hours
|
|
@@ -766,6 +940,16 @@ export async function akmImprove(options = {}) {
|
|
|
766
940
|
// run past the declared budget.
|
|
767
941
|
// References: Anthropic *Building Effective Agents* (2024); CoALA §5 (arXiv:2309.02427).
|
|
768
942
|
const budgetAbortController = new AbortController();
|
|
943
|
+
// Attach a live `remainingBudgetMs` getter to the signal so sub-callers
|
|
944
|
+
// (e.g. consolidate.ts cold-start budget estimation) can read the remaining
|
|
945
|
+
// wall-clock budget without needing an extra plumbing parameter. The property
|
|
946
|
+
// is computed at access time via a getter so it always reflects the actual
|
|
947
|
+
// elapsed time rather than a stale snapshot taken at arm time.
|
|
948
|
+
Object.defineProperty(budgetAbortController.signal, "remainingBudgetMs", {
|
|
949
|
+
get: () => Math.max(0, budgetMs - (Date.now() - startMs)),
|
|
950
|
+
enumerable: false,
|
|
951
|
+
configurable: true,
|
|
952
|
+
});
|
|
769
953
|
// Declared in the outer scope so the `finally` can clear the timer even if a
|
|
770
954
|
// throw occurs before/after it is armed. Defaults to a no-op until armed.
|
|
771
955
|
let clearBudgetTimer = () => { };
|
|
@@ -777,6 +961,9 @@ export async function akmImprove(options = {}) {
|
|
|
777
961
|
// Pinned to the boundary snapshot so the fallback per-call `appendEvent`
|
|
778
962
|
// opens (when the long-lived handle below fails to open) never re-read env.
|
|
779
963
|
let eventsCtx = { dbPath: resolvedStateDbPath };
|
|
964
|
+
// #576: clears the per-run LLM usage sink. Defaults to a no-op until the sink
|
|
965
|
+
// is installed inside the try; the `finally` always calls it.
|
|
966
|
+
let disposeLlmUsageSink = () => { };
|
|
780
967
|
try {
|
|
781
968
|
// H7 (#566): arm the budget watchdog. `armBudgetWatchdog` captures both the
|
|
782
969
|
// budget timer and the hard-kill timer it schedules on exhaustion, returning
|
|
@@ -796,84 +983,216 @@ export async function akmImprove(options = {}) {
|
|
|
796
983
|
// still pinned to the boundary-resolved path, never a live env re-read.
|
|
797
984
|
eventsCtx = { dbPath: resolvedStateDbPath };
|
|
798
985
|
}
|
|
799
|
-
//
|
|
986
|
+
// #576: persist per-call LLM usage telemetry for this run as `llm_usage`
|
|
987
|
+
// events, reusing the same boundary-pinned events context (and long-lived
|
|
988
|
+
// handle when available). Disposed in `finally` so the sink never leaks
|
|
989
|
+
// across runs. Wrapping is best-effort end to end — see usage-telemetry.ts.
|
|
990
|
+
disposeLlmUsageSink = installLlmUsagePersistence(eventsCtx);
|
|
991
|
+
// 2026-05-27: emit an `improve_skipped` audit event for refs the planner
|
|
800
992
|
// pre-filtered (reflect AND distill both refuse them under the active
|
|
801
|
-
// profile).
|
|
802
|
-
//
|
|
803
|
-
//
|
|
804
|
-
//
|
|
805
|
-
|
|
993
|
+
// profile). Emitted as a single summary event (count only) rather than one
|
|
994
|
+
// event per ref (#592) — the per-ref loop caused O(n) sequential state.db
|
|
995
|
+
// writes that consumed ~500 s on a 9 000-ref stash. No downstream consumer
|
|
996
|
+
// needs the per-ref audit trail: health's skip histogram reads the
|
|
997
|
+
// `profile_filtered_all_passes` counters from `improve_completed` metadata.
|
|
998
|
+
if (profileFilteredRefs.length > 0) {
|
|
806
999
|
appendEvent({
|
|
807
1000
|
eventType: "improve_skipped",
|
|
808
|
-
ref:
|
|
809
|
-
metadata: {
|
|
1001
|
+
ref: undefined,
|
|
1002
|
+
metadata: {
|
|
1003
|
+
reason: "profile_filtered_all_passes",
|
|
1004
|
+
count: profileFilteredRefs.length,
|
|
1005
|
+
},
|
|
810
1006
|
}, eventsCtx);
|
|
811
1007
|
}
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
1008
|
+
// #616 — bounded multi-cycle phasing. The prep->loop->post-loop sequence is
|
|
1009
|
+
// wrapped in an N-cycle loop. Each cycle re-runs ensureIndex +
|
|
1010
|
+
// collectEligibleRefs (via runIndexAndCollect) so gate-accepted output of
|
|
1011
|
+
// cycle N becomes selectable input to cycle N+1. The per-stage process locks
|
|
1012
|
+
// (consolidate / reflect-distill) are acquired+released INSIDE each cycle,
|
|
1013
|
+
// exactly as the single-pass path did. For maxCycles:1 the loop runs once and
|
|
1014
|
+
// every accumulator below collapses to the single-cycle value (sum-of-one,
|
|
1015
|
+
// concat-of-one, last==only) => BYTE-IDENTICAL to pre-#616.
|
|
1016
|
+
//
|
|
1017
|
+
// Accumulators (see CONSTRAINTS / aggregation plan in #616): SUM the count
|
|
1018
|
+
// fields and durations; CONCAT the array fields; LAST-WINS for point-in-time
|
|
1019
|
+
// objects (the final cycle's value reflects the converged state).
|
|
1020
|
+
let cyclesRun = 0;
|
|
1021
|
+
// Last-wins point-in-time values (assigned every cycle; the final cycle wins).
|
|
1022
|
+
let preparation;
|
|
1023
|
+
let memoryRefsForInference = new Set();
|
|
1024
|
+
let consolidation;
|
|
1025
|
+
let memoryInference;
|
|
1026
|
+
let graphExtraction;
|
|
1027
|
+
let stalenessDetection;
|
|
1028
|
+
let recombination;
|
|
1029
|
+
let proceduralCompilation;
|
|
1030
|
+
// Summed counters/durations.
|
|
1031
|
+
let prepGateCount = 0;
|
|
1032
|
+
let prepGateFailedCount = 0;
|
|
1033
|
+
let reflectsWithErrorContext = 0;
|
|
1034
|
+
let loopGateCount = 0;
|
|
1035
|
+
let loopGateFailedCount = 0;
|
|
1036
|
+
let postLoopGateCount = 0;
|
|
1037
|
+
let postLoopGateFailedCount = 0;
|
|
1038
|
+
let memoryInferenceDurationMs = 0;
|
|
1039
|
+
let graphExtractionDurationMs = 0;
|
|
1040
|
+
let orphansPurged;
|
|
1041
|
+
let proposalsExpired;
|
|
1042
|
+
// Concatenated arrays.
|
|
1043
|
+
const allWarnings = [];
|
|
1044
|
+
let deadUrls;
|
|
1045
|
+
const finalActions = [];
|
|
1046
|
+
for (let cycleIndex = 0; cycleIndex < maxCycles; cycleIndex++) {
|
|
1047
|
+
// #616 budget gate: never start a NEW cycle once the run's wall-clock
|
|
1048
|
+
// budget is exhausted (or the run was aborted). Cycle 0 ALWAYS runs so
|
|
1049
|
+
// maxCycles:1 is byte-identical regardless of budget.
|
|
1050
|
+
if (cycleIndex > 0) {
|
|
1051
|
+
const remaining = budgetAbortController.signal.remainingBudgetMs;
|
|
1052
|
+
if (budgetAbortController.signal.aborted || (remaining !== undefined && remaining <= 0)) {
|
|
1053
|
+
break;
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
// Re-run ensureIndex + collectEligibleRefs + memory-cleanup recompute for
|
|
1057
|
+
// cycles 2+ (cycle 1 already ran them in the first try above). This makes
|
|
1058
|
+
// cycle N's gate-promoted proposals visible to this cycle's ref selection.
|
|
1059
|
+
if (cycleIndex > 0) {
|
|
1060
|
+
await runIndexAndCollect();
|
|
1061
|
+
// Re-emit the profile-filtered audit summary for this cycle's selection.
|
|
1062
|
+
if (profileFilteredRefs.length > 0) {
|
|
1063
|
+
appendEvent({
|
|
1064
|
+
eventType: "improve_skipped",
|
|
1065
|
+
ref: undefined,
|
|
1066
|
+
metadata: { reason: "profile_filtered_all_passes", count: profileFilteredRefs.length },
|
|
1067
|
+
}, eventsCtx);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
// #607: acquire consolidate.lock for the preparation stage (consolidate,
|
|
1071
|
+
// ensureIndex, extract all write index.db). Released immediately after.
|
|
1072
|
+
const consolidateLPath = processLockPath(lockBaseDir, "consolidate");
|
|
1073
|
+
const consolidatePrepAcquired = tryAcquireProcessLock(consolidateLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
|
|
1074
|
+
preparation = await runImprovePreparationStageImpl({
|
|
1075
|
+
scope,
|
|
1076
|
+
options,
|
|
1077
|
+
plannedRefs,
|
|
1078
|
+
memoryCleanupPlan,
|
|
1079
|
+
primaryStashDir,
|
|
1080
|
+
memorySummary,
|
|
1081
|
+
reindexFn,
|
|
1082
|
+
startMs,
|
|
1083
|
+
budgetMs,
|
|
1084
|
+
eventsCtx,
|
|
1085
|
+
initialCleanupWarnings: preEnsureCleanupWarnings,
|
|
1086
|
+
improveProfile,
|
|
1087
|
+
budgetSignal: budgetAbortController.signal,
|
|
1088
|
+
});
|
|
1089
|
+
if (consolidatePrepAcquired)
|
|
1090
|
+
releaseProcessLock(consolidateLPath);
|
|
1091
|
+
prepGateCount += preparation.gateAutoAcceptedCount;
|
|
1092
|
+
prepGateFailedCount += preparation.gateAutoAcceptFailedCount;
|
|
1093
|
+
// D6: pre-load all proposal_rejected events from the last 30 days once,
|
|
1094
|
+
// so the per-asset loop can use a Map lookup instead of N DB round trips.
|
|
1095
|
+
const REJECTED_PROPOSAL_WINDOW_MS = daysToMs(30);
|
|
1096
|
+
const rejectedProposalSince = new Date(Date.now() - REJECTED_PROPOSAL_WINDOW_MS).toISOString();
|
|
1097
|
+
const allRejectedProposalEvents = readEvents({ type: "proposal_rejected", since: rejectedProposalSince }).events;
|
|
1098
|
+
const rejectedProposalsByRef = new Map();
|
|
1099
|
+
for (const e of allRejectedProposalEvents) {
|
|
1100
|
+
if (e.ref && (!rejectedProposalsByRef.has(e.ref) || e.ts > (rejectedProposalsByRef.get(e.ref)?.ts ?? ""))) {
|
|
1101
|
+
rejectedProposalsByRef.set(e.ref, e);
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
// #607: acquire reflect-distill.lock for the loop stage (reflect + distill
|
|
1105
|
+
// both write proposals to state.db). Released immediately after.
|
|
1106
|
+
const reflectDistillLPath = processLockPath(lockBaseDir, "reflectDistill");
|
|
1107
|
+
const reflectDistillAcquired = tryAcquireProcessLock(reflectDistillLPath, PROCESS_LOCK_DEFS.reflectDistill.staleAfterMs, options.skipIfLocked, "reflect-distill") === "acquired";
|
|
1108
|
+
const loopResult = await runImproveLoopStageImpl({
|
|
1109
|
+
scope,
|
|
1110
|
+
options,
|
|
1111
|
+
primaryStashDir,
|
|
1112
|
+
reflectFn,
|
|
1113
|
+
distillFn,
|
|
1114
|
+
loopRefs: preparation.loopRefs,
|
|
1115
|
+
actions: preparation.actions,
|
|
1116
|
+
signalBearingSet: preparation.signalBearingSet,
|
|
1117
|
+
distillCooledRefs: preparation.distillCooledRefs,
|
|
1118
|
+
distillOnlyRefs: preparation.distillOnlyRefs,
|
|
1119
|
+
recentErrors: preparation.recentErrors,
|
|
1120
|
+
rejectedProposalsByRef,
|
|
1121
|
+
utilityMap: preparation.utilityMap,
|
|
1122
|
+
startMs,
|
|
1123
|
+
budgetMs,
|
|
1124
|
+
eventsCtx,
|
|
1125
|
+
improveProfile,
|
|
1126
|
+
budgetSignal: budgetAbortController.signal,
|
|
1127
|
+
});
|
|
1128
|
+
if (reflectDistillAcquired)
|
|
1129
|
+
releaseProcessLock(reflectDistillLPath);
|
|
1130
|
+
const loopGateCountThisCycle = loopResult.gateAutoAcceptedCount;
|
|
1131
|
+
reflectsWithErrorContext += loopResult.reflectsWithErrorContext;
|
|
1132
|
+
loopGateCount += loopResult.gateAutoAcceptedCount;
|
|
1133
|
+
loopGateFailedCount += loopResult.gateAutoAcceptFailedCount;
|
|
1134
|
+
memoryRefsForInference = loopResult.memoryRefsForInference;
|
|
1135
|
+
// #551: consolidation now runs in the preparation stage (before extract);
|
|
1136
|
+
// its result and run-flag are read from `preparation`, not the post-loop.
|
|
1137
|
+
consolidation = preparation.consolidation;
|
|
1138
|
+
// #607: acquire consolidate.lock for the post-loop stage (memoryInference +
|
|
1139
|
+
// graphExtraction both write index.db). Released immediately after.
|
|
1140
|
+
const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
|
|
1141
|
+
const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
|
|
1142
|
+
const postLoopResult = await runImprovePostLoopStageImpl({
|
|
1143
|
+
scope,
|
|
1144
|
+
options,
|
|
1145
|
+
primaryStashDir,
|
|
1146
|
+
actionableRefs: preparation.actionableRefs,
|
|
1147
|
+
appliedCleanup: preparation.appliedCleanup,
|
|
1148
|
+
cleanupWarnings: preparation.cleanupWarnings,
|
|
1149
|
+
memoryRefsForInference,
|
|
1150
|
+
reindexFn,
|
|
1151
|
+
eventsCtx,
|
|
1152
|
+
budgetSignal: budgetAbortController.signal,
|
|
1153
|
+
improveProfile,
|
|
1154
|
+
consolidationRan: preparation.consolidationRan,
|
|
1155
|
+
});
|
|
1156
|
+
if (consolidatePostAcquired)
|
|
1157
|
+
releaseProcessLock(consolidatePostLPath);
|
|
1158
|
+
const postLoopGateCountThisCycle = postLoopResult.gateAutoAcceptedCount;
|
|
1159
|
+
// Last-wins point-in-time objects.
|
|
1160
|
+
memoryInference = postLoopResult.memoryInference;
|
|
1161
|
+
graphExtraction = postLoopResult.graphExtraction;
|
|
1162
|
+
stalenessDetection = postLoopResult.stalenessDetection;
|
|
1163
|
+
recombination = postLoopResult.recombination;
|
|
1164
|
+
proceduralCompilation = postLoopResult.proceduralCompilation;
|
|
1165
|
+
// Summed counters/durations.
|
|
1166
|
+
postLoopGateCount += postLoopResult.gateAutoAcceptedCount;
|
|
1167
|
+
postLoopGateFailedCount += postLoopResult.gateAutoAcceptFailedCount;
|
|
1168
|
+
memoryInferenceDurationMs += postLoopResult.memoryInferenceDurationMs;
|
|
1169
|
+
graphExtractionDurationMs += postLoopResult.graphExtractionDurationMs;
|
|
1170
|
+
if (postLoopResult.orphansPurged !== undefined) {
|
|
1171
|
+
orphansPurged = (orphansPurged ?? 0) + postLoopResult.orphansPurged;
|
|
1172
|
+
}
|
|
1173
|
+
if (postLoopResult.proposalsExpired !== undefined) {
|
|
1174
|
+
proposalsExpired = (proposalsExpired ?? 0) + postLoopResult.proposalsExpired;
|
|
1175
|
+
}
|
|
1176
|
+
// Concatenated arrays.
|
|
1177
|
+
allWarnings.push(...postLoopResult.allWarnings);
|
|
1178
|
+
if (postLoopResult.deadUrls !== undefined) {
|
|
1179
|
+
deadUrls = [...(deadUrls ?? []), ...postLoopResult.deadUrls];
|
|
1180
|
+
}
|
|
1181
|
+
const maintenanceActions = postLoopResult.maintenanceActions;
|
|
1182
|
+
if (maintenanceActions && maintenanceActions.length > 0) {
|
|
1183
|
+
finalActions.push(...preparation.actions, ...maintenanceActions);
|
|
1184
|
+
}
|
|
1185
|
+
else {
|
|
1186
|
+
finalActions.push(...preparation.actions);
|
|
1187
|
+
}
|
|
1188
|
+
cyclesRun++;
|
|
1189
|
+
// #616 fixed-point stop: a cycle that produced ZERO gate-accepted proposals
|
|
1190
|
+
// (summed across prep + loop + post-loop) would feed cycle N+1 an identical
|
|
1191
|
+
// ref set, so end the loop here rather than spin a pointless next cycle.
|
|
1192
|
+
const gateAcceptedThisCycle = preparation.gateAutoAcceptedCount + loopGateCountThisCycle + postLoopGateCountThisCycle;
|
|
1193
|
+
if (gateAcceptedThisCycle === 0)
|
|
1194
|
+
break;
|
|
1195
|
+
}
|
|
877
1196
|
const result = {
|
|
878
1197
|
schemaVersion: 1,
|
|
879
1198
|
ok: true,
|
|
@@ -932,17 +1251,19 @@ export async function akmImprove(options = {}) {
|
|
|
932
1251
|
...(memoryInferenceDurationMs > 0 ? { memoryInferenceDurationMs } : {}),
|
|
933
1252
|
...(graphExtractionDurationMs > 0 ? { graphExtractionDurationMs } : {}),
|
|
934
1253
|
...(stalenessDetection ? { stalenessDetection } : {}),
|
|
1254
|
+
...(recombination ? { recombination } : {}),
|
|
1255
|
+
...(proceduralCompilation ? { proceduralCompilation } : {}),
|
|
935
1256
|
...(orphansPurged !== undefined ? { orphansPurged } : {}),
|
|
936
1257
|
...(proposalsExpired !== undefined && proposalsExpired > 0 ? { proposalsExpired } : {}),
|
|
937
1258
|
reflectCooldownActions: finalActions.filter((a) => a.mode === "reflect-cooldown").length,
|
|
938
1259
|
reflectSkippedActions: finalActions.filter((a) => a.mode === "reflect-skipped").length,
|
|
939
1260
|
reflectGuardRejectedActions: finalActions.filter((a) => a.mode === "reflect-guard-rejected").length,
|
|
940
1261
|
...(() => {
|
|
941
|
-
const t =
|
|
1262
|
+
const t = prepGateCount + loopGateCount + postLoopGateCount;
|
|
942
1263
|
return t > 0 ? { gateAutoAcceptedCount: t } : {};
|
|
943
1264
|
})(),
|
|
944
1265
|
...(() => {
|
|
945
|
-
const f =
|
|
1266
|
+
const f = prepGateFailedCount + loopGateFailedCount + postLoopGateFailedCount;
|
|
946
1267
|
return f > 0 ? { gateAutoAcceptFailedCount: f } : {};
|
|
947
1268
|
})(),
|
|
948
1269
|
...(triageDrain
|
|
@@ -955,6 +1276,10 @@ export async function akmImprove(options = {}) {
|
|
|
955
1276
|
},
|
|
956
1277
|
}
|
|
957
1278
|
: {}),
|
|
1279
|
+
...(preparation.proactiveMaintenance ? { proactiveMaintenance: preparation.proactiveMaintenance } : {}),
|
|
1280
|
+
// #616 — report cycles run only when >1 so the default single-pass
|
|
1281
|
+
// serialized envelope stays byte-identical to pre-#616 (AC1).
|
|
1282
|
+
...(cyclesRun > 1 ? { cyclesRun } : {}),
|
|
958
1283
|
...(options.runId !== undefined ? { runId: options.runId } : {}),
|
|
959
1284
|
};
|
|
960
1285
|
if (!result.dryRun)
|
|
@@ -1031,18 +1356,18 @@ export async function akmImprove(options = {}) {
|
|
|
1031
1356
|
throw err;
|
|
1032
1357
|
}
|
|
1033
1358
|
finally {
|
|
1359
|
+
// #576: clear the per-run LLM usage sink BEFORE closing `eventsDb` below, so
|
|
1360
|
+
// no late sink invocation can write through a closed handle.
|
|
1361
|
+
disposeLlmUsageSink();
|
|
1034
1362
|
// O-1 (#364): Clear the budget abort timer so it does not keep the event
|
|
1035
1363
|
// loop alive after the run completes.
|
|
1036
1364
|
clearBudgetTimer();
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
// The normal path released the lock above; drop the process.exit backstop so
|
|
1044
|
-
// it does not fire later (or accumulate across repeated in-process calls).
|
|
1045
|
-
process.removeListener("exit", releaseLockOnExit);
|
|
1365
|
+
// #607: release any per-process locks still held (backstop for error paths;
|
|
1366
|
+
// the normal path already released each lock after its stage completed).
|
|
1367
|
+
releaseAllProcessLocks();
|
|
1368
|
+
// Drop the process.exit backstop so it does not fire later (or accumulate
|
|
1369
|
+
// across repeated in-process calls).
|
|
1370
|
+
process.removeAllListeners("exit");
|
|
1046
1371
|
// I1: close the long-lived state.db connection opened at the top of the run.
|
|
1047
1372
|
try {
|
|
1048
1373
|
eventsDb?.close();
|
|
@@ -1155,6 +1480,11 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1155
1480
|
memoryInferenceDurationMs: durations.memoryInferenceDurationMs,
|
|
1156
1481
|
graphExtractionExtractedFiles: result.graphExtraction?.quality.extractedFiles ?? 0,
|
|
1157
1482
|
graphExtractionDurationMs: durations.graphExtractionDurationMs,
|
|
1483
|
+
// Layer-2 proactive-maintenance coverage (0 when the process is disabled
|
|
1484
|
+
// or the run was ref-scoped) so a scheduled sweep's reach is trackable.
|
|
1485
|
+
proactiveSelected: result.proactiveMaintenance?.selected ?? 0,
|
|
1486
|
+
proactiveDueTotal: result.proactiveMaintenance?.dueTotal ?? 0,
|
|
1487
|
+
proactiveNeverReflected: result.proactiveMaintenance?.neverReflected ?? 0,
|
|
1158
1488
|
// New metrics for tuning the improve loop.
|
|
1159
1489
|
...(durations.totalDurationMs !== undefined ? { durationMs: durations.totalDurationMs } : {}),
|
|
1160
1490
|
...(durations.warningCount !== undefined ? { warningCount: durations.warningCount } : {}),
|
|
@@ -1193,7 +1523,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1193
1523
|
* need, so the fix is non-invasive and provably correct.
|
|
1194
1524
|
*/
|
|
1195
1525
|
async function runConsolidationPass(args) {
|
|
1196
|
-
const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx } = args;
|
|
1526
|
+
const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx, budgetSignal, runBudgetMs } = args;
|
|
1197
1527
|
const baseConfig = options.config ?? loadConfig();
|
|
1198
1528
|
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
1199
1529
|
const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
|
|
@@ -1337,6 +1667,7 @@ async function runConsolidationPass(args) {
|
|
|
1337
1667
|
stashDir: primaryStashDir,
|
|
1338
1668
|
config: consolidationConfig,
|
|
1339
1669
|
eventsCtx,
|
|
1670
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
1340
1671
|
}, { minimumThreshold: 95 });
|
|
1341
1672
|
if (consolidateDisabledByProfile) {
|
|
1342
1673
|
info("[improve] consolidation skipped (disabled by improve profile)");
|
|
@@ -1357,7 +1688,7 @@ async function runConsolidationPass(args) {
|
|
|
1357
1688
|
info(`[improve] consolidation skipped (pool ${eligiblePoolSize} < minPoolSize ${minPoolSize})`);
|
|
1358
1689
|
}
|
|
1359
1690
|
else if (!consolidationOnCooldown) {
|
|
1360
|
-
consolidation = await akmConsolidate({
|
|
1691
|
+
consolidation = await withLlmStage("consolidate", () => akmConsolidate({
|
|
1361
1692
|
...options.consolidateOptions,
|
|
1362
1693
|
config: consolidationConfig,
|
|
1363
1694
|
stashDir: options.stashDir,
|
|
@@ -1365,14 +1696,22 @@ async function runConsolidationPass(args) {
|
|
|
1365
1696
|
// Tie consolidate proposals back to this improve invocation so
|
|
1366
1697
|
// accept-rate-per-run aggregation works. Mirrors reflect/propose/extract.
|
|
1367
1698
|
sourceRun: `consolidate-${Date.now()}`,
|
|
1368
|
-
//
|
|
1369
|
-
//
|
|
1370
|
-
//
|
|
1371
|
-
//
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1699
|
+
// Pass profile-configured options. incrementalSince narrows the pool to
|
|
1700
|
+
// recently-changed memories + graph neighbours — use this for frequent
|
|
1701
|
+
// passes (quick-shredder). Leave absent in the nightly default profile for
|
|
1702
|
+
// a full-pool sweep that catches stale-but-unmerged duplicates.
|
|
1703
|
+
incrementalSince: improveProfile?.processes?.consolidate?.incrementalSince,
|
|
1704
|
+
limit: improveProfile?.processes?.consolidate?.limit,
|
|
1705
|
+
neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
|
|
1375
1706
|
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
1707
|
+
// #617 — deterministic near-duplicate dedup pre-pass. DEFAULT OFF; only
|
|
1708
|
+
// runs when the profile explicitly sets `consolidate.dedup.enabled`.
|
|
1709
|
+
dedup: improveProfile?.processes?.consolidate?.dedup,
|
|
1710
|
+
// #581 — judged-state cache. DEFAULT OFF; only engages when the profile
|
|
1711
|
+
// explicitly sets `consolidate.judgedCache.enabled`. Skips memories
|
|
1712
|
+
// judged-unchanged since their last judge so one run sweeps the full
|
|
1713
|
+
// corpus instead of narrowing to a time-window slice.
|
|
1714
|
+
judgedCache: improveProfile?.processes?.consolidate?.judgedCache,
|
|
1376
1715
|
// Honor profile.autoAccept (already merged into options.autoAccept at the
|
|
1377
1716
|
// top of akmImprove). The CLI parser always supplies 90 when --auto-accept
|
|
1378
1717
|
// is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
|
|
@@ -1380,7 +1719,14 @@ async function runConsolidationPass(args) {
|
|
|
1380
1719
|
// options.consolidateOptions.autoAccept (if explicitly provided by caller)
|
|
1381
1720
|
// still wins because the spread above runs first.
|
|
1382
1721
|
autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
|
|
1383
|
-
|
|
1722
|
+
// WS-3a: forward budget signal for graceful abort on timeout, and pass
|
|
1723
|
+
// the profile's p90 estimate for cold-start budget reduction.
|
|
1724
|
+
signal: budgetSignal,
|
|
1725
|
+
p90ChunkSecondsDefault: improveProfile?.processes?.consolidate?.p90ChunkSecondsDefault,
|
|
1726
|
+
// WS-5: pass total run budget so perfTelemetry.estimatedBudgetFractionUsed
|
|
1727
|
+
// can flag when consolidation alone exceeded the budget.
|
|
1728
|
+
runBudgetMs,
|
|
1729
|
+
}));
|
|
1384
1730
|
{
|
|
1385
1731
|
const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
|
|
1386
1732
|
try {
|
|
@@ -1400,7 +1746,14 @@ async function runConsolidationPass(args) {
|
|
|
1400
1746
|
appendEvent({
|
|
1401
1747
|
eventType: "consolidate_completed",
|
|
1402
1748
|
ref: "memory:_consolidation",
|
|
1403
|
-
metadata: {
|
|
1749
|
+
metadata: {
|
|
1750
|
+
processed: consolidation.processed,
|
|
1751
|
+
merged: consolidation.merged,
|
|
1752
|
+
deleted: consolidation.deleted,
|
|
1753
|
+
contradicted: consolidation.contradicted,
|
|
1754
|
+
failedChunks: consolidation.failedChunks ?? 0,
|
|
1755
|
+
durationMs: consolidation.durationMs,
|
|
1756
|
+
},
|
|
1404
1757
|
}, eventsCtx);
|
|
1405
1758
|
}
|
|
1406
1759
|
}
|
|
@@ -1417,10 +1770,21 @@ async function runConsolidationPass(args) {
|
|
|
1417
1770
|
}
|
|
1418
1771
|
// D9: track whether consolidation wrote any data so graph extraction can reindex if needed
|
|
1419
1772
|
const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
|
|
1773
|
+
// WS-4: Per-phase threshold auto-tune for the consolidate phase.
|
|
1774
|
+
// Persists result for the NEXT run's makeGateConfig to read.
|
|
1775
|
+
const consolidateTuneDbPath = eventsCtx?.dbPath;
|
|
1776
|
+
if (options.autoAccept !== undefined && consolidateTuneDbPath) {
|
|
1777
|
+
try {
|
|
1778
|
+
maybeAutoTuneThreshold(consolidateGateCfg.phaseThreshold ?? options.autoAccept, consolidationConfig, consolidateTuneDbPath, undefined, "consolidate");
|
|
1779
|
+
}
|
|
1780
|
+
catch (err) {
|
|
1781
|
+
warn(`[improve] calibration auto-tune (consolidate) skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1420
1784
|
return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
1421
1785
|
}
|
|
1422
1786
|
async function runImprovePreparationStage(args) {
|
|
1423
|
-
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, } = args;
|
|
1787
|
+
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, budgetSignal, } = args;
|
|
1424
1788
|
const actions = [];
|
|
1425
1789
|
const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
|
|
1426
1790
|
// Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
|
|
@@ -1457,6 +1821,8 @@ async function runImprovePreparationStage(args) {
|
|
|
1457
1821
|
memorySummary,
|
|
1458
1822
|
improveProfile,
|
|
1459
1823
|
eventsCtx,
|
|
1824
|
+
budgetSignal,
|
|
1825
|
+
runBudgetMs: budgetMs,
|
|
1460
1826
|
});
|
|
1461
1827
|
// Phase 0.4 — session-extract pass.
|
|
1462
1828
|
//
|
|
@@ -1467,7 +1833,9 @@ async function runImprovePreparationStage(args) {
|
|
|
1467
1833
|
// / `akm feedback` invocations. Replaces the akm-plugin session-checkpoint
|
|
1468
1834
|
// hook with an on-demand pull pipeline.
|
|
1469
1835
|
//
|
|
1470
|
-
// Default-on; opt out via `
|
|
1836
|
+
// Default-on; opt out via the ACTIVE profile's `processes.extract.enabled: false`
|
|
1837
|
+
// (#593: the gate respects the resolved improve profile, not just the
|
|
1838
|
+
// hardcoded `default` profile path the legacy feature flag reads).
|
|
1471
1839
|
// Each available harness gets one call with the default --since window;
|
|
1472
1840
|
// already-seen sessions (tracked in state.db.extract_sessions_seen) are
|
|
1473
1841
|
// skipped automatically so re-runs don't burn LLM calls on unchanged data.
|
|
@@ -1487,6 +1855,7 @@ async function runImprovePreparationStage(args) {
|
|
|
1487
1855
|
stashDir: primaryStashDir,
|
|
1488
1856
|
config: extractConfig,
|
|
1489
1857
|
eventsCtx,
|
|
1858
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
1490
1859
|
});
|
|
1491
1860
|
// #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
|
|
1492
1861
|
// already done upstream; here we elide every akmExtract/processSession call)
|
|
@@ -1501,7 +1870,14 @@ async function runImprovePreparationStage(args) {
|
|
|
1501
1870
|
const EXTRACT_DEFAULT_MIN_NEW_SESSIONS = 0;
|
|
1502
1871
|
const configuredMinNewSessions = extractConfig.profiles?.improve?.default?.processes?.extract?.minNewSessions;
|
|
1503
1872
|
const minNewSessions = typeof configuredMinNewSessions === "number" ? configuredMinNewSessions : EXTRACT_DEFAULT_MIN_NEW_SESSIONS;
|
|
1504
|
-
|
|
1873
|
+
// #593/#594: the ACTIVE resolved improve profile is the single source of
|
|
1874
|
+
// truth for whether extract runs. (Previously this also ANDed in the legacy
|
|
1875
|
+
// `session_extraction` feature flag, which only reads
|
|
1876
|
+
// `profiles.improve.default.processes.extract.enabled`; that made the default
|
|
1877
|
+
// profile a global kill switch, so a non-default profile enabling extract was
|
|
1878
|
+
// silently overridden. The default profile is now just another profile.)
|
|
1879
|
+
// `akmExtract` re-checks the same active profile internally via `improveProfile`.
|
|
1880
|
+
if (resolveProcessEnabled("extract", improveProfile)) {
|
|
1505
1881
|
const availableHarnesses = options.extractHarnesses ?? getAvailableHarnesses();
|
|
1506
1882
|
// The guard engages only when minNewSessions > 0; 0 disables it entirely.
|
|
1507
1883
|
let belowMinNewSessions = false;
|
|
@@ -1532,15 +1908,18 @@ async function runImprovePreparationStage(args) {
|
|
|
1532
1908
|
extractResults = [];
|
|
1533
1909
|
for (const h of availableHarnesses) {
|
|
1534
1910
|
try {
|
|
1535
|
-
const result = await akmExtract({
|
|
1911
|
+
const result = await withLlmStage("session-extraction", () => akmExtract({
|
|
1536
1912
|
type: h.name,
|
|
1537
1913
|
...(primaryStashDir !== undefined ? { stashDir: primaryStashDir } : {}),
|
|
1538
1914
|
config: extractConfig,
|
|
1915
|
+
// Thread the ACTIVE profile so extract's internal gate + per-process
|
|
1916
|
+
// config read the running profile, not always `default`.
|
|
1917
|
+
improveProfile,
|
|
1539
1918
|
dryRun: options.dryRun ?? false,
|
|
1540
1919
|
...(options.extractHarnesses ? { harnesses: options.extractHarnesses } : {}),
|
|
1541
1920
|
// C2: pin extract's skip-tracking state.db open to the boundary path.
|
|
1542
1921
|
...(eventsCtx?.dbPath ? { stateDbPath: eventsCtx.dbPath } : {}),
|
|
1543
|
-
});
|
|
1922
|
+
}));
|
|
1544
1923
|
extractResults.push(result);
|
|
1545
1924
|
{
|
|
1546
1925
|
const gr = await runAutoAcceptGate(primaryStashDir
|
|
@@ -1631,7 +2010,13 @@ async function runImprovePreparationStage(args) {
|
|
|
1631
2010
|
const validationFailures = [];
|
|
1632
2011
|
for (const candidate of postCleanupRefs) {
|
|
1633
2012
|
try {
|
|
1634
|
-
|
|
2013
|
+
// #591: use the path pre-resolved at planning time when it is still on
|
|
2014
|
+
// disk — a serial async DB lookup per ref cost ~500 s on a 9 000-ref
|
|
2015
|
+
// stash. Fall back to findAssetFilePath only for refs that bypassed
|
|
2016
|
+
// collectEligibleRefs' index scan or whose file moved since planning.
|
|
2017
|
+
const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
|
|
2018
|
+
? candidate.filePath
|
|
2019
|
+
: await findAssetFilePath(candidate.ref, options.stashDir);
|
|
1635
2020
|
if (!filePath) {
|
|
1636
2021
|
validationFailures.push({ ref: candidate.ref, reason: "file not found on disk" });
|
|
1637
2022
|
continue;
|
|
@@ -1757,10 +2142,19 @@ async function runImprovePreparationStage(args) {
|
|
|
1757
2142
|
// refs that fail the distill signal-delta gate).
|
|
1758
2143
|
// distillOnlyRefs — reflect blocked but distill signal-delta passes
|
|
1759
2144
|
// AND ref is a distill candidate.
|
|
1760
|
-
//
|
|
1761
|
-
//
|
|
2145
|
+
// noFeedbackPool — neither signal-delta gate passes *and* the ref has
|
|
2146
|
+
// no recent feedback signal at all. These are NOT
|
|
2147
|
+
// skipped here: they are handed to the high-retrieval
|
|
2148
|
+
// fallback (P0-A) below so frequently-retrieved but
|
|
2149
|
+
// never-rated assets can still be improved. Only refs
|
|
2150
|
+
// that P0-A declines are ultimately fully skipped.
|
|
2151
|
+
// fullySkippedCount — has stale feedback but no signal delta → genuine
|
|
2152
|
+
// skip (counted, aggregated event emitted post-loop),
|
|
2153
|
+
// excluded from sort.
|
|
1762
2154
|
const eligibleRefs = [];
|
|
1763
2155
|
const distillOnlyRefs = [];
|
|
2156
|
+
// Zero-(recent-)feedback refs deferred to the P0-A high-retrieval fallback.
|
|
2157
|
+
const noFeedbackPool = [];
|
|
1764
2158
|
let fullySkippedCount = 0;
|
|
1765
2159
|
// O-2 (#365): explicit --scope <ref> bypasses every gate (user intent wins).
|
|
1766
2160
|
const scopeRefBypass = scope.mode === "ref";
|
|
@@ -1798,57 +2192,134 @@ async function runImprovePreparationStage(args) {
|
|
|
1798
2192
|
// Reflect blocked but distill passes → distill-only bucket.
|
|
1799
2193
|
distillOnlyRefs.push(r);
|
|
1800
2194
|
}
|
|
2195
|
+
else if (!latestFeedbackTs.has(r.ref)) {
|
|
2196
|
+
// Neither signal-delta gate passes AND there is no recent feedback signal
|
|
2197
|
+
// at all. Rather than skip outright, defer to the high-retrieval fallback
|
|
2198
|
+
// (P0-A) below: a never-rated-but-frequently-retrieved asset is exactly
|
|
2199
|
+
// what that path is meant to rescue. Refs P0-A declines are skipped there.
|
|
2200
|
+
noFeedbackPool.push(r);
|
|
2201
|
+
}
|
|
1801
2202
|
else {
|
|
1802
|
-
//
|
|
2203
|
+
// Has feedback on record but no signal delta since the last proposal —
|
|
2204
|
+
// genuinely fully skipped. Counted here; a single aggregated
|
|
2205
|
+
// improve_skipped event is emitted after the loop (mirrors
|
|
2206
|
+
// profile_filtered_all_passes) instead of one event per ref.
|
|
1803
2207
|
fullySkippedCount++;
|
|
1804
2208
|
actions.push({
|
|
1805
2209
|
ref: r.ref,
|
|
1806
2210
|
mode: "distill-skipped",
|
|
1807
2211
|
result: { ok: true, reason: "no new signal since last proposal" },
|
|
1808
2212
|
});
|
|
1809
|
-
appendEvent({ eventType: "improve_skipped", ref: r.ref, metadata: { reason: "no_new_signal" } }, eventsCtx);
|
|
1810
2213
|
}
|
|
1811
2214
|
}
|
|
2215
|
+
// Emit ONE aggregated skip event for the fully-skipped bucket rather than one
|
|
2216
|
+
// improve_skipped event per ref (#592 pattern, mirrors
|
|
2217
|
+
// profile_filtered_all_passes above). The per-ref loop previously produced
|
|
2218
|
+
// ~11K state.db writes per run on a large stash, the dominant contributor to
|
|
2219
|
+
// 900 s timeouts. The in-memory `actions` log keeps the per-ref detail for the
|
|
2220
|
+
// run summary; no downstream consumer needs a per-ref DB audit trail (health's
|
|
2221
|
+
// skip histogram reads the `no_new_signal` counter from the count field).
|
|
2222
|
+
if (fullySkippedCount > 0) {
|
|
2223
|
+
appendEvent({
|
|
2224
|
+
eventType: "improve_skipped",
|
|
2225
|
+
ref: undefined,
|
|
2226
|
+
metadata: {
|
|
2227
|
+
reason: "no_new_signal",
|
|
2228
|
+
count: fullySkippedCount,
|
|
2229
|
+
},
|
|
2230
|
+
}, eventsCtx);
|
|
2231
|
+
}
|
|
1812
2232
|
// ── Phase 4: signal/feedback/utility/sort on the reduced set ──────────────
|
|
1813
|
-
// Everything from here works
|
|
1814
|
-
//
|
|
1815
|
-
//
|
|
2233
|
+
// Everything from here works on (eligibleRefs ∪ distillOnlyRefs) plus the
|
|
2234
|
+
// deferred noFeedbackPool that may be rescued by the high-retrieval fallback
|
|
2235
|
+
// (P0-A). The fully-skipped bucket has already been routed and its aggregated
|
|
2236
|
+
// event emitted; we deliberately avoid spending DB/CPU on refs that the
|
|
2237
|
+
// signal-delta gate rejected with feedback already on record.
|
|
1816
2238
|
const processableRefs = [...eligibleRefs, ...distillOnlyRefs];
|
|
2239
|
+
// Refs eligible for the high-retrieval fallback (P0-A): the signal-delta
|
|
2240
|
+
// partition above could not place these in a reflect/distill bucket, but they
|
|
2241
|
+
// may still qualify if they have been retrieved often enough. Two disjoint
|
|
2242
|
+
// sources feed this set:
|
|
2243
|
+
// 1. noFeedbackPool — refs with no recent feedback that the partition loop
|
|
2244
|
+
// deliberately deferred here (otherwise they would never reach P0-A).
|
|
2245
|
+
// 2. processableRefs entries that turn out to carry no recent feedback
|
|
2246
|
+
// *signal* once feedbackSummary is computed below.
|
|
2247
|
+
// (1) is added here; (2) is folded in after feedbackSummary is built.
|
|
1817
2248
|
// Gap 6: only surface feedback signals from the last 30 days so that
|
|
1818
2249
|
// ancient one-off feedback events don't permanently lock an asset into
|
|
1819
2250
|
// every improve run. Assets with only stale signals fall through to the
|
|
1820
2251
|
// high-retrieval path (P0-A) or are skipped until new signals arrive.
|
|
1821
2252
|
// (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
|
|
1822
2253
|
// Phase 2 above for the signal-delta gate; we reuse them here.)
|
|
1823
|
-
// Pre-compute feedback summary per ref in a
|
|
1824
|
-
//
|
|
1825
|
-
//
|
|
2254
|
+
// Pre-compute feedback summary per ref in a SINGLE bulk read so we don't
|
|
2255
|
+
// open state.db once per asset (which caused 5000+ accumulated FDs and a
|
|
2256
|
+
// 2-hour runaway on a 13K-asset stash). Pattern mirrors buildLatestFeedbackTsMap
|
|
2257
|
+
// above: one readEvents() call fetches ALL feedback events, then we aggregate
|
|
2258
|
+
// in-memory by ref — O(1) DB opens regardless of candidate set size.
|
|
2259
|
+
// Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
|
|
2260
|
+
// ratios are available for any noFeedbackPool ref that P0-A rescues below.
|
|
2261
|
+
//
|
|
2262
|
+
// Behavioral note: positive/negative COUNTS are all-time (same as the old
|
|
2263
|
+
// per-ref readEvents call which had no `since` filter); hasSignal is bounded
|
|
2264
|
+
// to feedbackSinceCutoff (same as the old inline `(e.ts ?? "") >= cutoff` guard).
|
|
1826
2265
|
const feedbackSummary = new Map();
|
|
1827
|
-
|
|
1828
|
-
const
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
(
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
2266
|
+
{
|
|
2267
|
+
const feedbackCandidateSet = new Set([...processableRefs, ...noFeedbackPool].map((r) => r.ref));
|
|
2268
|
+
if (feedbackCandidateSet.size > 0) {
|
|
2269
|
+
// Fetch ALL feedback events in one query (no ref filter, no since filter =
|
|
2270
|
+
// single full table scan). Filtering per-ref in memory avoids N sequential
|
|
2271
|
+
// state.db opens — the dominant FD-leak path on large stashes.
|
|
2272
|
+
const { events: allFeedbackEvents } = readEvents({ type: "feedback" }, eventsCtx);
|
|
2273
|
+
for (const e of allFeedbackEvents) {
|
|
2274
|
+
const ref = e.ref;
|
|
2275
|
+
if (!ref || !feedbackCandidateSet.has(ref))
|
|
2276
|
+
continue;
|
|
2277
|
+
const entry = feedbackSummary.get(ref) ?? { hasSignal: false, positive: 0, negative: 0 };
|
|
2278
|
+
const meta = e.metadata;
|
|
2279
|
+
// hasSignal: only count feedback events within the 30-day window.
|
|
2280
|
+
if (!entry.hasSignal &&
|
|
2281
|
+
(e.ts ?? "") >= feedbackSinceCutoff &&
|
|
2282
|
+
meta !== undefined &&
|
|
2283
|
+
(typeof meta.signal === "string" || typeof meta.note === "string")) {
|
|
2284
|
+
entry.hasSignal = true;
|
|
2285
|
+
}
|
|
2286
|
+
// positive/negative: all-time counts (no since filter, matching prior behaviour).
|
|
2287
|
+
if (meta?.signal === "positive")
|
|
2288
|
+
entry.positive++;
|
|
2289
|
+
else if (meta?.signal === "negative")
|
|
2290
|
+
entry.negative++;
|
|
2291
|
+
feedbackSummary.set(ref, entry);
|
|
2292
|
+
}
|
|
2293
|
+
// Ensure every candidate has an entry (even refs with zero feedback events).
|
|
2294
|
+
for (const ref of feedbackCandidateSet) {
|
|
2295
|
+
if (!feedbackSummary.has(ref)) {
|
|
2296
|
+
feedbackSummary.set(ref, { hasSignal: false, positive: 0, negative: 0 });
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
1845
2300
|
}
|
|
1846
2301
|
const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
|
|
1847
2302
|
// P0-A: also surface zero-feedback assets that have been retrieved many times.
|
|
1848
2303
|
const RETRIEVAL_COUNT_THRESHOLD = options.minRetrievalCount ?? 5;
|
|
1849
2304
|
const signalBearingSet = new Set(signalFiltered.map((r) => r.ref));
|
|
1850
|
-
|
|
2305
|
+
// Zero-feedback candidates for P0-A: processableRefs without a recent signal,
|
|
2306
|
+
// plus the deferred noFeedbackPool. Dedupe by ref (the two sources are
|
|
2307
|
+
// disjoint by construction, but guard against overlap defensively).
|
|
2308
|
+
const noFeedbackSeen = new Set();
|
|
2309
|
+
const noFeedbackCandidates = [];
|
|
2310
|
+
for (const r of [...processableRefs.filter((r) => !signalBearingSet.has(r.ref)), ...noFeedbackPool]) {
|
|
2311
|
+
if (noFeedbackSeen.has(r.ref))
|
|
2312
|
+
continue;
|
|
2313
|
+
noFeedbackSeen.add(r.ref);
|
|
2314
|
+
noFeedbackCandidates.push(r);
|
|
2315
|
+
}
|
|
1851
2316
|
let highRetrievalRefs = [];
|
|
2317
|
+
// Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
|
|
2318
|
+
// proactive-maintenance selector below can reuse them without a second DB pass.
|
|
2319
|
+
// Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
|
|
2320
|
+
// step 2: recency is MANDATORY — never pinned to floor).
|
|
2321
|
+
let retrievalCounts = new Map();
|
|
2322
|
+
let lastUseMsForProactive = new Map();
|
|
1852
2323
|
let dbForRetrieval;
|
|
1853
2324
|
try {
|
|
1854
2325
|
dbForRetrieval = openExistingDatabase();
|
|
@@ -1856,15 +2327,29 @@ async function runImprovePreparationStage(args) {
|
|
|
1856
2327
|
if (showEventCount === 0) {
|
|
1857
2328
|
warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
|
|
1858
2329
|
}
|
|
1859
|
-
|
|
2330
|
+
// Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
|
|
2331
|
+
// Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
|
|
2332
|
+
// had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
|
|
2333
|
+
// to 0 regardless of actual use. Two assets of the same type — one
|
|
2334
|
+
// heavily-retrieved, one never-touched — would receive identical rankScores.
|
|
2335
|
+
// Fix (WS-1 blocker 3): union the feedback pool into the lookup.
|
|
2336
|
+
const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
|
|
2337
|
+
retrievalCounts = getRetrievalCounts(dbForRetrieval, allCandidateRefs);
|
|
2338
|
+
lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
|
|
1860
2339
|
// High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
|
|
1861
|
-
// ref qualifies exactly once — when
|
|
1862
|
-
//
|
|
1863
|
-
//
|
|
1864
|
-
//
|
|
1865
|
-
//
|
|
1866
|
-
//
|
|
1867
|
-
|
|
2340
|
+
// ref qualifies exactly once — when it has actually been retrieved
|
|
2341
|
+
// (retrievalCount ≥ 1) AND retrievalCount ≥ threshold AND no prior reflect
|
|
2342
|
+
// proposal exists for it. Once a reflect proposal is on record, subsequent
|
|
2343
|
+
// re-eligibility requires explicit feedback (which flows through the normal
|
|
2344
|
+
// signal-delta gate above). The explicit `> 0` guard keeps a threshold of 0
|
|
2345
|
+
// from rescuing genuinely never-retrieved assets — the fallback is for
|
|
2346
|
+
// *retrieved* assets, not silent ones. Tracking growth in retrieval count
|
|
2347
|
+
// would require persisting the count in proposal metadata; deferred to a
|
|
2348
|
+
// follow-up.
|
|
2349
|
+
highRetrievalRefs = noFeedbackCandidates.filter((r) => {
|
|
2350
|
+
const count = retrievalCounts.get(r.ref) ?? 0;
|
|
2351
|
+
return count > 0 && count >= RETRIEVAL_COUNT_THRESHOLD && !lastReflectProposalTs.has(r.ref);
|
|
2352
|
+
});
|
|
1868
2353
|
}
|
|
1869
2354
|
catch (err) {
|
|
1870
2355
|
rethrowIfTestIsolationError(err);
|
|
@@ -1874,6 +2359,129 @@ async function runImprovePreparationStage(args) {
|
|
|
1874
2359
|
if (dbForRetrieval)
|
|
1875
2360
|
closeDatabase(dbForRetrieval);
|
|
1876
2361
|
}
|
|
2362
|
+
// ── Layer 2: PROACTIVE MAINTENANCE SELECTOR (third eligibility source) ─────
|
|
2363
|
+
// The signal-delta gate and P0-A only surface assets with fresh feedback or a
|
|
2364
|
+
// raw-retrieval spike. Neither revisits a stable, high-value asset on a
|
|
2365
|
+
// schedule, so on a quiet stash useful assets drift stale and are never
|
|
2366
|
+
// refreshed. When the `proactiveMaintenance` process is enabled (DEFAULT OFF)
|
|
2367
|
+
// and the run is whole-stash / type scope, this selector ranks the eligible
|
|
2368
|
+
// population by a composite maintenance priority, gates on staleness ("due"),
|
|
2369
|
+
// bounds to top-N, and folds the winners into the SAME candidate set the other
|
|
2370
|
+
// two sources feed — so they flow through the existing #580 empty-diff /
|
|
2371
|
+
// cosmetic suppression and additive-distill gates. It adds no new mutation
|
|
2372
|
+
// logic of its own. The due gate doubles as the rotation cooldown: a freshly
|
|
2373
|
+
// reflected asset is excluded until it ages back past `dueDays`, so successive
|
|
2374
|
+
// runs rotate through the due pool rather than re-selecting the same heads.
|
|
2375
|
+
let proactiveRefs = [];
|
|
2376
|
+
let proactiveMaintenanceSummary;
|
|
2377
|
+
const proactiveEnabled = scope.mode !== "ref" && resolveProcessEnabled("proactiveMaintenance", improveProfile);
|
|
2378
|
+
if (proactiveEnabled) {
|
|
2379
|
+
const pmCfg = improveProfile.processes?.proactiveMaintenance;
|
|
2380
|
+
const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
|
|
2381
|
+
const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
|
|
2382
|
+
// Candidate population: the zero-feedback / non-signal pool — exactly the
|
|
2383
|
+
// assets the other two sources would NOT pick this run. Exclude any P0-A
|
|
2384
|
+
// rescued this run so we never double-select the same ref.
|
|
2385
|
+
const alreadySelected = new Set(highRetrievalRefs.map((r) => r.ref));
|
|
2386
|
+
const pmCandidates = noFeedbackCandidates.filter((r) => !alreadySelected.has(r.ref));
|
|
2387
|
+
const selection = selectProactiveMaintenanceRefs({
|
|
2388
|
+
candidates: pmCandidates,
|
|
2389
|
+
lastReflectTs: lastReflectProposalTs,
|
|
2390
|
+
lastDistillTs: lastDistillProposalTs,
|
|
2391
|
+
retrievalCounts,
|
|
2392
|
+
// WS-1: wire lastUseMs so the recency decay term is genuine (plan §step 2).
|
|
2393
|
+
lastUseMs: lastUseMsForProactive,
|
|
2394
|
+
sizeBytesOf: (r) => {
|
|
2395
|
+
const fp = r.filePath;
|
|
2396
|
+
if (!fp)
|
|
2397
|
+
return undefined;
|
|
2398
|
+
try {
|
|
2399
|
+
return fs.statSync(fp).size;
|
|
2400
|
+
}
|
|
2401
|
+
catch {
|
|
2402
|
+
return undefined;
|
|
2403
|
+
}
|
|
2404
|
+
},
|
|
2405
|
+
dueDays,
|
|
2406
|
+
maxPerRun,
|
|
2407
|
+
});
|
|
2408
|
+
proactiveRefs = selection.selected;
|
|
2409
|
+
proactiveMaintenanceSummary = {
|
|
2410
|
+
selected: selection.selected.length,
|
|
2411
|
+
dueTotal: selection.dueTotal,
|
|
2412
|
+
neverReflected: selection.neverReflected,
|
|
2413
|
+
};
|
|
2414
|
+
// Aggregated observability event (never per-ref — avoids the event flood the
|
|
2415
|
+
// Layer-1 work eliminated). Mirrors the `no_new_signal` aggregation pattern.
|
|
2416
|
+
appendEvent({
|
|
2417
|
+
eventType: "proactive_selected",
|
|
2418
|
+
ref: undefined,
|
|
2419
|
+
metadata: {
|
|
2420
|
+
count: selection.selected.length,
|
|
2421
|
+
dueTotal: selection.dueTotal,
|
|
2422
|
+
neverReflected: selection.neverReflected,
|
|
2423
|
+
},
|
|
2424
|
+
}, eventsCtx);
|
|
2425
|
+
if (selection.selected.length > 0) {
|
|
2426
|
+
info(`[improve] proactive maintenance selected ${selection.selected.length}/${selection.dueTotal} due refs ` +
|
|
2427
|
+
`(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
// ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
|
|
2431
|
+
// Zero-feedback refs whose encoding_salience (set at distill time by
|
|
2432
|
+
// scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
|
|
2433
|
+
// into the improve run even without retrieval or feedback signal. This rescues
|
|
2434
|
+
// newly distilled assets that the stash has not yet surfaced to users.
|
|
2435
|
+
//
|
|
2436
|
+
// Cap: at most 10% of the effective run limit so the lane cannot crowd out
|
|
2437
|
+
// reactive feedback. Requires state.db to have an asset_salience row — refs
|
|
2438
|
+
// without a row (pre-#608 assets still on the type-weight stub) are skipped.
|
|
2439
|
+
const highSalienceRefs = [];
|
|
2440
|
+
const salienceCfg = (options.config ?? loadConfig()).improve?.salience;
|
|
2441
|
+
const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
|
|
2442
|
+
const proactiveAndRetrievalSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
|
|
2443
|
+
{
|
|
2444
|
+
let dbForHighSalience;
|
|
2445
|
+
try {
|
|
2446
|
+
dbForHighSalience = openStateDatabase(eventsCtx?.dbPath);
|
|
2447
|
+
const effectiveLimit = options.limit ?? 10;
|
|
2448
|
+
const highSalienceCap = Math.max(1, Math.floor(effectiveLimit * 0.1));
|
|
2449
|
+
const candidates = noFeedbackCandidates.filter((r) => !proactiveAndRetrievalSet.has(r.ref));
|
|
2450
|
+
for (const r of candidates) {
|
|
2451
|
+
if (highSalienceRefs.length >= highSalienceCap)
|
|
2452
|
+
break;
|
|
2453
|
+
const row = getAssetSalience(dbForHighSalience, r.ref);
|
|
2454
|
+
if (row && row.encoding_salience >= salienceThreshold) {
|
|
2455
|
+
highSalienceRefs.push(r);
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2459
|
+
catch (err) {
|
|
2460
|
+
rethrowIfTestIsolationError(err);
|
|
2461
|
+
// best-effort: if DB unavailable, highSalienceRefs stays empty
|
|
2462
|
+
}
|
|
2463
|
+
finally {
|
|
2464
|
+
if (dbForHighSalience)
|
|
2465
|
+
dbForHighSalience.close();
|
|
2466
|
+
}
|
|
2467
|
+
}
|
|
2468
|
+
// Record an in-memory skip action for every zero-feedback ref that the
|
|
2469
|
+
// partition loop deferred to P0-A but P0-A then declined (retrievalCount below
|
|
2470
|
+
// threshold, or a prior reflect proposal already on record). These never make
|
|
2471
|
+
// it into mergedRefs, so without this they would silently vanish from the run
|
|
2472
|
+
// summary. No DB event is written here — these refs carry no signal at all, so
|
|
2473
|
+
// there is nothing for the skip histogram to aggregate; the action log alone
|
|
2474
|
+
// preserves the per-ref audit trail (mirrors the fully-skipped action above).
|
|
2475
|
+
const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs, ...highSalienceRefs].map((r) => r.ref));
|
|
2476
|
+
for (const r of noFeedbackPool) {
|
|
2477
|
+
if (rescuedSet.has(r.ref))
|
|
2478
|
+
continue;
|
|
2479
|
+
actions.push({
|
|
2480
|
+
ref: r.ref,
|
|
2481
|
+
mode: "distill-skipped",
|
|
2482
|
+
result: { ok: true, reason: "no new signal since last proposal" },
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
1877
2485
|
// If the user explicitly scoped to a single ref, always act on it —
|
|
1878
2486
|
// skip the signal/retrieval filter entirely. The filter exists to avoid
|
|
1879
2487
|
// noisy "improve everything" runs; it should not gate an intentional
|
|
@@ -1883,39 +2491,622 @@ async function runImprovePreparationStage(args) {
|
|
|
1883
2491
|
// or sufficient retrievals). A stash with no signals has 0 eligible refs —
|
|
1884
2492
|
// usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
|
|
1885
2493
|
// to bring them into the eligible pool.
|
|
1886
|
-
|
|
1887
|
-
|
|
2494
|
+
// Layer-2 proactive refs join the eligible set alongside feedback-signal and
|
|
2495
|
+
// high-retrieval (P0-A) refs. The four sources are disjoint by construction
|
|
2496
|
+
// (proactive draws from noFeedbackCandidates with the P0-A picks removed, and
|
|
2497
|
+
// high-salience draws from the remainder), but dedupe defensively so a ref can
|
|
2498
|
+
// never enter the loop twice. `requireFeedbackSignal` still suppresses all
|
|
2499
|
+
// fallback sources for callers that want feedback-only runs.
|
|
2500
|
+
const signalAndRetrievalRefs = dedupeRefs([
|
|
2501
|
+
...signalFiltered,
|
|
2502
|
+
...highRetrievalRefs,
|
|
2503
|
+
...proactiveRefs,
|
|
2504
|
+
...highSalienceRefs,
|
|
2505
|
+
]);
|
|
2506
|
+
let mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
|
|
2507
|
+
// ── Attribution tagging: stamp each ref with the eligibility lane that
|
|
2508
|
+
// selected it ──────────────────────────────────────────────────────────────
|
|
2509
|
+
// Every reflect/distill proposal must record WHICH lane chose its source asset
|
|
2510
|
+
// so downstream accept/reject/revert/retrieval outcomes can be sliced by lane
|
|
2511
|
+
// (does the PROACTIVE lane produce value vs the reactive lanes?). We build the
|
|
2512
|
+
// lane map here — the one place all four lanes are known — and stamp it onto
|
|
2513
|
+
// each ImproveEligibleRef object. Because the ref objects are shared by
|
|
2514
|
+
// reference across buckets, the stamp travels with the ref through the sort,
|
|
2515
|
+
// disk-check, and loop stages down to the reflect/distill event emit sites and
|
|
2516
|
+
// createProposal calls. See EligibilitySource for the lane vocabulary.
|
|
2517
|
+
//
|
|
2518
|
+
// Precedence (prefer the most specific reactive signal):
|
|
2519
|
+
// scope > signal-delta > high-retrieval > proactive > high-salience
|
|
2520
|
+
// A ref with real feedback is attributed to feedback even if it was also due
|
|
2521
|
+
// for proactive maintenance or had high encoding salience. We apply lanes
|
|
2522
|
+
// weakest-first so the strongest overwrites; the explicit --scope <ref> bypass
|
|
2523
|
+
// wins outright (user intent).
|
|
2524
|
+
const eligibilitySourceByRef = new Map();
|
|
2525
|
+
for (const r of highSalienceRefs)
|
|
2526
|
+
eligibilitySourceByRef.set(r.ref, "high-salience");
|
|
2527
|
+
for (const r of proactiveRefs)
|
|
2528
|
+
eligibilitySourceByRef.set(r.ref, "proactive");
|
|
2529
|
+
for (const r of highRetrievalRefs)
|
|
2530
|
+
eligibilitySourceByRef.set(r.ref, "high-retrieval");
|
|
2531
|
+
for (const r of signalFiltered)
|
|
2532
|
+
eligibilitySourceByRef.set(r.ref, "signal-delta");
|
|
2533
|
+
if (scope.mode === "ref") {
|
|
2534
|
+
// O-2 (#365): explicit --scope <ref> bypass — every ref in processableRefs
|
|
2535
|
+
// arrived via the scopeRefBypass branch, so attribute the whole set to scope.
|
|
2536
|
+
for (const r of processableRefs)
|
|
2537
|
+
eligibilitySourceByRef.set(r.ref, "scope");
|
|
2538
|
+
}
|
|
2539
|
+
for (const r of mergedRefs) {
|
|
2540
|
+
// "unknown" is a genuine fallback, never a silent alias for signal-delta:
|
|
2541
|
+
// only refs we truly cannot attribute land here (none in practice, since
|
|
2542
|
+
// mergedRefs is always a subset of the four lanes above).
|
|
2543
|
+
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
2544
|
+
}
|
|
2545
|
+
// WS-1 — Unified salience vector (S1 seam).
|
|
2546
|
+
//
|
|
2547
|
+
// The legacy sort combined three independent formulas (utility EMA, negative-only
|
|
2548
|
+
// ratio / symmetric-valence magnitude, and the proactive-maintenance priority
|
|
2549
|
+
// formula). WS-1 converges them into one `computeSalience()` call per ref, with
|
|
2550
|
+
// three independently-stored sub-scores and one documented rankScore projection.
|
|
2551
|
+
//
|
|
2552
|
+
// Migration note: if a profile still has `symmetricValence` set, emit a one-time
|
|
2553
|
+
// warning — its behaviour (symmetric |valence| attention) is now always-on as
|
|
2554
|
+
// part of the salience vector, so the knob is a no-op and will be removed in 0.10.
|
|
2555
|
+
if (improveProfile.symmetricValence === true) {
|
|
2556
|
+
warn("[improve] Profile option 'symmetricValence' is deprecated (WS-1 salience vector). " +
|
|
2557
|
+
"Symmetric valence is now always active; remove the option from your improve profile.");
|
|
2558
|
+
}
|
|
2559
|
+
// Fetch last-use timestamps from the index DB for the full merged set so the
|
|
2560
|
+
// recency term in retrievalSalience is genuinely decayable (plan §WS-1 step 2).
|
|
2561
|
+
// This reuses the index DB opened earlier for retrieval counts; a separate
|
|
2562
|
+
// lightweight open is used here to avoid holding the connection longer than needed.
|
|
2563
|
+
let lastUseMsByRef = new Map();
|
|
2564
|
+
// utilityMap is kept for backward-compatible observability (health report reads it).
|
|
1888
2565
|
const utilityMap = buildUtilityMap(mergedRefs);
|
|
1889
|
-
|
|
1890
|
-
const feedbackRatios = new Map();
|
|
1891
|
-
for (const ref of mergedRefs) {
|
|
1892
|
-
const summary = feedbackSummary.get(ref.ref);
|
|
1893
|
-
const positive = summary?.positive ?? 0;
|
|
1894
|
-
const negative = summary?.negative ?? 0;
|
|
1895
|
-
const total = positive + negative;
|
|
1896
|
-
// ratio = negative proportion (high = needs more improvement)
|
|
1897
|
-
feedbackRatios.set(ref.ref, total > 0 ? negative / total : 0);
|
|
1898
|
-
}
|
|
1899
|
-
// Sort: combine utility (desc) with feedback negativity (desc) — high-negative assets rank higher
|
|
1900
|
-
const sorted = [...mergedRefs].sort((a, b) => {
|
|
1901
|
-
const utilA = utilityMap.get(a.ref) ?? 0;
|
|
1902
|
-
const utilB = utilityMap.get(b.ref) ?? 0;
|
|
1903
|
-
const ratioA = feedbackRatios.get(a.ref) ?? 0;
|
|
1904
|
-
const ratioB = feedbackRatios.get(b.ref) ?? 0;
|
|
1905
|
-
// Combined score: 70% utility, 30% negative ratio
|
|
1906
|
-
const scoreA = utilA * 0.7 + ratioA * 0.3;
|
|
1907
|
-
const scoreB = utilB * 0.7 + ratioB * 0.3;
|
|
1908
|
-
return scoreB - scoreA;
|
|
1909
|
-
});
|
|
1910
|
-
// Phase 0: surface coverage gaps from zero-result search queries
|
|
1911
|
-
let coverageGaps = [];
|
|
2566
|
+
let dbForSalience;
|
|
1912
2567
|
try {
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
2568
|
+
dbForSalience = openExistingDatabase();
|
|
2569
|
+
lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs.map((r) => r.ref));
|
|
2570
|
+
}
|
|
2571
|
+
catch (err) {
|
|
2572
|
+
rethrowIfTestIsolationError(err);
|
|
2573
|
+
// best-effort: if DB unavailable, recency term stays at floor (lastUseMs=0)
|
|
2574
|
+
}
|
|
2575
|
+
finally {
|
|
2576
|
+
if (dbForSalience)
|
|
2577
|
+
closeDatabase(dbForSalience);
|
|
2578
|
+
}
|
|
2579
|
+
// ── WS-2 Outcome loop ─────────────────────────────────────────────────────
|
|
2580
|
+
//
|
|
2581
|
+
// Update asset_outcome for every ref in the merged set BEFORE computing the
|
|
2582
|
+
// salience vector so the updated outcome_score feeds outcomeSalience this run.
|
|
2583
|
+
//
|
|
2584
|
+
// Inputs per ref:
|
|
2585
|
+
// - currentRetrievalCount: from retrievalCounts (index DB)
|
|
2586
|
+
// - lastRetrievedAt: from lastUseMsByRef (utility_scores.last_used_at)
|
|
2587
|
+
// - negativeFeedbackCount: cumulative negatives from feedbackSummary
|
|
2588
|
+
// - acceptedChangeCount: accepted proposals for this ref (state.db)
|
|
2589
|
+
// - valence: net valence from computeValenceScore(feedbackSummary.get(ref))
|
|
2590
|
+
// - utilityScore: from utilityMap (for warm-start seed on new rows)
|
|
2591
|
+
//
|
|
2592
|
+
// Best-effort: outcome failures never block the salience or ranking pass.
|
|
2593
|
+
const outcomeSalienceByRef = new Map();
|
|
2594
|
+
try {
|
|
2595
|
+
const outcomeDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
|
|
2596
|
+
const ownsOutcomeDb = !eventsCtx?.db;
|
|
2597
|
+
try {
|
|
2598
|
+
// Count accepted proposals per ref in one pass (avoid N separate queries).
|
|
2599
|
+
// Scoped to primaryStashDir when available so multi-stash installs don't
|
|
2600
|
+
// inflate counts with proposals from other stashes.
|
|
2601
|
+
const acceptedCountByRef = new Map();
|
|
2602
|
+
try {
|
|
2603
|
+
const acceptedProposals = listStateProposals(outcomeDb, {
|
|
2604
|
+
status: "accepted",
|
|
2605
|
+
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
2606
|
+
});
|
|
2607
|
+
for (const p of acceptedProposals) {
|
|
2608
|
+
acceptedCountByRef.set(p.ref, (acceptedCountByRef.get(p.ref) ?? 0) + 1);
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
catch {
|
|
2612
|
+
// best-effort: if proposals query fails, accepted counts stay at 0
|
|
2613
|
+
}
|
|
2614
|
+
// Update each ref's outcome row and collect the resulting outcome scores.
|
|
2615
|
+
const rawOutcomeScores = new Map();
|
|
2616
|
+
const nowForOutcome = Date.now();
|
|
2617
|
+
for (const r of mergedRefs) {
|
|
2618
|
+
const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
|
|
2619
|
+
const valenceResult = computeValenceScore(fb);
|
|
2620
|
+
try {
|
|
2621
|
+
const result = updateAssetOutcome(outcomeDb, {
|
|
2622
|
+
ref: r.ref,
|
|
2623
|
+
currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
|
|
2624
|
+
lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
|
|
2625
|
+
acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
|
|
2626
|
+
negativeFeedbackCount: fb.negative,
|
|
2627
|
+
valence: valenceResult.valence,
|
|
2628
|
+
utilityScore: utilityMap.get(r.ref),
|
|
2629
|
+
now: nowForOutcome,
|
|
2630
|
+
});
|
|
2631
|
+
rawOutcomeScores.set(r.ref, result.outcomeScore);
|
|
2632
|
+
}
|
|
2633
|
+
catch {
|
|
2634
|
+
// best-effort per-ref: skip this ref's outcome update on failure
|
|
2635
|
+
}
|
|
2636
|
+
}
|
|
2637
|
+
// Compute stash-wide max outcome_score for normalisation (diversity floor).
|
|
2638
|
+
// Read ALL rows (not just this run's batch) so the normalisation is
|
|
2639
|
+
// stash-relative, not pool-relative.
|
|
2640
|
+
let maxOutcomeScore = 0;
|
|
2641
|
+
try {
|
|
2642
|
+
const allOutcomes = getAllAssetOutcomes(outcomeDb);
|
|
2643
|
+
for (const row of allOutcomes) {
|
|
2644
|
+
if (row.outcome_score > maxOutcomeScore)
|
|
2645
|
+
maxOutcomeScore = row.outcome_score;
|
|
2646
|
+
}
|
|
2647
|
+
// Proxy-adequacy tripwire: emit a health event if outcome_score is
|
|
2648
|
+
// negatively correlated with accepted_change_rate (inverted proxy).
|
|
2649
|
+
const adequacy = computeProxyAdequacy(allOutcomes);
|
|
2650
|
+
if (adequacy.isInverted) {
|
|
2651
|
+
appendEvent({
|
|
2652
|
+
eventType: "outcome_proxy_inverted",
|
|
2653
|
+
ref: undefined,
|
|
2654
|
+
metadata: {
|
|
2655
|
+
correlation: adequacy.correlation,
|
|
2656
|
+
n: adequacy.n,
|
|
2657
|
+
note: "corr(outcome_score, accepted_change_rate) < −0.3: high-outcome_score assets have LOW accepted-change rates — the proxy's 'doing well' signal is inverted, so the coarse retrieval-delta signal is no longer trustworthy and the 0.10+ rich in-session signal is no longer deferrable. See plan §WS-2 proxy-adequacy tripwire.",
|
|
2658
|
+
},
|
|
2659
|
+
}, eventsCtx);
|
|
2660
|
+
}
|
|
2661
|
+
}
|
|
2662
|
+
catch {
|
|
2663
|
+
// best-effort: tripwire failure never blocks ranking
|
|
2664
|
+
}
|
|
2665
|
+
// Convert raw outcome scores → normalised outcomeSalience values in [0,1].
|
|
2666
|
+
for (const [ref, score] of rawOutcomeScores) {
|
|
2667
|
+
const normalised = outcomeScoreToSalience(score, maxOutcomeScore);
|
|
2668
|
+
outcomeSalienceByRef.set(ref, normalised);
|
|
2669
|
+
}
|
|
2670
|
+
// Also fetch outcome scores for refs NOT updated this run (stale or absent)
|
|
2671
|
+
// so the outcomeSalience read path works for all refs in the batch.
|
|
2672
|
+
const missingRefs = mergedRefs.map((r) => r.ref).filter((ref) => !rawOutcomeScores.has(ref));
|
|
2673
|
+
if (missingRefs.length > 0) {
|
|
2674
|
+
const storedScores = getOutcomeScoresByRef(outcomeDb, missingRefs);
|
|
2675
|
+
for (const [ref, score] of storedScores) {
|
|
2676
|
+
outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
|
|
2677
|
+
}
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
finally {
|
|
2681
|
+
if (ownsOutcomeDb)
|
|
2682
|
+
outcomeDb.close();
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
catch (err) {
|
|
2686
|
+
rethrowIfTestIsolationError(err);
|
|
2687
|
+
// best-effort: outcome failures never block salience computation
|
|
2688
|
+
}
|
|
2689
|
+
// Compute the salience vector for every ref in the merged set.
|
|
2690
|
+
// retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
|
|
2691
|
+
// so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
|
|
2692
|
+
// outcomeSalienceByRef is populated by WS-2 above (or empty on first run).
|
|
2693
|
+
//
|
|
2694
|
+
// Part-V gate: read the operator opt-in flag from config. Default false
|
|
2695
|
+
// (WS-1 parity weights) until the maintainer runs scripts/akm-eval and sets
|
|
2696
|
+
// improve.salience.outcomeWeightEnabled: true in the config.
|
|
2697
|
+
const salienceConfig = (options.config ?? loadConfig()).improve?.salience;
|
|
2698
|
+
const outcomeWeightEnabled = salienceConfig?.outcomeWeightEnabled === true;
|
|
2699
|
+
const salienceMap = new Map();
|
|
2700
|
+
const nowForSalience = Date.now();
|
|
2701
|
+
for (const r of mergedRefs) {
|
|
2702
|
+
const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
|
|
2703
|
+
const sizeBytes = (() => {
|
|
2704
|
+
const fp = r.filePath;
|
|
2705
|
+
if (!fp)
|
|
2706
|
+
return undefined;
|
|
2707
|
+
try {
|
|
2708
|
+
return fs.statSync(fp).size;
|
|
2709
|
+
}
|
|
2710
|
+
catch {
|
|
2711
|
+
return undefined;
|
|
2712
|
+
}
|
|
2713
|
+
})();
|
|
2714
|
+
const vector = computeSalience({
|
|
2715
|
+
ref: r.ref,
|
|
2716
|
+
type,
|
|
2717
|
+
retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
|
|
2718
|
+
lastUseMs: lastUseMsByRef.get(r.ref),
|
|
2719
|
+
utilityScore: utilityMap.get(r.ref),
|
|
2720
|
+
outcomeSalience: outcomeSalienceByRef.get(r.ref),
|
|
2721
|
+
sizeBytes,
|
|
2722
|
+
now: nowForSalience,
|
|
2723
|
+
outcomeWeightEnabled,
|
|
2724
|
+
});
|
|
2725
|
+
salienceMap.set(r.ref, vector);
|
|
2726
|
+
}
|
|
2727
|
+
// Persist salience vectors to state.db (best-effort, non-blocking).
|
|
2728
|
+
// The canonical store enables WS-3 homeostatic demotion and WS-2 outcome reads.
|
|
2729
|
+
//
|
|
2730
|
+
// Forgetting-safety report (plan §WS-1 step 7) — stash-wide rank comparison:
|
|
2731
|
+
//
|
|
2732
|
+
// BEFORE persisting the new rankScores, read ALL existing rows from state.db
|
|
2733
|
+
// (not just the per-run candidate pool). This gives stash-wide rank positions so
|
|
2734
|
+
// the top-200/below-500 thresholds are meaningful.
|
|
2735
|
+
//
|
|
2736
|
+
// Two distinct scenarios:
|
|
2737
|
+
//
|
|
2738
|
+
// A. First WS-1 run (table empty): the old stash-wide combinedEligibilityScore
|
|
2739
|
+
// ordering was never persisted in state.db (asset_salience is a new WS-1 table).
|
|
2740
|
+
// However, the old formula's inputs are available in-scope for every candidate
|
|
2741
|
+
// in the current pool: utility comes from utilityMap and the attention term
|
|
2742
|
+
// from feedbackSummary (positive/negative counts). We reconstruct the old
|
|
2743
|
+
// combinedEligibilityScore = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT
|
|
2744
|
+
// for every ref in salienceMap and rank them, giving a candidate-pool-scoped
|
|
2745
|
+
// old ordering. This is a partial reconstruction (only current-pool refs, not
|
|
2746
|
+
// stash-wide), but it is the most faithful comparison possible at cutover and
|
|
2747
|
+
// allows the top-200→below-500 forgetting guard to fire if the formula change
|
|
2748
|
+
// dramatically reorders the candidate pool.
|
|
2749
|
+
// See docs/design/improve-reconciliation-plan.md §WS-1 step 7 — the stash-wide
|
|
2750
|
+
// ordering was unreconstructable (no prior state.db snapshot), so this candidate-
|
|
2751
|
+
// pool partial reconstruction is the documented resolution for the first-run case.
|
|
2752
|
+
// Emit `improve_salience_first_run` to mark the cutover moment and include the
|
|
2753
|
+
// reconstructed comparison result in the metadata.
|
|
2754
|
+
//
|
|
2755
|
+
// B. Subsequent runs (table has rows): use ALL existing rows as old ranks, merge
|
|
2756
|
+
// them with the current run's salienceMap updates for new ranks, and call
|
|
2757
|
+
// buildRankChangeReport with stash-wide positions. This detects real rank drift
|
|
2758
|
+
// — e.g. a retrieval-pattern shift causing a previously top-200 asset to slip
|
|
2759
|
+
// below position 500.
|
|
2760
|
+
//
|
|
2761
|
+
// Measurement-protocol deferral (plan §269, Part-V):
|
|
2762
|
+
// The Part-V T0 baseline (scripts/akm-eval + health report) and the throughput/
|
|
2763
|
+
// quality gate are deferred pending owner sign-off. Full measurement requires a
|
|
2764
|
+
// before/after `akm health` report. Owner-acknowledged deferral: WS-2 landing
|
|
2765
|
+
// will re-introduce outcome salience and trigger the full re-tuning pass at that
|
|
2766
|
+
// time. salience.ts already accepts outcomeSalience directly as an input
|
|
2767
|
+
// (see SalienceInputs.outcomeSalience); no separate hook is needed.
|
|
2768
|
+
//
|
|
2769
|
+
// Forgetting-safety collection: populated inside scenario B below, consumed
|
|
2770
|
+
// after the try/catch to union candidates into mergedRefs before the sort.
|
|
2771
|
+
// Only refs from a real pre-existing ordering (scenario B) are collected;
|
|
2772
|
+
// empty on scenario A or when no candidates dropped below the threshold.
|
|
2773
|
+
let pendingForgettingRefs = [];
|
|
2774
|
+
try {
|
|
2775
|
+
const stateDb = openStateDatabase(eventsCtx?.dbPath);
|
|
2776
|
+
try {
|
|
2777
|
+
// Step 7: stash-wide rank-change report BEFORE overwriting the table.
|
|
2778
|
+
//
|
|
2779
|
+
// Load ALL existing rows so rank positions are stash-relative, not pool-relative.
|
|
2780
|
+
const existingAllScores = getAllRankScores(stateDb);
|
|
2781
|
+
if (existingAllScores.size === 0) {
|
|
2782
|
+
// Scenario A: first WS-1 run — table empty.
|
|
2783
|
+
//
|
|
2784
|
+
// Reconstruct the old combinedEligibilityScore ordering for the current
|
|
2785
|
+
// candidate pool using inputs that are already in-scope: utility from
|
|
2786
|
+
// utilityMap and the attention term from feedbackSummary (positive/negative
|
|
2787
|
+
// counts). Old formula: score = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT.
|
|
2788
|
+
//
|
|
2789
|
+
// Limitation: this covers only the current-run candidate pool, not the full
|
|
2790
|
+
// stash. The stash-wide ordering was never persisted (asset_salience is a new
|
|
2791
|
+
// WS-1 table), so this is the most faithful comparison possible at cutover.
|
|
2792
|
+
// See docs/design/improve-reconciliation-plan.md §WS-1 step 7.
|
|
2793
|
+
const reconstructedOldScores = new Map();
|
|
2794
|
+
for (const ref of salienceMap.keys()) {
|
|
2795
|
+
const utility = utilityMap.get(ref) ?? 0;
|
|
2796
|
+
const fb = feedbackSummary.get(ref) ?? { positive: 0, negative: 0 };
|
|
2797
|
+
const attention = computeValenceScore(fb).attention;
|
|
2798
|
+
reconstructedOldScores.set(ref, utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT);
|
|
2799
|
+
}
|
|
2800
|
+
// Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
|
|
2801
|
+
const toRanks = (scores) => {
|
|
2802
|
+
const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
|
|
2803
|
+
return new Map(sorted.map(([ref], i) => [ref, i + 1]));
|
|
2804
|
+
};
|
|
2805
|
+
const oldRanks = toRanks(reconstructedOldScores);
|
|
2806
|
+
const newRanks = toRanks(new Map([...salienceMap.entries()].map(([ref, v]) => [ref, v.rankScore])));
|
|
2807
|
+
const firstRunReport = buildRankChangeReport(oldRanks, newRanks);
|
|
2808
|
+
if (firstRunReport.forgettingCandidates.length > 0) {
|
|
2809
|
+
warn(`[improve/salience] WS-1 first-run rank-change report: ${firstRunReport.forgettingCandidates.length} asset(s) fell from top-200 to below position 500 (cutover formula change). ` +
|
|
2810
|
+
`Top drops: ${firstRunReport.forgettingCandidates
|
|
2811
|
+
.slice(0, 5)
|
|
2812
|
+
.map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
|
|
2813
|
+
.join(", ")}`);
|
|
2814
|
+
pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
|
|
2815
|
+
}
|
|
2816
|
+
appendEvent({
|
|
2817
|
+
eventType: "improve_salience_first_run",
|
|
2818
|
+
ref: undefined,
|
|
2819
|
+
metadata: {
|
|
2820
|
+
candidateCount: salienceMap.size,
|
|
2821
|
+
note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); see improve-reconciliation-plan.md §WS-1 step 7",
|
|
2822
|
+
forgettingCandidates: firstRunReport.forgettingCandidates.length,
|
|
2823
|
+
topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
|
|
2824
|
+
ref: e.ref,
|
|
2825
|
+
oldRank: e.oldRank,
|
|
2826
|
+
newRank: e.newRank,
|
|
2827
|
+
})),
|
|
2828
|
+
},
|
|
2829
|
+
}, eventsCtx);
|
|
2830
|
+
}
|
|
2831
|
+
else {
|
|
2832
|
+
// Scenario B: subsequent run — compare stash-wide old vs. new ranks.
|
|
2833
|
+
//
|
|
2834
|
+
// Build new scores by merging the full table with this run's updates.
|
|
2835
|
+
// Refs in salienceMap override their stored value; refs not in this run
|
|
2836
|
+
// retain their stored value unchanged. This gives a complete stash-wide
|
|
2837
|
+
// picture of what the new ordering looks like after this run.
|
|
2838
|
+
const mergedNewScores = new Map(existingAllScores);
|
|
2839
|
+
for (const [ref, vector] of salienceMap) {
|
|
2840
|
+
mergedNewScores.set(ref, vector.rankScore);
|
|
2841
|
+
}
|
|
2842
|
+
// Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
|
|
2843
|
+
const toRanks = (scores) => {
|
|
2844
|
+
const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
|
|
2845
|
+
return new Map(sorted.map(([ref], i) => [ref, i + 1]));
|
|
2846
|
+
};
|
|
2847
|
+
const oldRanks = toRanks(existingAllScores);
|
|
2848
|
+
const newRanks = toRanks(mergedNewScores);
|
|
2849
|
+
const report = buildRankChangeReport(oldRanks, newRanks);
|
|
2850
|
+
if (report.forgettingCandidates.length > 0) {
|
|
2851
|
+
warn(`[improve/salience] WS-1 rank-change report: ${report.forgettingCandidates.length} asset(s) fell from top-200 to below position 500. ` +
|
|
2852
|
+
`Top drops: ${report.forgettingCandidates
|
|
2853
|
+
.slice(0, 5)
|
|
2854
|
+
.map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
|
|
2855
|
+
.join(", ")}`);
|
|
2856
|
+
// Collect refs for protective consolidation pass (plan §WS-1 step 7).
|
|
2857
|
+
// These are force-included in the candidate pool (mergedRefs) after
|
|
2858
|
+
// this try block, bypassing cooldown/signal-delta gating.
|
|
2859
|
+
pendingForgettingRefs = report.forgettingCandidates.map((e) => e.ref);
|
|
2860
|
+
}
|
|
2861
|
+
appendEvent({
|
|
2862
|
+
eventType: "improve_salience_rank_change",
|
|
2863
|
+
ref: undefined,
|
|
2864
|
+
metadata: {
|
|
2865
|
+
stashSize: existingAllScores.size,
|
|
2866
|
+
totalChanged: report.allChanges.length,
|
|
2867
|
+
forgettingCandidates: report.forgettingCandidates.length,
|
|
2868
|
+
topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
|
|
2869
|
+
ref: e.ref,
|
|
2870
|
+
oldRank: e.oldRank,
|
|
2871
|
+
newRank: e.newRank,
|
|
2872
|
+
})),
|
|
2873
|
+
},
|
|
2874
|
+
}, eventsCtx);
|
|
2875
|
+
}
|
|
2876
|
+
for (const [ref, vector] of salienceMap) {
|
|
2877
|
+
upsertAssetSalience(stateDb, ref, vector, nowForSalience);
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
finally {
|
|
2881
|
+
stateDb.close();
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
catch (err) {
|
|
2885
|
+
rethrowIfTestIsolationError(err);
|
|
2886
|
+
// best-effort: salience persistence failure never blocks ranking
|
|
2887
|
+
}
|
|
2888
|
+
// ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
|
|
2889
|
+
// Forgetting candidates detected in scenario B are force-injected into
|
|
2890
|
+
// mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
|
|
2891
|
+
// signal-delta gating. Any ref already present in mergedRefs keeps its
|
|
2892
|
+
// existing eligibilitySource (stronger reactive signals win); refs not yet in
|
|
2893
|
+
// the pool are synthesised as minimal ImproveEligibleRef stubs and labelled
|
|
2894
|
+
// 'forgetting-safety' so S5/WS-5 can slice by lane. The dedupeRefs call
|
|
2895
|
+
// ensures no ref can enter the loop twice.
|
|
2896
|
+
if (pendingForgettingRefs.length > 0 && scope.mode !== "ref") {
|
|
2897
|
+
const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
|
|
2898
|
+
const newForgettingRefs = [];
|
|
2899
|
+
for (const ref of pendingForgettingRefs) {
|
|
2900
|
+
if (!existingRefSet.has(ref)) {
|
|
2901
|
+
// Ref not already in the candidate pool — synthesise a stub so it
|
|
2902
|
+
// participates in the reflect/distill loop with proper attribution.
|
|
2903
|
+
newForgettingRefs.push({ ref, reason: "scope-type", eligibilitySource: "forgetting-safety" });
|
|
2904
|
+
}
|
|
2905
|
+
// Always stamp the lane in the attribution map (overwrites weaker lanes;
|
|
2906
|
+
// stronger reactive signals — scope/signal-delta/high-retrieval/proactive
|
|
2907
|
+
// — are written after this block so they take precedence).
|
|
2908
|
+
eligibilitySourceByRef.set(ref, "forgetting-safety");
|
|
2909
|
+
}
|
|
2910
|
+
if (newForgettingRefs.length > 0) {
|
|
2911
|
+
mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
|
|
2912
|
+
}
|
|
2913
|
+
// Re-stamp attribution for any refs whose lane needs updating.
|
|
2914
|
+
// Precedence (weakest → strongest, each overwrites the previous):
|
|
2915
|
+
// proactive < high-retrieval < forgetting-safety < signal-delta
|
|
2916
|
+
// Scope mode is already excluded by the outer guard (`scope.mode !== "ref"`).
|
|
2917
|
+
// forgetting-safety sits above proactive and high-retrieval so that a ref
|
|
2918
|
+
// flagged as a forgetting candidate is always visible to S5/WS-5 as such,
|
|
2919
|
+
// even when it was also due for a proactive maintenance run. signal-delta
|
|
2920
|
+
// overrides forgetting-safety because a ref with fresh feedback is reactive
|
|
2921
|
+
// and doesn't need the protective pass label for measurement purposes.
|
|
2922
|
+
for (const r of highSalienceRefs)
|
|
2923
|
+
eligibilitySourceByRef.set(r.ref, "high-salience");
|
|
2924
|
+
for (const r of proactiveRefs)
|
|
2925
|
+
eligibilitySourceByRef.set(r.ref, "proactive");
|
|
2926
|
+
for (const r of highRetrievalRefs)
|
|
2927
|
+
eligibilitySourceByRef.set(r.ref, "high-retrieval");
|
|
2928
|
+
// Apply forgetting-safety OVER proactive, high-retrieval, and high-salience
|
|
2929
|
+
// (already stamped in the loop above via
|
|
2930
|
+
// `eligibilitySourceByRef.set(ref, "forgetting-safety")`). No-op here: the
|
|
2931
|
+
// set() calls above for proactive/high-retrieval/high-salience overwrite the
|
|
2932
|
+
// earlier forgetting-safety stamp — so we re-apply forgetting-safety now for
|
|
2933
|
+
// those refs that are both forgetting candidates AND in another fallback lane.
|
|
2934
|
+
for (const ref of pendingForgettingRefs) {
|
|
2935
|
+
eligibilitySourceByRef.set(ref, "forgetting-safety");
|
|
2936
|
+
}
|
|
2937
|
+
// signal-delta is the strongest reactive signal and overrides forgetting-safety.
|
|
2938
|
+
for (const r of signalFiltered)
|
|
2939
|
+
eligibilitySourceByRef.set(r.ref, "signal-delta");
|
|
2940
|
+
// Update eligibilitySource on the ref objects themselves for any refs whose
|
|
2941
|
+
// lane changed (covers both new stubs and pre-existing refs).
|
|
2942
|
+
for (const r of mergedRefs) {
|
|
2943
|
+
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2946
|
+
// ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
|
|
2947
|
+
// Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
|
|
2948
|
+
// revisited even with zero reactive signal (no feedback, no retrieval) and
|
|
2949
|
+
// regardless of cooldown — exactly like the forgetting-safety lane, replay is
|
|
2950
|
+
// injected AFTER cooldown/signal-delta partitioning so it bypasses those gates.
|
|
2951
|
+
//
|
|
2952
|
+
// Strictly additive: the replay slice is appended AFTER the --limit fresh slice
|
|
2953
|
+
// (see the loopRefs partition below), so it can never shrink the fresh-ref set.
|
|
2954
|
+
// Replay is the WEAKEST lane — it only stamps refs no other lane already claimed,
|
|
2955
|
+
// and budget is spent only on refs not already in mergedRefs (so a stronger lane
|
|
2956
|
+
// never has its budget wasted or its label overwritten).
|
|
2957
|
+
//
|
|
2958
|
+
// Default replayBudget=0 ⇒ this whole block is a no-op (no DB open, no event,
|
|
2959
|
+
// no mergedRefs mutation), preserving byte-identical pre-#610 selection behavior.
|
|
2960
|
+
const replayBudget = (options.config ?? loadConfig()).improve?.salience?.replayBudget ?? 0;
|
|
2961
|
+
const replayRefSet = new Set();
|
|
2962
|
+
if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
|
|
2963
|
+
let replayDb;
|
|
2964
|
+
try {
|
|
2965
|
+
replayDb = openStateDatabase(eventsCtx?.dbPath);
|
|
2966
|
+
const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
|
|
2967
|
+
const allRankScores = getAllRankScores(replayDb);
|
|
2968
|
+
// Candidate universe = every salience row NOT already in the pool, ordered by
|
|
2969
|
+
// rank_score desc with a deterministic ref-string tie-break (mirrors the main
|
|
2970
|
+
// sort). Converged refs (consecutive_no_ops >= dampener threshold) are fully
|
|
2971
|
+
// EXCLUDED — a stronger skip than the dampener (which only halves order).
|
|
2972
|
+
let convergedSkipped = 0;
|
|
2973
|
+
const candidates = [];
|
|
2974
|
+
for (const [ref, rankScore] of allRankScores) {
|
|
2975
|
+
if (alreadyInPool.has(ref))
|
|
2976
|
+
continue;
|
|
2977
|
+
const noOps = getConsecutiveNoOps(replayDb, ref);
|
|
2978
|
+
if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
|
|
2979
|
+
convergedSkipped++;
|
|
2980
|
+
continue;
|
|
2981
|
+
}
|
|
2982
|
+
candidates.push({ ref, rankScore });
|
|
2983
|
+
}
|
|
2984
|
+
candidates.sort((a, b) => b.rankScore !== a.rankScore ? b.rankScore - a.rankScore : a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0);
|
|
2985
|
+
const candidatePool = candidates.length;
|
|
2986
|
+
const selected = candidates.slice(0, replayBudget);
|
|
2987
|
+
const newReplayRefs = [];
|
|
2988
|
+
for (const { ref } of selected) {
|
|
2989
|
+
replayRefSet.add(ref);
|
|
2990
|
+
// Synthesise a stub (mirror the forgetting-safety stub). Resolve the
|
|
2991
|
+
// backing file from the planned-ref pool so the downstream existsSync
|
|
2992
|
+
// guard keeps the ref (a replay candidate from asset_salience whose file
|
|
2993
|
+
// is gone correctly drops out). Only refs present in the indexed pool can
|
|
2994
|
+
// be revisited — refs without a planned entry get no filePath and are
|
|
2995
|
+
// dropped by the disk check, which is the desired behavior.
|
|
2996
|
+
const planned = plannedRefs.find((p) => p.ref === ref);
|
|
2997
|
+
newReplayRefs.push({
|
|
2998
|
+
ref,
|
|
2999
|
+
reason: "scope-type",
|
|
3000
|
+
eligibilitySource: "replay",
|
|
3001
|
+
...(planned?.filePath ? { filePath: planned.filePath } : {}),
|
|
3002
|
+
});
|
|
3003
|
+
// Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
|
|
3004
|
+
if (!salienceMap.has(ref)) {
|
|
3005
|
+
salienceMap.set(ref, {
|
|
3006
|
+
encoding: 0,
|
|
3007
|
+
outcome: 0,
|
|
3008
|
+
retrieval: 0,
|
|
3009
|
+
rankScore: allRankScores.get(ref) ?? 0,
|
|
3010
|
+
});
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
if (newReplayRefs.length > 0) {
|
|
3014
|
+
mergedRefs = dedupeRefs([...mergedRefs, ...newReplayRefs]);
|
|
3015
|
+
// Replay is the WEAKEST lane: stamp 'replay' ONLY for refs not already
|
|
3016
|
+
// keyed by a stronger lane.
|
|
3017
|
+
for (const ref of replayRefSet) {
|
|
3018
|
+
if (!eligibilitySourceByRef.has(ref))
|
|
3019
|
+
eligibilitySourceByRef.set(ref, "replay");
|
|
3020
|
+
}
|
|
3021
|
+
for (const r of mergedRefs) {
|
|
3022
|
+
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
3023
|
+
}
|
|
3024
|
+
}
|
|
3025
|
+
// Aggregated observability event (never per-ref).
|
|
3026
|
+
appendEvent({
|
|
3027
|
+
eventType: "improve_replay_selected",
|
|
3028
|
+
ref: undefined,
|
|
3029
|
+
metadata: {
|
|
3030
|
+
count: newReplayRefs.length,
|
|
3031
|
+
budget: replayBudget,
|
|
3032
|
+
convergedSkipped,
|
|
3033
|
+
candidatePool,
|
|
3034
|
+
},
|
|
3035
|
+
}, eventsCtx);
|
|
3036
|
+
}
|
|
3037
|
+
catch (err) {
|
|
3038
|
+
rethrowIfTestIsolationError(err);
|
|
3039
|
+
// best-effort: if DB unavailable, replayRefSet stays empty
|
|
3040
|
+
}
|
|
3041
|
+
finally {
|
|
3042
|
+
if (replayDb)
|
|
3043
|
+
replayDb.close();
|
|
3044
|
+
}
|
|
3045
|
+
}
|
|
3046
|
+
// Build no-op map for consolidation-selection dampener (plan §WS-1 step 8).
|
|
3047
|
+
// Reads consecutive_no_ops from the SAME pinned db handle used elsewhere in
|
|
3048
|
+
// this function. The effective score is used ONLY for processing/selection
|
|
3049
|
+
// order — the persisted rank_score in asset_salience is never mutated here.
|
|
3050
|
+
const noOpMap = new Map();
|
|
3051
|
+
try {
|
|
3052
|
+
const noOpDb = eventsCtx?.db ?? (eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
|
|
3053
|
+
if (noOpDb) {
|
|
3054
|
+
const ownsNoOpDb = !eventsCtx?.db;
|
|
3055
|
+
try {
|
|
3056
|
+
for (const r of mergedRefs) {
|
|
3057
|
+
noOpMap.set(r.ref, getConsecutiveNoOps(noOpDb, r.ref));
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
finally {
|
|
3061
|
+
if (ownsNoOpDb)
|
|
3062
|
+
noOpDb.close();
|
|
3063
|
+
}
|
|
3064
|
+
}
|
|
3065
|
+
}
|
|
3066
|
+
catch {
|
|
3067
|
+
// best-effort: dampener failure never blocks selection
|
|
3068
|
+
}
|
|
3069
|
+
// Sort by effective selection score (desc), with explicit ref-string tie-break
|
|
3070
|
+
// for determinism. The effective score applies the consolidation-selection
|
|
3071
|
+
// dampener: assets that have been repeatedly skipped (consecutive_no_ops >=
|
|
3072
|
+
// THRESHOLD) are penalised by FACTOR so they sort after peers with similar
|
|
3073
|
+
// rankScore. The persisted rank_score is left unchanged — this is the whole
|
|
3074
|
+
// point of the dampener (stable assets stay fully retrievable).
|
|
3075
|
+
//
|
|
3076
|
+
// WIRING NOTE (plan §WS-1 step 8 / "consolidation-selection" disambiguation):
|
|
3077
|
+
// "consolidation-selection" in the plan refers to THIS reflect/distill
|
|
3078
|
+
// eligibility ordering — i.e. which assets are chosen for the reflect/distill
|
|
3079
|
+
// LLM pass — NOT to akmConsolidate (the cluster-merge phase at ~line 1994,
|
|
3080
|
+
// which runs earlier and never reads noOpMap). The no-op counter originates
|
|
3081
|
+
// from no-change reflect / quality-rejected distill outcomes; the dampener
|
|
3082
|
+
// suppresses repeated LLM attempts on those same assets without touching their
|
|
3083
|
+
// persisted rank_score (so they remain fully retrievable).
|
|
3084
|
+
//
|
|
3085
|
+
// This is the ONLY ranking path — negativeOnlyRatio and the legacy
|
|
3086
|
+
// symmetricValence branch are replaced. The three eligibilitySource lanes
|
|
3087
|
+
// (signal-delta / high-retrieval / proactive) survive as labels (set above).
|
|
3088
|
+
const effectiveScore = (ref) => {
|
|
3089
|
+
const rankScore = salienceMap.get(ref)?.rankScore ?? 0;
|
|
3090
|
+
const noOps = noOpMap.get(ref) ?? 0;
|
|
3091
|
+
return noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rankScore * SALIENCE_NO_OP_DAMPEN_FACTOR : rankScore;
|
|
3092
|
+
};
|
|
3093
|
+
const sorted = [...mergedRefs].sort((a, b) => {
|
|
3094
|
+
const scoreA = effectiveScore(a.ref);
|
|
3095
|
+
const scoreB = effectiveScore(b.ref);
|
|
3096
|
+
if (scoreB !== scoreA)
|
|
3097
|
+
return scoreB - scoreA;
|
|
3098
|
+
// Stable tie-break: deterministic regardless of input ordering.
|
|
3099
|
+
return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
|
|
3100
|
+
});
|
|
3101
|
+
// Phase 0: surface coverage gaps from zero-result search queries
|
|
3102
|
+
let coverageGaps = [];
|
|
3103
|
+
try {
|
|
3104
|
+
const dbForGaps = openExistingDatabase();
|
|
3105
|
+
try {
|
|
3106
|
+
coverageGaps = getZeroResultSearches(dbForGaps);
|
|
3107
|
+
}
|
|
3108
|
+
finally {
|
|
3109
|
+
closeDatabase(dbForGaps);
|
|
1919
3110
|
}
|
|
1920
3111
|
}
|
|
1921
3112
|
catch (err) {
|
|
@@ -1939,15 +3130,32 @@ async function runImprovePreparationStage(args) {
|
|
|
1939
3130
|
const assetMissingOnDisk = [];
|
|
1940
3131
|
const existsCheckedActionable = [];
|
|
1941
3132
|
for (const candidate of sorted) {
|
|
1942
|
-
|
|
3133
|
+
// #591: prefer the path pre-resolved at planning time (synchronous
|
|
3134
|
+
// existsSync) over a serial async DB lookup per ref.
|
|
3135
|
+
const filePath = candidate.filePath && fs.existsSync(candidate.filePath)
|
|
3136
|
+
? candidate.filePath
|
|
3137
|
+
: await findAssetFilePath(candidate.ref, options.stashDir);
|
|
1943
3138
|
if (filePath && fs.existsSync(filePath)) {
|
|
1944
3139
|
existsCheckedActionable.push(candidate);
|
|
1945
3140
|
}
|
|
1946
3141
|
else {
|
|
1947
3142
|
assetMissingOnDisk.push(candidate.ref);
|
|
1948
|
-
appendEvent({ eventType: "improve_skipped", ref: candidate.ref, metadata: { reason: "asset_missing_on_disk" } }, eventsCtx);
|
|
1949
3143
|
}
|
|
1950
3144
|
}
|
|
3145
|
+
// #592 audit: one summary event instead of one per missing ref. Normally
|
|
3146
|
+
// tiny, but a stash deletion racing the run could make this O(n) sequential
|
|
3147
|
+
// state.db writes. `refs` is capped so the metadata row stays bounded.
|
|
3148
|
+
if (assetMissingOnDisk.length > 0) {
|
|
3149
|
+
appendEvent({
|
|
3150
|
+
eventType: "improve_skipped",
|
|
3151
|
+
ref: undefined,
|
|
3152
|
+
metadata: {
|
|
3153
|
+
reason: "asset_missing_on_disk",
|
|
3154
|
+
count: assetMissingOnDisk.length,
|
|
3155
|
+
refs: assetMissingOnDisk.slice(0, 50),
|
|
3156
|
+
},
|
|
3157
|
+
}, eventsCtx);
|
|
3158
|
+
}
|
|
1951
3159
|
const actionableRefs = existsCheckedActionable;
|
|
1952
3160
|
// Re-split actionableRefs (sorted) into reflect-path vs distill-only-path while
|
|
1953
3161
|
// preserving sort order. distillOnlyRefs participate in the sort so --limit
|
|
@@ -1964,8 +3172,22 @@ async function runImprovePreparationStage(args) {
|
|
|
1964
3172
|
}
|
|
1965
3173
|
}
|
|
1966
3174
|
// ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
|
|
3175
|
+
//
|
|
3176
|
+
// #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
|
|
3177
|
+
// fresh slice. Without this split, a high-rankScore replay ref could sort above
|
|
3178
|
+
// a fresh ref in the single combined slice and STEAL its slot (violating AC2).
|
|
3179
|
+
// We partition into the replay lane vs the rest, apply --limit to the
|
|
3180
|
+
// non-replay (fresh) refs only, then APPEND up to `replayBudget` replay refs
|
|
3181
|
+
// after the fresh slice. Sort order within each partition is preserved.
|
|
3182
|
+
//
|
|
3183
|
+
// Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
|
|
3184
|
+
// replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
|
|
3185
|
+
// `replayLoop.slice(0, 0) === []` — byte-identical.
|
|
1967
3186
|
const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
|
|
1968
|
-
const
|
|
3187
|
+
const replayLoop = allLoopRefs.filter((r) => r.eligibilitySource === "replay");
|
|
3188
|
+
const nonReplayLoop = allLoopRefs.filter((r) => r.eligibilitySource !== "replay");
|
|
3189
|
+
const baseLoop = options.limit ? nonReplayLoop.slice(0, options.limit) : nonReplayLoop;
|
|
3190
|
+
const loopRefs = [...baseLoop, ...replayLoop.slice(0, replayBudget)];
|
|
1969
3191
|
// Update the returned distillOnlyRefs to the sorted order so callers see the
|
|
1970
3192
|
// ranked view (loop stage uses it as a Set so order is irrelevant, but the
|
|
1971
3193
|
// shape change keeps downstream consumers consistent).
|
|
@@ -1976,7 +3198,7 @@ async function runImprovePreparationStage(args) {
|
|
|
1976
3198
|
`(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
|
|
1977
3199
|
}
|
|
1978
3200
|
if (signalAndRetrievalRefs.length > 0) {
|
|
1979
|
-
info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval)`);
|
|
3201
|
+
info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
|
|
1980
3202
|
}
|
|
1981
3203
|
if (validationFailureRefs.size > 0) {
|
|
1982
3204
|
info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
|
|
@@ -1987,6 +3209,17 @@ async function runImprovePreparationStage(args) {
|
|
|
1987
3209
|
const deferredCount = actionableRefs.length - loopRefs.length;
|
|
1988
3210
|
info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
|
|
1989
3211
|
(options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
|
|
3212
|
+
// WS-4: Per-phase threshold auto-tune for the extract phase.
|
|
3213
|
+
// Persists result for the NEXT run's makeGateConfig to read.
|
|
3214
|
+
const extractTuneDbPath = eventsCtx?.dbPath;
|
|
3215
|
+
if (options.autoAccept !== undefined && extractTuneDbPath) {
|
|
3216
|
+
try {
|
|
3217
|
+
maybeAutoTuneThreshold(extractGateCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), extractTuneDbPath, undefined, "extract");
|
|
3218
|
+
}
|
|
3219
|
+
catch (err) {
|
|
3220
|
+
warn(`[improve] calibration auto-tune (extract) skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
3221
|
+
}
|
|
3222
|
+
}
|
|
1990
3223
|
return {
|
|
1991
3224
|
actions,
|
|
1992
3225
|
cleanupWarnings,
|
|
@@ -2008,6 +3241,7 @@ async function runImprovePreparationStage(args) {
|
|
|
2008
3241
|
gateAutoAcceptFailedCount,
|
|
2009
3242
|
consolidation: consolidationPass.consolidation,
|
|
2010
3243
|
consolidationRan: consolidationPass.consolidationRan,
|
|
3244
|
+
...(proactiveMaintenanceSummary ? { proactiveMaintenance: proactiveMaintenanceSummary } : {}),
|
|
2011
3245
|
};
|
|
2012
3246
|
}
|
|
2013
3247
|
async function runImproveLoopStage(args) {
|
|
@@ -2016,6 +3250,14 @@ async function runImproveLoopStage(args) {
|
|
|
2016
3250
|
// receives only its fair share of the wall-clock budget.
|
|
2017
3251
|
const remainingBudgetMs = () => Math.max(0, budgetMs - (Date.now() - startMs));
|
|
2018
3252
|
const RECENT_ERRORS_CAP = 3;
|
|
3253
|
+
// requirePlannedRefs guard: when the distill profile sets this flag, skip
|
|
3254
|
+
// distill for distill-only refs if the reflect phase produced no planned refs.
|
|
3255
|
+
// Prevents the distill loop from generating hundreds of distill-skipped events
|
|
3256
|
+
// on quiet passes (all refs on reflect cooldown, no new signal to distill).
|
|
3257
|
+
const requirePlannedRefs = improveProfile?.processes?.distill?.requirePlannedRefs === true;
|
|
3258
|
+
const _distillOnlyRefNames = new Set(distillOnlyRefs.map((r) => r.ref));
|
|
3259
|
+
const hasReflectEligibleRefs = loopRefs.some((r) => !_distillOnlyRefNames.has(r.ref));
|
|
3260
|
+
const skipDistillDueToRequirePlannedRefs = requirePlannedRefs && !hasReflectEligibleRefs;
|
|
2019
3261
|
// R-2 / #389: Self-Consistency multi-sample voting helpers.
|
|
2020
3262
|
// Wang et al. arXiv:2203.11171 — N=3 samples beat single-shot on reasoning tasks.
|
|
2021
3263
|
const SC_THRESHOLD = options.selfConsistencyThreshold ?? 0.7;
|
|
@@ -2097,6 +3339,10 @@ async function runImproveLoopStage(args) {
|
|
|
2097
3339
|
stashDir: primaryStashDir,
|
|
2098
3340
|
config: options.config ?? loadConfig(),
|
|
2099
3341
|
eventsCtx,
|
|
3342
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
3343
|
+
// candidateCount drives the exploration budget. loopRefs is the per-phase
|
|
3344
|
+
// set for reflect/distill; pass it so exploration budget is proportional.
|
|
3345
|
+
candidateCount: loopRefs.length,
|
|
2100
3346
|
});
|
|
2101
3347
|
const distillGateCfg = makeGateConfig("distill", {
|
|
2102
3348
|
globalThreshold: options.autoAccept,
|
|
@@ -2104,6 +3350,8 @@ async function runImproveLoopStage(args) {
|
|
|
2104
3350
|
stashDir: primaryStashDir,
|
|
2105
3351
|
config: options.config ?? loadConfig(),
|
|
2106
3352
|
eventsCtx,
|
|
3353
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
3354
|
+
candidateCount: loopRefs.length,
|
|
2107
3355
|
});
|
|
2108
3356
|
for (const planned of loopRefs) {
|
|
2109
3357
|
if (Date.now() - startMs >= budgetMs) {
|
|
@@ -2176,6 +3424,9 @@ async function runImproveLoopStage(args) {
|
|
|
2176
3424
|
eventSource: "improve",
|
|
2177
3425
|
...(reflectBudgetMs > 0 ? { timeoutMs: reflectBudgetMs } : {}),
|
|
2178
3426
|
...(reflectProfileRunner ? { runner: reflectProfileRunner } : {}),
|
|
3427
|
+
// Attribution: carry the eligibility lane so reflect stamps it on
|
|
3428
|
+
// the reflect_invoked event and the persisted proposal.
|
|
3429
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
2179
3430
|
};
|
|
2180
3431
|
// R-2 / #389: Self-consistency multi-sample voting for high-utility refs.
|
|
2181
3432
|
// Self-Consistency arXiv:2203.11171 — N=3 samples beat single-shot quality.
|
|
@@ -2188,9 +3439,11 @@ async function runImproveLoopStage(args) {
|
|
|
2188
3439
|
if (remainingBudgetMs() <= 0)
|
|
2189
3440
|
break;
|
|
2190
3441
|
// draftMode: skip DB write so each sample doesn't create a proposal.
|
|
2191
|
-
samples.push(await reflectFn({ ...reflectCallArgs, draftMode: true }));
|
|
3442
|
+
samples.push(await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true })));
|
|
2192
3443
|
}
|
|
2193
|
-
const winner = pickMajorityVote(samples.length > 0
|
|
3444
|
+
const winner = pickMajorityVote(samples.length > 0
|
|
3445
|
+
? samples
|
|
3446
|
+
: [await withLlmStage("reflect", () => reflectFn({ ...reflectCallArgs, draftMode: true }))]);
|
|
2194
3447
|
// Persist only the majority-vote winner as a single real proposal.
|
|
2195
3448
|
if (winner.ok && primaryStashDir) {
|
|
2196
3449
|
const persistResult = createProposal(primaryStashDir, {
|
|
@@ -2198,6 +3451,9 @@ async function runImproveLoopStage(args) {
|
|
|
2198
3451
|
source: "reflect",
|
|
2199
3452
|
sourceRun: `reflect-sc-${Date.now()}`,
|
|
2200
3453
|
payload: winner.proposal.payload,
|
|
3454
|
+
// Attribution: the self-consistency path persists the winner here
|
|
3455
|
+
// (draftMode skips reflect's own createProposal), so stamp the lane.
|
|
3456
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
2201
3457
|
});
|
|
2202
3458
|
reflectResult = isProposalSkipped(persistResult)
|
|
2203
3459
|
? {
|
|
@@ -2215,7 +3471,7 @@ async function runImproveLoopStage(args) {
|
|
|
2215
3471
|
}
|
|
2216
3472
|
}
|
|
2217
3473
|
else {
|
|
2218
|
-
reflectResult = await reflectFn(reflectCallArgs);
|
|
3474
|
+
reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs));
|
|
2219
3475
|
}
|
|
2220
3476
|
const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
|
|
2221
3477
|
// Content-policy guard hits (reflect size-rail rejections) are NOT
|
|
@@ -2232,6 +3488,12 @@ async function runImproveLoopStage(args) {
|
|
|
2232
3488
|
// user's stack were this case; see review §1a row "Reflect refused
|
|
2233
3489
|
// asset type".
|
|
2234
3490
|
const isTypeRefused = !reflectResult.ok && reflectResult.reason === "unsupported_type";
|
|
3491
|
+
// Noise-gate suppression (#580): the candidate edit was an empty
|
|
3492
|
+
// diff or a cosmetic-only reformat of the current asset. Like
|
|
3493
|
+
// `unsupported_type`, this is a deterministic skip — not an LLM
|
|
3494
|
+
// fault — so it routes to the `reflect-skipped` bucket and stays
|
|
3495
|
+
// out of recentErrors/avoidPatterns.
|
|
3496
|
+
const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
|
|
2235
3497
|
actions.push({
|
|
2236
3498
|
ref: planned.ref,
|
|
2237
3499
|
mode: reflectResult.ok
|
|
@@ -2240,18 +3502,19 @@ async function runImproveLoopStage(args) {
|
|
|
2240
3502
|
? "reflect-cooldown"
|
|
2241
3503
|
: isGuardReject
|
|
2242
3504
|
? "reflect-guard-rejected"
|
|
2243
|
-
: isTypeRefused
|
|
3505
|
+
: isTypeRefused || isNoChange
|
|
2244
3506
|
? "reflect-skipped"
|
|
2245
3507
|
: "reflect-failed",
|
|
2246
3508
|
result: reflectResult,
|
|
2247
3509
|
});
|
|
2248
|
-
// Cooldown skips, guard rejects,
|
|
2249
|
-
// failures — do not pollute recentErrors with them
|
|
2250
|
-
// injected as `avoidPatterns` into the next reflect
|
|
2251
|
-
// rejects ARE worth showing the LLM as a learn-signal
|
|
2252
|
-
// iteration sees "your last expansion was too large";
|
|
2253
|
-
//
|
|
2254
|
-
|
|
3510
|
+
// Cooldown skips, guard rejects, type-refused skips, and noise-gate
|
|
3511
|
+
// skips are not failures — do not pollute recentErrors with them
|
|
3512
|
+
// (those get injected as `avoidPatterns` into the next reflect
|
|
3513
|
+
// prompt). Guard rejects ARE worth showing the LLM as a learn-signal
|
|
3514
|
+
// so the next iteration sees "your last expansion was too large";
|
|
3515
|
+
// type-refused and no-change are deterministic and add no learning
|
|
3516
|
+
// signal.
|
|
3517
|
+
if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
|
|
2255
3518
|
const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
|
|
2256
3519
|
pushRecentError("reflect", errMsg);
|
|
2257
3520
|
}
|
|
@@ -2266,6 +3529,28 @@ async function runImproveLoopStage(args) {
|
|
|
2266
3529
|
reason: reflectResult.ok ? undefined : reflectResult.reason,
|
|
2267
3530
|
},
|
|
2268
3531
|
}, eventsCtx);
|
|
3532
|
+
// Plasticity counter (plan §WS-1 step 8): record no-ops so the
|
|
3533
|
+
// WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
|
|
3534
|
+
// repeatedly-silent assets during consolidation-selection.
|
|
3535
|
+
// A no_change reflect means the LLM was invoked but found nothing to
|
|
3536
|
+
// improve — the asset is stable. Track it. A successful reflect means
|
|
3537
|
+
// the asset changed; reset the counter so the dampener lifts.
|
|
3538
|
+
if (isNoChange && eventsCtx?.db) {
|
|
3539
|
+
try {
|
|
3540
|
+
recordNoOp(eventsCtx.db, planned.ref);
|
|
3541
|
+
}
|
|
3542
|
+
catch {
|
|
3543
|
+
// best-effort: plasticity counter failure never blocks the run
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
else if (reflectResult.ok && eventsCtx?.db) {
|
|
3547
|
+
try {
|
|
3548
|
+
resetConsecutiveNoOps(eventsCtx.db, planned.ref);
|
|
3549
|
+
}
|
|
3550
|
+
catch {
|
|
3551
|
+
// best-effort
|
|
3552
|
+
}
|
|
3553
|
+
}
|
|
2269
3554
|
if (reflectResult.ok) {
|
|
2270
3555
|
const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
|
|
2271
3556
|
gateAutoAcceptedCount += reflectGr.promoted.length;
|
|
@@ -2304,6 +3589,18 @@ async function runImproveLoopStage(args) {
|
|
|
2304
3589
|
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
2305
3590
|
continue;
|
|
2306
3591
|
}
|
|
3592
|
+
// requirePlannedRefs guard: skip distill for distill-only refs when no
|
|
3593
|
+
// reflect-eligible refs were planned this run, preventing mass skip events.
|
|
3594
|
+
if (skipDistillDueToRequirePlannedRefs && isDistillOnly) {
|
|
3595
|
+
actions.push({
|
|
3596
|
+
ref: planned.ref,
|
|
3597
|
+
mode: "distill-skipped",
|
|
3598
|
+
result: { ok: true, reason: "require_planned_refs" },
|
|
3599
|
+
});
|
|
3600
|
+
completedCount++;
|
|
3601
|
+
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
3602
|
+
continue;
|
|
3603
|
+
}
|
|
2307
3604
|
// See `isDistillCandidateRef` — excludes `lesson:*` (and anything else in
|
|
2308
3605
|
// DISTILL_REFUSED_INPUT_TYPES) so distill never gets queued for an input
|
|
2309
3606
|
// it will refuse.
|
|
@@ -2373,11 +3670,14 @@ async function runImproveLoopStage(args) {
|
|
|
2373
3670
|
}
|
|
2374
3671
|
}
|
|
2375
3672
|
}
|
|
2376
|
-
const distillResult = await distillFn({
|
|
3673
|
+
const distillResult = await withLlmStage("distill", () => distillFn({
|
|
2377
3674
|
ref: planned.ref,
|
|
2378
3675
|
...(parsedPlannedRef.type === "memory" ? { proposalKind: "auto" } : {}),
|
|
2379
3676
|
...(options.stashDir ? { stashDir: options.stashDir } : {}),
|
|
2380
|
-
|
|
3677
|
+
// Attribution: carry the eligibility lane so distill stamps it on the
|
|
3678
|
+
// distill_invoked event and the persisted proposal.
|
|
3679
|
+
...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
|
|
3680
|
+
}));
|
|
2381
3681
|
actions.push({ ref: planned.ref, mode: "distill", result: distillResult });
|
|
2382
3682
|
if (distillResult.outcome === "queued" && distillResult.proposal) {
|
|
2383
3683
|
const distillGr = await runAutoAcceptGate([{ proposalId: distillResult.proposal.id, confidence: distillResult.proposal.confidence }], distillGateCfg);
|
|
@@ -2389,6 +3689,23 @@ async function runImproveLoopStage(args) {
|
|
|
2389
3689
|
if (!promotedToKnowledge)
|
|
2390
3690
|
memoryRefsForInference.add(planned.ref);
|
|
2391
3691
|
}
|
|
3692
|
+
// Plasticity counter (plan §WS-1 step 8) for the distill path.
|
|
3693
|
+
// quality_rejected: the LLM ran but produced output that didn't pass the
|
|
3694
|
+
// quality gate — the asset is not yielding useful distill output.
|
|
3695
|
+
// queued: a proposal was produced; reset the no-op counter.
|
|
3696
|
+
if (eventsCtx?.db) {
|
|
3697
|
+
try {
|
|
3698
|
+
if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
|
|
3699
|
+
recordNoOp(eventsCtx.db, planned.ref);
|
|
3700
|
+
}
|
|
3701
|
+
else if (distillResult.outcome === "queued") {
|
|
3702
|
+
resetConsecutiveNoOps(eventsCtx.db, planned.ref);
|
|
3703
|
+
}
|
|
3704
|
+
}
|
|
3705
|
+
catch {
|
|
3706
|
+
// best-effort: plasticity counter failure never blocks the run
|
|
3707
|
+
}
|
|
3708
|
+
}
|
|
2392
3709
|
if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
|
|
2393
3710
|
const slug = planned.ref
|
|
2394
3711
|
.replace(/[^a-z0-9]/gi, "-")
|
|
@@ -2455,6 +3772,26 @@ async function runImproveLoopStage(args) {
|
|
|
2455
3772
|
completedCount++;
|
|
2456
3773
|
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
2457
3774
|
}
|
|
3775
|
+
// WS-4: Per-phase threshold auto-tune — runs AFTER the loop so the gate
|
|
3776
|
+
// has processed all candidates for this run. Persists each phase's tuned
|
|
3777
|
+
// threshold to state.db for the NEXT run's makeGateConfig to read.
|
|
3778
|
+
// Best-effort: a tune failure must never fail the improve run.
|
|
3779
|
+
const stateDbPathForTune = eventsCtx?.dbPath;
|
|
3780
|
+
if (options.autoAccept !== undefined && stateDbPathForTune) {
|
|
3781
|
+
const phaseGateCfgMap = {
|
|
3782
|
+
reflect: reflectGateCfg,
|
|
3783
|
+
distill: distillGateCfg,
|
|
3784
|
+
};
|
|
3785
|
+
for (const phase of ["reflect", "distill"]) {
|
|
3786
|
+
const phaseCfg = phaseGateCfgMap[phase];
|
|
3787
|
+
try {
|
|
3788
|
+
maybeAutoTuneThreshold(phaseCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), stateDbPathForTune, undefined, phase);
|
|
3789
|
+
}
|
|
3790
|
+
catch (err) {
|
|
3791
|
+
warn(`[improve] calibration auto-tune (${phase}) skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
3792
|
+
}
|
|
3793
|
+
}
|
|
3794
|
+
}
|
|
2458
3795
|
return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
2459
3796
|
}
|
|
2460
3797
|
async function runImprovePostLoopStage(args) {
|
|
@@ -2498,9 +3835,70 @@ async function runImprovePostLoopStage(args) {
|
|
|
2498
3835
|
// best-effort
|
|
2499
3836
|
}
|
|
2500
3837
|
}
|
|
3838
|
+
// #609 — recombine / synthesize pass. Whole-corpus cross-episodic
|
|
3839
|
+
// generalization. Runs in the post-loop stage under consolidate.lock (it
|
|
3840
|
+
// reads the consolidated corpus and writes proposals). Opt-in: gated on the
|
|
3841
|
+
// `recombine` process being enabled, whole-stash / type scope (never `ref`),
|
|
3842
|
+
// and not a dry run. Mirrors the proactiveMaintenance opt-in wiring.
|
|
3843
|
+
let recombination;
|
|
3844
|
+
if (primaryStashDir &&
|
|
3845
|
+
improveProfile &&
|
|
3846
|
+
resolveProcessEnabled("recombine", improveProfile) &&
|
|
3847
|
+
scope.mode !== "ref" &&
|
|
3848
|
+
!options.dryRun) {
|
|
3849
|
+
const recombineFn = options.recombineFn ?? akmRecombine;
|
|
3850
|
+
try {
|
|
3851
|
+
recombination = await recombineFn({
|
|
3852
|
+
stashDir: primaryStashDir,
|
|
3853
|
+
config: options.config ?? loadConfig(),
|
|
3854
|
+
...(options.runId ? { sourceRun: options.runId } : {}),
|
|
3855
|
+
...(budgetSignal ? { signal: budgetSignal } : {}),
|
|
3856
|
+
...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
|
|
3857
|
+
eligibilitySource: "recombine",
|
|
3858
|
+
...(eventsCtx ? { ctx: eventsCtx } : {}),
|
|
3859
|
+
minClusterSize: improveProfile.processes?.recombine?.minClusterSize,
|
|
3860
|
+
maxClustersPerRun: improveProfile.processes?.recombine?.maxClustersPerRun,
|
|
3861
|
+
relatednessSource: improveProfile.processes?.recombine?.relatednessSource,
|
|
3862
|
+
confirmThreshold: improveProfile.processes?.recombine?.confirmThreshold,
|
|
3863
|
+
});
|
|
3864
|
+
}
|
|
3865
|
+
catch (e) {
|
|
3866
|
+
allWarnings.push(`recombine: ${String(e)}`);
|
|
3867
|
+
}
|
|
3868
|
+
}
|
|
3869
|
+
// #615 — procedural-compilation pass. Detects recurring successful ordered
|
|
3870
|
+
// action sequences and compiles them into workflow proposals. Opt-in: gated
|
|
3871
|
+
// on the `procedural` process being enabled, whole-stash / type scope (never
|
|
3872
|
+
// `ref`), and not a dry run. Mirrors the recombine opt-in wiring.
|
|
3873
|
+
let proceduralCompilation;
|
|
3874
|
+
if (primaryStashDir &&
|
|
3875
|
+
improveProfile &&
|
|
3876
|
+
resolveProcessEnabled("procedural", improveProfile) &&
|
|
3877
|
+
scope.mode !== "ref" &&
|
|
3878
|
+
!options.dryRun) {
|
|
3879
|
+
const proceduralFn = options.proceduralFn ?? akmProcedural;
|
|
3880
|
+
try {
|
|
3881
|
+
proceduralCompilation = await proceduralFn({
|
|
3882
|
+
stashDir: primaryStashDir,
|
|
3883
|
+
config: options.config ?? loadConfig(),
|
|
3884
|
+
...(options.runId ? { sourceRun: options.runId } : {}),
|
|
3885
|
+
...(budgetSignal ? { signal: budgetSignal } : {}),
|
|
3886
|
+
...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
|
|
3887
|
+
eligibilitySource: "procedural",
|
|
3888
|
+
...(eventsCtx ? { ctx: eventsCtx } : {}),
|
|
3889
|
+
minRecurrence: improveProfile.processes?.procedural?.minRecurrence,
|
|
3890
|
+
maxProposalsPerRun: improveProfile.processes?.procedural?.maxProposalsPerRun,
|
|
3891
|
+
});
|
|
3892
|
+
}
|
|
3893
|
+
catch (e) {
|
|
3894
|
+
allWarnings.push(`procedural: ${String(e)}`);
|
|
3895
|
+
}
|
|
3896
|
+
}
|
|
2501
3897
|
return {
|
|
2502
3898
|
allWarnings,
|
|
2503
3899
|
deadUrls,
|
|
3900
|
+
...(recombination ? { recombination } : {}),
|
|
3901
|
+
...(proceduralCompilation ? { proceduralCompilation } : {}),
|
|
2504
3902
|
...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
|
|
2505
3903
|
...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
|
|
2506
3904
|
...(maintenanceResult.stalenessDetection ? { stalenessDetection: maintenanceResult.stalenessDetection } : {}),
|
|
@@ -2518,7 +3916,9 @@ async function runImprovePostLoopStage(args) {
|
|
|
2518
3916
|
};
|
|
2519
3917
|
}
|
|
2520
3918
|
// TODO(refactor): mutates the passed-in `allWarnings` array as a hidden side channel. Return warnings in ImproveMaintenanceResult and merge in caller — invasive signature change deferred to next refactor pass.
|
|
2521
|
-
|
|
3919
|
+
// Exported for tests (#584/#585 DB-locking regression coverage); production
|
|
3920
|
+
// callers reach it only through akmImprove → runImprovePostLoopStage.
|
|
3921
|
+
export async function runImproveMaintenancePasses(args) {
|
|
2522
3922
|
const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, consolidationRan, budgetSignal, eventsCtx, improveProfile, } = args;
|
|
2523
3923
|
if (!primaryStashDir)
|
|
2524
3924
|
return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
|
|
@@ -2537,276 +3937,347 @@ async function runImproveMaintenancePasses(args) {
|
|
|
2537
3937
|
let graphExtractionDurationMs = 0;
|
|
2538
3938
|
let orphansPurged = 0;
|
|
2539
3939
|
let proposalsExpired = 0;
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
// Fix: always run the pass when the feature is enabled; let the pass's
|
|
2552
|
-
// own `collectPendingMemories` + `isPendingMemory` predicate find
|
|
2553
|
-
// candidates from the filesystem-of-truth. The this-run set is still
|
|
2554
|
-
// logged as a hint but no longer used as a filter.
|
|
2555
|
-
const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
|
|
2556
|
-
if (memoryInferenceDisabledByProfile) {
|
|
2557
|
-
info("[improve] memory inference skipped (disabled by improve profile)");
|
|
3940
|
+
const openIndexDb = () => openDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
|
|
3941
|
+
// #584: reindexFn opens its own write handle on the same index.db WAL file.
|
|
3942
|
+
// Holding our handle across that call produced SQLITE_BUSY / "database is
|
|
3943
|
+
// locked" failures in production, so the handle is closed BEFORE every
|
|
3944
|
+
// reindex and reopened after — the fresh handle also sees the post-reindex
|
|
3945
|
+
// state that graph extraction and staleness detection below rely on. The
|
|
3946
|
+
// reopen runs in `finally` so a failed reindex still leaves a usable handle.
|
|
3947
|
+
const reindexWithIndexDbReleased = async (stashDir) => {
|
|
3948
|
+
if (db) {
|
|
3949
|
+
closeDatabase(db);
|
|
3950
|
+
db = undefined;
|
|
2558
3951
|
}
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
info(hintRefs > 0
|
|
2562
|
-
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
2563
|
-
: "[improve] memory inference starting (discovering pending parents)");
|
|
2564
|
-
const inferenceStart = Date.now();
|
|
2565
|
-
try {
|
|
2566
|
-
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
2567
|
-
memoryInference = await memoryInferenceFn({
|
|
2568
|
-
config,
|
|
2569
|
-
sources,
|
|
2570
|
-
signal: budgetSignal,
|
|
2571
|
-
db,
|
|
2572
|
-
reEnrich: false,
|
|
2573
|
-
onProgress: (event) => {
|
|
2574
|
-
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
2575
|
-
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
2576
|
-
},
|
|
2577
|
-
});
|
|
2578
|
-
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2579
|
-
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
2580
|
-
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
2581
|
-
}
|
|
2582
|
-
catch (err) {
|
|
2583
|
-
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
2584
|
-
allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2585
|
-
}
|
|
3952
|
+
try {
|
|
3953
|
+
await reindexFn({ stashDir });
|
|
2586
3954
|
}
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
3955
|
+
finally {
|
|
3956
|
+
db = openIndexDb();
|
|
3957
|
+
}
|
|
3958
|
+
};
|
|
3959
|
+
await withIndexWriterLease({ purpose: "improve-maintenance", signal: budgetSignal }, async () => {
|
|
3960
|
+
try {
|
|
3961
|
+
db = openIndexDb();
|
|
3962
|
+
// Memory inference candidate-discovery (post-Item 9 fix from
|
|
3963
|
+
// memory:akm-improve-critical-review-2026-05-20). Previously this pass
|
|
3964
|
+
// was gated on memoryRefsForInference.size > 0 AND passed those refs as a
|
|
3965
|
+
// candidateRefs filter. But memoryRefsForInference is populated from refs
|
|
3966
|
+
// distilled THIS RUN — by the time that happens, those parents are
|
|
3967
|
+
// already split (`inferenceProcessed: true`) and `isPendingMemory` excludes
|
|
3968
|
+
// them. The genuinely-pending parents in the stash never entered the
|
|
3969
|
+
// filter. Result: 0/0/0 for 25 consecutive runs.
|
|
3970
|
+
//
|
|
3971
|
+
// Fix: always run the pass when the feature is enabled; let the pass's
|
|
3972
|
+
// own `collectPendingMemories` + `isPendingMemory` predicate find
|
|
3973
|
+
// candidates from the filesystem-of-truth. The this-run set is still
|
|
3974
|
+
// logged as a hint but no longer used as a filter.
|
|
3975
|
+
const memoryInferenceDisabledByProfile = improveProfile?.processes?.memoryInference?.enabled === false;
|
|
3976
|
+
const minPendingCount = improveProfile?.processes?.memoryInference?.minPendingCount;
|
|
3977
|
+
const pendingBelowMinCount = (() => {
|
|
3978
|
+
if (!primaryStashDir || minPendingCount === undefined || minPendingCount <= 0)
|
|
3979
|
+
return false;
|
|
3980
|
+
const pending = collectPendingMemories(primaryStashDir).length;
|
|
3981
|
+
if (pending < minPendingCount) {
|
|
3982
|
+
info(`[improve] memory inference skipped (${pending} pending < minPendingCount ${minPendingCount})`);
|
|
3983
|
+
return true;
|
|
3984
|
+
}
|
|
3985
|
+
return false;
|
|
3986
|
+
})();
|
|
3987
|
+
if (memoryInferenceDisabledByProfile) {
|
|
3988
|
+
info("[improve] memory inference skipped (disabled by improve profile)");
|
|
2593
3989
|
}
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
}
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
const extractionStart = Date.now();
|
|
2620
|
-
try {
|
|
2621
|
-
// D9: if consolidation ran but memory inference did not reindex, force a reindex
|
|
2622
|
-
// so graph extraction sees current DB state after consolidation writes.
|
|
2623
|
-
if (consolidationRan && !reindexedAfterInference) {
|
|
2624
|
-
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
2625
|
-
try {
|
|
2626
|
-
await reindexFn({ stashDir: primaryStashDir });
|
|
2627
|
-
reindexedAfterInference = true;
|
|
2628
|
-
info("[improve] reindex after consolidation complete");
|
|
2629
|
-
}
|
|
2630
|
-
catch (err) {
|
|
2631
|
-
allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2632
|
-
}
|
|
3990
|
+
else if (pendingBelowMinCount) {
|
|
3991
|
+
// skipped — message already emitted above
|
|
3992
|
+
}
|
|
3993
|
+
else {
|
|
3994
|
+
const hintRefs = memoryRefsForInference.size;
|
|
3995
|
+
info(hintRefs > 0
|
|
3996
|
+
? `[improve] memory inference starting (${hintRefs} hint refs touched this run; pass discovers all pending)`
|
|
3997
|
+
: "[improve] memory inference starting (discovering pending parents)");
|
|
3998
|
+
const inferenceStart = Date.now();
|
|
3999
|
+
try {
|
|
4000
|
+
// O-1 (#364): pass budget signal so a hung inference call is cancelled.
|
|
4001
|
+
memoryInference = await withLlmStage("memory-inference", () => memoryInferenceFn({
|
|
4002
|
+
config,
|
|
4003
|
+
sources,
|
|
4004
|
+
signal: budgetSignal,
|
|
4005
|
+
db,
|
|
4006
|
+
reEnrich: false,
|
|
4007
|
+
onProgress: (event) => {
|
|
4008
|
+
const current = event.currentRef ? ` ${event.currentRef}` : "";
|
|
4009
|
+
info(`[improve] memory inference ${event.processed}/${event.total}${current} (written ${event.writtenFacts}, skipped ${event.skippedNoFacts})`);
|
|
4010
|
+
},
|
|
4011
|
+
}));
|
|
4012
|
+
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
4013
|
+
actions.push({ ref: "memory:_inference", mode: "memory-inference", result: memoryInference });
|
|
4014
|
+
info(`[improve] memory inference complete (${memoryInference.writtenFacts} facts written from ${memoryInference.splitParents} parents)`);
|
|
2633
4015
|
}
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
}
|
|
2638
|
-
// Resolve touched refs to absolute file paths. Skipped for fullScan
|
|
2639
|
-
// (candidatePaths stays undefined → extractor processes all files).
|
|
2640
|
-
let candidatePaths;
|
|
2641
|
-
if (!graphExtractionFullScan) {
|
|
2642
|
-
candidatePaths = new Set();
|
|
2643
|
-
if (primaryStashDir && touchedRefs.size > 0) {
|
|
2644
|
-
const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
|
|
2645
|
-
const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
|
|
2646
|
-
for (const p of resolved) {
|
|
2647
|
-
if (typeof p === "string" && p.length > 0)
|
|
2648
|
-
candidatePaths.add(p);
|
|
2649
|
-
}
|
|
2650
|
-
}
|
|
4016
|
+
catch (err) {
|
|
4017
|
+
memoryInferenceDurationMs = Date.now() - inferenceStart;
|
|
4018
|
+
allWarnings.push(`memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2651
4019
|
}
|
|
2652
|
-
const progressHandler = (event) => {
|
|
2653
|
-
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
2654
|
-
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
2655
|
-
};
|
|
2656
|
-
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
2657
|
-
graphExtraction = await graphExtractionFn({
|
|
2658
|
-
config,
|
|
2659
|
-
sources,
|
|
2660
|
-
signal: budgetSignal,
|
|
2661
|
-
db,
|
|
2662
|
-
reEnrich: false,
|
|
2663
|
-
onProgress: progressHandler,
|
|
2664
|
-
options: { candidatePaths },
|
|
2665
|
-
});
|
|
2666
|
-
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
2667
|
-
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
2668
|
-
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
2669
|
-
}
|
|
2670
|
-
catch (err) {
|
|
2671
|
-
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
2672
|
-
allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2673
4020
|
}
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
|
|
2684
|
-
orphansPurged = purgeResult.rejected;
|
|
2685
|
-
if (purgeResult.rejected > 0) {
|
|
2686
|
-
info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
|
|
4021
|
+
if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
|
|
4022
|
+
info("[improve] reindexing after memory inference writes");
|
|
4023
|
+
try {
|
|
4024
|
+
await reindexWithIndexDbReleased(primaryStashDir);
|
|
4025
|
+
reindexedAfterInference = true;
|
|
4026
|
+
info("[improve] reindex after memory inference complete");
|
|
4027
|
+
}
|
|
4028
|
+
catch (err) {
|
|
4029
|
+
allWarnings.push(`reindex after memory inference failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2687
4030
|
}
|
|
2688
|
-
appendEvent({
|
|
2689
|
-
eventType: "proposal_orphan_purge",
|
|
2690
|
-
ref: "proposals:_orphan-purge",
|
|
2691
|
-
metadata: {
|
|
2692
|
-
checked: purgeResult.checked,
|
|
2693
|
-
rejected: purgeResult.rejected,
|
|
2694
|
-
durationMs: purgeResult.durationMs,
|
|
2695
|
-
byType: purgeResult.byType,
|
|
2696
|
-
orphans: purgeResult.orphans.map((o) => o.ref),
|
|
2697
|
-
},
|
|
2698
|
-
}, eventsCtx);
|
|
2699
4031
|
}
|
|
2700
|
-
|
|
2701
|
-
|
|
4032
|
+
const graphEnabled = isProcessEnabled("index", "graph_extraction", config);
|
|
4033
|
+
const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
|
|
4034
|
+
const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
|
|
4035
|
+
// #624 P2: optional incremental high-signal-first cap. Unset = process all
|
|
4036
|
+
// eligible (byte-identical to today; no ranking/slice).
|
|
4037
|
+
const graphExtractionTopN = improveProfile?.processes?.graphExtraction?.topN;
|
|
4038
|
+
// Build the set of refs actually touched this run.
|
|
4039
|
+
const touchedRefs = new Set();
|
|
4040
|
+
for (const r of args.actionableRefs)
|
|
4041
|
+
touchedRefs.add(r.ref);
|
|
4042
|
+
for (const r of memoryRefsForInference)
|
|
4043
|
+
touchedRefs.add(r);
|
|
4044
|
+
// INVARIANT: graph extraction normally runs only on files touched by
|
|
4045
|
+
// actionable refs (candidatePaths). Full-corpus scans are opt-in via
|
|
4046
|
+
// profile.processes.graphExtraction.fullScan = true (used by the
|
|
4047
|
+
// `graph-refresh` built-in profile and its weekly scheduled task).
|
|
4048
|
+
// The empty-Set fallback is intentional when no refs were touched —
|
|
4049
|
+
// the extractor's filter rejects every file and returns empty, keeping
|
|
4050
|
+
// the pass invoked so the action is recorded and tests stay exercised.
|
|
4051
|
+
if (graphExtractionDisabledByProfile) {
|
|
4052
|
+
info("[improve] graph extraction skipped (disabled by improve profile)");
|
|
2702
4053
|
}
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
4054
|
+
else if (sources.length > 0 && graphEnabled) {
|
|
4055
|
+
info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
|
|
4056
|
+
const extractionStart = Date.now();
|
|
4057
|
+
try {
|
|
4058
|
+
// D9: if consolidation ran but memory inference did not reindex, force a reindex
|
|
4059
|
+
// so graph extraction sees current DB state after consolidation writes.
|
|
4060
|
+
if (consolidationRan && !reindexedAfterInference) {
|
|
4061
|
+
info("[improve] reindexing after consolidation (graph extraction needs current state)");
|
|
4062
|
+
try {
|
|
4063
|
+
await reindexWithIndexDbReleased(primaryStashDir);
|
|
4064
|
+
reindexedAfterInference = true;
|
|
4065
|
+
info("[improve] reindex after consolidation complete");
|
|
4066
|
+
}
|
|
4067
|
+
catch (err) {
|
|
4068
|
+
allWarnings.push(`reindex after consolidation failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
4069
|
+
}
|
|
4070
|
+
}
|
|
4071
|
+
// #584: no close/reopen needed here — reindexWithIndexDbReleased
|
|
4072
|
+
// already swapped in a fresh post-reindex handle.
|
|
4073
|
+
// Resolve touched refs to absolute file paths. Skipped for fullScan
|
|
4074
|
+
// (candidatePaths stays undefined → extractor processes all files).
|
|
4075
|
+
let candidatePaths;
|
|
4076
|
+
if (!graphExtractionFullScan) {
|
|
4077
|
+
candidatePaths = new Set();
|
|
4078
|
+
if (primaryStashDir && touchedRefs.size > 0) {
|
|
4079
|
+
const writableDirSet = new Set(getWritableStashDirs(primaryStashDir).map((d) => path.resolve(d)));
|
|
4080
|
+
const resolved = await Promise.all([...touchedRefs].map((ref) => findAssetFilePath(ref, primaryStashDir, writableDirSet).catch(() => null)));
|
|
4081
|
+
for (const p of resolved) {
|
|
4082
|
+
if (typeof p === "string" && p.length > 0)
|
|
4083
|
+
candidatePaths.add(p);
|
|
4084
|
+
}
|
|
4085
|
+
}
|
|
4086
|
+
}
|
|
4087
|
+
const progressHandler = (event) => {
|
|
4088
|
+
const current = event.currentPath ? ` ${path.basename(event.currentPath)}` : "";
|
|
4089
|
+
info(`[improve] graph extraction ${event.processed}/${event.total}${current} (extracted ${event.extracted}, entities ${event.totalEntities}, relations ${event.totalRelations})`);
|
|
4090
|
+
};
|
|
4091
|
+
// O-1 (#364): pass budget signal so a hung graph extraction call is cancelled.
|
|
4092
|
+
graphExtraction = await withLlmStage("graph-extraction", () => graphExtractionFn({
|
|
4093
|
+
config,
|
|
4094
|
+
sources,
|
|
4095
|
+
signal: budgetSignal,
|
|
4096
|
+
db,
|
|
4097
|
+
reEnrich: false,
|
|
4098
|
+
onProgress: progressHandler,
|
|
4099
|
+
options: { candidatePaths, ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}) },
|
|
4100
|
+
}));
|
|
4101
|
+
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
4102
|
+
actions.push({ ref: "graph:_artifact", mode: "graph-extraction", result: graphExtraction });
|
|
4103
|
+
info(`[improve] graph extraction complete (${graphExtraction.quality.extractedFiles} files, ${graphExtraction.quality.entityCount} entities, ${graphExtraction.quality.relationCount} relations)`);
|
|
4104
|
+
}
|
|
4105
|
+
catch (err) {
|
|
4106
|
+
graphExtractionDurationMs = Date.now() - extractionStart;
|
|
4107
|
+
allWarnings.push(`graph extraction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2714
4108
|
}
|
|
2715
|
-
appendEvent({
|
|
2716
|
-
eventType: "proposal_expiration_pass",
|
|
2717
|
-
ref: "proposals:_expiration",
|
|
2718
|
-
metadata: {
|
|
2719
|
-
checked: expireResult.checked,
|
|
2720
|
-
expired: expireResult.expired,
|
|
2721
|
-
durationMs: expireResult.durationMs,
|
|
2722
|
-
retentionDays: expireResult.retentionDays,
|
|
2723
|
-
expiredProposals: expireResult.expiredProposals,
|
|
2724
|
-
},
|
|
2725
|
-
}, eventsCtx);
|
|
2726
4109
|
}
|
|
2727
|
-
|
|
2728
|
-
|
|
4110
|
+
else if (sources.length > 0 && !graphEnabled) {
|
|
4111
|
+
info("[improve] graph extraction skipped (features.index.graph_extraction is disabled)");
|
|
2729
4112
|
}
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
// without this trim, state.db is a permanent append-only log. Config key
|
|
2735
|
-
// `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
|
|
2736
|
-
// window. `purgeOldEvents()` opens its own state.db handle separate from
|
|
2737
|
-
// the index `db` above (different SQLite file).
|
|
2738
|
-
{
|
|
2739
|
-
const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
|
|
2740
|
-
if (retentionDays > 0) {
|
|
2741
|
-
let stateDb;
|
|
4113
|
+
// Orphan proposal purge — reject pending reflect proposals whose target
|
|
4114
|
+
// asset no longer exists on disk. Runs after graph extraction so newly
|
|
4115
|
+
// promoted assets from accept flows during this run are already present.
|
|
4116
|
+
if (primaryStashDir) {
|
|
2742
4117
|
try {
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
stateDb = openStateDatabase(eventsCtx?.dbPath);
|
|
2748
|
-
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
2749
|
-
if (purgedCount > 0) {
|
|
2750
|
-
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
4118
|
+
const purgeResult = purgeOrphanProposals(primaryStashDir, sources.map((s) => s.path));
|
|
4119
|
+
orphansPurged = purgeResult.rejected;
|
|
4120
|
+
if (purgeResult.rejected > 0) {
|
|
4121
|
+
info(`[improve] orphan purge: ${purgeResult.rejected}/${purgeResult.checked} orphaned proposals rejected (${purgeResult.durationMs}ms)`);
|
|
2751
4122
|
}
|
|
2752
4123
|
appendEvent({
|
|
2753
|
-
eventType: "
|
|
2754
|
-
ref: "
|
|
2755
|
-
metadata: {
|
|
4124
|
+
eventType: "proposal_orphan_purge",
|
|
4125
|
+
ref: "proposals:_orphan-purge",
|
|
4126
|
+
metadata: {
|
|
4127
|
+
checked: purgeResult.checked,
|
|
4128
|
+
rejected: purgeResult.rejected,
|
|
4129
|
+
durationMs: purgeResult.durationMs,
|
|
4130
|
+
byType: purgeResult.byType,
|
|
4131
|
+
orphans: purgeResult.orphans.map((o) => o.ref),
|
|
4132
|
+
},
|
|
2756
4133
|
}, eventsCtx);
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
4134
|
+
}
|
|
4135
|
+
catch (err) {
|
|
4136
|
+
allWarnings.push(`orphan purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
4137
|
+
}
|
|
4138
|
+
// Phase 6B (Advantage D6b): expire pending proposals that have aged past
|
|
4139
|
+
// the retention window. Runs AFTER orphan purge so we never double-archive
|
|
4140
|
+
// a proposal that orphan-purge already moved. `expireStaleProposals` emits
|
|
4141
|
+
// its own per-proposal `proposal_expired` events; we additionally emit a
|
|
4142
|
+
// single roll-up event here for parity with the orphan-purge surface.
|
|
4143
|
+
try {
|
|
4144
|
+
const expireResult = expireStaleProposals(primaryStashDir, config);
|
|
4145
|
+
proposalsExpired = expireResult.expired;
|
|
4146
|
+
if (expireResult.expired > 0) {
|
|
4147
|
+
info(`[improve] expiration: ${expireResult.expired}/${expireResult.checked} pending proposals expired ` +
|
|
4148
|
+
`(retention=${expireResult.retentionDays}d, ${expireResult.durationMs}ms)`);
|
|
2764
4149
|
}
|
|
2765
4150
|
appendEvent({
|
|
2766
|
-
eventType: "
|
|
2767
|
-
ref: "
|
|
2768
|
-
metadata: {
|
|
4151
|
+
eventType: "proposal_expiration_pass",
|
|
4152
|
+
ref: "proposals:_expiration",
|
|
4153
|
+
metadata: {
|
|
4154
|
+
checked: expireResult.checked,
|
|
4155
|
+
expired: expireResult.expired,
|
|
4156
|
+
durationMs: expireResult.durationMs,
|
|
4157
|
+
retentionDays: expireResult.retentionDays,
|
|
4158
|
+
expiredProposals: expireResult.expiredProposals,
|
|
4159
|
+
},
|
|
2769
4160
|
}, eventsCtx);
|
|
2770
4161
|
}
|
|
2771
4162
|
catch (err) {
|
|
2772
|
-
allWarnings.push(`
|
|
4163
|
+
allWarnings.push(`proposal expiration failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2773
4164
|
}
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
4165
|
+
}
|
|
4166
|
+
// Fix #2 (observability 0.8.0): trim the events table in state.db so it
|
|
4167
|
+
// doesn't grow unbounded. `akm health` writes a `health_probe` row on every
|
|
4168
|
+
// invocation, and every command surface emits at least one event besides —
|
|
4169
|
+
// without this trim, state.db is a permanent append-only log. Config key
|
|
4170
|
+
// `improve.eventRetentionDays` (default 90, set 0 to disable) controls the
|
|
4171
|
+
// window. The purge runs against state.db (a different SQLite file from
|
|
4172
|
+
// the index `db` above).
|
|
4173
|
+
{
|
|
4174
|
+
const retentionDays = typeof config.improve?.eventRetentionDays === "number" ? config.improve.eventRetentionDays : 90;
|
|
4175
|
+
if (retentionDays > 0) {
|
|
4176
|
+
// #585: reuse the long-lived eventsCtx.db connection when akmImprove
|
|
4177
|
+
// opened one — opening a second state.db write connection while
|
|
4178
|
+
// eventsDb is still live made two simultaneous writers contend on the
|
|
4179
|
+
// same WAL file ("database is locked"). Only the eventsCtx.dbPath
|
|
4180
|
+
// fallback path (state.db failed to open up-front) opens — and then
|
|
4181
|
+
// owns and closes — its own handle. C2 still holds: the fallback uses
|
|
4182
|
+
// the boundary-pinned path, never a live `process.env` re-read.
|
|
4183
|
+
const ownsStateDb = !eventsCtx?.db;
|
|
4184
|
+
let stateDb;
|
|
4185
|
+
try {
|
|
4186
|
+
stateDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
|
|
4187
|
+
const purgedCount = purgeOldEvents(stateDb, retentionDays);
|
|
4188
|
+
if (purgedCount > 0) {
|
|
4189
|
+
info(`[improve] events purge: ${purgedCount} event(s) older than ${retentionDays}d removed from state.db`);
|
|
2778
4190
|
}
|
|
2779
|
-
|
|
2780
|
-
|
|
4191
|
+
appendEvent({
|
|
4192
|
+
eventType: "events_purged",
|
|
4193
|
+
ref: "events:_purge",
|
|
4194
|
+
metadata: { purgedCount, retentionDays },
|
|
4195
|
+
}, eventsCtx);
|
|
4196
|
+
// improve_runs uses the same retention window as events — both are
|
|
4197
|
+
// observability/audit data, both grow append-only, both have a
|
|
4198
|
+
// dedicated purge helper. Mirroring the events purge here means a
|
|
4199
|
+
// single retention knob (improve.eventRetentionDays) governs both.
|
|
4200
|
+
const improveRunsPurged = purgeOldImproveRuns(stateDb, retentionDays);
|
|
4201
|
+
if (improveRunsPurged > 0) {
|
|
4202
|
+
info(`[improve] improve_runs purge: ${improveRunsPurged} run(s) older than ${retentionDays}d removed from state.db`);
|
|
4203
|
+
}
|
|
4204
|
+
appendEvent({
|
|
4205
|
+
eventType: "improve_runs_purged",
|
|
4206
|
+
ref: "improve_runs:_purge",
|
|
4207
|
+
metadata: { purgedCount: improveRunsPurged, retentionDays },
|
|
4208
|
+
}, eventsCtx);
|
|
4209
|
+
}
|
|
4210
|
+
catch (err) {
|
|
4211
|
+
allWarnings.push(`events purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
4212
|
+
}
|
|
4213
|
+
finally {
|
|
4214
|
+
if (ownsStateDb && stateDb) {
|
|
4215
|
+
try {
|
|
4216
|
+
stateDb.close();
|
|
4217
|
+
}
|
|
4218
|
+
catch {
|
|
4219
|
+
// best-effort
|
|
4220
|
+
}
|
|
4221
|
+
}
|
|
4222
|
+
}
|
|
4223
|
+
// task_logs in logs.db (#579) shares the same retention window as
|
|
4224
|
+
// events/improve_runs — all three are observability data governed by
|
|
4225
|
+
// the single improve.eventRetentionDays knob. Separate try/finally
|
|
4226
|
+
// because logs.db is a different file: a locked/missing logs.db must
|
|
4227
|
+
// not block the state.db purges above.
|
|
4228
|
+
let logsDb;
|
|
4229
|
+
try {
|
|
4230
|
+
logsDb = openLogsDatabase();
|
|
4231
|
+
const taskLogsPurged = purgeOldTaskLogs(logsDb, retentionDays);
|
|
4232
|
+
if (taskLogsPurged > 0) {
|
|
4233
|
+
info(`[improve] task_logs purge: ${taskLogsPurged} log line(s) older than ${retentionDays}d removed from logs.db`);
|
|
4234
|
+
}
|
|
4235
|
+
appendEvent({
|
|
4236
|
+
eventType: "task_logs_purged",
|
|
4237
|
+
ref: "task_logs:_purge",
|
|
4238
|
+
metadata: { purgedCount: taskLogsPurged, retentionDays },
|
|
4239
|
+
}, eventsCtx);
|
|
4240
|
+
}
|
|
4241
|
+
catch (err) {
|
|
4242
|
+
allWarnings.push(`task_logs purge failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
4243
|
+
}
|
|
4244
|
+
finally {
|
|
4245
|
+
if (logsDb) {
|
|
4246
|
+
try {
|
|
4247
|
+
logsDb.close();
|
|
4248
|
+
}
|
|
4249
|
+
catch {
|
|
4250
|
+
// best-effort
|
|
4251
|
+
}
|
|
2781
4252
|
}
|
|
2782
4253
|
}
|
|
2783
4254
|
}
|
|
2784
4255
|
}
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
4256
|
+
// Phase 4A (staleness detection). Activates the `deprecated` belief-state
|
|
4257
|
+
// machinery shipped in Phase 1A. Default OFF — gated by
|
|
4258
|
+
// `features.index.staleness_detection.enabled`. Runs after orphan purge
|
|
4259
|
+
// and before the URL check (which lives in the outer caller).
|
|
4260
|
+
if (sources.length > 0) {
|
|
4261
|
+
try {
|
|
4262
|
+
stalenessDetection = await withLlmStage("staleness-detection", () => stalenessDetectionFn({ config, sources, signal: budgetSignal, db }));
|
|
4263
|
+
if (stalenessDetection.considered > 0) {
|
|
4264
|
+
info(`[improve] staleness detection complete (considered ${stalenessDetection.considered}, ` +
|
|
4265
|
+
`deprecated ${stalenessDetection.deprecated}, confirmed ${stalenessDetection.confirmed}, ` +
|
|
4266
|
+
`skipped ${stalenessDetection.skipped}, ${stalenessDetection.durationMs}ms)`);
|
|
4267
|
+
}
|
|
4268
|
+
for (const w of stalenessDetection.warnings)
|
|
4269
|
+
allWarnings.push(`[improve] staleness detection: ${w}`);
|
|
4270
|
+
}
|
|
4271
|
+
catch (err) {
|
|
4272
|
+
allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2797
4273
|
}
|
|
2798
|
-
for (const w of stalenessDetection.warnings)
|
|
2799
|
-
allWarnings.push(`[improve] staleness detection: ${w}`);
|
|
2800
|
-
}
|
|
2801
|
-
catch (err) {
|
|
2802
|
-
allWarnings.push(`staleness detection failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2803
4274
|
}
|
|
2804
4275
|
}
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
}
|
|
4276
|
+
finally {
|
|
4277
|
+
if (db)
|
|
4278
|
+
closeDatabase(db);
|
|
4279
|
+
}
|
|
4280
|
+
});
|
|
2810
4281
|
return {
|
|
2811
4282
|
...(memoryInference ? { memoryInference } : {}),
|
|
2812
4283
|
...(graphExtraction ? { graphExtraction } : {}),
|