akm-cli 0.9.0-beta.11 → 0.9.0-beta.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +163 -0
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +5 -1
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/templates/html/health.html +25 -27
- package/dist/cli.js +2 -2
- package/dist/commands/agent/contribute-cli.js +16 -3
- package/dist/commands/feedback-cli.js +48 -44
- package/dist/commands/health/html-report.js +140 -16
- package/dist/commands/health.js +277 -1
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate.js +595 -105
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +119 -64
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +115 -1
- package/dist/commands/improve/extract-prompt.js +32 -1
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +210 -30
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +80 -7
- package/dist/commands/improve/improve-profiles.js +8 -0
- package/dist/commands/improve/improve.js +991 -61
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/proactive-maintenance.js +9 -35
- package/dist/commands/improve/procedural.js +409 -0
- package/dist/commands/improve/recombine.js +488 -0
- package/dist/commands/improve/reflect.js +20 -1
- package/dist/commands/improve/related-sessions.js +120 -0
- package/dist/commands/improve/salience.js +386 -0
- package/dist/commands/improve/triage.js +95 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +34 -1
- package/dist/commands/lint/index.js +30 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +5 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/validators/proposals.js +4 -0
- package/dist/commands/read/curate.js +284 -86
- package/dist/commands/read/search-cli.js +7 -0
- package/dist/commands/read/search.js +1 -0
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/config/config-schema.js +211 -3
- package/dist/core/config/config.js +2 -2
- package/dist/core/logs-db.js +4 -3
- package/dist/core/state-db.js +555 -29
- package/dist/indexer/db/db.js +250 -27
- package/dist/indexer/db/graph-db.js +81 -86
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/passes/memory-inference.js +10 -3
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +15 -4
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/integrations/harnesses/claude/session-log.js +10 -0
- package/dist/integrations/harnesses/opencode/session-log.js +9 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/embedder.js +27 -3
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/graph-extract.js +2 -1
- package/dist/llm/memory-infer.js +4 -8
- package/dist/llm/metadata-enhance.js +9 -1
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/text/helpers.js +9 -0
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +1025 -567
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +435 -269
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +1 -0
- package/package.json +5 -4
|
@@ -14,7 +14,7 @@ import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "
|
|
|
14
14
|
import { classifyImproveAction } from "../../core/improve-types.js";
|
|
15
15
|
import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
|
|
16
16
|
import { getDbPath, getStateDbPathInDataDir } from "../../core/paths.js";
|
|
17
|
-
import { openStateDatabase, purgeOldEvents, purgeOldImproveRuns } from "../../core/state-db.js";
|
|
17
|
+
import { listProposalGateDecisions, listStateProposals, openStateDatabase, persistPhaseThreshold, purgeOldEvents, purgeOldImproveRuns, } from "../../core/state-db.js";
|
|
18
18
|
import { info, warn } from "../../core/warn.js";
|
|
19
19
|
import { closeDatabase, getAllEntries, getEntryCount, getRetrievalCounts, getUtilityScoresByIds, getZeroResultSearches, openDatabase, openExistingDatabase, } from "../../indexer/db/db.js";
|
|
20
20
|
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
@@ -38,17 +38,23 @@ import { resolveDrainPolicy } from "../proposal/drain-policies.js";
|
|
|
38
38
|
import { createProposal, expireStaleProposals, getProposal, isProposalSkipped, listProposals, purgeOrphanProposals, } from "../proposal/validators/proposals.js";
|
|
39
39
|
import { runSchemaRepairPass } from "../sources/schema-repair.js";
|
|
40
40
|
import { checkDeadUrls } from "../url-checker.js";
|
|
41
|
+
import { computeThresholdAutoTune, gateDecisionsToSamples, summarizeCalibration, } from "./calibration.js";
|
|
41
42
|
import { akmConsolidate } from "./consolidate.js";
|
|
42
43
|
import { akmDistill, deriveLessonRef, isDistillRefusedInputType } from "./distill.js";
|
|
43
44
|
import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
|
|
44
45
|
import { countEvalCases, writeEvalCase } from "./eval-cases.js";
|
|
45
46
|
import { akmExtract, countNewExtractCandidates } from "./extract.js";
|
|
47
|
+
import { computeValenceScore, FEEDBACK_WEIGHT, UTILITY_WEIGHT } from "./feedback-valence.js";
|
|
46
48
|
import { makeGateConfig, resolveExtractConfidence, runAutoAcceptGate } from "./improve-auto-accept.js";
|
|
47
49
|
import { isProfileFilteredForAllPasses, resolveImproveProfile, resolveProcessEnabled, shouldSkipRef, } from "./improve-profiles.js";
|
|
48
50
|
import { detectAndWriteContradictions } from "./memory/memory-contradiction-detect.js";
|
|
49
51
|
import { analyzeMemoryCleanup, applyMemoryCleanup } from "./memory/memory-improve.js";
|
|
52
|
+
import { computeProxyAdequacy, getAllAssetOutcomes, getOutcomeScoresByRef, outcomeScoreToSalience, updateAssetOutcome, } from "./outcome-loop.js";
|
|
50
53
|
import { DEFAULT_DUE_DAYS, DEFAULT_MAX_PER_RUN, selectProactiveMaintenanceRefs } from "./proactive-maintenance.js";
|
|
54
|
+
import { akmProcedural } from "./procedural.js";
|
|
55
|
+
import { akmRecombine } from "./recombine.js";
|
|
51
56
|
import { akmReflect } from "./reflect.js";
|
|
57
|
+
import { buildRankChangeReport, computeSalience, getAllRankScores, getAssetSalience, getConsecutiveNoOps, getLastUseMsByRef, recordNoOp, resetConsecutiveNoOps, SALIENCE_NO_OP_DAMPEN_FACTOR, SALIENCE_NO_OP_DAMPEN_THRESHOLD, upsertAssetSalience, } from "./salience.js";
|
|
52
58
|
// #607 Lock Decomposition: fine-grained per-process locks replace the single
|
|
53
59
|
// `improve.lock`. Three independent locks allow concurrent improve runs when
|
|
54
60
|
// they touch different subsystems (e.g. quick-shredder consolidate can run
|
|
@@ -569,6 +575,107 @@ export function armBudgetWatchdog(budgetMs, controller, deps) {
|
|
|
569
575
|
}
|
|
570
576
|
};
|
|
571
577
|
}
|
|
578
|
+
/**
|
|
579
|
+
* #612 / WS-4 — bounded, opt-in per-phase auto-accept threshold auto-tune.
|
|
580
|
+
*
|
|
581
|
+
* Reads `improve.calibration` from config. When `autoTune` is enabled, computes
|
|
582
|
+
* the calibration of recent gate decisions, derives a bounded threshold
|
|
583
|
+
* adjustment (clamped into the configured band, capped per step), logs it, and
|
|
584
|
+
* records a `calibration_autotune` event. Returns the new threshold (integer
|
|
585
|
+
* 0-100) when an adjustment was made, or `undefined` to leave the caller's
|
|
586
|
+
* threshold unchanged.
|
|
587
|
+
*
|
|
588
|
+
* WS-4 change: accepts an optional `phase` parameter. When provided, the tuned
|
|
589
|
+
* threshold is persisted to `improve_gate_thresholds` (state.db Migration 012)
|
|
590
|
+
* keyed by phase so `makeGateConfig` can read it back on the next run and each
|
|
591
|
+
* phase maintains its own calibrated threshold rather than a shared global.
|
|
592
|
+
*
|
|
593
|
+
* WS-4 ceiling: `maxThreshold` defaults to 85 (not 100) to prevent the gate
|
|
594
|
+
* converging to pure exploitation and shutting down Gap-3/4 novelty throughput.
|
|
595
|
+
*
|
|
596
|
+
* DEFAULT OFF: with no `improve.calibration` block (or `autoTune: false`) this
|
|
597
|
+
* returns `undefined` immediately, so the gate threshold is unchanged and
|
|
598
|
+
* behaviour is byte-identical to today.
|
|
599
|
+
*/
|
|
600
|
+
export function maybeAutoTuneThreshold(currentThreshold, config, stateDbPath, ctx, phase) {
|
|
601
|
+
const cal = config.improve?.calibration;
|
|
602
|
+
if (!cal?.autoTune)
|
|
603
|
+
return undefined;
|
|
604
|
+
// WS-4: default maxThreshold is now 85 (ceiling to prevent pure exploitation).
|
|
605
|
+
// Callers that explicitly set maxThreshold in config override this default.
|
|
606
|
+
const tuneConfig = {
|
|
607
|
+
autoTune: true,
|
|
608
|
+
minThreshold: cal.minThreshold ?? 0,
|
|
609
|
+
maxThreshold: cal.maxThreshold ?? 85,
|
|
610
|
+
maxStep: cal.maxStep ?? 5,
|
|
611
|
+
minSamples: cal.minSamples ?? 20,
|
|
612
|
+
targetAcceptRate: cal.targetAcceptRate ?? 0.9,
|
|
613
|
+
};
|
|
614
|
+
// Defensive: an inverted band disables tuning rather than clamping to nonsense.
|
|
615
|
+
if (tuneConfig.minThreshold > tuneConfig.maxThreshold)
|
|
616
|
+
return undefined;
|
|
617
|
+
const db = openStateDatabase(stateDbPath);
|
|
618
|
+
let summary;
|
|
619
|
+
try {
|
|
620
|
+
const allDecisions = listProposalGateDecisions(db);
|
|
621
|
+
// WS-4 fix: when called with a phase label, restrict calibration to that
|
|
622
|
+
// phase's decision pool so a reflect-dominated run cannot tighten the
|
|
623
|
+
// consolidate gate (or vice-versa). The gate field is `improve:<phase>`,
|
|
624
|
+
// matching what improve-auto-accept.ts stamps at line ~163.
|
|
625
|
+
const gateLabel = phase ? `improve:${phase}` : undefined;
|
|
626
|
+
const decisions = gateLabel ? allDecisions.filter((d) => d.gate === gateLabel) : allDecisions;
|
|
627
|
+
summary = summarizeCalibration(gateDecisionsToSamples(decisions));
|
|
628
|
+
}
|
|
629
|
+
finally {
|
|
630
|
+
db.close();
|
|
631
|
+
}
|
|
632
|
+
const result = computeThresholdAutoTune(currentThreshold, summary, tuneConfig);
|
|
633
|
+
if (!result.adjusted)
|
|
634
|
+
return undefined;
|
|
635
|
+
const appendEventFn = ctx?.appendEventFn ?? appendEvent;
|
|
636
|
+
const phaseLabel = phase ?? "global";
|
|
637
|
+
info(`[improve] calibration auto-tune (${phaseLabel}): threshold ${result.previousThreshold} -> ${result.newThreshold} ` +
|
|
638
|
+
`(${result.reason}; samples=${summary.samples}, acceptRate=${summary.overallAcceptRate}, ` +
|
|
639
|
+
`gap=${summary.calibrationGap}, band=[${tuneConfig.minThreshold},${tuneConfig.maxThreshold}])`);
|
|
640
|
+
try {
|
|
641
|
+
appendEventFn({
|
|
642
|
+
eventType: "calibration_autotune",
|
|
643
|
+
ref: "improve:calibration",
|
|
644
|
+
metadata: {
|
|
645
|
+
phase: phaseLabel,
|
|
646
|
+
previousThreshold: result.previousThreshold,
|
|
647
|
+
newThreshold: result.newThreshold,
|
|
648
|
+
delta: result.delta,
|
|
649
|
+
reason: result.reason,
|
|
650
|
+
samples: summary.samples,
|
|
651
|
+
overallAcceptRate: summary.overallAcceptRate,
|
|
652
|
+
calibrationGap: summary.calibrationGap,
|
|
653
|
+
minThreshold: tuneConfig.minThreshold,
|
|
654
|
+
maxThreshold: tuneConfig.maxThreshold,
|
|
655
|
+
},
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
catch (err) {
|
|
659
|
+
warn(`[improve] calibration auto-tune event not recorded: ${err instanceof Error ? err.message : String(err)}`);
|
|
660
|
+
}
|
|
661
|
+
// WS-4: Persist the per-phase threshold so makeGateConfig reads it on the
|
|
662
|
+
// next run. Best-effort — a write failure must not abort the improve run.
|
|
663
|
+
if (phase) {
|
|
664
|
+
try {
|
|
665
|
+
const persistDb = openStateDatabase(stateDbPath);
|
|
666
|
+
try {
|
|
667
|
+
persistPhaseThreshold(persistDb, phase, result.newThreshold);
|
|
668
|
+
}
|
|
669
|
+
finally {
|
|
670
|
+
persistDb.close();
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
catch (err) {
|
|
674
|
+
warn(`[improve] calibration auto-tune: failed to persist phase threshold for ${phase}: ${err instanceof Error ? err.message : String(err)}`);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
return result.newThreshold;
|
|
678
|
+
}
|
|
572
679
|
export async function akmImprove(options = {}) {
|
|
573
680
|
const scope = resolveImproveScope(options.scope);
|
|
574
681
|
const reflectFn = options.reflectFn ?? akmReflect;
|
|
@@ -606,6 +713,23 @@ export async function akmImprove(options = {}) {
|
|
|
606
713
|
// timeout root cause). Because beforeEach runs synchronously, env is still the
|
|
607
714
|
// calling test's own at this point; we capture it before yielding the loop.
|
|
608
715
|
const resolvedStateDbPath = getStateDbPathInDataDir();
|
|
716
|
+
// #612 / WS-4 — bounded, OPT-IN per-phase auto-accept threshold auto-tune.
|
|
717
|
+
// DEFAULT OFF: `autoTune: false` (or absent) is a complete no-op.
|
|
718
|
+
//
|
|
719
|
+
// WS-4 change: thresholds are now PER PHASE. The old single global mutation
|
|
720
|
+
// of `options.autoAccept` is retired — it caused every phase to share one
|
|
721
|
+
// calibration signal, so a reflect-dominated run could tighten the consolidate
|
|
722
|
+
// gate (or vice-versa). Instead:
|
|
723
|
+
// - Each `makeGateConfig` call reads the phase's stored threshold from
|
|
724
|
+
// state.db (Migration 012) and uses it as `phaseThreshold`, overriding
|
|
725
|
+
// the `globalThreshold` (= options.autoAccept) for that phase.
|
|
726
|
+
// - Per-phase `maybeAutoTuneThreshold` calls fire AFTER each phase's gate
|
|
727
|
+
// has run and persist the new threshold to state.db for the NEXT run.
|
|
728
|
+
// - `options.autoAccept` stays unchanged (it is the operator-supplied
|
|
729
|
+
// baseline, not a mutable run-time state).
|
|
730
|
+
//
|
|
731
|
+
// The global tune call is intentionally removed here. See per-phase calls
|
|
732
|
+
// below (near each makeGateConfig / runAutoAcceptGate block).
|
|
609
733
|
// #607 Lock decomposition: three per-process locks replace the single
|
|
610
734
|
// `improve.lock`. Each process acquires only the lock(s) it needs, so
|
|
611
735
|
// quick-shredder consolidate can run alongside daily reflect+distill.
|
|
@@ -792,6 +916,16 @@ export async function akmImprove(options = {}) {
|
|
|
792
916
|
// run past the declared budget.
|
|
793
917
|
// References: Anthropic *Building Effective Agents* (2024); CoALA §5 (arXiv:2309.02427).
|
|
794
918
|
const budgetAbortController = new AbortController();
|
|
919
|
+
// Attach a live `remainingBudgetMs` getter to the signal so sub-callers
|
|
920
|
+
// (e.g. consolidate.ts cold-start budget estimation) can read the remaining
|
|
921
|
+
// wall-clock budget without needing an extra plumbing parameter. The property
|
|
922
|
+
// is computed at access time via a getter so it always reflects the actual
|
|
923
|
+
// elapsed time rather than a stale snapshot taken at arm time.
|
|
924
|
+
Object.defineProperty(budgetAbortController.signal, "remainingBudgetMs", {
|
|
925
|
+
get: () => Math.max(0, budgetMs - (Date.now() - startMs)),
|
|
926
|
+
enumerable: false,
|
|
927
|
+
configurable: true,
|
|
928
|
+
});
|
|
795
929
|
// Declared in the outer scope so the `finally` can clear the timer even if a
|
|
796
930
|
// throw occurs before/after it is armed. Defaults to a no-op until armed.
|
|
797
931
|
let clearBudgetTimer = () => { };
|
|
@@ -864,6 +998,7 @@ export async function akmImprove(options = {}) {
|
|
|
864
998
|
eventsCtx,
|
|
865
999
|
initialCleanupWarnings: preEnsureCleanupWarnings,
|
|
866
1000
|
improveProfile,
|
|
1001
|
+
budgetSignal: budgetAbortController.signal,
|
|
867
1002
|
});
|
|
868
1003
|
if (consolidatePrepAcquired)
|
|
869
1004
|
releaseProcessLock(consolidateLPath);
|
|
@@ -910,7 +1045,7 @@ export async function akmImprove(options = {}) {
|
|
|
910
1045
|
// graphExtraction both write index.db). Released immediately after.
|
|
911
1046
|
const consolidatePostLPath = processLockPath(lockBaseDir, "consolidate");
|
|
912
1047
|
const consolidatePostAcquired = tryAcquireProcessLock(consolidatePostLPath, PROCESS_LOCK_DEFS.consolidate.staleAfterMs, options.skipIfLocked, "consolidate") === "acquired";
|
|
913
|
-
const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, } = await runImprovePostLoopStage({
|
|
1048
|
+
const { allWarnings, deadUrls, memoryInference, graphExtraction, stalenessDetection, maintenanceActions, memoryInferenceDurationMs, graphExtractionDurationMs, orphansPurged, proposalsExpired, gateAutoAcceptedCount: postLoopGateCount, gateAutoAcceptFailedCount: postLoopGateFailedCount, recombination, proceduralCompilation, } = await runImprovePostLoopStage({
|
|
914
1049
|
scope,
|
|
915
1050
|
options,
|
|
916
1051
|
primaryStashDir,
|
|
@@ -987,6 +1122,8 @@ export async function akmImprove(options = {}) {
|
|
|
987
1122
|
...(memoryInferenceDurationMs > 0 ? { memoryInferenceDurationMs } : {}),
|
|
988
1123
|
...(graphExtractionDurationMs > 0 ? { graphExtractionDurationMs } : {}),
|
|
989
1124
|
...(stalenessDetection ? { stalenessDetection } : {}),
|
|
1125
|
+
...(recombination ? { recombination } : {}),
|
|
1126
|
+
...(proceduralCompilation ? { proceduralCompilation } : {}),
|
|
990
1127
|
...(orphansPurged !== undefined ? { orphansPurged } : {}),
|
|
991
1128
|
...(proposalsExpired !== undefined && proposalsExpired > 0 ? { proposalsExpired } : {}),
|
|
992
1129
|
reflectCooldownActions: finalActions.filter((a) => a.mode === "reflect-cooldown").length,
|
|
@@ -1254,7 +1391,7 @@ function emitImproveCompletedEvent(result, durations, eventsCtx) {
|
|
|
1254
1391
|
* need, so the fix is non-invasive and provably correct.
|
|
1255
1392
|
*/
|
|
1256
1393
|
async function runConsolidationPass(args) {
|
|
1257
|
-
const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx } = args;
|
|
1394
|
+
const { options, primaryStashDir, memorySummary, improveProfile, eventsCtx, budgetSignal, runBudgetMs } = args;
|
|
1258
1395
|
const baseConfig = options.config ?? loadConfig();
|
|
1259
1396
|
const MEMORY_VOLUME_THRESHOLD = options.memoryVolumeConsolidationThreshold ?? 100;
|
|
1260
1397
|
const hasLlm = !!(baseConfig.defaults?.llm || baseConfig.defaults?.agent);
|
|
@@ -1398,6 +1535,7 @@ async function runConsolidationPass(args) {
|
|
|
1398
1535
|
stashDir: primaryStashDir,
|
|
1399
1536
|
config: consolidationConfig,
|
|
1400
1537
|
eventsCtx,
|
|
1538
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
1401
1539
|
}, { minimumThreshold: 95 });
|
|
1402
1540
|
if (consolidateDisabledByProfile) {
|
|
1403
1541
|
info("[improve] consolidation skipped (disabled by improve profile)");
|
|
@@ -1434,6 +1572,14 @@ async function runConsolidationPass(args) {
|
|
|
1434
1572
|
limit: improveProfile?.processes?.consolidate?.limit,
|
|
1435
1573
|
neighborsPerChanged: improveProfile?.processes?.consolidate?.neighborsPerChanged,
|
|
1436
1574
|
maxChunkSize: improveProfile?.processes?.consolidate?.maxChunkSize,
|
|
1575
|
+
// #617 — deterministic near-duplicate dedup pre-pass. DEFAULT OFF; only
|
|
1576
|
+
// runs when the profile explicitly sets `consolidate.dedup.enabled`.
|
|
1577
|
+
dedup: improveProfile?.processes?.consolidate?.dedup,
|
|
1578
|
+
// #581 — judged-state cache. DEFAULT OFF; only engages when the profile
|
|
1579
|
+
// explicitly sets `consolidate.judgedCache.enabled`. Skips memories
|
|
1580
|
+
// judged-unchanged since their last judge so one run sweeps the full
|
|
1581
|
+
// corpus instead of narrowing to a time-window slice.
|
|
1582
|
+
judgedCache: improveProfile?.processes?.consolidate?.judgedCache,
|
|
1437
1583
|
// Honor profile.autoAccept (already merged into options.autoAccept at the
|
|
1438
1584
|
// top of akmImprove). The CLI parser always supplies 90 when --auto-accept
|
|
1439
1585
|
// is absent, so ?? 90 is not needed here and would prevent --auto-accept=false
|
|
@@ -1441,6 +1587,13 @@ async function runConsolidationPass(args) {
|
|
|
1441
1587
|
// options.consolidateOptions.autoAccept (if explicitly provided by caller)
|
|
1442
1588
|
// still wins because the spread above runs first.
|
|
1443
1589
|
autoAccept: options.consolidateOptions?.autoAccept ?? options.autoAccept,
|
|
1590
|
+
// WS-3a: forward budget signal for graceful abort on timeout, and pass
|
|
1591
|
+
// the profile's p90 estimate for cold-start budget reduction.
|
|
1592
|
+
signal: budgetSignal,
|
|
1593
|
+
p90ChunkSecondsDefault: improveProfile?.processes?.consolidate?.p90ChunkSecondsDefault,
|
|
1594
|
+
// WS-5: pass total run budget so perfTelemetry.estimatedBudgetFractionUsed
|
|
1595
|
+
// can flag when consolidation alone exceeded the budget.
|
|
1596
|
+
runBudgetMs,
|
|
1444
1597
|
}));
|
|
1445
1598
|
{
|
|
1446
1599
|
const consolidateGr = await runAutoAcceptGate(consolidation.promoted.map((proposalId) => {
|
|
@@ -1485,10 +1638,21 @@ async function runConsolidationPass(args) {
|
|
|
1485
1638
|
}
|
|
1486
1639
|
// D9: track whether consolidation wrote any data so graph extraction can reindex if needed
|
|
1487
1640
|
const consolidationRan = !consolidateDisabledByProfile && !poolBelowMinSize && !consolidationOnCooldown && consolidation.processed > 0;
|
|
1641
|
+
// WS-4: Per-phase threshold auto-tune for the consolidate phase.
|
|
1642
|
+
// Persists result for the NEXT run's makeGateConfig to read.
|
|
1643
|
+
const consolidateTuneDbPath = eventsCtx?.dbPath;
|
|
1644
|
+
if (options.autoAccept !== undefined && consolidateTuneDbPath) {
|
|
1645
|
+
try {
|
|
1646
|
+
maybeAutoTuneThreshold(consolidateGateCfg.phaseThreshold ?? options.autoAccept, consolidationConfig, consolidateTuneDbPath, undefined, "consolidate");
|
|
1647
|
+
}
|
|
1648
|
+
catch (err) {
|
|
1649
|
+
warn(`[improve] calibration auto-tune (consolidate) skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1488
1652
|
return { consolidation, consolidationRan, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
1489
1653
|
}
|
|
1490
1654
|
async function runImprovePreparationStage(args) {
|
|
1491
|
-
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, } = args;
|
|
1655
|
+
const { scope, options, plannedRefs, memoryCleanupPlan, primaryStashDir, memorySummary, reindexFn, startMs, budgetMs, eventsCtx, initialCleanupWarnings, improveProfile, budgetSignal, } = args;
|
|
1492
1656
|
const actions = [];
|
|
1493
1657
|
const cleanupWarnings = initialCleanupWarnings ? [...initialCleanupWarnings] : [];
|
|
1494
1658
|
// Phase 0 — MEMORY.md budget check (200-line cap; warn at 180)
|
|
@@ -1525,6 +1689,8 @@ async function runImprovePreparationStage(args) {
|
|
|
1525
1689
|
memorySummary,
|
|
1526
1690
|
improveProfile,
|
|
1527
1691
|
eventsCtx,
|
|
1692
|
+
budgetSignal,
|
|
1693
|
+
runBudgetMs: budgetMs,
|
|
1528
1694
|
});
|
|
1529
1695
|
// Phase 0.4 — session-extract pass.
|
|
1530
1696
|
//
|
|
@@ -1557,6 +1723,7 @@ async function runImprovePreparationStage(args) {
|
|
|
1557
1723
|
stashDir: primaryStashDir,
|
|
1558
1724
|
config: extractConfig,
|
|
1559
1725
|
eventsCtx,
|
|
1726
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
1560
1727
|
});
|
|
1561
1728
|
// #554 minNewSessions gate: skip the entire extract pass (ensureIndex was
|
|
1562
1729
|
// already done upstream; here we elide every akmExtract/processSession call)
|
|
@@ -1952,32 +2119,52 @@ async function runImprovePreparationStage(args) {
|
|
|
1952
2119
|
// high-retrieval path (P0-A) or are skipped until new signals arrive.
|
|
1953
2120
|
// (FEEDBACK_SIGNAL_WINDOW_DAYS / feedbackSinceCutoff are already defined in
|
|
1954
2121
|
// Phase 2 above for the signal-delta gate; we reuse them here.)
|
|
1955
|
-
// Pre-compute feedback summary per ref in a
|
|
1956
|
-
//
|
|
1957
|
-
//
|
|
2122
|
+
// Pre-compute feedback summary per ref in a SINGLE bulk read so we don't
|
|
2123
|
+
// open state.db once per asset (which caused 5000+ accumulated FDs and a
|
|
2124
|
+
// 2-hour runaway on a 13K-asset stash). Pattern mirrors buildLatestFeedbackTsMap
|
|
2125
|
+
// above: one readEvents() call fetches ALL feedback events, then we aggregate
|
|
2126
|
+
// in-memory by ref — O(1) DB opens regardless of candidate set size.
|
|
1958
2127
|
// Cover processableRefs *and* the deferred noFeedbackPool so utility/feedback
|
|
1959
2128
|
// ratios are available for any noFeedbackPool ref that P0-A rescues below.
|
|
2129
|
+
//
|
|
2130
|
+
// Behavioral note: positive/negative COUNTS are all-time (same as the old
|
|
2131
|
+
// per-ref readEvents call which had no `since` filter); hasSignal is bounded
|
|
2132
|
+
// to feedbackSinceCutoff (same as the old inline `(e.ts ?? "") >= cutoff` guard).
|
|
1960
2133
|
const feedbackSummary = new Map();
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
(
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
2134
|
+
{
|
|
2135
|
+
const feedbackCandidateSet = new Set([...processableRefs, ...noFeedbackPool].map((r) => r.ref));
|
|
2136
|
+
if (feedbackCandidateSet.size > 0) {
|
|
2137
|
+
// Fetch ALL feedback events in one query (no ref filter, no since filter =
|
|
2138
|
+
// single full table scan). Filtering per-ref in memory avoids N sequential
|
|
2139
|
+
// state.db opens — the dominant FD-leak path on large stashes.
|
|
2140
|
+
const { events: allFeedbackEvents } = readEvents({ type: "feedback" }, eventsCtx);
|
|
2141
|
+
for (const e of allFeedbackEvents) {
|
|
2142
|
+
const ref = e.ref;
|
|
2143
|
+
if (!ref || !feedbackCandidateSet.has(ref))
|
|
2144
|
+
continue;
|
|
2145
|
+
const entry = feedbackSummary.get(ref) ?? { hasSignal: false, positive: 0, negative: 0 };
|
|
2146
|
+
const meta = e.metadata;
|
|
2147
|
+
// hasSignal: only count feedback events within the 30-day window.
|
|
2148
|
+
if (!entry.hasSignal &&
|
|
2149
|
+
(e.ts ?? "") >= feedbackSinceCutoff &&
|
|
2150
|
+
meta !== undefined &&
|
|
2151
|
+
(typeof meta.signal === "string" || typeof meta.note === "string")) {
|
|
2152
|
+
entry.hasSignal = true;
|
|
2153
|
+
}
|
|
2154
|
+
// positive/negative: all-time counts (no since filter, matching prior behaviour).
|
|
2155
|
+
if (meta?.signal === "positive")
|
|
2156
|
+
entry.positive++;
|
|
2157
|
+
else if (meta?.signal === "negative")
|
|
2158
|
+
entry.negative++;
|
|
2159
|
+
feedbackSummary.set(ref, entry);
|
|
2160
|
+
}
|
|
2161
|
+
// Ensure every candidate has an entry (even refs with zero feedback events).
|
|
2162
|
+
for (const ref of feedbackCandidateSet) {
|
|
2163
|
+
if (!feedbackSummary.has(ref)) {
|
|
2164
|
+
feedbackSummary.set(ref, { hasSignal: false, positive: 0, negative: 0 });
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
1981
2168
|
}
|
|
1982
2169
|
const signalFiltered = processableRefs.filter((candidate) => feedbackSummary.get(candidate.ref)?.hasSignal === true);
|
|
1983
2170
|
// P0-A: also surface zero-feedback assets that have been retrieved many times.
|
|
@@ -1997,7 +2184,10 @@ async function runImprovePreparationStage(args) {
|
|
|
1997
2184
|
let highRetrievalRefs = [];
|
|
1998
2185
|
// Retrieval counts for the zero-feedback pool, hoisted so the Layer-2
|
|
1999
2186
|
// proactive-maintenance selector below can reuse them without a second DB pass.
|
|
2187
|
+
// Also fetch lastUseMs here for the proactive-maintenance recency term (plan §WS-1
|
|
2188
|
+
// step 2: recency is MANDATORY — never pinned to floor).
|
|
2000
2189
|
let retrievalCounts = new Map();
|
|
2190
|
+
let lastUseMsForProactive = new Map();
|
|
2001
2191
|
let dbForRetrieval;
|
|
2002
2192
|
try {
|
|
2003
2193
|
dbForRetrieval = openExistingDatabase();
|
|
@@ -2005,7 +2195,15 @@ async function runImprovePreparationStage(args) {
|
|
|
2005
2195
|
if (showEventCount === 0) {
|
|
2006
2196
|
warn("Warning: show events not yet in usage_events — zero-feedback fallback will match only search-retrieved assets.");
|
|
2007
2197
|
}
|
|
2008
|
-
|
|
2198
|
+
// Fetch retrieval counts for ALL candidates — not only the zero-feedback pool.
|
|
2199
|
+
// Previously only noFeedbackCandidates were looked up, so feedback-bearing refs
|
|
2200
|
+
// had retrievalFreq=0 in computeSalience(), collapsing their retrievalSalience
|
|
2201
|
+
// to 0 regardless of actual use. Two assets of the same type — one
|
|
2202
|
+
// heavily-retrieved, one never-touched — would receive identical rankScores.
|
|
2203
|
+
// Fix (WS-1 blocker 3): union the feedback pool into the lookup.
|
|
2204
|
+
const allCandidateRefs = [...new Set([...signalFiltered, ...noFeedbackCandidates].map((r) => r.ref))];
|
|
2205
|
+
retrievalCounts = getRetrievalCounts(dbForRetrieval, allCandidateRefs);
|
|
2206
|
+
lastUseMsForProactive = getLastUseMsByRef(dbForRetrieval, noFeedbackCandidates.map((r) => r.ref));
|
|
2009
2207
|
// High-retrieval signal-delta (simplified rule, 0.8.0): a no-feedback
|
|
2010
2208
|
// ref qualifies exactly once — when it has actually been retrieved
|
|
2011
2209
|
// (retrievalCount ≥ 1) AND retrievalCount ≥ threshold AND no prior reflect
|
|
@@ -2049,7 +2247,6 @@ async function runImprovePreparationStage(args) {
|
|
|
2049
2247
|
const pmCfg = improveProfile.processes?.proactiveMaintenance;
|
|
2050
2248
|
const dueDays = pmCfg?.dueDays ?? DEFAULT_DUE_DAYS;
|
|
2051
2249
|
const maxPerRun = pmCfg?.maxPerRun ?? pmCfg?.limit ?? DEFAULT_MAX_PER_RUN;
|
|
2052
|
-
const importanceWeights = pmCfg?.importanceWeights;
|
|
2053
2250
|
// Candidate population: the zero-feedback / non-signal pool — exactly the
|
|
2054
2251
|
// assets the other two sources would NOT pick this run. Exclude any P0-A
|
|
2055
2252
|
// rescued this run so we never double-select the same ref.
|
|
@@ -2060,6 +2257,8 @@ async function runImprovePreparationStage(args) {
|
|
|
2060
2257
|
lastReflectTs: lastReflectProposalTs,
|
|
2061
2258
|
lastDistillTs: lastDistillProposalTs,
|
|
2062
2259
|
retrievalCounts,
|
|
2260
|
+
// WS-1: wire lastUseMs so the recency decay term is genuine (plan §step 2).
|
|
2261
|
+
lastUseMs: lastUseMsForProactive,
|
|
2063
2262
|
sizeBytesOf: (r) => {
|
|
2064
2263
|
const fp = r.filePath;
|
|
2065
2264
|
if (!fp)
|
|
@@ -2073,7 +2272,6 @@ async function runImprovePreparationStage(args) {
|
|
|
2073
2272
|
},
|
|
2074
2273
|
dueDays,
|
|
2075
2274
|
maxPerRun,
|
|
2076
|
-
importanceWeights,
|
|
2077
2275
|
});
|
|
2078
2276
|
proactiveRefs = selection.selected;
|
|
2079
2277
|
proactiveMaintenanceSummary = {
|
|
@@ -2097,6 +2295,44 @@ async function runImprovePreparationStage(args) {
|
|
|
2097
2295
|
`(${selection.neverReflected} never reflected, dueDays=${dueDays}, maxPerRun=${maxPerRun})`);
|
|
2098
2296
|
}
|
|
2099
2297
|
}
|
|
2298
|
+
// ── Layer 3: HIGH-SALIENCE ADMISSION GATE (#608) ──────────────────────────
|
|
2299
|
+
// Zero-feedback refs whose encoding_salience (set at distill time by
|
|
2300
|
+
// scoreEncodingSalience) exceeds the configured salienceThreshold are admitted
|
|
2301
|
+
// into the improve run even without retrieval or feedback signal. This rescues
|
|
2302
|
+
// newly distilled assets that the stash has not yet surfaced to users.
|
|
2303
|
+
//
|
|
2304
|
+
// Cap: at most 10% of the effective run limit so the lane cannot crowd out
|
|
2305
|
+
// reactive feedback. Requires state.db to have an asset_salience row — refs
|
|
2306
|
+
// without a row (pre-#608 assets still on the type-weight stub) are skipped.
|
|
2307
|
+
const highSalienceRefs = [];
|
|
2308
|
+
const salienceCfg = (options.config ?? loadConfig()).improve?.salience;
|
|
2309
|
+
const salienceThreshold = salienceCfg?.salienceThreshold ?? 0.75;
|
|
2310
|
+
const proactiveAndRetrievalSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
|
|
2311
|
+
{
|
|
2312
|
+
let dbForHighSalience;
|
|
2313
|
+
try {
|
|
2314
|
+
dbForHighSalience = openStateDatabase(eventsCtx?.dbPath);
|
|
2315
|
+
const effectiveLimit = options.limit ?? 10;
|
|
2316
|
+
const highSalienceCap = Math.max(1, Math.floor(effectiveLimit * 0.1));
|
|
2317
|
+
const candidates = noFeedbackCandidates.filter((r) => !proactiveAndRetrievalSet.has(r.ref));
|
|
2318
|
+
for (const r of candidates) {
|
|
2319
|
+
if (highSalienceRefs.length >= highSalienceCap)
|
|
2320
|
+
break;
|
|
2321
|
+
const row = getAssetSalience(dbForHighSalience, r.ref);
|
|
2322
|
+
if (row && row.encoding_salience >= salienceThreshold) {
|
|
2323
|
+
highSalienceRefs.push(r);
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
2327
|
+
catch (err) {
|
|
2328
|
+
rethrowIfTestIsolationError(err);
|
|
2329
|
+
// best-effort: if DB unavailable, highSalienceRefs stays empty
|
|
2330
|
+
}
|
|
2331
|
+
finally {
|
|
2332
|
+
if (dbForHighSalience)
|
|
2333
|
+
dbForHighSalience.close();
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2100
2336
|
// Record an in-memory skip action for every zero-feedback ref that the
|
|
2101
2337
|
// partition loop deferred to P0-A but P0-A then declined (retrievalCount below
|
|
2102
2338
|
// threshold, or a prior reflect proposal already on record). These never make
|
|
@@ -2104,7 +2340,7 @@ async function runImprovePreparationStage(args) {
|
|
|
2104
2340
|
// summary. No DB event is written here — these refs carry no signal at all, so
|
|
2105
2341
|
// there is nothing for the skip histogram to aggregate; the action log alone
|
|
2106
2342
|
// preserves the per-ref audit trail (mirrors the fully-skipped action above).
|
|
2107
|
-
const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs].map((r) => r.ref));
|
|
2343
|
+
const rescuedSet = new Set([...highRetrievalRefs, ...proactiveRefs, ...highSalienceRefs].map((r) => r.ref));
|
|
2108
2344
|
for (const r of noFeedbackPool) {
|
|
2109
2345
|
if (rescuedSet.has(r.ref))
|
|
2110
2346
|
continue;
|
|
@@ -2124,12 +2360,18 @@ async function runImprovePreparationStage(args) {
|
|
|
2124
2360
|
// usage is the gate. Run `akm feedback <ref> --positive` or retrieve assets
|
|
2125
2361
|
// to bring them into the eligible pool.
|
|
2126
2362
|
// Layer-2 proactive refs join the eligible set alongside feedback-signal and
|
|
2127
|
-
// high-retrieval (P0-A) refs. The
|
|
2128
|
-
// (proactive draws from noFeedbackCandidates with the P0-A picks removed
|
|
2129
|
-
// dedupe defensively so a ref can
|
|
2130
|
-
//
|
|
2131
|
-
|
|
2132
|
-
const
|
|
2363
|
+
// high-retrieval (P0-A) refs. The four sources are disjoint by construction
|
|
2364
|
+
// (proactive draws from noFeedbackCandidates with the P0-A picks removed, and
|
|
2365
|
+
// high-salience draws from the remainder), but dedupe defensively so a ref can
|
|
2366
|
+
// never enter the loop twice. `requireFeedbackSignal` still suppresses all
|
|
2367
|
+
// fallback sources for callers that want feedback-only runs.
|
|
2368
|
+
const signalAndRetrievalRefs = dedupeRefs([
|
|
2369
|
+
...signalFiltered,
|
|
2370
|
+
...highRetrievalRefs,
|
|
2371
|
+
...proactiveRefs,
|
|
2372
|
+
...highSalienceRefs,
|
|
2373
|
+
]);
|
|
2374
|
+
let mergedRefs = scope.mode === "ref" ? processableRefs : options.requireFeedbackSignal ? signalFiltered : signalAndRetrievalRefs;
|
|
2133
2375
|
// ── Attribution tagging: stamp each ref with the eligibility lane that
|
|
2134
2376
|
// selected it ──────────────────────────────────────────────────────────────
|
|
2135
2377
|
// Every reflect/distill proposal must record WHICH lane chose its source asset
|
|
@@ -2142,11 +2384,14 @@ async function runImprovePreparationStage(args) {
|
|
|
2142
2384
|
// createProposal calls. See EligibilitySource for the lane vocabulary.
|
|
2143
2385
|
//
|
|
2144
2386
|
// Precedence (prefer the most specific reactive signal):
|
|
2145
|
-
// scope > signal-delta > high-retrieval > proactive
|
|
2387
|
+
// scope > signal-delta > high-retrieval > proactive > high-salience
|
|
2146
2388
|
// A ref with real feedback is attributed to feedback even if it was also due
|
|
2147
|
-
// for proactive maintenance. We apply lanes
|
|
2148
|
-
// overwrites; the explicit --scope <ref> bypass
|
|
2389
|
+
// for proactive maintenance or had high encoding salience. We apply lanes
|
|
2390
|
+
// weakest-first so the strongest overwrites; the explicit --scope <ref> bypass
|
|
2391
|
+
// wins outright (user intent).
|
|
2149
2392
|
const eligibilitySourceByRef = new Map();
|
|
2393
|
+
for (const r of highSalienceRefs)
|
|
2394
|
+
eligibilitySourceByRef.set(r.ref, "high-salience");
|
|
2150
2395
|
for (const r of proactiveRefs)
|
|
2151
2396
|
eligibilitySourceByRef.set(r.ref, "proactive");
|
|
2152
2397
|
for (const r of highRetrievalRefs)
|
|
@@ -2165,27 +2410,561 @@ async function runImprovePreparationStage(args) {
|
|
|
2165
2410
|
// mergedRefs is always a subset of the four lanes above).
|
|
2166
2411
|
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
2167
2412
|
}
|
|
2413
|
+
// WS-1 — Unified salience vector (S1 seam).
|
|
2414
|
+
//
|
|
2415
|
+
// The legacy sort combined three independent formulas (utility EMA, negative-only
|
|
2416
|
+
// ratio / symmetric-valence magnitude, and the proactive-maintenance priority
|
|
2417
|
+
// formula). WS-1 converges them into one `computeSalience()` call per ref, with
|
|
2418
|
+
// three independently-stored sub-scores and one documented rankScore projection.
|
|
2419
|
+
//
|
|
2420
|
+
// Migration note: if a profile still has `symmetricValence` set, emit a one-time
|
|
2421
|
+
// warning — its behaviour (symmetric |valence| attention) is now always-on as
|
|
2422
|
+
// part of the salience vector, so the knob is a no-op and will be removed in 0.10.
|
|
2423
|
+
if (improveProfile.symmetricValence === true) {
|
|
2424
|
+
warn("[improve] Profile option 'symmetricValence' is deprecated (WS-1 salience vector). " +
|
|
2425
|
+
"Symmetric valence is now always active; remove the option from your improve profile.");
|
|
2426
|
+
}
|
|
2427
|
+
// Fetch last-use timestamps from the index DB for the full merged set so the
|
|
2428
|
+
// recency term in retrievalSalience is genuinely decayable (plan §WS-1 step 2).
|
|
2429
|
+
// This reuses the index DB opened earlier for retrieval counts; a separate
|
|
2430
|
+
// lightweight open is used here to avoid holding the connection longer than needed.
|
|
2431
|
+
let lastUseMsByRef = new Map();
|
|
2432
|
+
// utilityMap is kept for backward-compatible observability (health report reads it).
|
|
2168
2433
|
const utilityMap = buildUtilityMap(mergedRefs);
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
//
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2434
|
+
let dbForSalience;
|
|
2435
|
+
try {
|
|
2436
|
+
dbForSalience = openExistingDatabase();
|
|
2437
|
+
lastUseMsByRef = getLastUseMsByRef(dbForSalience, mergedRefs.map((r) => r.ref));
|
|
2438
|
+
}
|
|
2439
|
+
catch (err) {
|
|
2440
|
+
rethrowIfTestIsolationError(err);
|
|
2441
|
+
// best-effort: if DB unavailable, recency term stays at floor (lastUseMs=0)
|
|
2442
|
+
}
|
|
2443
|
+
finally {
|
|
2444
|
+
if (dbForSalience)
|
|
2445
|
+
closeDatabase(dbForSalience);
|
|
2446
|
+
}
|
|
2447
|
+
// ── WS-2 Outcome loop ─────────────────────────────────────────────────────
|
|
2448
|
+
//
|
|
2449
|
+
// Update asset_outcome for every ref in the merged set BEFORE computing the
|
|
2450
|
+
// salience vector so the updated outcome_score feeds outcomeSalience this run.
|
|
2451
|
+
//
|
|
2452
|
+
// Inputs per ref:
|
|
2453
|
+
// - currentRetrievalCount: from retrievalCounts (index DB)
|
|
2454
|
+
// - lastRetrievedAt: from lastUseMsByRef (utility_scores.last_used_at)
|
|
2455
|
+
// - negativeFeedbackCount: cumulative negatives from feedbackSummary
|
|
2456
|
+
// - acceptedChangeCount: accepted proposals for this ref (state.db)
|
|
2457
|
+
// - valence: net valence from computeValenceScore(feedbackSummary.get(ref))
|
|
2458
|
+
// - utilityScore: from utilityMap (for warm-start seed on new rows)
|
|
2459
|
+
//
|
|
2460
|
+
// Best-effort: outcome failures never block the salience or ranking pass.
|
|
2461
|
+
const outcomeSalienceByRef = new Map();
|
|
2462
|
+
try {
|
|
2463
|
+
const outcomeDb = eventsCtx?.db ?? openStateDatabase(eventsCtx?.dbPath);
|
|
2464
|
+
const ownsOutcomeDb = !eventsCtx?.db;
|
|
2465
|
+
try {
|
|
2466
|
+
// Count accepted proposals per ref in one pass (avoid N separate queries).
|
|
2467
|
+
// Scoped to primaryStashDir when available so multi-stash installs don't
|
|
2468
|
+
// inflate counts with proposals from other stashes.
|
|
2469
|
+
const acceptedCountByRef = new Map();
|
|
2470
|
+
try {
|
|
2471
|
+
const acceptedProposals = listStateProposals(outcomeDb, {
|
|
2472
|
+
status: "accepted",
|
|
2473
|
+
...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
|
|
2474
|
+
});
|
|
2475
|
+
for (const p of acceptedProposals) {
|
|
2476
|
+
acceptedCountByRef.set(p.ref, (acceptedCountByRef.get(p.ref) ?? 0) + 1);
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
catch {
|
|
2480
|
+
// best-effort: if proposals query fails, accepted counts stay at 0
|
|
2481
|
+
}
|
|
2482
|
+
// Update each ref's outcome row and collect the resulting outcome scores.
|
|
2483
|
+
const rawOutcomeScores = new Map();
|
|
2484
|
+
const nowForOutcome = Date.now();
|
|
2485
|
+
for (const r of mergedRefs) {
|
|
2486
|
+
const fb = feedbackSummary.get(r.ref) ?? { positive: 0, negative: 0 };
|
|
2487
|
+
const valenceResult = computeValenceScore(fb);
|
|
2488
|
+
try {
|
|
2489
|
+
const result = updateAssetOutcome(outcomeDb, {
|
|
2490
|
+
ref: r.ref,
|
|
2491
|
+
currentRetrievalCount: retrievalCounts.get(r.ref) ?? 0,
|
|
2492
|
+
lastRetrievedAt: lastUseMsByRef.get(r.ref) ?? 0,
|
|
2493
|
+
acceptedChangeCount: acceptedCountByRef.get(r.ref) ?? 0,
|
|
2494
|
+
negativeFeedbackCount: fb.negative,
|
|
2495
|
+
valence: valenceResult.valence,
|
|
2496
|
+
utilityScore: utilityMap.get(r.ref),
|
|
2497
|
+
now: nowForOutcome,
|
|
2498
|
+
});
|
|
2499
|
+
rawOutcomeScores.set(r.ref, result.outcomeScore);
|
|
2500
|
+
}
|
|
2501
|
+
catch {
|
|
2502
|
+
// best-effort per-ref: skip this ref's outcome update on failure
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
// Compute stash-wide max outcome_score for normalisation (diversity floor).
|
|
2506
|
+
// Read ALL rows (not just this run's batch) so the normalisation is
|
|
2507
|
+
// stash-relative, not pool-relative.
|
|
2508
|
+
let maxOutcomeScore = 0;
|
|
2509
|
+
try {
|
|
2510
|
+
const allOutcomes = getAllAssetOutcomes(outcomeDb);
|
|
2511
|
+
for (const row of allOutcomes) {
|
|
2512
|
+
if (row.outcome_score > maxOutcomeScore)
|
|
2513
|
+
maxOutcomeScore = row.outcome_score;
|
|
2514
|
+
}
|
|
2515
|
+
// Proxy-adequacy tripwire: emit a health event if outcome_score is
|
|
2516
|
+
// negatively correlated with accepted_change_rate (inverted proxy).
|
|
2517
|
+
const adequacy = computeProxyAdequacy(allOutcomes);
|
|
2518
|
+
if (adequacy.isInverted) {
|
|
2519
|
+
appendEvent({
|
|
2520
|
+
eventType: "outcome_proxy_inverted",
|
|
2521
|
+
ref: undefined,
|
|
2522
|
+
metadata: {
|
|
2523
|
+
correlation: adequacy.correlation,
|
|
2524
|
+
n: adequacy.n,
|
|
2525
|
+
note: "corr(outcome_score, accepted_change_rate) < −0.3: high-outcome_score assets have LOW accepted-change rates — the proxy's 'doing well' signal is inverted, so the coarse retrieval-delta signal is no longer trustworthy and the 0.10+ rich in-session signal is no longer deferrable. See plan §WS-2 proxy-adequacy tripwire.",
|
|
2526
|
+
},
|
|
2527
|
+
}, eventsCtx);
|
|
2528
|
+
}
|
|
2529
|
+
}
|
|
2530
|
+
catch {
|
|
2531
|
+
// best-effort: tripwire failure never blocks ranking
|
|
2532
|
+
}
|
|
2533
|
+
// Convert raw outcome scores → normalised outcomeSalience values in [0,1].
|
|
2534
|
+
for (const [ref, score] of rawOutcomeScores) {
|
|
2535
|
+
const normalised = outcomeScoreToSalience(score, maxOutcomeScore);
|
|
2536
|
+
outcomeSalienceByRef.set(ref, normalised);
|
|
2537
|
+
}
|
|
2538
|
+
// Also fetch outcome scores for refs NOT updated this run (stale or absent)
|
|
2539
|
+
// so the outcomeSalience read path works for all refs in the batch.
|
|
2540
|
+
const missingRefs = mergedRefs.map((r) => r.ref).filter((ref) => !rawOutcomeScores.has(ref));
|
|
2541
|
+
if (missingRefs.length > 0) {
|
|
2542
|
+
const storedScores = getOutcomeScoresByRef(outcomeDb, missingRefs);
|
|
2543
|
+
for (const [ref, score] of storedScores) {
|
|
2544
|
+
outcomeSalienceByRef.set(ref, outcomeScoreToSalience(score, maxOutcomeScore));
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
finally {
|
|
2549
|
+
if (ownsOutcomeDb)
|
|
2550
|
+
outcomeDb.close();
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2553
|
+
catch (err) {
|
|
2554
|
+
rethrowIfTestIsolationError(err);
|
|
2555
|
+
// best-effort: outcome failures never block salience computation
|
|
2556
|
+
}
|
|
2557
|
+
// Compute the salience vector for every ref in the merged set.
|
|
2558
|
+
// retrievalCounts now covers the full candidate set (feedback-bearing + zero-feedback)
|
|
2559
|
+
// so feedback refs get their genuine retrieval frequency, not a 0-floor fallback.
|
|
2560
|
+
// outcomeSalienceByRef is populated by WS-2 above (or empty on first run).
|
|
2561
|
+
//
|
|
2562
|
+
// Part-V gate: read the operator opt-in flag from config. Default false
|
|
2563
|
+
// (WS-1 parity weights) until the maintainer runs scripts/akm-eval and sets
|
|
2564
|
+
// improve.salience.outcomeWeightEnabled: true in the config.
|
|
2565
|
+
const salienceConfig = (options.config ?? loadConfig()).improve?.salience;
|
|
2566
|
+
const outcomeWeightEnabled = salienceConfig?.outcomeWeightEnabled === true;
|
|
2567
|
+
const salienceMap = new Map();
|
|
2568
|
+
const nowForSalience = Date.now();
|
|
2569
|
+
for (const r of mergedRefs) {
|
|
2570
|
+
const type = r.ref.includes(":") ? r.ref.slice(0, r.ref.indexOf(":")) : "";
|
|
2571
|
+
const sizeBytes = (() => {
|
|
2572
|
+
const fp = r.filePath;
|
|
2573
|
+
if (!fp)
|
|
2574
|
+
return undefined;
|
|
2575
|
+
try {
|
|
2576
|
+
return fs.statSync(fp).size;
|
|
2577
|
+
}
|
|
2578
|
+
catch {
|
|
2579
|
+
return undefined;
|
|
2580
|
+
}
|
|
2581
|
+
})();
|
|
2582
|
+
const vector = computeSalience({
|
|
2583
|
+
ref: r.ref,
|
|
2584
|
+
type,
|
|
2585
|
+
retrievalFreq: retrievalCounts.get(r.ref) ?? 0,
|
|
2586
|
+
lastUseMs: lastUseMsByRef.get(r.ref),
|
|
2587
|
+
utilityScore: utilityMap.get(r.ref),
|
|
2588
|
+
outcomeSalience: outcomeSalienceByRef.get(r.ref),
|
|
2589
|
+
sizeBytes,
|
|
2590
|
+
now: nowForSalience,
|
|
2591
|
+
outcomeWeightEnabled,
|
|
2592
|
+
});
|
|
2593
|
+
salienceMap.set(r.ref, vector);
|
|
2594
|
+
}
|
|
2595
|
+
// Persist salience vectors to state.db (best-effort, non-blocking).
|
|
2596
|
+
// The canonical store enables WS-3 homeostatic demotion and WS-2 outcome reads.
|
|
2597
|
+
//
|
|
2598
|
+
// Forgetting-safety report (plan §WS-1 step 7) — stash-wide rank comparison:
|
|
2599
|
+
//
|
|
2600
|
+
// BEFORE persisting the new rankScores, read ALL existing rows from state.db
|
|
2601
|
+
// (not just the per-run candidate pool). This gives stash-wide rank positions so
|
|
2602
|
+
// the top-200/below-500 thresholds are meaningful.
|
|
2603
|
+
//
|
|
2604
|
+
// Two distinct scenarios:
|
|
2605
|
+
//
|
|
2606
|
+
// A. First WS-1 run (table empty): the old stash-wide combinedEligibilityScore
|
|
2607
|
+
// ordering was never persisted in state.db (asset_salience is a new WS-1 table).
|
|
2608
|
+
// However, the old formula's inputs are available in-scope for every candidate
|
|
2609
|
+
// in the current pool: utility comes from utilityMap and the attention term
|
|
2610
|
+
// from feedbackSummary (positive/negative counts). We reconstruct the old
|
|
2611
|
+
// combinedEligibilityScore = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT
|
|
2612
|
+
// for every ref in salienceMap and rank them, giving a candidate-pool-scoped
|
|
2613
|
+
// old ordering. This is a partial reconstruction (only current-pool refs, not
|
|
2614
|
+
// stash-wide), but it is the most faithful comparison possible at cutover and
|
|
2615
|
+
// allows the top-200→below-500 forgetting guard to fire if the formula change
|
|
2616
|
+
// dramatically reorders the candidate pool.
|
|
2617
|
+
// See docs/design/improve-reconciliation-plan.md §WS-1 step 7 — the stash-wide
|
|
2618
|
+
// ordering was unreconstructable (no prior state.db snapshot), so this candidate-
|
|
2619
|
+
// pool partial reconstruction is the documented resolution for the first-run case.
|
|
2620
|
+
// Emit `improve_salience_first_run` to mark the cutover moment and include the
|
|
2621
|
+
// reconstructed comparison result in the metadata.
|
|
2622
|
+
//
|
|
2623
|
+
// B. Subsequent runs (table has rows): use ALL existing rows as old ranks, merge
|
|
2624
|
+
// them with the current run's salienceMap updates for new ranks, and call
|
|
2625
|
+
// buildRankChangeReport with stash-wide positions. This detects real rank drift
|
|
2626
|
+
// — e.g. a retrieval-pattern shift causing a previously top-200 asset to slip
|
|
2627
|
+
// below position 500.
|
|
2628
|
+
//
|
|
2629
|
+
// Measurement-protocol deferral (plan §269, Part-V):
|
|
2630
|
+
// The Part-V T0 baseline (scripts/akm-eval + health report) and the throughput/
|
|
2631
|
+
// quality gate are deferred pending owner sign-off. Full measurement requires a
|
|
2632
|
+
// before/after `akm health` report. Owner-acknowledged deferral: WS-2 landing
|
|
2633
|
+
// will re-introduce outcome salience and trigger the full re-tuning pass at that
|
|
2634
|
+
// time. salience.ts already accepts outcomeSalience directly as an input
|
|
2635
|
+
// (see SalienceInputs.outcomeSalience); no separate hook is needed.
|
|
2636
|
+
//
|
|
2637
|
+
// Forgetting-safety collection: populated inside scenario B below, consumed
|
|
2638
|
+
// after the try/catch to union candidates into mergedRefs before the sort.
|
|
2639
|
+
// Only refs from a real pre-existing ordering (scenario B) are collected;
|
|
2640
|
+
// empty on scenario A or when no candidates dropped below the threshold.
|
|
2641
|
+
let pendingForgettingRefs = [];
|
|
2642
|
+
try {
|
|
2643
|
+
const stateDb = openStateDatabase(eventsCtx?.dbPath);
|
|
2644
|
+
try {
|
|
2645
|
+
// Step 7: stash-wide rank-change report BEFORE overwriting the table.
|
|
2646
|
+
//
|
|
2647
|
+
// Load ALL existing rows so rank positions are stash-relative, not pool-relative.
|
|
2648
|
+
const existingAllScores = getAllRankScores(stateDb);
|
|
2649
|
+
if (existingAllScores.size === 0) {
|
|
2650
|
+
// Scenario A: first WS-1 run — table empty.
|
|
2651
|
+
//
|
|
2652
|
+
// Reconstruct the old combinedEligibilityScore ordering for the current
|
|
2653
|
+
// candidate pool using inputs that are already in-scope: utility from
|
|
2654
|
+
// utilityMap and the attention term from feedbackSummary (positive/negative
|
|
2655
|
+
// counts). Old formula: score = utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT.
|
|
2656
|
+
//
|
|
2657
|
+
// Limitation: this covers only the current-run candidate pool, not the full
|
|
2658
|
+
// stash. The stash-wide ordering was never persisted (asset_salience is a new
|
|
2659
|
+
// WS-1 table), so this is the most faithful comparison possible at cutover.
|
|
2660
|
+
// See docs/design/improve-reconciliation-plan.md §WS-1 step 7.
|
|
2661
|
+
const reconstructedOldScores = new Map();
|
|
2662
|
+
for (const ref of salienceMap.keys()) {
|
|
2663
|
+
const utility = utilityMap.get(ref) ?? 0;
|
|
2664
|
+
const fb = feedbackSummary.get(ref) ?? { positive: 0, negative: 0 };
|
|
2665
|
+
const attention = computeValenceScore(fb).attention;
|
|
2666
|
+
reconstructedOldScores.set(ref, utility * UTILITY_WEIGHT + attention * FEEDBACK_WEIGHT);
|
|
2667
|
+
}
|
|
2668
|
+
// Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
|
|
2669
|
+
const toRanks = (scores) => {
|
|
2670
|
+
const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
|
|
2671
|
+
return new Map(sorted.map(([ref], i) => [ref, i + 1]));
|
|
2672
|
+
};
|
|
2673
|
+
const oldRanks = toRanks(reconstructedOldScores);
|
|
2674
|
+
const newRanks = toRanks(new Map([...salienceMap.entries()].map(([ref, v]) => [ref, v.rankScore])));
|
|
2675
|
+
const firstRunReport = buildRankChangeReport(oldRanks, newRanks);
|
|
2676
|
+
if (firstRunReport.forgettingCandidates.length > 0) {
|
|
2677
|
+
warn(`[improve/salience] WS-1 first-run rank-change report: ${firstRunReport.forgettingCandidates.length} asset(s) fell from top-200 to below position 500 (cutover formula change). ` +
|
|
2678
|
+
`Top drops: ${firstRunReport.forgettingCandidates
|
|
2679
|
+
.slice(0, 5)
|
|
2680
|
+
.map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
|
|
2681
|
+
.join(", ")}`);
|
|
2682
|
+
pendingForgettingRefs = firstRunReport.forgettingCandidates.map((e) => e.ref);
|
|
2683
|
+
}
|
|
2684
|
+
appendEvent({
|
|
2685
|
+
eventType: "improve_salience_first_run",
|
|
2686
|
+
ref: undefined,
|
|
2687
|
+
metadata: {
|
|
2688
|
+
candidateCount: salienceMap.size,
|
|
2689
|
+
note: "first WS-1 salience run — partial reconstruction of old combinedEligibilityScore ordering for candidate pool (stash-wide ordering not available); see improve-reconciliation-plan.md §WS-1 step 7",
|
|
2690
|
+
forgettingCandidates: firstRunReport.forgettingCandidates.length,
|
|
2691
|
+
topDrops: firstRunReport.forgettingCandidates.slice(0, 10).map((e) => ({
|
|
2692
|
+
ref: e.ref,
|
|
2693
|
+
oldRank: e.oldRank,
|
|
2694
|
+
newRank: e.newRank,
|
|
2695
|
+
})),
|
|
2696
|
+
},
|
|
2697
|
+
}, eventsCtx);
|
|
2698
|
+
}
|
|
2699
|
+
else {
|
|
2700
|
+
// Scenario B: subsequent run — compare stash-wide old vs. new ranks.
|
|
2701
|
+
//
|
|
2702
|
+
// Build new scores by merging the full table with this run's updates.
|
|
2703
|
+
// Refs in salienceMap override their stored value; refs not in this run
|
|
2704
|
+
// retain their stored value unchanged. This gives a complete stash-wide
|
|
2705
|
+
// picture of what the new ordering looks like after this run.
|
|
2706
|
+
const mergedNewScores = new Map(existingAllScores);
|
|
2707
|
+
for (const [ref, vector] of salienceMap) {
|
|
2708
|
+
mergedNewScores.set(ref, vector.rankScore);
|
|
2709
|
+
}
|
|
2710
|
+
// Assign 1-indexed rank positions sorted by score desc (tie-break: ref asc).
|
|
2711
|
+
const toRanks = (scores) => {
|
|
2712
|
+
const sorted = [...scores.entries()].sort(([refA, a], [refB, b]) => b !== a ? b - a : refA < refB ? -1 : refA > refB ? 1 : 0);
|
|
2713
|
+
return new Map(sorted.map(([ref], i) => [ref, i + 1]));
|
|
2714
|
+
};
|
|
2715
|
+
const oldRanks = toRanks(existingAllScores);
|
|
2716
|
+
const newRanks = toRanks(mergedNewScores);
|
|
2717
|
+
const report = buildRankChangeReport(oldRanks, newRanks);
|
|
2718
|
+
if (report.forgettingCandidates.length > 0) {
|
|
2719
|
+
warn(`[improve/salience] WS-1 rank-change report: ${report.forgettingCandidates.length} asset(s) fell from top-200 to below position 500. ` +
|
|
2720
|
+
`Top drops: ${report.forgettingCandidates
|
|
2721
|
+
.slice(0, 5)
|
|
2722
|
+
.map((e) => `${e.ref} (#${e.oldRank}→#${e.newRank})`)
|
|
2723
|
+
.join(", ")}`);
|
|
2724
|
+
// Collect refs for protective consolidation pass (plan §WS-1 step 7).
|
|
2725
|
+
// These are force-included in the candidate pool (mergedRefs) after
|
|
2726
|
+
// this try block, bypassing cooldown/signal-delta gating.
|
|
2727
|
+
pendingForgettingRefs = report.forgettingCandidates.map((e) => e.ref);
|
|
2728
|
+
}
|
|
2729
|
+
appendEvent({
|
|
2730
|
+
eventType: "improve_salience_rank_change",
|
|
2731
|
+
ref: undefined,
|
|
2732
|
+
metadata: {
|
|
2733
|
+
stashSize: existingAllScores.size,
|
|
2734
|
+
totalChanged: report.allChanges.length,
|
|
2735
|
+
forgettingCandidates: report.forgettingCandidates.length,
|
|
2736
|
+
topDrops: report.forgettingCandidates.slice(0, 10).map((e) => ({
|
|
2737
|
+
ref: e.ref,
|
|
2738
|
+
oldRank: e.oldRank,
|
|
2739
|
+
newRank: e.newRank,
|
|
2740
|
+
})),
|
|
2741
|
+
},
|
|
2742
|
+
}, eventsCtx);
|
|
2743
|
+
}
|
|
2744
|
+
for (const [ref, vector] of salienceMap) {
|
|
2745
|
+
upsertAssetSalience(stateDb, ref, vector, nowForSalience);
|
|
2746
|
+
}
|
|
2747
|
+
}
|
|
2748
|
+
finally {
|
|
2749
|
+
stateDb.close();
|
|
2750
|
+
}
|
|
2751
|
+
}
|
|
2752
|
+
catch (err) {
|
|
2753
|
+
rethrowIfTestIsolationError(err);
|
|
2754
|
+
// best-effort: salience persistence failure never blocks ranking
|
|
2755
|
+
}
|
|
2756
|
+
// ── Protective consolidation pass (plan §WS-1 step 7) ─────────────────────
|
|
2757
|
+
// Forgetting candidates detected in scenario B are force-injected into
|
|
2758
|
+
// mergedRefs here, BEFORE the effectiveScore sort, bypassing cooldown and
|
|
2759
|
+
// signal-delta gating. Any ref already present in mergedRefs keeps its
|
|
2760
|
+
// existing eligibilitySource (stronger reactive signals win); refs not yet in
|
|
2761
|
+
// the pool are synthesised as minimal ImproveEligibleRef stubs and labelled
|
|
2762
|
+
// 'forgetting-safety' so S5/WS-5 can slice by lane. The dedupeRefs call
|
|
2763
|
+
// ensures no ref can enter the loop twice.
|
|
2764
|
+
if (pendingForgettingRefs.length > 0 && scope.mode !== "ref") {
|
|
2765
|
+
const existingRefSet = new Set(mergedRefs.map((r) => r.ref));
|
|
2766
|
+
const newForgettingRefs = [];
|
|
2767
|
+
for (const ref of pendingForgettingRefs) {
|
|
2768
|
+
if (!existingRefSet.has(ref)) {
|
|
2769
|
+
// Ref not already in the candidate pool — synthesise a stub so it
|
|
2770
|
+
// participates in the reflect/distill loop with proper attribution.
|
|
2771
|
+
newForgettingRefs.push({ ref, reason: "scope-type", eligibilitySource: "forgetting-safety" });
|
|
2772
|
+
}
|
|
2773
|
+
// Always stamp the lane in the attribution map (overwrites weaker lanes;
|
|
2774
|
+
// stronger reactive signals — scope/signal-delta/high-retrieval/proactive
|
|
2775
|
+
// — are written after this block so they take precedence).
|
|
2776
|
+
eligibilitySourceByRef.set(ref, "forgetting-safety");
|
|
2777
|
+
}
|
|
2778
|
+
if (newForgettingRefs.length > 0) {
|
|
2779
|
+
mergedRefs = dedupeRefs([...mergedRefs, ...newForgettingRefs]);
|
|
2780
|
+
}
|
|
2781
|
+
// Re-stamp attribution for any refs whose lane needs updating.
|
|
2782
|
+
// Precedence (weakest → strongest, each overwrites the previous):
|
|
2783
|
+
// proactive < high-retrieval < forgetting-safety < signal-delta
|
|
2784
|
+
// Scope mode is already excluded by the outer guard (`scope.mode !== "ref"`).
|
|
2785
|
+
// forgetting-safety sits above proactive and high-retrieval so that a ref
|
|
2786
|
+
// flagged as a forgetting candidate is always visible to S5/WS-5 as such,
|
|
2787
|
+
// even when it was also due for a proactive maintenance run. signal-delta
|
|
2788
|
+
// overrides forgetting-safety because a ref with fresh feedback is reactive
|
|
2789
|
+
// and doesn't need the protective pass label for measurement purposes.
|
|
2790
|
+
for (const r of highSalienceRefs)
|
|
2791
|
+
eligibilitySourceByRef.set(r.ref, "high-salience");
|
|
2792
|
+
for (const r of proactiveRefs)
|
|
2793
|
+
eligibilitySourceByRef.set(r.ref, "proactive");
|
|
2794
|
+
for (const r of highRetrievalRefs)
|
|
2795
|
+
eligibilitySourceByRef.set(r.ref, "high-retrieval");
|
|
2796
|
+
// Apply forgetting-safety OVER proactive, high-retrieval, and high-salience
|
|
2797
|
+
// (already stamped in the loop above via
|
|
2798
|
+
// `eligibilitySourceByRef.set(ref, "forgetting-safety")`). No-op here: the
|
|
2799
|
+
// set() calls above for proactive/high-retrieval/high-salience overwrite the
|
|
2800
|
+
// earlier forgetting-safety stamp — so we re-apply forgetting-safety now for
|
|
2801
|
+
// those refs that are both forgetting candidates AND in another fallback lane.
|
|
2802
|
+
for (const ref of pendingForgettingRefs) {
|
|
2803
|
+
eligibilitySourceByRef.set(ref, "forgetting-safety");
|
|
2804
|
+
}
|
|
2805
|
+
// signal-delta is the strongest reactive signal and overrides forgetting-safety.
|
|
2806
|
+
for (const r of signalFiltered)
|
|
2807
|
+
eligibilitySourceByRef.set(r.ref, "signal-delta");
|
|
2808
|
+
// Update eligibilitySource on the ref objects themselves for any refs whose
|
|
2809
|
+
// lane changed (covers both new stubs and pre-existing refs).
|
|
2810
|
+
for (const r of mergedRefs) {
|
|
2811
|
+
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
2812
|
+
}
|
|
2813
|
+
}
|
|
2814
|
+
// ── REPLAY SELECTION layer (#610) ─────────────────────────────────────────
|
|
2815
|
+
// Bounded, ADDITIVE replay budget: up to `replayBudget` top-salience refs are
|
|
2816
|
+
// revisited even with zero reactive signal (no feedback, no retrieval) and
|
|
2817
|
+
// regardless of cooldown — exactly like the forgetting-safety lane, replay is
|
|
2818
|
+
// injected AFTER cooldown/signal-delta partitioning so it bypasses those gates.
|
|
2819
|
+
//
|
|
2820
|
+
// Strictly additive: the replay slice is appended AFTER the --limit fresh slice
|
|
2821
|
+
// (see the loopRefs partition below), so it can never shrink the fresh-ref set.
|
|
2822
|
+
// Replay is the WEAKEST lane — it only stamps refs no other lane already claimed,
|
|
2823
|
+
// and budget is spent only on refs not already in mergedRefs (so a stronger lane
|
|
2824
|
+
// never has its budget wasted or its label overwritten).
|
|
2825
|
+
//
|
|
2826
|
+
// Default replayBudget=0 ⇒ this whole block is a no-op (no DB open, no event,
|
|
2827
|
+
// no mergedRefs mutation), preserving byte-identical pre-#610 selection behavior.
|
|
2828
|
+
const replayBudget = (options.config ?? loadConfig()).improve?.salience?.replayBudget ?? 0;
|
|
2829
|
+
const replayRefSet = new Set();
|
|
2830
|
+
if (replayBudget > 0 && scope.mode !== "ref" && !options.requireFeedbackSignal) {
|
|
2831
|
+
let replayDb;
|
|
2832
|
+
try {
|
|
2833
|
+
replayDb = openStateDatabase(eventsCtx?.dbPath);
|
|
2834
|
+
const alreadyInPool = new Set(mergedRefs.map((r) => r.ref));
|
|
2835
|
+
const allRankScores = getAllRankScores(replayDb);
|
|
2836
|
+
// Candidate universe = every salience row NOT already in the pool, ordered by
|
|
2837
|
+
// rank_score desc with a deterministic ref-string tie-break (mirrors the main
|
|
2838
|
+
// sort). Converged refs (consecutive_no_ops >= dampener threshold) are fully
|
|
2839
|
+
// EXCLUDED — a stronger skip than the dampener (which only halves order).
|
|
2840
|
+
let convergedSkipped = 0;
|
|
2841
|
+
const candidates = [];
|
|
2842
|
+
for (const [ref, rankScore] of allRankScores) {
|
|
2843
|
+
if (alreadyInPool.has(ref))
|
|
2844
|
+
continue;
|
|
2845
|
+
const noOps = getConsecutiveNoOps(replayDb, ref);
|
|
2846
|
+
if (noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD) {
|
|
2847
|
+
convergedSkipped++;
|
|
2848
|
+
continue;
|
|
2849
|
+
}
|
|
2850
|
+
candidates.push({ ref, rankScore });
|
|
2851
|
+
}
|
|
2852
|
+
candidates.sort((a, b) => b.rankScore !== a.rankScore ? b.rankScore - a.rankScore : a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0);
|
|
2853
|
+
const candidatePool = candidates.length;
|
|
2854
|
+
const selected = candidates.slice(0, replayBudget);
|
|
2855
|
+
const newReplayRefs = [];
|
|
2856
|
+
for (const { ref } of selected) {
|
|
2857
|
+
replayRefSet.add(ref);
|
|
2858
|
+
// Synthesise a stub (mirror the forgetting-safety stub). Resolve the
|
|
2859
|
+
// backing file from the planned-ref pool so the downstream existsSync
|
|
2860
|
+
// guard keeps the ref (a replay candidate from asset_salience whose file
|
|
2861
|
+
// is gone correctly drops out). Only refs present in the indexed pool can
|
|
2862
|
+
// be revisited — refs without a planned entry get no filePath and are
|
|
2863
|
+
// dropped by the disk check, which is the desired behavior.
|
|
2864
|
+
const planned = plannedRefs.find((p) => p.ref === ref);
|
|
2865
|
+
newReplayRefs.push({
|
|
2866
|
+
ref,
|
|
2867
|
+
reason: "scope-type",
|
|
2868
|
+
eligibilitySource: "replay",
|
|
2869
|
+
...(planned?.filePath ? { filePath: planned.filePath } : {}),
|
|
2870
|
+
});
|
|
2871
|
+
// Seed the salienceMap so the sort/effectiveScore can rank the replay ref.
|
|
2872
|
+
if (!salienceMap.has(ref)) {
|
|
2873
|
+
salienceMap.set(ref, {
|
|
2874
|
+
encoding: 0,
|
|
2875
|
+
outcome: 0,
|
|
2876
|
+
retrieval: 0,
|
|
2877
|
+
rankScore: allRankScores.get(ref) ?? 0,
|
|
2878
|
+
});
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
if (newReplayRefs.length > 0) {
|
|
2882
|
+
mergedRefs = dedupeRefs([...mergedRefs, ...newReplayRefs]);
|
|
2883
|
+
// Replay is the WEAKEST lane: stamp 'replay' ONLY for refs not already
|
|
2884
|
+
// keyed by a stronger lane.
|
|
2885
|
+
for (const ref of replayRefSet) {
|
|
2886
|
+
if (!eligibilitySourceByRef.has(ref))
|
|
2887
|
+
eligibilitySourceByRef.set(ref, "replay");
|
|
2888
|
+
}
|
|
2889
|
+
for (const r of mergedRefs) {
|
|
2890
|
+
r.eligibilitySource = eligibilitySourceByRef.get(r.ref) ?? "unknown";
|
|
2891
|
+
}
|
|
2892
|
+
}
|
|
2893
|
+
// Aggregated observability event (never per-ref).
|
|
2894
|
+
appendEvent({
|
|
2895
|
+
eventType: "improve_replay_selected",
|
|
2896
|
+
ref: undefined,
|
|
2897
|
+
metadata: {
|
|
2898
|
+
count: newReplayRefs.length,
|
|
2899
|
+
budget: replayBudget,
|
|
2900
|
+
convergedSkipped,
|
|
2901
|
+
candidatePool,
|
|
2902
|
+
},
|
|
2903
|
+
}, eventsCtx);
|
|
2904
|
+
}
|
|
2905
|
+
catch (err) {
|
|
2906
|
+
rethrowIfTestIsolationError(err);
|
|
2907
|
+
// best-effort: if DB unavailable, replayRefSet stays empty
|
|
2908
|
+
}
|
|
2909
|
+
finally {
|
|
2910
|
+
if (replayDb)
|
|
2911
|
+
replayDb.close();
|
|
2912
|
+
}
|
|
2913
|
+
}
|
|
2914
|
+
// Build no-op map for consolidation-selection dampener (plan §WS-1 step 8).
|
|
2915
|
+
// Reads consecutive_no_ops from the SAME pinned db handle used elsewhere in
|
|
2916
|
+
// this function. The effective score is used ONLY for processing/selection
|
|
2917
|
+
// order — the persisted rank_score in asset_salience is never mutated here.
|
|
2918
|
+
const noOpMap = new Map();
|
|
2919
|
+
try {
|
|
2920
|
+
const noOpDb = eventsCtx?.db ?? (eventsCtx?.dbPath ? openStateDatabase(eventsCtx.dbPath) : null);
|
|
2921
|
+
if (noOpDb) {
|
|
2922
|
+
const ownsNoOpDb = !eventsCtx?.db;
|
|
2923
|
+
try {
|
|
2924
|
+
for (const r of mergedRefs) {
|
|
2925
|
+
noOpMap.set(r.ref, getConsecutiveNoOps(noOpDb, r.ref));
|
|
2926
|
+
}
|
|
2927
|
+
}
|
|
2928
|
+
finally {
|
|
2929
|
+
if (ownsNoOpDb)
|
|
2930
|
+
noOpDb.close();
|
|
2931
|
+
}
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
catch {
|
|
2935
|
+
// best-effort: dampener failure never blocks selection
|
|
2936
|
+
}
|
|
2937
|
+
// Sort by effective selection score (desc), with explicit ref-string tie-break
|
|
2938
|
+
// for determinism. The effective score applies the consolidation-selection
|
|
2939
|
+
// dampener: assets that have been repeatedly skipped (consecutive_no_ops >=
|
|
2940
|
+
// THRESHOLD) are penalised by FACTOR so they sort after peers with similar
|
|
2941
|
+
// rankScore. The persisted rank_score is left unchanged — this is the whole
|
|
2942
|
+
// point of the dampener (stable assets stay fully retrievable).
|
|
2943
|
+
//
|
|
2944
|
+
// WIRING NOTE (plan §WS-1 step 8 / "consolidation-selection" disambiguation):
|
|
2945
|
+
// "consolidation-selection" in the plan refers to THIS reflect/distill
|
|
2946
|
+
// eligibility ordering — i.e. which assets are chosen for the reflect/distill
|
|
2947
|
+
// LLM pass — NOT to akmConsolidate (the cluster-merge phase at ~line 1994,
|
|
2948
|
+
// which runs earlier and never reads noOpMap). The no-op counter originates
|
|
2949
|
+
// from no-change reflect / quality-rejected distill outcomes; the dampener
|
|
2950
|
+
// suppresses repeated LLM attempts on those same assets without touching their
|
|
2951
|
+
// persisted rank_score (so they remain fully retrievable).
|
|
2952
|
+
//
|
|
2953
|
+
// This is the ONLY ranking path — negativeOnlyRatio and the legacy
|
|
2954
|
+
// symmetricValence branch are replaced. The three eligibilitySource lanes
|
|
2955
|
+
// (signal-delta / high-retrieval / proactive) survive as labels (set above).
|
|
2956
|
+
const effectiveScore = (ref) => {
|
|
2957
|
+
const rankScore = salienceMap.get(ref)?.rankScore ?? 0;
|
|
2958
|
+
const noOps = noOpMap.get(ref) ?? 0;
|
|
2959
|
+
return noOps >= SALIENCE_NO_OP_DAMPEN_THRESHOLD ? rankScore * SALIENCE_NO_OP_DAMPEN_FACTOR : rankScore;
|
|
2960
|
+
};
|
|
2180
2961
|
const sorted = [...mergedRefs].sort((a, b) => {
|
|
2181
|
-
const
|
|
2182
|
-
const
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
//
|
|
2186
|
-
|
|
2187
|
-
const scoreB = utilB * 0.7 + ratioB * 0.3;
|
|
2188
|
-
return scoreB - scoreA;
|
|
2962
|
+
const scoreA = effectiveScore(a.ref);
|
|
2963
|
+
const scoreB = effectiveScore(b.ref);
|
|
2964
|
+
if (scoreB !== scoreA)
|
|
2965
|
+
return scoreB - scoreA;
|
|
2966
|
+
// Stable tie-break: deterministic regardless of input ordering.
|
|
2967
|
+
return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
|
|
2189
2968
|
});
|
|
2190
2969
|
// Phase 0: surface coverage gaps from zero-result search queries
|
|
2191
2970
|
let coverageGaps = [];
|
|
@@ -2261,8 +3040,22 @@ async function runImprovePreparationStage(args) {
|
|
|
2261
3040
|
}
|
|
2262
3041
|
}
|
|
2263
3042
|
// ── Phase 5: --limit applies to the post-cooldown actionable set ──────────
|
|
3043
|
+
//
|
|
3044
|
+
// #610 ADDITIVITY: replay-lane refs are budgeted SEPARATELY from the --limit
|
|
3045
|
+
// fresh slice. Without this split, a high-rankScore replay ref could sort above
|
|
3046
|
+
// a fresh ref in the single combined slice and STEAL its slot (violating AC2).
|
|
3047
|
+
// We partition into the replay lane vs the rest, apply --limit to the
|
|
3048
|
+
// non-replay (fresh) refs only, then APPEND up to `replayBudget` replay refs
|
|
3049
|
+
// after the fresh slice. Sort order within each partition is preserved.
|
|
3050
|
+
//
|
|
3051
|
+
// Default replayBudget=0 reduces this to the exact pre-#610 expression: with no
|
|
3052
|
+
// replay refs, `nonReplayLoop === allLoopRefs`, so `baseLoop === old slice` and
|
|
3053
|
+
// `replayLoop.slice(0, 0) === []` — byte-identical.
|
|
2264
3054
|
const allLoopRefs = [...reflectAndDistillRefsAfterSort, ...distillOnlyRefsAfterSort];
|
|
2265
|
-
const
|
|
3055
|
+
const replayLoop = allLoopRefs.filter((r) => r.eligibilitySource === "replay");
|
|
3056
|
+
const nonReplayLoop = allLoopRefs.filter((r) => r.eligibilitySource !== "replay");
|
|
3057
|
+
const baseLoop = options.limit ? nonReplayLoop.slice(0, options.limit) : nonReplayLoop;
|
|
3058
|
+
const loopRefs = [...baseLoop, ...replayLoop.slice(0, replayBudget)];
|
|
2266
3059
|
// Update the returned distillOnlyRefs to the sorted order so callers see the
|
|
2267
3060
|
// ranked view (loop stage uses it as a Set so order is irrelevant, but the
|
|
2268
3061
|
// shape change keeps downstream consumers consistent).
|
|
@@ -2273,7 +3066,7 @@ async function runImprovePreparationStage(args) {
|
|
|
2273
3066
|
`(${fullySkippedCount} fully skipped, ${distillOnlyRefs.length} routed to distill-only)`);
|
|
2274
3067
|
}
|
|
2275
3068
|
if (signalAndRetrievalRefs.length > 0) {
|
|
2276
|
-
info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval)`);
|
|
3069
|
+
info(`[improve] ${signalAndRetrievalRefs.length} refs with usage signals (${signalFiltered.length} feedback, ${highRetrievalRefs.length} high-retrieval${replayRefSet.size > 0 ? `, ${replayRefSet.size} replay` : ""})`);
|
|
2277
3070
|
}
|
|
2278
3071
|
if (validationFailureRefs.size > 0) {
|
|
2279
3072
|
info(`[improve] ${validationFailureRefs.size} with validation failures excluded`);
|
|
@@ -2284,6 +3077,17 @@ async function runImprovePreparationStage(args) {
|
|
|
2284
3077
|
const deferredCount = actionableRefs.length - loopRefs.length;
|
|
2285
3078
|
info(`[improve] ${actionableRefs.length} actionable; ${loopRefs.length} will be processed` +
|
|
2286
3079
|
(options.limit && deferredCount > 0 ? ` (--limit ${options.limit} applied; ${deferredCount} deferred)` : ""));
|
|
3080
|
+
// WS-4: Per-phase threshold auto-tune for the extract phase.
|
|
3081
|
+
// Persists result for the NEXT run's makeGateConfig to read.
|
|
3082
|
+
const extractTuneDbPath = eventsCtx?.dbPath;
|
|
3083
|
+
if (options.autoAccept !== undefined && extractTuneDbPath) {
|
|
3084
|
+
try {
|
|
3085
|
+
maybeAutoTuneThreshold(extractGateCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), extractTuneDbPath, undefined, "extract");
|
|
3086
|
+
}
|
|
3087
|
+
catch (err) {
|
|
3088
|
+
warn(`[improve] calibration auto-tune (extract) skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
2287
3091
|
return {
|
|
2288
3092
|
actions,
|
|
2289
3093
|
cleanupWarnings,
|
|
@@ -2403,6 +3207,10 @@ async function runImproveLoopStage(args) {
|
|
|
2403
3207
|
stashDir: primaryStashDir,
|
|
2404
3208
|
config: options.config ?? loadConfig(),
|
|
2405
3209
|
eventsCtx,
|
|
3210
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
3211
|
+
// candidateCount drives the exploration budget. loopRefs is the per-phase
|
|
3212
|
+
// set for reflect/distill; pass it so exploration budget is proportional.
|
|
3213
|
+
candidateCount: loopRefs.length,
|
|
2406
3214
|
});
|
|
2407
3215
|
const distillGateCfg = makeGateConfig("distill", {
|
|
2408
3216
|
globalThreshold: options.autoAccept,
|
|
@@ -2410,6 +3218,8 @@ async function runImproveLoopStage(args) {
|
|
|
2410
3218
|
stashDir: primaryStashDir,
|
|
2411
3219
|
config: options.config ?? loadConfig(),
|
|
2412
3220
|
eventsCtx,
|
|
3221
|
+
stateDbPath: eventsCtx?.dbPath,
|
|
3222
|
+
candidateCount: loopRefs.length,
|
|
2413
3223
|
});
|
|
2414
3224
|
for (const planned of loopRefs) {
|
|
2415
3225
|
if (Date.now() - startMs >= budgetMs) {
|
|
@@ -2587,6 +3397,28 @@ async function runImproveLoopStage(args) {
|
|
|
2587
3397
|
reason: reflectResult.ok ? undefined : reflectResult.reason,
|
|
2588
3398
|
},
|
|
2589
3399
|
}, eventsCtx);
|
|
3400
|
+
// Plasticity counter (plan §WS-1 step 8): record no-ops so the
|
|
3401
|
+
// WS-1 selection comparator (effectiveScore, ~line 3073) can dampen
|
|
3402
|
+
// repeatedly-silent assets during consolidation-selection.
|
|
3403
|
+
// A no_change reflect means the LLM was invoked but found nothing to
|
|
3404
|
+
// improve — the asset is stable. Track it. A successful reflect means
|
|
3405
|
+
// the asset changed; reset the counter so the dampener lifts.
|
|
3406
|
+
if (isNoChange && eventsCtx?.db) {
|
|
3407
|
+
try {
|
|
3408
|
+
recordNoOp(eventsCtx.db, planned.ref);
|
|
3409
|
+
}
|
|
3410
|
+
catch {
|
|
3411
|
+
// best-effort: plasticity counter failure never blocks the run
|
|
3412
|
+
}
|
|
3413
|
+
}
|
|
3414
|
+
else if (reflectResult.ok && eventsCtx?.db) {
|
|
3415
|
+
try {
|
|
3416
|
+
resetConsecutiveNoOps(eventsCtx.db, planned.ref);
|
|
3417
|
+
}
|
|
3418
|
+
catch {
|
|
3419
|
+
// best-effort
|
|
3420
|
+
}
|
|
3421
|
+
}
|
|
2590
3422
|
if (reflectResult.ok) {
|
|
2591
3423
|
const reflectGr = await runAutoAcceptGate([{ proposalId: reflectResult.proposal.id, confidence: reflectResult.proposal.confidence }], reflectGateCfg);
|
|
2592
3424
|
gateAutoAcceptedCount += reflectGr.promoted.length;
|
|
@@ -2725,6 +3557,23 @@ async function runImproveLoopStage(args) {
|
|
|
2725
3557
|
if (!promotedToKnowledge)
|
|
2726
3558
|
memoryRefsForInference.add(planned.ref);
|
|
2727
3559
|
}
|
|
3560
|
+
// Plasticity counter (plan §WS-1 step 8) for the distill path.
|
|
3561
|
+
// quality_rejected: the LLM ran but produced output that didn't pass the
|
|
3562
|
+
// quality gate — the asset is not yielding useful distill output.
|
|
3563
|
+
// queued: a proposal was produced; reset the no-op counter.
|
|
3564
|
+
if (eventsCtx?.db) {
|
|
3565
|
+
try {
|
|
3566
|
+
if (distillResult.outcome === "quality_rejected" || distillResult.outcome === "skipped") {
|
|
3567
|
+
recordNoOp(eventsCtx.db, planned.ref);
|
|
3568
|
+
}
|
|
3569
|
+
else if (distillResult.outcome === "queued") {
|
|
3570
|
+
resetConsecutiveNoOps(eventsCtx.db, planned.ref);
|
|
3571
|
+
}
|
|
3572
|
+
}
|
|
3573
|
+
catch {
|
|
3574
|
+
// best-effort: plasticity counter failure never blocks the run
|
|
3575
|
+
}
|
|
3576
|
+
}
|
|
2728
3577
|
if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
|
|
2729
3578
|
const slug = planned.ref
|
|
2730
3579
|
.replace(/[^a-z0-9]/gi, "-")
|
|
@@ -2791,6 +3640,26 @@ async function runImproveLoopStage(args) {
|
|
|
2791
3640
|
completedCount++;
|
|
2792
3641
|
info(`[improve] ${completedCount}/${loopRefs.length} ${planned.ref}`);
|
|
2793
3642
|
}
|
|
3643
|
+
// WS-4: Per-phase threshold auto-tune — runs AFTER the loop so the gate
|
|
3644
|
+
// has processed all candidates for this run. Persists each phase's tuned
|
|
3645
|
+
// threshold to state.db for the NEXT run's makeGateConfig to read.
|
|
3646
|
+
// Best-effort: a tune failure must never fail the improve run.
|
|
3647
|
+
const stateDbPathForTune = eventsCtx?.dbPath;
|
|
3648
|
+
if (options.autoAccept !== undefined && stateDbPathForTune) {
|
|
3649
|
+
const phaseGateCfgMap = {
|
|
3650
|
+
reflect: reflectGateCfg,
|
|
3651
|
+
distill: distillGateCfg,
|
|
3652
|
+
};
|
|
3653
|
+
for (const phase of ["reflect", "distill"]) {
|
|
3654
|
+
const phaseCfg = phaseGateCfgMap[phase];
|
|
3655
|
+
try {
|
|
3656
|
+
maybeAutoTuneThreshold(phaseCfg.phaseThreshold ?? options.autoAccept, options.config ?? loadConfig(), stateDbPathForTune, undefined, phase);
|
|
3657
|
+
}
|
|
3658
|
+
catch (err) {
|
|
3659
|
+
warn(`[improve] calibration auto-tune (${phase}) skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
3660
|
+
}
|
|
3661
|
+
}
|
|
3662
|
+
}
|
|
2794
3663
|
return { reflectsWithErrorContext, memoryRefsForInference, gateAutoAcceptedCount, gateAutoAcceptFailedCount };
|
|
2795
3664
|
}
|
|
2796
3665
|
async function runImprovePostLoopStage(args) {
|
|
@@ -2834,9 +3703,70 @@ async function runImprovePostLoopStage(args) {
|
|
|
2834
3703
|
// best-effort
|
|
2835
3704
|
}
|
|
2836
3705
|
}
|
|
3706
|
+
// #609 — recombine / synthesize pass. Whole-corpus cross-episodic
|
|
3707
|
+
// generalization. Runs in the post-loop stage under consolidate.lock (it
|
|
3708
|
+
// reads the consolidated corpus and writes proposals). Opt-in: gated on the
|
|
3709
|
+
// `recombine` process being enabled, whole-stash / type scope (never `ref`),
|
|
3710
|
+
// and not a dry run. Mirrors the proactiveMaintenance opt-in wiring.
|
|
3711
|
+
let recombination;
|
|
3712
|
+
if (primaryStashDir &&
|
|
3713
|
+
improveProfile &&
|
|
3714
|
+
resolveProcessEnabled("recombine", improveProfile) &&
|
|
3715
|
+
scope.mode !== "ref" &&
|
|
3716
|
+
!options.dryRun) {
|
|
3717
|
+
const recombineFn = options.recombineFn ?? akmRecombine;
|
|
3718
|
+
try {
|
|
3719
|
+
recombination = await recombineFn({
|
|
3720
|
+
stashDir: primaryStashDir,
|
|
3721
|
+
config: options.config ?? loadConfig(),
|
|
3722
|
+
...(options.runId ? { sourceRun: options.runId } : {}),
|
|
3723
|
+
...(budgetSignal ? { signal: budgetSignal } : {}),
|
|
3724
|
+
...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
|
|
3725
|
+
eligibilitySource: "recombine",
|
|
3726
|
+
...(eventsCtx ? { ctx: eventsCtx } : {}),
|
|
3727
|
+
minClusterSize: improveProfile.processes?.recombine?.minClusterSize,
|
|
3728
|
+
maxClustersPerRun: improveProfile.processes?.recombine?.maxClustersPerRun,
|
|
3729
|
+
relatednessSource: improveProfile.processes?.recombine?.relatednessSource,
|
|
3730
|
+
confirmThreshold: improveProfile.processes?.recombine?.confirmThreshold,
|
|
3731
|
+
});
|
|
3732
|
+
}
|
|
3733
|
+
catch (e) {
|
|
3734
|
+
allWarnings.push(`recombine: ${String(e)}`);
|
|
3735
|
+
}
|
|
3736
|
+
}
|
|
3737
|
+
// #615 — procedural-compilation pass. Detects recurring successful ordered
|
|
3738
|
+
// action sequences and compiles them into workflow proposals. Opt-in: gated
|
|
3739
|
+
// on the `procedural` process being enabled, whole-stash / type scope (never
|
|
3740
|
+
// `ref`), and not a dry run. Mirrors the recombine opt-in wiring.
|
|
3741
|
+
let proceduralCompilation;
|
|
3742
|
+
if (primaryStashDir &&
|
|
3743
|
+
improveProfile &&
|
|
3744
|
+
resolveProcessEnabled("procedural", improveProfile) &&
|
|
3745
|
+
scope.mode !== "ref" &&
|
|
3746
|
+
!options.dryRun) {
|
|
3747
|
+
const proceduralFn = options.proceduralFn ?? akmProcedural;
|
|
3748
|
+
try {
|
|
3749
|
+
proceduralCompilation = await proceduralFn({
|
|
3750
|
+
stashDir: primaryStashDir,
|
|
3751
|
+
config: options.config ?? loadConfig(),
|
|
3752
|
+
...(options.runId ? { sourceRun: options.runId } : {}),
|
|
3753
|
+
...(budgetSignal ? { signal: budgetSignal } : {}),
|
|
3754
|
+
...(options.autoAccept !== undefined ? { autoAccept: options.autoAccept } : {}),
|
|
3755
|
+
eligibilitySource: "procedural",
|
|
3756
|
+
...(eventsCtx ? { ctx: eventsCtx } : {}),
|
|
3757
|
+
minRecurrence: improveProfile.processes?.procedural?.minRecurrence,
|
|
3758
|
+
maxProposalsPerRun: improveProfile.processes?.procedural?.maxProposalsPerRun,
|
|
3759
|
+
});
|
|
3760
|
+
}
|
|
3761
|
+
catch (e) {
|
|
3762
|
+
allWarnings.push(`procedural: ${String(e)}`);
|
|
3763
|
+
}
|
|
3764
|
+
}
|
|
2837
3765
|
return {
|
|
2838
3766
|
allWarnings,
|
|
2839
3767
|
deadUrls,
|
|
3768
|
+
...(recombination ? { recombination } : {}),
|
|
3769
|
+
...(proceduralCompilation ? { proceduralCompilation } : {}),
|
|
2840
3770
|
...(maintenanceResult.memoryInference ? { memoryInference: maintenanceResult.memoryInference } : {}),
|
|
2841
3771
|
...(maintenanceResult.graphExtraction ? { graphExtraction: maintenanceResult.graphExtraction } : {}),
|
|
2842
3772
|
...(maintenanceResult.stalenessDetection ? { stalenessDetection: maintenanceResult.stalenessDetection } : {}),
|