akm-cli 0.9.16 → 0.9.17-alpha.1

package/dist/commands/improve/loop-stages.js

@@ -7,12 +7,13 @@ import { parseRefInput } from "../../core/asset/resolve-ref.js";
 import { daysToMs } from "../../core/common.js";
 import { DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE, loadConfig } from "../../core/config/config.js";
 import { UsageError } from "../../core/errors.js";
-import { appendEvent } from "../../core/events.js";
+import { appendEvent, readEvents } from "../../core/events.js";
 import { openLogsDatabase, purgeOldTaskLogs } from "../../core/logs-db.js";
 import { getDbPath, getTaskLogDir } from "../../core/paths.js";
 import { withStateDb } from "../../core/state-db.js";
 import { info } from "../../core/warn.js";
 import { DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES, runGraphExtractionPass, } from "../../indexer/graph/graph-extraction.js";
+import { indexWrittenAssets } from "../../indexer/index-written-assets.js";
 import { deriveWritableBundleIds } from "../../indexer/installations.js";
 import { collectPendingMemories, runMemoryInferencePass, } from "../../indexer/passes/memory-inference.js";
 import { resolveSourceEntries } from "../../indexer/search/search-source.js";
@@ -22,21 +23,23 @@ import { purgeOldCycleMetrics } from "../../storage/repositories/canaries-reposi
 import { purgeOldEvents } from "../../storage/repositories/events-repository.js";
 import { purgeOldImproveRuns } from "../../storage/repositories/improve-runs-repository.js";
 import { closeDatabase, openIndexDatabase } from "../../storage/repositories/index-connection.js";
-import { getEntryByRef } from "../../storage/repositories/index-entries-repository.js";
+import { getLiveRefSnapshot, isRefLiveInSnapshot, } from "../../storage/repositories/index-entries-repository.js";
 import { clearAssetOutcomeMissing, countAssetOutcomeMissing, deleteAssetOutcomeMissingBefore, listAssetOutcomeMissingState, stampAssetOutcomeMissing, } from "../../storage/repositories/outcome-repository.js";
 import { clearAssetSalienceMissing, countAssetSalienceMissing, deleteAssetSalienceMissingBefore, listAssetSalienceMissingState, stampAssetSalienceMissing, } from "../../storage/repositories/salience-repository.js";
+import { readFreelistInfo, vacuumStateDbIfReclaimable } from "../../storage/state-db-integrity.js";
 import { purgeOldTaskLogFiles } from "../../tasks/run/task-log.js";
-import { expireStaleProposals, listProposals, purgeOrphanProposals } from "../proposal/repository.js";
+import { checkProposalGuard, expireStaleProposals, listProposals, purgeOrphanProposals } from "../proposal/repository.js";
 import { checkDeadUrls } from "../url-checker.js";
 import { DEFAULT_RETENTION_DAYS as CYCLE_METRICS_RETENTION_DAYS, runCollapseDetector } from "./collapse-detector.js";
-import { deriveLessonRef } from "./distill.js";
+import { defaultLookup, deriveLessonRef } from "./distill.js";
+import { wouldPromoteMemoryToKnowledge } from "./distill/promote-memory.js";
 import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
 // Eligibility / candidate-selection predicates live in ./eligibility.
 import { findAssetFilePath, isDistillCandidateRef } from "./eligibility.js";
-import { writeEvalCase } from "./eval-cases.js";
 import { shouldSkipRef } from "./improve-strategies.js";
+import { readOnlyEventsContext } from "./reflect.js";
 import { recordNoOp, resetConsecutiveNoOps } from "./salience.js";
-import { errMessage, refSlug } from "./shared.js";
+import { errMessage } from "./shared.js";
 import { bareImproveRef, durableImproveRef } from "./source-identity.js";
 // ── improve loop / post-loop / maintenance stages ───────────────────
 // The cycle stages run by akmImprove, extracted from improve.ts.
@@ -179,6 +182,8 @@ async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
             // O-1 (#364): pass remaining budget as timeoutMs so the agent spawn is
             // bounded by the wall-clock deadline rather than the default per-profile timeout.
             const reflectBudgetMs = env.remainingBudgetMs();
+            const reflectEngine = resolvedPlan.processes.reflect.runner?.engine;
+            const reflectTarget = options.sourceName && primaryStashDir ? { source: options.sourceName, root: primaryStashDir } : undefined;
             // Re-enter canonical named-engine lowering with the config snapshot and
             // process profile frozen into the invocation plan. The loop never injects
             // a RunnerSpec seam or observes later caller-config mutations.
@@ -192,9 +197,7 @@ async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
                 ...(improveProfile ? { improveProfile } : {}),
                 config: resolvedPlan.config,
                 ...(primaryStashDir ? { stashDir: primaryStashDir } : {}),
-                ...(options.sourceName && primaryStashDir
-                    ? { target: { source: options.sourceName, root: primaryStashDir } }
-                    : {}),
+                ...(reflectTarget ? { target: reflectTarget } : {}),
                 ...(reflectErrors.length > 0 ? { avoidPatterns: [...reflectErrors] } : {}),
                 eventSource: "improve",
                 // #639 — resolve the low-value filter from the ACTIVE improve profile
@@ -208,10 +211,73 @@ async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
                 // the reflect_invoked event and the persisted proposal.
                 ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
             };
-            const reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs), {
-                engine: resolvedPlan.processes.reflect.runner?.engine,
-                process: "reflect",
-            });
+            // R9 (tier2-0917): the fingerprint/rejection-backoff guard `createProposal`
+            // runs AFTER reflect's ~39s generation + judge is computable from inputs
+            // available before dispatch. Check it here first — on a hit, skip the LLM
+            // call entirely and synthesize the same "cooldown" envelope reflect.ts's
+            // createProposal-skip branch returns, so everything below (mode
+            // classification, improve_reflect_outcome, plasticity) is unchanged. This
+            // pre-check is an optimisation only — createProposal's post-generation
+            // check stays authoritative (see checkProposalGuard's doc comment).
+            const guardStash = primaryStashDir ?? options.stashDir;
+            const guardSkip = guardStash
+                ? checkProposalGuard({
+                    stash: guardStash,
+                    ref: planned.ref,
+                    source: "reflect",
+                    ...(reflectTarget ? { target: reflectTarget } : {}),
+                    ...(reflectEngine ? { modelId: reflectEngine } : {}),
+                })
+                : undefined;
+            let reflectResult;
+            if (guardSkip) {
+                // Mirror reflect.ts's buildReflectEventEmitters().emitInvoked(): the
+                // signal-delta cursor (buildLatestProposalTsMap) reads `reflect_invoked`
+                // events regardless of outcome, so it must still advance for this ref
+                // even though reflectFn was never called.
+                appendEvent({
+                    eventType: "reflect_invoked",
+                    ref: planned.itemRef ?? durableImproveRef(planned.ref),
+                    metadata: {
+                        ...(options.task ? { task: options.task } : {}),
+                        ...(reflectEngine ? { engine: reflectEngine } : {}),
+                        ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
+                    },
+                }, eventsCtx);
+                // Mirror reflect.ts's buildReflectEventEmitters().emitFailed(): every
+                // reflect_invoked must be paired with a reflect_completed so observers
+                // building closed-loop telemetry see balanced invoke/complete pairs.
+                // reflectFn is never called on this path, so reflect.ts's own
+                // emitFailed (fired from its post-generation cooldown branch) never
+                // runs either — this is the pre-generation guard's own pairing.
+                appendEvent({
+                    eventType: "reflect_completed",
+                    ref: planned.itemRef ?? durableImproveRef(planned.ref),
+                    metadata: {
+                        source: "reflect",
+                        ok: false,
+                        reason: "cooldown",
+                        subreason: "pre_generation_guard",
+                        proposalSkipReason: guardSkip.reason,
+                        ...(guardSkip.existingProposalId ? { existingProposalId: guardSkip.existingProposalId } : {}),
+                    },
+                }, eventsCtx);
+                reflectResult = {
+                    schemaVersion: 2,
+                    ok: false,
+                    reason: "cooldown",
+                    error: `Proposal skipped (${guardSkip.reason}): ${guardSkip.message}`,
+                    ref: planned.ref,
+                    ...(reflectEngine ? { engine: reflectEngine } : {}),
+                    exitCode: null,
+                };
+            }
+            else {
+                reflectResult = await withLlmStage("reflect", () => reflectFn(reflectCallArgs), {
+                    engine: reflectEngine,
+                    process: "reflect",
+                });
+            }
             const isCooldown = !reflectResult.ok && reflectResult.reason === "cooldown";
             // Content-policy guard hits (reflect size-rail rejections) are NOT
             // LLM faults — the agent responded fine, the downstream guard
@@ -233,6 +299,13 @@ async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
             // fault — so it routes to the `reflect-skipped` bucket and stays
             // out of recentErrors/avoidPatterns.
             const isNoChange = !reflectResult.ok && reflectResult.reason === "no_change";
+            // Quality-gate rejection (R3): the judge rejected an otherwise
+            // well-parsed proposal. Stays in the `reflect-failed` bucket for
+            // metrics continuity, but — like the deterministic skips above —
+            // must not be injected into recentErrors/avoidPatterns: the judge's
+            // rejection text is not a reusable "avoid this pattern" lesson, and
+            // feeding it back in poisoned later prompts in the same run.
+            const isQualityRejected = !reflectResult.ok && reflectResult.reason === "quality_rejected";
             tally.actions.push({
                 ref: planned.ref,
                 mode: reflectResult.ok
@@ -246,14 +319,15 @@ async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
                                 : "reflect-failed",
                 result: reflectResult,
             });
-            // Cooldown skips, guard rejects, type-refused skips, and noise-gate
-            // skips are not failures — do not pollute recentErrors with them
-            // (those get injected as `avoidPatterns` into the next reflect
-            // prompt). Guard rejects ARE worth showing the LLM as a learn-signal
-            // so the next iteration sees "your last expansion was too large";
-            // type-refused and no-change are deterministic and add no learning
+            // Cooldown skips, guard rejects, type-refused skips, noise-gate
+            // skips, and quality-gate rejections are not failures — do not
+            // pollute recentErrors with them (those get injected as
+            // `avoidPatterns` into the next reflect prompt). Guard rejects ARE
+            // worth showing the LLM as a learn-signal so the next iteration sees
+            // "your last expansion was too large"; type-refused, no-change, and
+            // quality-rejected are deterministic/judge-side and add no learning
             // signal.
-            if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange) {
+            if (!reflectResult.ok && !isCooldown && !isTypeRefused && !isNoChange && !isQualityRejected) {
                 const errMsg = reflectResult.error ?? reflectResult.reason ?? "unknown reflect error";
                 tally.recentErrorPushes.push({ originator: "reflect", message: errMsg });
             }
@@ -315,7 +389,7 @@ async function runLoopReflectPass(planned, isDistillOnly, env, tally) {
  * that was a `continue` in the old inline loop body is an early `return` here.
  */
 async function runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env, tally) {
-    const { options, primaryStashDir, eventsCtx, improveProfile } = env;
+    const { options, primaryStashDir, eventsCtx, improveProfile, resolvedPlan } = env;
     const hasRecentFeedbackSignal = env.signalBearingSet.has(planned.ref);
     const explicitRefScope = env.scope.mode === "ref";
     // Profile gate: apply the full type-filter / raw-wiki / disabled rules to
@@ -405,6 +479,97 @@ async function runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env,
                     return;
                 }
             }
+            // R9 extension (r2-6, tier2-0917; PRECHECK, tier3-0917): the
+            // fingerprint/rejection-backoff guard `createProposal` runs AFTER
+            // distill's ~generation + judge is computable from inputs available
+            // before dispatch — mirror the reflect pre-check above so a guard hit
+            // skips the LLM call entirely. Distill's real `createProposal` call
+            // always targets the derived lesson/knowledge ref (`effectiveLessonRef`
+            // in distill.ts), never the input ref.
+            // Which ref that is: for every non-memory distill-candidate type,
+            // `targetKind` defaults to "lesson" (distill.ts ~L882, `invokeDistill
+            // AndRecord` above only ever sets `proposalKind: "auto"` for memory
+            // refs) and is never overridden to "knowledge", so lessonRef is the
+            // ONLY real target. For memory refs (`proposalKind: "auto"`), the
+            // target is decided at dispatch by `planMemoryKnowledgePromotion`
+            // (knowledgeRef via promotion, lessonRef as fallback) — that decision
+            // IS cheap and LLM-free (a deterministic score over the asset content
+            // + its feedback history, plus one lookup for an existing knowledge
+            // file), so it is pre-checked exactly via `wouldPromoteMemoryToKnowledge`,
+            // a thin wrapper that delegates to `planMemoryKnowledgePromotion`
+            // itself so this can never drift from distill's real decision. A
+            // guard hit on the ref distill would NOT have targeted must never
+            // suppress a legitimate dispatch.
+            // §23.6 fingerprint model-id term: distill resolves models, not
+            // engines (unlike reflect), so this must match `distillRunner?.
+            // connection.model` in distill.ts, not the engine name.
+            const distillModelId = resolvedPlan.processes.distill.runner?.connection.model;
+            let realTargetRef = lessonRef;
+            if (parsedPlannedRef.type === "memory") {
+                // distill.ts's real dispatch (akmDistill) always derives
+                // durableInputRef from options.ref alone (durableImproveRef(inputRef),
+                // never itemRef) and reads/scores content via that ref
+                // (loadAndScoreInputSalience's `lookup(durableInputRef)`); mirror
+                // that here so the pre-check can never read/score a different file
+                // than the real dispatch would. itemRef is preferred only for the
+                // feedback-events query, matching readDistillFeedback's
+                // `ref: options.itemRef ?? durableInputRef`.
+                const durableInputRef = durableImproveRef(planned.ref);
+                const feedbackRef = planned.itemRef ?? durableInputRef;
+                const lookup = (ref) => defaultLookup(ref, dedupeStashDir);
+                const filePath = await lookup(durableInputRef);
+                const assetContent = filePath && fs.existsSync(filePath) ? fs.readFileSync(filePath, "utf8") : null;
+                // PRECHECK (tier3-0917-r3, r3-4): reuse the loop's long-lived
+                // eventsCtx.db handle when one is open, instead of opening a fresh
+                // read-only state.db connection per memory ref (R25). Degrades to
+                // the previous readOnly-open when no live handle is present (e.g.
+                // this function invoked without a run-scoped eventsCtx), via the
+                // same readOnlyEventsContext helper reflect.ts's read call sites use.
+                const { events: feedbackEvents } = readEvents({ ref: feedbackRef, type: "feedback" }, readOnlyEventsContext(eventsCtx));
+                const promotesToKnowledge = await wouldPromoteMemoryToKnowledge({
+                    inputRef: planned.ref,
+                    durableInputRef,
+                    assetContent,
+                    feedbackEvents,
+                    config: options.config ?? loadConfig(),
+                    stash: dedupeStashDir,
+                    lookup,
+                });
+                if (promotesToKnowledge)
+                    realTargetRef = knowledgeRef;
+            }
+            const guardSkip = checkProposalGuard({
+                stash: dedupeStashDir,
+                ref: realTargetRef,
+                source: "distill",
+                ...(distillModelId ? { modelId: distillModelId } : {}),
+            });
+            if (guardSkip) {
+                tally.actions.push({
+                    ref: planned.ref,
+                    mode: "distill-skipped",
+                    result: { ok: true, reason: guardSkip.reason },
+                });
+                // Mirror distill.ts's own proposal-skip branch (the post-generation
+                // guard `createProposal` hits): emit `distill_invoked` with a
+                // `skipped` outcome so the signal-delta cursor
+                // (buildLatestProposalTsMap, eligibility.ts) advances for this ref
+                // even though distillFn was never called.
+                appendEvent({
+                    eventType: "distill_invoked",
+                    // Use item_ref when resolved, otherwise the input conceptId —
+                    // matches distill.ts's own distill_invoked key.
+                    ref: planned.itemRef ?? durableImproveRef(planned.ref),
+                    metadata: {
+                        outcome: "skipped",
+                        proposalRef: realTargetRef,
+                        message: guardSkip.message,
+                        skipReason: guardSkip.reason,
+                        ...(planned.eligibilitySource ? { eligibilitySource: planned.eligibilitySource } : {}),
+                    },
+                }, eventsCtx);
+                return;
+            }
         }
         await invokeDistillAndRecord(planned, parsedPlannedRef, env, tally);
     }
@@ -423,8 +588,7 @@ async function runLoopDistillPass(planned, parsedPlannedRef, isDistillOnly, env,
 }
 /**
  * The distill invocation for one ref that passed every gate: the `distillFn`
- * call, memory-inference queueing, plasticity counters, and the
- * quality-rejected / proposal-rejected eval-case writes.
+ * call, memory-inference queueing, and plasticity counters.
  */
 async function invokeDistillAndRecord(planned, parsedPlannedRef, env, tally) {
     const { options, primaryStashDir, distillFn, eventsCtx, improveProfile, resolvedPlan, budgetSignal } = env;
@@ -470,30 +634,6 @@ async function invokeDistillAndRecord(planned, parsedPlannedRef, env, tally) {
             // best-effort: plasticity counter failure never blocks the run
         }
     }
-    if (distillResult.outcome === "quality_rejected" && primaryStashDir) {
-        const slug = refSlug(planned.ref);
-        writeEvalCase(primaryStashDir, {
-            ref: planned.ref,
-            failureReason: distillResult.reason ?? "quality gate rejected",
-            assetType: parseRefInput(planned.ref).type ?? "unknown",
-            rejectedAt: Date.now(),
-            source: "distill_quality_rejected",
-            slug: `${slug}-${Date.now()}`,
-        });
-    }
-    // D6: use pre-loaded map instead of per-iteration DB query
-    const rejectedProposalEvent = env.rejectedProposalsByRef.get(planned.ref);
-    if (rejectedProposalEvent && primaryStashDir) {
-        const slug = refSlug(planned.ref);
-        writeEvalCase(primaryStashDir, {
-            ref: planned.ref,
-            failureReason: rejectedProposalEvent.metadata?.reason ?? "proposal rejected",
-            assetType: parseRefInput(planned.ref).type ?? "unknown",
-            rejectedAt: new Date(rejectedProposalEvent.ts).getTime(),
-            source: "proposal_rejected",
-            slug: `${slug}-rejected`,
-        });
-    }
 }
 /**
  * Wall-clock budget exhausted mid-loop (O-1 / #364): emit the improve_skipped
@@ -552,7 +692,7 @@ export async function runImproveLoopStage(args) {
     return { reflectsWithErrorContext, memoryRefsForInference };
 }
 export async function runImprovePostLoopStage(args) {
-    const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, reindexFn, eventsCtx, budgetSignal, improveProfile, resolvedPlan, consolidationRan, } = args;
+    const { scope, options, primaryStashDir, actionableRefs, appliedCleanup, cleanupWarnings, memoryRefsForInference, eventsCtx, budgetSignal, improveProfile, resolvedPlan, consolidationRan, } = args;
     const allWarnings = [...cleanupWarnings, ...(appliedCleanup?.warnings ?? [])];
     info("[improve] post-loop maintenance starting");
     const maintenanceResult = await runImproveMaintenancePasses({
@@ -561,8 +701,6 @@ export async function runImprovePostLoopStage(args) {
         actionableRefs,
         memoryRefsForInference,
         allWarnings,
-        reindexFn,
-        consolidationRan,
         // O-1 (#364): forward the budget signal to memory inference + graph extraction.
         budgetSignal,
         eventsCtx,
@@ -617,8 +755,7 @@ export async function runImprovePostLoopStage(args) {
         }
     }
     // ── R5: collapse/churn detector ────────────────────────────────────────────
-    // One snapshot per QUALIFYING cycle: consolidate processed work. Runs AFTER
-    // the maintenance reindex so FTS sees the post-merge index. Deterministic,
+    // One snapshot per QUALIFYING cycle: consolidate processed work. Deterministic,
     // observe-only, fail-open (the orchestrator catches everything) — and inert
     // on the ~9-in-10 default-profile runs that touch no merges.
     let cycleMetrics;
@@ -651,7 +788,7 @@ export async function runImprovePostLoopStage(args) {
 // Exported for tests (#584/#585 DB-locking regression coverage); production
 // callers reach it only through akmImprove → runImprovePostLoopStage.
 export async function runImproveMaintenancePasses(args) {
-    const { options, primaryStashDir, memoryRefsForInference, allWarnings, reindexFn, budgetSignal, eventsCtx } = args;
+    const { options, primaryStashDir, memoryRefsForInference, allWarnings, budgetSignal, eventsCtx } = args;
     if (!primaryStashDir)
         return { memoryInferenceDurationMs: 0, graphExtractionDurationMs: 0 };
     if (budgetSignal?.aborted)
@@ -662,21 +799,6 @@ export async function runImproveMaintenancePasses(args) {
     const graphExtractionFn = options.graphExtractionFn ?? runGraphExtractionPass;
     const openIndexDb = () => openIndexDatabase(getDbPath(), config.embedding?.dimension ? { embeddingDim: config.embedding.dimension } : undefined);
     const dbCell = {};
-    // #584: see the MaintenanceCtx.reindexWithIndexDbReleased doc — close before
-    // every reindex, reopen in `finally` so a failed reindex still leaves a
-    // usable handle in the cell.
-    const reindexWithIndexDbReleased = async (stashDir) => {
-        if (dbCell.current) {
-            closeDatabase(dbCell.current);
-            dbCell.current = undefined;
-        }
-        try {
-            await reindexFn({ stashDir, signal: budgetSignal });
-        }
-        finally {
-            dbCell.current = openIndexDb();
-        }
-    };
     const ctx = {
         config,
         sources,
@@ -687,12 +809,10 @@ export async function runImproveMaintenancePasses(args) {
         resolvedPlan: args.resolvedPlan,
         memoryInferenceFn,
         graphExtractionFn,
-        reindexWithIndexDbReleased,
     };
     const collected = await runMaintenancePassesUnderLease(ctx, dbCell, {
         actionableRefs: args.actionableRefs,
         memoryRefsForInference,
-        consolidationRan: args.consolidationRan,
         allWarnings,
         openIndexDb,
     });
@@ -717,7 +837,6 @@ export async function runImproveMaintenancePasses(args) {
 async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
     const { allWarnings } = args;
     const actions = [];
-    let reindexedAfterInference = false;
     try {
         dbCell.current = args.openIndexDb();
         const inference = await runMemoryInferenceMaintenancePass(ctx, dbCell, args.memoryRefsForInference);
@@ -725,22 +844,34 @@ async function runMaintenancePassesUnderLease(ctx, dbCell, args) {
             actions.push(inference.action);
         allWarnings.push(...inference.warnings);
         const memoryInference = inference.memoryInference;
-        if (memoryInference && (memoryInference.splitParents > 0 || memoryInference.writtenFacts > 0)) {
-            info("[improve] reindexing after memory inference writes");
+        // R78 (tier1-0917): index exactly the files memory inference wrote (derived children
+        // + rewritten parents) instead of a full reindex — typically one written
+        // fact per run, which used to pay a full-corpus reindex regardless.
+        if (memoryInference && memoryInference.writtenPaths.length > 0) {
+            info(`[improve] indexing ${memoryInference.writtenPaths.length} file(s) written by memory inference`);
             try {
-                await ctx.reindexWithIndexDbReleased(ctx.primaryStashDir);
-                reindexedAfterInference = true;
-                info("[improve] reindex after memory inference complete");
+                // #584: indexWrittenAssets opens its own write handle on the same
+                // index.db WAL file, so the maintenance handle must be closed first
+                // and a fresh one reopened after, even on failure.
+                if (dbCell.current) {
+                    closeDatabase(dbCell.current);
+                    dbCell.current = undefined;
+                }
+                try {
+                    await indexWrittenAssets(ctx.primaryStashDir, memoryInference.writtenPaths);
+                }
+                finally {
+                    dbCell.current = args.openIndexDb();
+                }
+                info("[improve] indexing after memory inference complete");
             }
             catch (err) {
-                allWarnings.push(`reindex after memory inference failed: ${errMessage(err)}`);
+                allWarnings.push(`indexing after memory inference failed: ${errMessage(err)}`);
             }
         }
         const graph = await runGraphExtractionMaintenancePass(ctx, dbCell, {
             actionableRefs: args.actionableRefs,
             memoryRefsForInference: args.memoryRefsForInference,
-            consolidationRan: args.consolidationRan,
-            reindexedAfterInference,
         });
         if (graph.action)
             actions.push(graph.action);
@@ -868,7 +999,6 @@ export async function runGraphExtractionMaintenancePass(ctx, dbCell, args) {
     let graphExtraction;
     let durationMs = 0;
     let action;
-    let reindexedAfterInference = args.reindexedAfterInference;
     const graphEnabled = resolvedPlan ? true : isProcessEnabled("index", "graph_extraction", config);
     const graphExtractionDisabledByProfile = improveProfile?.processes?.graphExtraction?.enabled === false;
     const graphExtractionFullScan = improveProfile?.processes?.graphExtraction?.fullScan === true;
@@ -879,6 +1009,7 @@ export async function runGraphExtractionMaintenancePass(ctx, dbCell, args) {
         ...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES,
     ];
     const graphExtractionBatchSize = improveProfile?.processes?.graphExtraction?.batchSize ?? DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE;
+    const graphExtractionMaxChunksPerAsset = improveProfile?.processes?.graphExtraction?.maxChunksPerAsset;
     // Build the set of refs actually touched this run.
     const touchedRefs = new Set();
     for (const r of args.actionableRefs)
@@ -892,21 +1023,6 @@ export async function runGraphExtractionMaintenancePass(ctx, dbCell, args) {
         info(`[improve] graph extraction starting${graphExtractionFullScan ? " (full-corpus scan)" : ""}`);
         const extractionStart = Date.now();
         try {
-            // D9: if consolidation ran but memory inference did not reindex, force a reindex
-            // so graph extraction sees current DB state after consolidation writes.
-            if (args.consolidationRan && !reindexedAfterInference) {
-                info("[improve] reindexing after consolidation (graph extraction needs current state)");
-                try {
-                    await ctx.reindexWithIndexDbReleased(primaryStashDir);
-                    reindexedAfterInference = true;
-                    info("[improve] reindex after consolidation complete");
-                }
-                catch (err) {
-                    warnings.push(`reindex after consolidation failed: ${errMessage(err)}`);
-                }
-            }
-            // #584: no close/reopen needed here — reindexWithIndexDbReleased
-            // already swapped in a fresh post-reindex handle.
             // Resolve touched refs to absolute file paths. Skipped for fullScan
             // (candidatePaths stays undefined → extractor processes all files).
             let candidatePaths;
@@ -943,6 +1059,9 @@ export async function runGraphExtractionMaintenancePass(ctx, dbCell, args) {
                     includeTypes: graphExtractionIncludeTypes,
                     batchSize: graphExtractionBatchSize,
                     ...(graphExtractionTopN != null ? { topN: graphExtractionTopN } : {}),
+                    ...(graphExtractionMaxChunksPerAsset != null
+                        ? { maxChunksPerAsset: graphExtractionMaxChunksPerAsset }
+                        : {}),
                 },
             }), { engine: resolvedPlan?.processes.graphExtraction.runner?.engine, process: "graphExtraction" });
             durationMs = Date.now() - extractionStart;
@@ -1089,6 +1208,13 @@ export function runRetentionPurgePass(ctx) {
                         metadata: { purgedCount: cycleMetricsPurged, retentionDays: cycleRetention },
                     }, eventsCtx);
                 }
+                // R0 step 3: opportunistic post-purge VACUUM. Reads the freelist
+                // off this same connection (no second state.db handle) and only
+                // runs when reclaimable space crosses STATE_DB_FREELIST_WARN_RATIO.
+                const vacuumOutcome = vacuumStateDbIfReclaimable(stateDb, readFreelistInfo(stateDb), eventsCtx);
+                if (vacuumOutcome.ran) {
+                    info(`[improve] state.db vacuum: ${vacuumOutcome.pagesBefore} -> ${vacuumOutcome.pagesAfter} pages`);
+                }
             }, { path: eventsCtx?.dbPath, borrowed: eventsCtx?.db });
         }
         catch (err) {
@@ -1165,12 +1291,13 @@ export const STATE_GC_GRACE_MS = daysToMs(7);
  * Resolve one state-table's stored `asset_ref` against the live index.
  *
  * "ref not present in entries.item_ref" is the authoritative-deletion
- * predicate (see the pass doc comment below), so this is a thin wrapper
- * around the same single-ref probe the rest of improve uses
- * (`getEntryByRef`, index-entries-repository.ts) — which already resolves
- * both storage spellings a write can produce (`salienceWriteKey`/
- * `outcomeWriteKey` = `itemRef ?? ref`): an exact bundle-qualified item_ref,
- * or a bare conceptId matched by suffix across all bundles.
+ * predicate (see the pass doc comment below), so this checks the same two
+ * spellings `getEntryByRef` (index-entries-repository.ts) resolves — an exact
+ * bundle-qualified item_ref, or a bare conceptId matched by suffix across all
+ * bundles — but against a prebuilt {@link LiveRefSnapshot}
+ * (`getLiveRefSnapshot`) instead of a database round trip per row: with up to
+ * a few thousand pending rows per run, one probe per row was the dominant
+ * cost R78 (tier1-0917).
  *
  * On top of that, falls back to the BARE conceptId form (`bareImproveRef` —
  * the same primitive `preparation.ts`'s `normalizeStoredKey` map is built
@@ -1182,11 +1309,11 @@ export const STATE_GC_GRACE_MS = daysToMs(7);
  * "never delete a live row" over "never miss a genuinely dead one" mirrors
  * `getEntryByRef`'s own bare-conceptId suffix-match trade-off.
  */
-function isStateRefLive(indexDb, storedRef) {
-    if (getEntryByRef(indexDb, storedRef) !== null)
+function isStateRefLive(snapshot, storedRef) {
+    if (isRefLiveInSnapshot(snapshot, storedRef))
         return true;
     const bare = bareImproveRef(storedRef);
-    return bare !== storedRef && getEntryByRef(indexDb, bare) !== null;
+    return bare !== storedRef && isRefLiveInSnapshot(snapshot, bare);
 }
 /**
  * Sweep ONE state table: stamp refs that just went unresolved, clear refs
@@ -1198,11 +1325,11 @@ function isStateRefLive(indexDb, storedRef) {
  * proof" (close-out plan, Workstream C).
  */
 function gcOneStateTable(args) {
-    const { refRows, indexDb, now, collect, stamp, clear, deleteOlderThan, countPending } = args;
+    const { refRows, liveRefs, now, collect, stamp, clear, deleteOlderThan, countPending } = args;
     const toStamp = [];
     const toClear = [];
     for (const row of refRows) {
-        const live = isStateRefLive(indexDb, row.asset_ref);
+        const live = isStateRefLive(liveRefs, row.asset_ref);
         if (!live && row.missing_since == null)
             toStamp.push(row.asset_ref);
         else if (live && row.missing_since != null)
@@ -1257,10 +1384,16 @@ export function runOrphanStateGcPass(ctx, dbCell) {
     let pending = 0;
     let collected = 0;
     try {
+        // R78 (tier1-0917): one query for every live item_ref, shared by both tables' sweeps
+        // below — replaces a `getEntryByRef` round trip per pending row. Inside
+        // the try so a schema mismatch (e.g. a DB version upgrade that dropped
+        // `entries`) degrades to the "orphan state GC failed" warning below
+        // instead of escaping this pass and failing the whole maintenance run.
+        const liveRefs = getLiveRefSnapshot(indexDb);
         withStateDb((stateDb) => {
             const salienceResult = gcOneStateTable({
                 refRows: listAssetSalienceMissingState(stateDb),
-                indexDb,
+                liveRefs,
                 now,
                 collect,
                 stamp: (refs, ts) => stampAssetSalienceMissing(stateDb, refs, ts),
@@ -1270,7 +1403,7 @@ export function runOrphanStateGcPass(ctx, dbCell) {
             });
             const outcomeResult = gcOneStateTable({
                 refRows: listAssetOutcomeMissingState(stateDb),
-                indexDb,
+                liveRefs,
                 now,
                 collect,
                 stamp: (refs, ts) => stampAssetOutcomeMissing(stateDb, refs, ts),