akm-cli 0.9.1 → 0.9.2-alpha.2

package/dist/commands/improve/distill.js

@@ -64,20 +64,19 @@ import { resolveStandardsContext } from "../../core/standards/resolve-standards-
 import { withStateDb } from "../../core/state-db.js";
 import { warnVerbose } from "../../core/warn.js";
 import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
-import { getDefaultLlmConfig } from "../../integrations/agent/engine-resolution.js";
-import { materializeLlmRunnerConnection, resolveImproveProcessRunner } from "../../integrations/agent/runner.js";
-import { chatCompletion } from "../../llm/client.js";
-import { callStructured } from "../../llm/structured-call.js";
-import { closeDatabase, openIndexDatabase } from "../../storage/repositories/index-connection.js";
+import { disposeLoweredExecutionDispatchLease, } from "../../integrations/agent/execution-lowering.js";
+import { callStructured, preflightStructuredLlmRunner } from "../../llm/structured-call.js";
+import { closeDatabase, openReadonlyExistingDatabase } from "../../storage/repositories/index-connection.js";
 import { getAllEntries } from "../../storage/repositories/index-entries-repository.js";
-import { isProposalSkipped, listProposals, proposalContent, } from "../proposal/repository.js";
+import { isProposalSkipped, listProposals, listProposalsReadOnly, proposalContent, } from "../proposal/repository.js";
 import { stripFrontmatterBody as stripBodyForFidelity } from "./content-hash.js";
 import { autoRepairLessonFrontmatter, autoSwapDescriptionWhenToUse, collectLessonQualityFindings, repairLessonDescriptionTruncation, } from "./distill/content-repair.js";
-import { promoteMemoryToKnowledge } from "./distill/promote-memory.js";
+import { memoryKnowledgePromotionRequiresDispatch, planMemoryKnowledgePromotion, promoteMemoryToKnowledge, } from "./distill/promote-memory.js";
 import { fetchTopSimilarLessons, persistOutputEncodingSalience, runLessonQualityJudge, writeQualityRejection, } from "./distill/quality-gate.js";
 import { buildClsContext, checkDistillFidelity } from "./distill-guards.js";
 import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
 import { buildRefVocabulary, scoreEncodingSalience } from "./encoding-salience.js";
+import { resolveImproveLlmExecution } from "./execution.js";
 import { resolveImproveStrategy, resolveProcessEnabled } from "./improve-strategies.js";
 import { emitProposal } from "./proposal-envelope.js";
 import { createRunContext, resolveRunStashDir } from "./run-context.js";
@@ -432,7 +431,7 @@ export function buildDistillPrompt(input) {
  * ref vocabulary the caller reuses when scoring the distilled OUTPUT (G4).
  */
 async function loadAndScoreInputSalience(args) {
-    const { inputRef, durableInputRef, salienceWriteKey, stash, config, outcomeWeightEnabled, lookup, ctx } = args;
+    const { inputRef, durableInputRef, salienceWriteKey, stash, outcomeWeightEnabled, lookup, ctx } = args;
     // Best-effort load: when the asset is not yet indexed we still proceed —
     // the LLM is asked to distil from "available signal" (feedback alone).
     let assetContent = null;
@@ -458,20 +457,21 @@ async function loadAndScoreInputSalience(args) {
     // reuses it when scoring the distilled OUTPUT at proposal creation (G4).
     let existingRefVocabulary = new Set();
     try {
-        const embCfg = config?.embedding;
-        const indexDb = openIndexDatabase(getDbPath(), embCfg?.dimension ? { embeddingDim: embCfg.dimension } : undefined);
-        try {
-            const allRefs = getAllEntries(indexDb).map((e) => e.entryKey);
-            existingRefVocabulary = buildRefVocabulary(allRefs);
-        }
-        finally {
-            closeDatabase(indexDb);
+        const indexDb = openReadonlyExistingDatabase(getDbPath(), { isolatedSnapshot: true });
+        if (indexDb) {
+            try {
+                const allRefs = getAllEntries(indexDb).map((e) => e.itemRef);
+                existingRefVocabulary = buildRefVocabulary(allRefs);
+            }
+            finally {
+                closeDatabase(indexDb);
+            }
         }
     }
     catch {
         // Index not available — novelty defaults to type-floor.
     }
-    if (assetContent && assetFilePath) {
+    if (args.persistSalience !== false && assetContent && assetFilePath) {
         try {
             const parsedRef = parseRefInput(inputRef);
             // G4: predictionError decays with revision count — the prior hardcoded
@@ -561,6 +561,92 @@ function refuseDisallowedDistillInput(args) {
         message,
     };
 }
+async function prepareDistillExecution(args) {
+    const { options, config, inputRef, durableInputRef, salienceWriteKey } = args;
+    const stash = resolveRunStashDir(options.stashDir);
+    const chat = options.chat;
+    const executionNotices = new Map();
+    const collectNotices = (notices) => {
+        for (const notice of notices)
+            executionNotices.set(JSON.stringify(notice), notice);
+    };
+    const resolvedExecution = !Object.hasOwn(options, "llmRunner")
+        ? resolveImproveLlmExecution({
+            config,
+            profile: options.improveProfile,
+            process: getImproveProcessConfig("distill", options.improveProfile),
+            processName: "distill",
+        })
+        : null;
+    if (resolvedExecution)
+        collectNotices(resolvedExecution.notices);
+    const distillRunner = Object.hasOwn(options, "llmRunner")
+        ? (options.llmRunner ?? undefined)
+        : resolvedExecution?.runner;
+    const withNotices = (result) => executionNotices.size > 0 ? { ...result, notices: Object.freeze([...executionNotices.values()]) } : result;
+    const lookup = options.lookupFn ?? ((ref) => defaultLookup(ref, stash));
+    const readEventsImpl = options.readEventsFn ??
+        ((readOptions) => readEvents(readOptions, { readOnly: true }));
+    const outcomeWeightEnabled = config.improve?.salience?.outcomeWeightEnabled !== false;
+    const fetchSimilarLessonsFn = options.fetchSimilarLessonsFn ?? ((query, n) => fetchTopSimilarLessons(query, n, options.stashDir));
+    const assetCtx = createRunContext({
+        stashDir: stash,
+        config,
+        eventsCtx: options.eventsCtx ?? {},
+        proposalsCtx: options.ctx ?? {},
+        chat,
+        getLlmRunner: () => distillRunner ?? null,
+        sourceRun: options.sourceRun ?? `distill-${Date.now()}`,
+        dryRun: false,
+        signal: options.signal,
+    }).withFreshAssetMemo();
+    const initialSalience = await loadAndScoreInputSalience({
+        inputRef,
+        durableInputRef,
+        salienceWriteKey,
+        stash,
+        config,
+        outcomeWeightEnabled,
+        lookup,
+        ctx: assetCtx,
+        persistSalience: false,
+    });
+    const assetState = { ...initialSalience };
+    const persistInputSalience = async () => {
+        const scored = await loadAndScoreInputSalience({
+            inputRef,
+            durableInputRef,
+            salienceWriteKey,
+            stash,
+            config,
+            outcomeWeightEnabled,
+            lookup,
+            ctx: assetCtx,
+        });
+        assetState.assetContent = scored.assetContent ?? assetState.assetContent;
+        assetState.existingRefVocabulary = scored.existingRefVocabulary;
+    };
+    const feedbackState = readDistillFeedback({ readEventsImpl, options, durableInputRef });
+    const feedback = feedbackState.filteredEvents.slice(-20).map((event) => ({
+        ts: event.ts,
+        eventType: event.eventType,
+        ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
+    }));
+    return {
+        stash,
+        chat,
+        collectNotices,
+        distillRunner,
+        withNotices,
+        lookup,
+        outcomeWeightEnabled,
+        fetchSimilarLessonsFn,
+        assetState,
+        persistInputSalience,
+        feedback,
+        ...feedbackState,
+    };
+}
 export async function akmDistill(options) {
     const inputRef = options.ref.trim();
     if (!inputRef) {
@@ -601,74 +687,30 @@ export async function akmDistill(options) {
     const refused = refuseDisallowedDistillInput({ options, parsedInputRef, inputRef, durableInputRef, eligMeta });
     if (refused)
         return refused;
-    const stash = resolveRunStashDir(options.stashDir);
-    const chat = options.chat ?? chatCompletion;
-    const distillLlm = Object.hasOwn(options, "llmConfig")
-        ? (options.llmConfig ?? undefined)
-        : (() => {
-            const runner = resolveImproveProcessRunner(options.improveProfile, "distill", config);
-            return runner ? materializeLlmRunnerConnection(runner) : getDefaultLlmConfig(config);
-        })();
-    const lookup = options.lookupFn ?? ((ref) => defaultLookup(ref, stash));
-    const readEventsImpl = options.readEventsFn ?? readEvents;
-    // R1 opt-out must flow into every computeSalience call this command makes so
-    // distill-written rank_score rows use the same weights as preparation's.
-    const outcomeWeightEnabled = config.improve?.salience?.outcomeWeightEnabled !== false;
-    // D-4 / #390: similar-lessons retrieval seam (test-injectable).
-    const fetchSimilarLessonsFn = options.fetchSimilarLessonsFn ?? ((query, n) => fetchTopSimilarLessons(query, n, options.stashDir));
-    const assetCtx = createRunContext({
-        stashDir: stash,
+    const prepared = await prepareDistillExecution({
+        options,
         config,
-        eventsCtx: options.eventsCtx ?? {},
-        proposalsCtx: options.ctx ?? {},
-        chat,
-        getLlmConfig: () => distillLlm ?? null,
-        sourceRun: options.sourceRun ?? `distill-${Date.now()}`,
-        dryRun: false,
-        signal: options.signal,
-    }).withFreshAssetMemo();
-    const { assetContent, existingRefVocabulary } = await loadAndScoreInputSalience({
         inputRef,
         durableInputRef,
         salienceWriteKey,
-        stash,
-        config,
-        outcomeWeightEnabled,
-        lookup,
-        ctx: assetCtx,
-    });
-    const { filteredEvents, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered } = readDistillFeedback({
-        readEventsImpl,
-        options,
-        durableInputRef,
     });
-    const feedback = filteredEvents.slice(-20).map((e) => ({
-        ts: e.ts,
-        eventType: e.eventType,
-        ...(e.metadata !== undefined ? { metadata: e.metadata } : {}),
-    }));
-    // Memory→knowledge promotion branch (D-1/#369). When the target ref is a
-    // reinforced memory, distill graduates it into a knowledge proposal instead
-    // of a lesson — the whole branch (LLM contradiction-merge, quality gate,
-    // proposal creation, event emit) lives in `promoteMemoryToKnowledge` and is
-    // terminal when it fires. A `null` return means "not a promotion candidate";
-    // fall through to the ordinary lesson/knowledge distillation path.
-    const promotionResult = await promoteMemoryToKnowledge({
+    const { stash, chat, collectNotices, distillRunner, withNotices, lookup, outcomeWeightEnabled, fetchSimilarLessonsFn, assetState, persistInputSalience, feedback, filteredEvents, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, } = prepared;
+    const promotionContext = {
         targetKind,
         inputRef,
         durableInputRef,
         ...(options.itemRef ? { itemRef: options.itemRef } : {}),
-        assetContent,
+        assetContent: assetState.assetContent,
         filteredEvents,
         config,
         strategy: options.improveProfile,
-        llmConfig: distillLlm,
+        llmRunner: distillRunner,
         signal: options.signal,
         chat,
         stash,
         lookup,
         fetchSimilarLessonsFn,
-        existingRefVocabulary,
+        existingRefVocabulary: assetState.existingRefVocabulary,
         outcomeWeightEnabled,
         eligMeta,
         eligibilitySource: options.eligibilitySource,
@@ -678,92 +720,120 @@ export async function akmDistill(options) {
         exclusionSetSize: exclusionSet.size,
         filteredFeedbackCount,
         feedbackFullyFiltered,
-    });
-    if (promotionResult)
-        return promotionResult;
-    const effectiveProposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
-    const effectiveLessonRef = effectiveProposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
-    const messages = await buildDistillMessages({
-        options,
-        stash,
-        inputRef,
-        assetContent,
-        feedback,
-        effectiveProposalKind,
-        effectiveLessonRef,
-        fetchSimilarLessonsFn,
-    });
-    const { raw, fallbackReason } = await runDistillLlmCall({
-        config,
-        options,
-        distillLlm,
-        messages,
-        effectiveProposalKind,
-    });
-    if (raw === null || raw.trim() === "") {
-        return distillEmptyResponseResult({
-            fallbackReason,
+        onNotices: collectNotices,
+    };
+    const promotionPlan = await planMemoryKnowledgePromotion(promotionContext);
+    let dispatchLease;
+    try {
+        // Memory→knowledge promotion branch (D-1/#369). When the target ref is a
+        // reinforced memory, distill graduates it into a knowledge proposal instead
+        // of a lesson — the whole branch (LLM contradiction-merge, quality gate,
+        // proposal creation, event emit) lives in `promoteMemoryToKnowledge` and is
+        // terminal when it fires. A `null` return means "not a promotion candidate";
+        // fall through to the ordinary lesson/knowledge distillation path.
+        if (promotionPlan && distillRunner && memoryKnowledgePromotionRequiresDispatch(promotionContext, promotionPlan)) {
+            dispatchLease = await preflightStructuredLlmRunner(distillRunner);
+        }
+        const promotionResult = await promoteMemoryToKnowledge({ ...promotionContext, ...(dispatchLease ? { lease: dispatchLease } : {}) }, promotionPlan);
+        if (promotionResult) {
+            await persistInputSalience();
+            return withNotices(promotionResult);
+        }
+        const effectiveProposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
+        const effectiveLessonRef = effectiveProposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
+        const messages = await buildDistillMessages({
+            options,
+            stash,
+            inputRef,
+            assetContent: assetState.assetContent,
+            feedback,
+            effectiveProposalKind,
+            effectiveLessonRef,
+            fetchSimilarLessonsFn,
+        });
+        if (!dispatchLease && distillRunner)
+            dispatchLease = await preflightStructuredLlmRunner(distillRunner);
+        const { raw, fallbackReason } = await runDistillLlmCall({
+            config,
+            options,
+            distillRunner,
+            lease: dispatchLease,
+            messages,
+            effectiveProposalKind,
+            onNotices: collectNotices,
+        });
+        await persistInputSalience();
+        if (raw === null || raw.trim() === "") {
+            return withNotices(distillEmptyResponseResult({
+                fallbackReason,
+                inputRef,
+                durableInputRef,
+                ...(options.itemRef ? { itemRef: options.itemRef } : {}),
+                effectiveLessonRef,
+                effectiveProposalKind,
+                exclusionSet,
+                filteredFeedbackCount,
+                feedbackFullyFiltered,
+                eligMeta,
+                eventsCtx: options.eventsCtx,
+            }));
+        }
+        const { content, descriptionSwapped } = assembleAndValidateDistillContent({
+            raw,
+            effectiveProposalKind,
             inputRef,
             durableInputRef,
             ...(options.itemRef ? { itemRef: options.itemRef } : {}),
             effectiveLessonRef,
-            effectiveProposalKind,
             exclusionSet,
             filteredFeedbackCount,
-            feedbackFullyFiltered,
             eligMeta,
             eventsCtx: options.eventsCtx,
         });
+        const gate = await applyDistillQualityGate({
+            config,
+            options,
+            content,
+            assetContent: assetState.assetContent,
+            chat,
+            distillRunner,
+            lease: dispatchLease,
+            fetchSimilarLessonsFn,
+            stash,
+            inputRef,
+            effectiveLessonRef,
+            exclusionSet,
+            filteredFeedbackCount,
+            feedbackFullyFiltered,
+            onNotices: collectNotices,
+        });
+        if ("rejection" in gate)
+            return withNotices(gate.rejection);
+        const lessonJudgeConfidence = gate.confidence;
+        return withNotices(await emitDistillLessonProposal({
+            content,
+            options,
+            distillRunner,
+            assetContent: assetState.assetContent,
+            inputRef,
+            durableInputRef,
+            effectiveLessonRef,
+            effectiveProposalKind,
+            stash,
+            exclusionSet,
+            filteredFeedbackCount,
+            feedbackFullyFiltered,
+            lessonJudgeConfidence,
+            existingRefVocabulary: assetState.existingRefVocabulary,
+            outcomeWeightEnabled,
+            descriptionSwapped,
+            eligMeta,
+        }));
+    }
+    finally {
+        if (dispatchLease)
+            disposeLoweredExecutionDispatchLease(dispatchLease);
     }
-    const { content, descriptionSwapped } = assembleAndValidateDistillContent({
-        raw,
-        effectiveProposalKind,
-        inputRef,
-        durableInputRef,
-        ...(options.itemRef ? { itemRef: options.itemRef } : {}),
-        effectiveLessonRef,
-        exclusionSet,
-        filteredFeedbackCount,
-        eligMeta,
-        eventsCtx: options.eventsCtx,
-    });
-    const gate = await applyDistillQualityGate({
-        config,
-        options,
-        content,
-        assetContent,
-        chat,
-        distillLlm,
-        fetchSimilarLessonsFn,
-        stash,
-        inputRef,
-        effectiveLessonRef,
-        exclusionSet,
-        filteredFeedbackCount,
-        feedbackFullyFiltered,
-    });
-    if ("rejection" in gate)
-        return gate.rejection;
-    const lessonJudgeConfidence = gate.confidence;
-    return emitDistillLessonProposal({
-        content,
-        options,
-        distillLlm,
-        assetContent,
-        inputRef,
-        durableInputRef,
-        effectiveLessonRef,
-        effectiveProposalKind,
-        stash,
-        exclusionSet,
-        filteredFeedbackCount,
-        feedbackFullyFiltered,
-        lessonJudgeConfidence,
-        existingRefVocabulary,
-        outcomeWeightEnabled,
-        descriptionSwapped,
-        eligMeta,
-    });
 }
 // ── Helpers ─────────────────────────────────────────────────────────────────
 /**
@@ -774,7 +844,7 @@ export async function akmDistill(options) {
  * event is byte-identical.
  */
 async function emitDistillLessonProposal(args) {
-    const { options, distillLlm, assetContent, inputRef, durableInputRef, effectiveLessonRef, effectiveProposalKind, stash, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, lessonJudgeConfidence, existingRefVocabulary, outcomeWeightEnabled, descriptionSwapped, eligMeta, } = args;
+    const { options, distillRunner, assetContent, inputRef, durableInputRef, effectiveLessonRef, effectiveProposalKind, stash, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, lessonJudgeConfidence, existingRefVocabulary, outcomeWeightEnabled, descriptionSwapped, eligMeta, } = args;
     let content = args.content;
     // WS-3b: Distill→source fidelity check (step 10).
     // When fidelityCheck.enabled, check the distill proposal against its cited
@@ -820,7 +890,7 @@ async function emitDistillLessonProposal(args) {
         // (profile/config fallback included), not the raw option — a standalone
         // `akm distill` run must fingerprint under the model that actually
         // generated the content, matching the promote-memory branch.
-        ...(distillLlm?.model ? { modelId: distillLlm.model } : {}),
+        ...(distillRunner?.connection.model ? { modelId: distillRunner.connection.model } : {}),
         source: "distill",
         ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
         payload: {
@@ -973,7 +1043,7 @@ function assembleAndValidateDistillContent(args) {
  * the fallback reason.
  */
 async function runDistillLlmCall(args) {
-    const { config, options, distillLlm, messages, effectiveProposalKind } = args;
+    const { config, options, distillRunner, lease, messages, effectiveProposalKind, onNotices } = args;
     const distillSchema = effectiveProposalKind === "knowledge" ? DISTILL_KNOWLEDGE_JSON_SCHEMA : DISTILL_LESSON_JSON_SCHEMA;
     let fallbackReason;
     const enabled = resolveProcessEnabled("distill", options.improveProfile ?? resolveImproveStrategy(undefined, config).config);
@@ -983,7 +1053,7 @@ async function runDistillLlmCall(args) {
         // emitting the distill_invoked event so we don't double-emit here.
         warnVerbose(`[akm] LLM fallback for ${feature}: ${reason}`);
     };
-    if (enabled && !distillLlm) {
+    if (enabled && !distillRunner) {
         // No LLM connection configured. At HEAD this threw a ConfigError inside
         // the gated fn and tryLlmFeature routed it through the "error" fallback;
         // reproduce that terminal state directly (the gate-disabled case above
@@ -996,16 +1066,20 @@ async function runDistillLlmCall(args) {
         feature: "distill",
         akmConfig: config,
         enabled,
-        // Safe: when the gate is open, distillLlm is defined (guard above); when
-        // it is closed, the transport never runs and config is never read.
-        config: distillLlm,
+        // Safe: when the gate is open, distillRunner is defined (guard above); when
+        // it is closed, the transport never runs and the runner is never read.
+        runner: distillRunner,
+        ...(lease ? { lease } : {}),
         messages,
         request: options.chat === undefined
             ? // Production path: pass the JSON schema so providers that honour
                 // `response_format: json_schema` enforce shape upstream. Providers
                 // that ignore the option fall through to the prompt-contract
                 // markdown path.
-                { responseSchema: distillSchema }
+                {
+                    responseSchema: distillSchema,
+                    ...(options.signal ? { signal: options.signal } : {}),
+                }
             : // Test seam: keep the injected fake as the transport; fakes never
                 // see the schema (they return markdown strings).
                 { chat: options.chat, ...(options.signal ? { signal: options.signal } : {}) },
@@ -1019,6 +1093,7 @@ async function runDistillLlmCall(args) {
         },
         fallback: null,
         onFallback: (evt) => recordFallback(evt.feature, evt.reason),
+        onNotices,
     });
     return { raw, fallbackReason };
 }
@@ -1080,7 +1155,7 @@ function distillEmptyResponseResult(args) {
  * from `akmDistill`.
  */
 async function applyDistillQualityGate(args) {
-    const { config, options, content, assetContent, chat, distillLlm, fetchSimilarLessonsFn, stash, inputRef, effectiveLessonRef, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, } = args;
+    const { config, options, content, assetContent, chat, distillRunner, lease, fetchSimilarLessonsFn, stash, inputRef, effectiveLessonRef, exclusionSet, filteredFeedbackCount, feedbackFullyFiltered, onNotices, } = args;
     if (!(options.improveProfile?.processes?.distill?.qualityGate?.enabled ?? true)) {
         return { confidence: undefined };
     }
@@ -1088,8 +1163,10 @@ async function applyDistillQualityGate(args) {
     const similarLessons = await fetchSimilarLessonsFn(content.slice(0, 500), 3);
     const judgeResult = await runLessonQualityJudge(config, content, assetContent ?? "", chat, {
         ...(similarLessons.length > 0 ? { similarLessons } : {}),
-        ...(distillLlm ? { llmConfig: distillLlm } : {}),
+        ...(distillRunner ? { llmRunner: distillRunner } : {}),
+        ...(lease ? { lease } : {}),
         ...(options.signal ? { signal: options.signal } : {}),
+        onNotices,
     });
     if (!judgeResult.pass) {
         if (judgeResult.reviewNeeded) {
@@ -1147,7 +1224,7 @@ async function buildDistillMessages(args) {
     const { options, stash, inputRef, assetContent, feedback, effectiveProposalKind, effectiveLessonRef, fetchSimilarLessonsFn, } = args;
     // Inject last 1–3 rejected proposals for this ref as Reflexion-style
     // verbal-RL context so the LLM avoids regenerating refused proposals.
-    const rejectedForRef = listProposals(stash, { ref: inputRef, status: "rejected", includeArchive: true })
+    const rejectedForRef = listProposalsReadOnly(stash, { ref: inputRef, status: "rejected", includeArchive: true }, options.ctx)
         .sort((a, b) => new Date(b.updatedAt ?? 0).getTime() - new Date(a.updatedAt ?? 0).getTime())
         .slice(0, MAX_REJECTED_PROPOSALS)
         .map((p) => ({