akm-cli 0.9.16 → 0.9.17-alpha.10

package/dist/commands/improve/consolidate/merge.js

@@ -1,149 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-export function isValidOp(op) {
-    if (typeof op !== "object" || op === null)
-        return false;
-    const o = op;
-    if (o.op === "merge") {
-        return typeof o.primary === "string" && Array.isArray(o.secondaries);
-    }
-    if (o.op === "delete") {
-        return typeof o.ref === "string";
-    }
-    if (o.op === "promote") {
-        return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
-    }
-    if (o.op === "contradict") {
-        return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
-    }
-    return false;
-}
-export function mergePlans(chunks, knownRefs) {
-    const mergeOps = new Map();
-    const deleteOps = new Map();
-    const promoteOps = new Map();
-    // C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
-    const contradictOps = new Map();
-    const warnings = [];
-    for (const chunk of chunks) {
-        for (const op of chunk) {
-            if (op.op === "merge") {
-                // Drop ops whose primary the LLM hallucinated (not in the loaded memory
-                // pool). Without this guard, a hallucinated primary flows all the way to
-                // Phase B where !memoryByRef.has(primary) fires and charges every real
-                // secondary with merge_primary_missing — masking LLM hallucinations as
-                // filter regressions in health metrics.
-                if (knownRefs && !knownRefs.has(op.primary)) {
-                    warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
-                    // Use a dedicated skip reason so dashboards can distinguish
-                    // hallucinated primaries from stale-DB regressions.
-                    // Secondaries are real refs; they are NOT charged here — they remain
-                    // available for other ops to claim.
-                    continue;
-                }
-                // Filter hallucinated secondaries while preserving real ones.
-                let mergeOp = op;
-                if (knownRefs) {
-                    const filteredSecondaries = op.secondaries.filter((sec) => {
-                        if (!knownRefs.has(sec)) {
-                            warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
-                            return false;
-                        }
-                        return true;
-                    });
-                    if (filteredSecondaries.length !== op.secondaries.length) {
-                        mergeOp = { ...op, secondaries: filteredSecondaries };
-                    }
-                }
-                if (mergeOp.secondaries.length > 1) {
-                    warnings.push(`Merge: ${mergeOp.primary} proposed multiple secondaries; retaining one secondary for review.`);
-                    mergeOp = { ...mergeOp, secondaries: mergeOp.secondaries.slice(0, 1) };
-                }
-                // merge wins over delete
-                if (deleteOps.has(mergeOp.primary)) {
-                    deleteOps.delete(mergeOp.primary);
-                }
-                for (const sec of mergeOp.secondaries) {
-                    if (deleteOps.has(sec))
-                        deleteOps.delete(sec);
-                }
-                mergeOps.set(mergeOp.primary, mergeOp);
-            }
-            else if (op.op === "delete") {
-                // merge and promote both win over delete. A promote is non-destructive
-                // (creates a proposal) but the source memory is counted in `promoted`;
-                // if a delete also fires, the ref lands in both `promoted` and
-                // `skipReasons`, breaking the invariant by +1.
-                if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
-                    deleteOps.set(op.ref, op);
-                }
-            }
-            else if (op.op === "promote") {
-                // C-2 / #381: when both a promote and a merge target the same ref,
-                // queue the promote FIRST rather than discarding it. The promote op
-                // routes through createProposal (the human-gated proposal queue), so
-                // it is non-destructive. The merge follows after the proposal is
-                // created. This preserves the human reviewer's ability to inspect the
-                // promotion before the source memory is merged/deleted.
-                // AGM K*8 — retain the maximally informative consistent subset.
-                promoteOps.set(op.ref, op);
-            }
-            else if (op.op === "contradict") {
-                // Deduplicate by ref+contradictedByRef pair.
-                const key = `${op.ref}|${op.contradictedByRef}`;
-                if (!contradictOps.has(key)) {
-                    contradictOps.set(key, op);
-                }
-            }
-        }
-    }
-    // Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
-    //
-    // 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
-    //    for secondaries that were already in deleteOps, but misses the case
-    //    where the delete chunk came first. A full sweep here fixes both orders.
-    //
-    // 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
-    //    only the first (insertion-order) retains it. Without this, a successful
-    //    merge credits A to mergedSecondaries and a later merge's emitMerge-
-    //    FailureSkips also charges A to skipReasons — double-counting A while
-    //    processed has it only once.
-    //
-    // 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
-    //    a secondary in another, remove A from the secondary list. Both merges
-    //    would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
-    //    breaking the invariant the same way.
-    // Also remove delete ops for any ref claimed by a promote op (handles the
-    // case where the delete chunk appeared before the promote chunk).
-    for (const ref of promoteOps.keys()) {
-        deleteOps.delete(ref);
-    }
-    const claimedSecondaries = new Set();
-    for (const mergeOp of mergeOps.values()) {
-        deleteOps.delete(mergeOp.primary);
-        mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
-            if (mergeOps.has(sec)) {
-                warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
-                return false;
-            }
-            if (claimedSecondaries.has(sec)) {
-                warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
-                return false;
-            }
-            claimedSecondaries.add(sec);
-            deleteOps.delete(sec);
-            return true;
-        });
-    }
-    // C-2 / #381: promote ops are ordered BEFORE merge ops so that the
-    // human-gated proposal queue entry is created before any destructive merge.
-    // Phase B processes ops in array order, so promote executes first.
-    const ops = [
-        ...promoteOps.values(),
-        ...mergeOps.values(),
-        ...deleteOps.values(),
-        ...contradictOps.values(),
-    ];
-    return { ops, warnings };
-}

package/dist/commands/improve/distill/promote-memory.js

@@ -1,291 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-/**
- * Memory→knowledge promotion branch for `akm distill`.
- *
- * This is an entire second command that used to be inlined inside `akmDistill`:
- * when a `memory:*` ref is reinforced enough (per the deterministic stability
- * heuristic in `distill-promotion-policy`), distill graduates it into a
- * `knowledge:*` proposal instead of a lesson. The branch owns its own LLM
- * contradiction-merge (mem0 ADD/UPDATE/NOOP), quality gate, proposal creation,
- * and `distill_invoked` event emit.
- *
- * {@link promoteMemoryToKnowledge} returns the finished {@link AkmDistillResult}
- * when the branch fired, or `null` when the ref is not a promotion candidate —
- * in which case the caller falls through to the ordinary lesson/knowledge LLM
- * path. Logic is byte-identical to the pre-extraction inline code.
- */
-import fs from "node:fs";
-import { parseFrontmatter } from "../../../core/asset/frontmatter.js";
-import { ConfigError } from "../../../core/errors.js";
-import { appendEvent } from "../../../core/events.js";
-import { parseEmbeddedJsonResponse } from "../../../core/parse.js";
-import { callStructured } from "../../../llm/structured-call.js";
-import { isProposalSkipped } from "../../proposal/repository.js";
-import { assessMemoryKnowledgePromotionCandidate } from "../distill-promotion-policy.js";
-import { emitProposal } from "../proposal-envelope.js";
-import { durableImproveRef } from "../source-identity.js";
-import { persistOutputEncodingSalience, runLessonQualityJudge, writeQualityRejection } from "./quality-gate.js";
-/** Read-only classification of the deterministic promotion path and destination conflict. */
-export async function planMemoryKnowledgePromotion(ctx) {
-    const durableInputRef = ctx.durableInputRef ?? ctx.inputRef;
-    const promotion = ctx.targetKind === "lesson"
-        ? null
-        : assessMemoryKnowledgePromotionCandidate({
-            inputRef: durableInputRef,
-            assetContent: ctx.assetContent,
-            feedbackEvents: ctx.filteredEvents.map((event) => ({
-                ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
-            })),
-        });
-    if (!(promotion?.promote && promotion.content && (ctx.targetKind === "knowledge" || ctx.targetKind === "auto"))) {
-        return null;
-    }
-    const existingKnowledgePath = await ctx.lookup(durableImproveRef(promotion.knowledgeRef));
-    let existingKnowledgeContent = null;
-    if (existingKnowledgePath && fs.existsSync(existingKnowledgePath)) {
-        try {
-            existingKnowledgeContent = fs.readFileSync(existingKnowledgePath, "utf8");
-        }
-        catch {
-            existingKnowledgeContent = null;
-        }
-    }
-    return { promotion: promotion, existingKnowledgeContent };
-}
-/** Whether a classified promotion will actually call merge generation and/or the judge. */
-export function memoryKnowledgePromotionRequiresDispatch(ctx, plan) {
-    const hasRunner = Boolean(ctx.llmRunner);
-    return ((Boolean(plan.existingKnowledgeContent) && hasRunner) ||
-        (ctx.strategy?.processes?.distill?.qualityGate?.enabled ?? true));
-}
-/**
- * Resolve the knowledge content to propose when a memory promotes to knowledge,
- * reconciling it with any existing knowledge file at the destination (D-1 / #369).
- *
- * When the destination already exists and an LLM is configured, follows the
- * mem0 ADD/UPDATE/NOOP pattern (arXiv:2504.19413 §3.2): ADD/UPDATE swap in the
- * merged content, NOOP short-circuits with a terminal "skipped" result. Without
- * an LLM the existing content is appended as reviewer-reference context so the
- * merge can be done by hand. Logic is byte-identical to the prior inline block.
- */
-async function resolveKnowledgePromotionContent(ctx, plan) {
-    const durableInputRef = ctx.durableInputRef ?? ctx.inputRef;
-    const baseContent = plan.promotion.content;
-    const knowledgeRef = plan.promotion.knowledgeRef;
-    let resolvedPromotionContent = baseContent;
-    const existingKnowledgeContent = plan.existingKnowledgeContent;
-    const runner = ctx.llmRunner;
-    if (existingKnowledgeContent && runner) {
-        // Existing content found: call LLM for contradiction-resolution merge.
-        const mergePrompt = [
-            "You are merging two versions of a knowledge document.",
-            "Existing content is already committed; new content comes from a memory distillation run.",
-            "Choose one of: ADD (combine both), UPDATE (replace existing with new), NOOP (keep existing unchanged).",
-            'Return ONLY valid JSON: {"action": "ADD"|"UPDATE"|"NOOP", "content": "<merged markdown if ADD/UPDATE, empty string if NOOP>"}',
-            "",
-            "## Existing knowledge content",
-            "```",
-            existingKnowledgeContent.slice(0, 3000),
-            "```",
-            "",
-            "## New content from distillation",
-            "```",
-            baseContent.slice(0, 3000),
-            "```",
-        ].join("\n");
-        try {
-            const mergeResponse = await callStructured({
-                feature: "distill",
-                runner,
-                ...(ctx.lease ? { lease: ctx.lease } : {}),
-                messages: [
-                    { role: "system", content: "Return only valid JSON. No prose." },
-                    { role: "user", content: mergePrompt },
-                ],
-                request: {
-                    ...(ctx.signal ? { signal: ctx.signal } : {}),
-                    ...(ctx.chat ? { chat: ctx.chat } : {}),
-                },
-                parse: (raw) => raw ?? "",
-                onError: () => "",
-                fallback: "",
-                ...(ctx.onNotices ? { onNotices: ctx.onNotices } : {}),
-            });
-            const mergeResult = parseEmbeddedJsonResponse(mergeResponse);
-            if (mergeResult?.action === "NOOP") {
-                // Existing content is authoritative — no update needed.
-                appendEvent({
-                    eventType: "distill_invoked",
-                    // Use item_ref when resolved, otherwise the input conceptId.
-                    ref: ctx.itemRef ?? durableInputRef,
-                    metadata: {
-                        outcome: "skipped",
-                        proposalRef: knowledgeRef,
-                        message: "D-1: LLM resolved destination conflict as NOOP — existing content kept",
-                        ...ctx.eligMeta,
-                    },
-                }, ctx.eventsCtx);
-                return {
-                    earlyResult: {
-                        schemaVersion: 1,
-                        ok: true,
-                        outcome: "skipped",
-                        inputRef: ctx.inputRef,
-                        proposalRef: knowledgeRef,
-                        skipReason: "conflict_noop",
-                        message: "Existing knowledge content unchanged (contradiction resolution: NOOP)",
-                    },
-                };
-            }
-            if (mergeResult?.action && (mergeResult.action === "ADD" || mergeResult.action === "UPDATE")) {
-                if (mergeResult.content?.trim()) {
-                    resolvedPromotionContent = mergeResult.content;
-                }
-            }
-        }
-        catch (error) {
-            if (error instanceof ConfigError)
-                throw error;
-            // LLM merge failed — fall through with the original promotion content.
-            // The reviewer will see both versions in the proposal diff.
-        }
-    }
-    else if (existingKnowledgeContent) {
-        // No LLM configured: include existing content as context in the proposal
-        // so the reviewer can do the contradiction resolution manually.
-        resolvedPromotionContent = [
-            baseContent,
-            "",
-            "---",
-            "<!-- D-1 / #369: Existing knowledge content is shown below for reviewer reference. -->",
-            "<!-- Review: decide whether to ADD (merge), UPDATE (replace), or NOOP (keep existing). -->",
-            "",
-            "## Existing content (for reviewer reference)",
-            "",
-            existingKnowledgeContent,
-        ].join("\n");
-    }
-    return { resolvedContent: resolvedPromotionContent };
-}
-/**
- * Run the memory→knowledge promotion branch. Returns the finished distill
- * result when promotion fired (all paths terminal), or `null` when the ref is
- * not a promotion candidate and the caller should continue to the ordinary
- * lesson/knowledge distillation path.
- */
-export async function promoteMemoryToKnowledge(ctx, planned) {
-    const { inputRef, assetContent, config, chat, stash, fetchSimilarLessonsFn, existingRefVocabulary, outcomeWeightEnabled, eligMeta, exclusionSetSize, filteredFeedbackCount, feedbackFullyFiltered, } = ctx;
-    const durableInputRef = ctx.durableInputRef ?? inputRef;
-    const promotionPlan = planned === undefined ? await planMemoryKnowledgePromotion(ctx) : planned;
-    if (!promotionPlan)
-        return null;
-    const promotion = promotionPlan.promotion;
-    // D-1 / #369: When the destination knowledge file already exists, route
-    // through the LLM for contradiction resolution instead of silently
-    // overwriting. Follows mem0 ADD/UPDATE/DELETE/NOOP pattern (arXiv:2504.19413 §3.2)
-    // and A-MEM dynamic linking (arXiv:2502.12110).
-    const merged = await resolveKnowledgePromotionContent(ctx, promotionPlan);
-    if ("earlyResult" in merged)
-        return merged.earlyResult;
-    const resolvedPromotionContent = merged.resolvedContent;
-    // Apply quality gate to fast-path knowledge promotion (Risk 4 fix).
-    // D-5 / #388: Three-band system — review_needed band queues to proposal
-    // queue with review_needed outcome rather than auto-rejecting.
-    let knowledgeJudgeConfidence;
-    if (ctx.strategy?.processes?.distill?.qualityGate?.enabled ?? true) {
-        // D-4 / #390: retrieve top-3 similar lessons for dedup check in judge.
-        const similarLessons = await fetchSimilarLessonsFn(resolvedPromotionContent.slice(0, 500), 3);
-        const judgeResult = await runLessonQualityJudge(config, resolvedPromotionContent, assetContent ?? "", chat, {
-            ...(similarLessons.length > 0 ? { similarLessons } : {}),
-            ...(ctx.llmRunner ? { llmRunner: ctx.llmRunner } : {}),
-            ...(ctx.lease ? { lease: ctx.lease } : {}),
-            ...(ctx.signal ? { signal: ctx.signal } : {}),
-            ...(ctx.onNotices ? { onNotices: ctx.onNotices } : {}),
-        });
-        if (!judgeResult.pass) {
-            if (judgeResult.reviewNeeded) {
-                // Uncertainty band (2.5–3.5): queue as review_needed instead of rejecting.
-                return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, { reviewNeeded: true }, ctx.eligibilitySource, ctx.eventsCtx);
-            }
-            return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, {}, ctx.eligibilitySource, ctx.eventsCtx);
-        }
-        // Normalize 1-5 judge score to [0, 1]. Only a real passing verdict reaches
-        // here (07 P0-2: the judge now fails CLOSED on no-LLM / timeout / parse
-        // failure, so those return pass:false and early-return above). The score>0
-        // guard defensively leaves confidence undefined for any non-positive score.
-        if (judgeResult.score > 0)
-            knowledgeJudgeConfidence = judgeResult.score / 5;
-    }
-    const knowledgeParsed = parseFrontmatter(resolvedPromotionContent);
-    const proposalResult = emitProposal({ stashDir: stash, proposalsCtx: ctx.proposalsCtx }, {
-        ref: promotion.knowledgeRef,
-        source: "distill",
-        // §23.6 fingerprint model-id term (WI-6.4).
-        ...(ctx.llmRunner?.connection.model ? { modelId: ctx.llmRunner.connection.model } : {}),
-        ...(ctx.sourceRun !== undefined ? { sourceRun: ctx.sourceRun } : {}),
-        payload: {
-            content: resolvedPromotionContent,
-            ...(Object.keys(knowledgeParsed.data).length > 0 ? { frontmatter: knowledgeParsed.data } : {}),
-        },
-        ...(knowledgeJudgeConfidence !== undefined ? { confidence: knowledgeJudgeConfidence } : {}),
-        // Attribution tagging: persist the eligibility lane on the proposal.
-        ...(ctx.eligibilitySource ? { eligibilitySource: ctx.eligibilitySource } : {}),
-    });
-    if (isProposalSkipped(proposalResult)) {
-        appendEvent({
-            eventType: "distill_invoked",
-            // Use item_ref when resolved, otherwise the input conceptId.
-            ref: ctx.itemRef ?? durableInputRef,
-            metadata: {
-                outcome: "skipped",
-                proposalRef: promotion.knowledgeRef,
-                message: proposalResult.message,
-                skipReason: proposalResult.reason,
-                ...eligMeta,
-            },
-        }, ctx.eventsCtx);
-        return {
-            schemaVersion: 1,
-            ok: true,
-            outcome: "skipped",
-            inputRef,
-            proposalRef: promotion.knowledgeRef,
-            skipReason: proposalResult.reason,
-            message: proposalResult.message,
-        };
-    }
-    const proposal = proposalResult;
-    // G4: content-score the distilled OUTPUT so it carries a real encoding
-    // salience (encoding_source='content') from creation.
-    persistOutputEncodingSalience(durableImproveRef(promotion.knowledgeRef), resolvedPromotionContent, existingRefVocabulary, outcomeWeightEnabled);
-    appendEvent({
-        eventType: "distill_invoked",
-        // Use item_ref when resolved, otherwise the input conceptId.
-        ref: ctx.itemRef ?? durableInputRef,
-        metadata: {
-            outcome: "queued",
-            proposalRef: promotion.knowledgeRef,
-            proposalKind: "knowledge",
-            proposalId: proposal.id,
-            // R3: judge verdicts are longitudinally queryable, not just a one-shot
-            // proposal.confidence write (normalized 1–5 score / 5).
-            ...(knowledgeJudgeConfidence !== undefined ? { judgeConfidence: knowledgeJudgeConfidence } : {}),
-            ...(ctx.sourceRun !== undefined ? { sourceRun: ctx.sourceRun } : {}),
-            ...(exclusionSetSize > 0 ? { filteredFeedbackCount } : {}),
-            ...eligMeta,
-        },
-    }, ctx.eventsCtx);
-    return {
-        schemaVersion: 1,
-        ok: true,
-        outcome: "queued",
-        inputRef,
-        proposalRef: promotion.knowledgeRef,
-        proposalKind: "knowledge",
-        proposalId: proposal.id,
-        proposal,
-        ...(exclusionSetSize > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
-    };
-}