akm-cli 0.9.17-alpha.3 → 0.9.17-alpha.4

package/dist/commands/improve/stage.js

@@ -0,0 +1,323 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { getImproveProcessConfig } from "../../core/config/config.js";
+import { ConfigError } from "../../core/errors.js";
+import { parseEmbeddedJsonResponse } from "../../core/parse.js";
+import { warn } from "../../core/warn.js";
+import { LlmCallError } from "../../llm/client.js";
+import { callStructured } from "../../llm/structured-call.js";
+import { withLlmStage } from "../../llm/usage-telemetry.js";
+import { isProceduralRejection } from "../proposal/proposal-types.js";
+import { createProposal, listProposalsReadOnly, proposalContentHash, recordGateDecision, } from "../proposal/repository.js";
+import { resolveImproveLlmExecution } from "./execution.js";
+/** Normalize an unknown thrown value to a message. */
+export function errMessage(e) {
+    return e instanceof Error ? e.message : String(e);
+}
+/** The lowering notices a stage's dispatches emitted, each once. */
+export function noticeSet(forward) {
+    const byKey = new Map();
+    const add = (notices) => {
+        for (const notice of notices)
+            byKey.set(JSON.stringify(notice), notice);
+        forward?.(notices);
+    };
+    const list = () => Object.freeze([...byKey.values()]);
+    return { add, list, fields: () => (byKey.size > 0 ? { notices: list() } : {}) };
+}
+/**
+ * A stage's LLM runner: the one the improve plan froze for it (an own
+ * `llmRunner` key, `null` meaning "none"), else the process engine cascade.
+ */
+export function stageRunner(frozen, config, profile, processName, onNotices) {
+    if (Object.hasOwn(frozen, "llmRunner"))
+        return frozen.llmRunner ?? undefined;
+    const resolved = resolveImproveLlmExecution({
+        config,
+        profile,
+        process: getImproveProcessConfig(processName, profile),
+        processName,
+    });
+    if (resolved)
+        onNotices?.(resolved.notices);
+    return resolved?.runner;
+}
+/**
+ * One model call. Provider trouble (transport error, timeout, a disabled
+ * feature) comes back as `{ ok: false }`; only a configuration failure throws.
+ */
+export async function callStage(call) {
+    const messages = [
+        ...(call.system ? [{ role: "system", content: call.system }] : []),
+        ...(call.history ?? []),
+        { role: "user", content: call.prompt },
+    ];
+    let failure;
+    try {
+        const raw = await callStructured({
+            feature: call.feature,
+            ...(call.gate
+                ? { akmConfig: call.gate.config, ...(call.gate.enabled !== undefined ? { enabled: call.gate.enabled } : {}) }
+                : {}),
+            runner: call.runner,
+            messages,
+            ...(call.request ? { request: call.request } : {}),
+            ...(call.onNotices ? { onNotices: call.onNotices } : {}),
+            parse: (r) => r ?? "",
+            onError: (_cls, err) => {
+                failure = { ok: false, reason: "error", error: errMessage(err) };
+                return undefined;
+            },
+            fallback: undefined,
+            onFallback: (event) => {
+                failure ??= { ok: false, reason: event.reason, ...(event.error ? { error: event.error.message } : {}) };
+            },
+        });
+        return raw === undefined ? (failure ?? { ok: false, reason: "error" }) : { ok: true, raw };
+    }
+    catch (err) {
+        if (err instanceof ConfigError)
+            throw err;
+        const timedOut = err instanceof LlmCallError && err.code === "timeout";
+        return { ok: false, reason: timedOut ? "timeout" : "error", error: errMessage(err) };
+    }
+}
+/** Attribute a stage's LLM calls to its process and planned engine (the usage report). */
+const STAGE_LABELS = {
+    reflect: "reflect",
+    distill: "distill",
+    consolidate: "consolidate",
+    extract: "session-extraction",
+    memoryInference: "memory-inference",
+    graphExtraction: "graph-extraction",
+    validation: "validation",
+};
+export function attributeStage(plan, process, fn) {
+    return withLlmStage(STAGE_LABELS[process], fn, { engine: plan?.processes[process].runner?.engine, process });
+}
+/** How many prior rejected proposals are shown to the model as "don't repeat this". */
+export const MAX_REJECTED_PROPOSALS = 3;
+/**
+ * Reflexion context: the newest reviewer rejections for `ref`. Procedural
+ * refusals (expiry, stale target, missing asset) are not judgements on the
+ * content and are left out. Reads never create state.db.
+ */
+export function rejectedProposalContext(stash, ref, ctx) {
+    if (!ref)
+        return [];
+    return listProposalsReadOnly(stash, { ref, status: "rejected", includeArchive: true }, ctx)
+        .filter((p) => !isProceduralRejection(p))
+        .sort((a, b) => new Date(b.updatedAt ?? 0).getTime() - new Date(a.updatedAt ?? 0).getTime())
+        .slice(0, MAX_REJECTED_PROPOSALS)
+        .map((p) => ({
+        ref: p.ref,
+        reason: p.review?.reason ?? "no reason given",
+        // `payload.content` is populated on every row, including legacy ones.
+        contentPreview: p.payload.content.slice(0, 500),
+    }));
+}
+// ── Mint ─────────────────────────────────────────────────────────────────────
+/**
+ * Create a stage's proposal. `judged` stamps a `staged` gate decision with the
+ * judged content's hash (the triage drain accepts it while the content still
+ * matches); `review` leaves it `deferred` for a human (`review_needed` in the
+ * improve ledger).
+ */
+export function mintProposal(stash, proposalsCtx, input, verdict = {}) {
+    const proposal = createProposal(stash, input, proposalsCtx);
+    if (verdict.review) {
+        return recordGateDecision(stash, proposal.id, { outcome: "deferred", ...verdict.review }, proposalsCtx) ?? proposal;
+    }
+    return verdict.judged ? stageJudgedProposal(stash, proposal, proposalsCtx) : proposal;
+}
+/**
+ * Stamp a proposal the quality judge passed. Best-effort: a failed stamp only
+ * means the triage drain judges it again.
+ */
+export function stageJudgedProposal(stash, proposal, proposalsCtx) {
+    try {
+        return (recordGateDecision(stash, proposal.id, {
+            outcome: "staged",
+            reason: "quality-judge",
+            gate: "quality-gate",
+            contentHash: proposalContentHash(proposal),
+        }, proposalsCtx) ?? proposal);
+    }
+    catch (error) {
+        warn(`[akm] failed to record the quality-judge pass for ${proposal.id}: ${errMessage(error)}`);
+        return proposal;
+    }
+}
+/** Lesson judge prompt; similar existing lessons let it mark near-duplicates down. */
+export function buildJudgePrompt(lessonContent, sourceContent, similarLessons) {
+    const lines = [
+        "You are evaluating a proposed lesson asset for an akm knowledge base.",
+        "",
+        "Score this lesson on each criterion from 1 (poor) to 5 (excellent):",
+        "1. NOVELTY: Does the lesson add information not already present in the source asset?",
+        "2. NON-REDUNDANCY: Is this lesson meaningfully different from what the source already says?",
+        "",
+        "Source asset content:",
+        "```",
+        sourceContent.slice(0, 2000),
+        "```",
+    ];
+    if (similarLessons && similarLessons.length > 0) {
+        lines.push("", "Existing similar lessons (top-3 by similarity). Rate lower if the proposed lesson is substantially similar to any of these:");
+        for (const sl of similarLessons)
+            lines.push(`\nExisting lesson ref: ${sl.ref}`, "```", sl.content.slice(0, 500), "```");
+    }
+    lines.push("", "Proposed lesson content:", "```", lessonContent.slice(0, 1000), "```", "", 'Return ONLY valid JSON, no prose: {"scores": {"novelty": <1-5 integer>, "nonRedundancy": <1-5 integer>}, "reason": "<one sentence>"}');
+    return lines.join("\n");
+}
+function boundedDocument(content, maxChars = 6000) {
+    if (content.length <= maxChars)
+        return content;
+    const half = Math.floor((maxChars - 80) / 2);
+    return `${content.slice(0, half)}\n\n[... middle omitted for bounded judge context ...]\n\n${content.slice(-half)}`;
+}
+function buildChangedRegion(sourceContent, candidateContent) {
+    const source = sourceContent.split("\n");
+    const candidate = candidateContent.split("\n");
+    let prefix = 0;
+    while (prefix < source.length && prefix < candidate.length && source[prefix] === candidate[prefix])
+        prefix++;
+    let suffix = 0;
+    while (suffix < source.length - prefix &&
+        suffix < candidate.length - prefix &&
+        source[source.length - 1 - suffix] === candidate[candidate.length - 1 - suffix]) {
+        suffix++;
+    }
+    const removed = source.slice(prefix, source.length - suffix).join("\n");
+    const added = candidate.slice(prefix, candidate.length - suffix).join("\n");
+    return boundedDocument(`Removed or replaced:\n${removed || "(none)"}\n\nAdded or replacement:\n${added || "(none)"}`);
+}
+/** Judge prompt for an in-place revision (overlap with the source is expected). */
+export function buildReflectJudgePrompt(candidateContent, sourceContent, feedback) {
+    return [
+        "You are evaluating a proposed revision to an existing akm asset.",
+        "",
+        "Score this revision on each criterion from 1 (poor) to 5 (excellent):",
+        "1. FEEDBACK ALIGNMENT: Does the revision address the supplied feedback or improve retrieval and clarity?",
+        "2. PRESERVATION: Does it retain the source's concrete facts, code, commands, examples, and structure without truncation?",
+        "3. QUALITY: Is the revision coherent, actionable, complete, and free of unsupported claims?",
+        "",
+        "Overlap with the source is expected and must not lower the score by itself; this is an in-place revision, not a new lesson.",
+        "",
+        "Feedback:",
+        "```",
+        (feedback.length > 0 ? feedback.join("\n") : "No explicit feedback supplied.").slice(0, 1000),
+        "```",
+        "",
+        "Source asset content:",
+        "```",
+        boundedDocument(sourceContent),
+        "```",
+        "",
+        "Proposed revision:",
+        "```",
+        boundedDocument(candidateContent),
+        "```",
+        "",
+        "Changed region:",
+        "```",
+        buildChangedRegion(sourceContent, candidateContent),
+        "```",
+        "",
+        'Return ONLY valid JSON, no prose: {"scores": {"feedbackAlignment": <1-5 integer>, "preservation": <1-5 integer>, "quality": <1-5 integer>}, "reason": "<one sentence>"}',
+    ].join("\n");
+}
+const LESSON_JUDGE_CRITERIA = ["novelty", "nonRedundancy"];
+const REFLECT_JUDGE_CRITERIA = ["feedbackAlignment", "preservation", "quality"];
+/**
+ * Read a judge response: the per-criterion shape (averaged here) or the older
+ * `{"score"}` shape. Only the expected criteria are read; any missing or
+ * out-of-range (1..5) value is a parse failure, extra keys are ignored.
+ */
+function parseJudgeResponse(raw, keys) {
+    const parsed = parseEmbeddedJsonResponse(raw);
+    if (!parsed || typeof parsed.reason !== "string")
+        return undefined;
+    const reason = parsed.reason;
+    const inRange = (value) => typeof value === "number" && Number.isFinite(value) && value >= 1 && value <= 5;
+    if (parsed.scores !== undefined) {
+        if (typeof parsed.scores !== "object" || parsed.scores === null || Array.isArray(parsed.scores))
+            return undefined;
+        const scores = parsed.scores;
+        const criteria = {};
+        for (const key of keys) {
+            const value = scores[key];
+            if (!inRange(value))
+                return undefined;
+            criteria[key] = value;
+        }
+        return { score: Object.values(criteria).reduce((a, b) => a + b, 0) / keys.length, reason, criteria };
+    }
+    return inRange(parsed.score) ? { score: parsed.score, reason } : undefined;
+}
+function judgeResponseSchema(keys) {
+    return {
+        type: "object",
+        required: ["scores", "reason"],
+        additionalProperties: false,
+        properties: {
+            scores: {
+                type: "object",
+                required: [...keys],
+                additionalProperties: false,
+                properties: Object.fromEntries(keys.map((key) => [key, { type: "integer", minimum: 1, maximum: 5 }])),
+            },
+            reason: { type: "string" },
+        },
+    };
+}
+/**
+ * The quality judge. Fails closed: no runner, an unparseable verdict or a
+ * provider failure never passes content. Bands: >= 3.5 pass, 2.5-3.5 review,
+ * < 2.5 reject. Temperature is pinned to 0 so verdicts do not flip.
+ */
+async function runQualityJudge(feature, config, prompt, keys, chat, options) {
+    const resolved = !options.runnerSelectionFrozen && !options.llmRunner
+        ? resolveImproveLlmExecution({ config, processName: `${feature}-judge` })
+        : null;
+    if (resolved)
+        options.onNotices?.(resolved.notices);
+    const runner = options.llmRunner ?? resolved?.runner;
+    if (!runner)
+        return { pass: false, score: -1, reason: "no LLM configured — cannot judge, failing closed" };
+    const outcome = await callStage({
+        feature,
+        runner,
+        system: "Return only valid JSON. No prose.",
+        prompt,
+        request: {
+            enableThinking: false,
+            temperature: 0,
+            responseSchema: judgeResponseSchema(keys),
+            ...(Object.hasOwn(options, "timeoutMs") ? { timeoutMs: options.timeoutMs } : {}),
+            ...(options.signal ? { signal: options.signal } : {}),
+            ...(chat ? { chat } : {}),
+        },
+        ...(options.onNotices ? { onNotices: options.onNotices } : {}),
+    });
+    if (!outcome.ok) {
+        return { pass: false, score: -1, reason: "judge timeout/error — routed to review", reviewNeeded: true };
+    }
+    const parsed = parseJudgeResponse(outcome.raw, keys);
+    if (!parsed)
+        return { pass: false, score: -1, reason: "judge parse failed — routed to review", reviewNeeded: true };
+    const { score, reason, criteria } = parsed;
+    const verdict = score >= 3.5 ? { pass: true } : score >= 2.5 ? { pass: false, reviewNeeded: true } : { pass: false };
+    return { ...verdict, score, reason, ...(criteria ? { criteria } : {}) };
+}
+/** Judge a proposed lesson (or knowledge promotion) against its source. */
+export function runLessonQualityJudge(config, lessonContent, sourceContent, chat, options = {}) {
+    const prompt = buildJudgePrompt(lessonContent, sourceContent, options.similarLessons);
+    return runQualityJudge("lesson_quality_gate", config, prompt, LESSON_JUDGE_CRITERIA, chat, options);
+}
+/** Judge an in-place reflect revision without new-lesson novelty criteria. */
+export function runReflectQualityJudge(config, candidateContent, sourceContent, feedback, chat, options = {}) {
+    const prompt = buildReflectJudgePrompt(candidateContent, sourceContent, feedback);
+    return runQualityJudge("proposal_quality_gate", config, prompt, REFLECT_JUDGE_CRITERIA, chat, options);
+}