akm-cli 0.7.5 → 0.8.0-rc.3

package/dist/commands/distill.js

@@ -46,17 +46,21 @@
  * be invoked from CI / automation without spinning up an agent harness.
  */
 import fs from "node:fs";
+import path from "node:path";
 import { parseAssetRef } from "../core/asset-ref";
-import { resolveStashDir } from "../core/common";
+import { resolveStashDir, timestampForFilename } from "../core/common";
 import { loadConfig } from "../core/config";
 import { ConfigError, UsageError } from "../core/errors";
 import { appendEvent, readEvents } from "../core/events";
 import { parseFrontmatter } from "../core/frontmatter";
 import { lintLessonContent } from "../core/lesson-lint";
+import { stripMarkdownFences } from "../core/markdown";
 import { createProposal } from "../core/proposals";
-import { lookup as indexerLookup } from "../indexer/indexer";
-import { chatCompletion } from "../llm/client";
-import { tryLlmFeature } from "../llm/feature-gate";
+import { warnVerbose } from "../core/warn";
+import { resolveAssetPath } from "../indexer/path-resolver";
+import { chatCompletion, parseEmbeddedJsonResponse } from "../llm/client";
+import { isLlmFeatureEnabled, tryLlmFeature } from "../llm/feature-gate";
+import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./distill-promotion-policy";
 // ── Lesson-ref derivation ───────────────────────────────────────────────────
 /** Derive the proposed lesson ref from the input ref. See module docblock. */
 export function deriveLessonRef(inputRef) {
@@ -75,7 +79,7 @@ export function deriveLessonRef(inputRef) {
     return `lesson:${safe}-lesson`;
 }
 // ── Prompt assembly ─────────────────────────────────────────────────────────
-const SYSTEM_PROMPT = [
+const LESSON_SYSTEM_PROMPT = [
     "You are the akm `distill` distiller.",
     "Given an asset and recent feedback events about it, produce a single",
     "concise *lesson* an agent should remember next time it works on this",
@@ -92,6 +96,28 @@ const SYSTEM_PROMPT = [
     "Both `description` and `when_to_use` MUST be non-empty single-line strings.",
     "Output ONLY the lesson file contents — no prose, no fences, no preamble.",
 ].join("\n");
+const KNOWLEDGE_SYSTEM_PROMPT = [
+    "You are the akm `distill` distiller.",
+    "Given an asset and recent feedback events about it, produce a concise",
+    "*knowledge* markdown document capturing the durable, reusable facts.",
+    "Prefer stable guidance over narrative recap.",
+    "If you include YAML frontmatter, keep it compatible with normal knowledge",
+    "assets (for example `description`, `tags`, `sources`, `observed_at`).",
+    "Include a meaningful markdown body.",
+    "Output ONLY the knowledge file contents — no prose, no fences, no preamble.",
+].join("\n");
+function validateKnowledgeContent(content, inputRef) {
+    const parsed = parseFrontmatter(content);
+    if (parsed.content.trim().length > 0)
+        return [];
+    return [
+        {
+            kind: "missing-body",
+            field: "body",
+            message: `Distilled knowledge for ${inputRef} must include a non-empty markdown body.`,
+        },
+    ];
+}
 /** Pure: build the user-prompt body. Exported for tests. */
 export function buildDistillPrompt(input) {
     const lines = [];
@@ -99,8 +125,9 @@ export function buildDistillPrompt(input) {
     lines.push("");
     lines.push("Asset content:");
     if (input.assetContent) {
+        const body = input.assetContent.trim().slice(0, 3000);
         lines.push("```");
-        lines.push(input.assetContent.trim());
+        lines.push(body);
         lines.push("```");
     }
     else {
@@ -118,9 +145,96 @@ export function buildDistillPrompt(input) {
         }
     }
     lines.push("");
-    lines.push("Produce the lesson markdown file now.");
+    lines.push(`Produce the ${input.proposalKind === "knowledge" ? "knowledge" : "lesson"} markdown file now.`);
     return lines.join("\n");
 }
+// ── LLM-as-judge quality gate (P2-B) ────────────────────────────────────────
+function buildJudgePrompt(lessonContent, sourceContent) {
+    return [
+        "You are evaluating a proposed lesson asset for an akm knowledge base.",
+        "",
+        "Score this lesson on each criterion from 1 (poor) to 5 (excellent):",
+        "1. NOVELTY: Does the lesson add information not already present in the source asset?",
+        "2. ACTIONABILITY: Can an agent follow this lesson without additional context?",
+        "3. NON-REDUNDANCY: Is this lesson meaningfully different from what the source already says?",
+        "",
+        "Source asset content:",
+        "```",
+        sourceContent.slice(0, 2000),
+        "```",
+        "",
+        "Proposed lesson content:",
+        "```",
+        lessonContent.slice(0, 1000),
+        "```",
+        "",
+        'Return ONLY valid JSON, no prose: {"score": <average score 1-5 as float>, "reason": "<one sentence>"}',
+    ].join("\n");
+}
+async function runLessonQualityJudge(config, lessonContent, sourceContent, chat) {
+    if (!config.llm) {
+        return { pass: true, score: -1, reason: "no LLM configured — passing through" };
+    }
+    const judgeLlmConfig = config.llm.judgeModel ? { ...config.llm, model: config.llm.judgeModel } : config.llm;
+    const JUDGE_TIMEOUT_MS = 8_000;
+    try {
+        const raw = await Promise.race([
+            chat(judgeLlmConfig, [
+                { role: "system", content: "Return only valid JSON. No prose." },
+                { role: "user", content: buildJudgePrompt(lessonContent, sourceContent) },
+            ]),
+            new Promise((_, reject) => setTimeout(() => reject(new Error("judge timeout")), JUDGE_TIMEOUT_MS)),
+        ]);
+        const parsed = parseEmbeddedJsonResponse(raw);
+        if (!parsed || typeof parsed.score !== "number") {
+            return { pass: true, score: -1, reason: "judge parse failed — passing through" };
+        }
+        return { pass: parsed.score >= 3, score: parsed.score, reason: parsed.reason ?? "" };
+    }
+    catch {
+        return { pass: true, score: -1, reason: "judge failed — passing through" };
+    }
+}
+// ── Quality-rejection helper ─────────────────────────────────────────────────
+/**
+ * Write a rejected lesson to `.akm/distill-rejected/`, append a `distill_invoked`
+ * quality-rejected event, and return the `quality_rejected` envelope.
+ *
+ * @param stash     - Root stash directory.
+ * @param inputRef  - The original input ref (for the event).
+ * @param lessonRef - The proposed lesson/knowledge ref.
+ * @param content   - The raw content that failed the quality gate.
+ * @param score     - Quality score from the judge.
+ * @param reason    - Human-readable rejection reason.
+ * @param extraMeta - Optional additional metadata for the event.
+ */
+function writeQualityRejection(stash, inputRef, lessonRef, content, score, reason, extraMeta = {}) {
+    const rejectDir = path.join(stash, ".akm", "distill-rejected");
+    fs.mkdirSync(rejectDir, { recursive: true });
+    const ts = timestampForFilename();
+    fs.writeFileSync(path.join(rejectDir, `${ts}-${lessonRef}.md`), `---\nscore: ${score}\nreason: ${reason}\n---\n\n${content}`, "utf8");
+    appendEvent({
+        eventType: "distill_invoked",
+        ref: inputRef,
+        metadata: {
+            outcome: "quality_rejected",
+            lessonRef,
+            score,
+            reason,
+            ...extraMeta,
+        },
+    });
+    return {
+        schemaVersion: 1,
+        ok: true,
+        outcome: "quality_rejected",
+        inputRef,
+        lessonRef,
+        score,
+        reason,
+        ...extraMeta,
+    };
+}
 // ── Main entry point ────────────────────────────────────────────────────────
 /**
  * Run a single bounded distillation pass for `ref`. Always emits exactly one
@@ -134,7 +248,7 @@ export async function akmDistill(options) {
     }
     // Validate the ref shape up front so a typo never reaches the LLM.
     parseAssetRef(inputRef);
-    const lessonRef = deriveLessonRef(inputRef);
+    const targetKind = options.proposalKind ?? "lesson";
     const config = options.config ?? loadConfig();
     const stash = options.stashDir ?? resolveStashDir();
     const chat = options.chat ?? chatCompletion;
@@ -175,9 +289,64 @@ export async function akmDistill(options) {
         eventType: e.eventType,
         ...(e.metadata !== undefined ? { metadata: e.metadata } : {}),
     }));
-    const userPrompt = buildDistillPrompt({ inputRef, assetContent, feedback });
+    const promotion = targetKind === "lesson"
+        ? null
+        : assessMemoryKnowledgePromotionCandidate({
+            inputRef,
+            assetContent,
+            feedbackEvents: filteredEvents.map((event) => ({
+                ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
+            })),
+        });
+    if (promotion?.promote && promotion.content && (targetKind === "knowledge" || targetKind === "auto")) {
+        // Apply quality gate to fast-path knowledge promotion (Risk 4 fix).
+        if (isLlmFeatureEnabled(config, "lesson_quality_gate")) {
+            const judgeResult = await runLessonQualityJudge(config, promotion.content, assetContent ?? "", chat);
+            if (!judgeResult.pass) {
+                return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, promotion.content, judgeResult.score, judgeResult.reason);
+            }
+        }
+        const knowledgeParsed = parseFrontmatter(promotion.content);
+        const proposal = createProposal(stash, {
+            ref: promotion.knowledgeRef,
+            source: "distill",
+            ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
+            payload: {
+                content: promotion.content,
+                ...(Object.keys(knowledgeParsed.data).length > 0 ? { frontmatter: knowledgeParsed.data } : {}),
+            },
+        }, options.ctx);
+        appendEvent({
+            eventType: "distill_invoked",
+            ref: inputRef,
+            metadata: {
+                outcome: "queued",
+                lessonRef: promotion.knowledgeRef,
+                proposalRef: promotion.knowledgeRef,
+                proposalKind: "knowledge",
+                proposalId: proposal.id,
+                ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
+                ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
+            },
+        });
+        return {
+            schemaVersion: 1,
+            ok: true,
+            outcome: "queued",
+            inputRef,
+            lessonRef: promotion.knowledgeRef,
+            proposalRef: promotion.knowledgeRef,
+            proposalKind: "knowledge",
+            proposalId: proposal.id,
+            proposal,
+            ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
+        };
+    }
+    const effectiveProposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
+    const effectiveLessonRef = effectiveProposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
+    const userPrompt = buildDistillPrompt({ inputRef, assetContent, feedback, proposalKind: effectiveProposalKind });
     const messages = [
-        { role: "system", content: SYSTEM_PROMPT },
+        { role: "system", content: effectiveProposalKind === "knowledge" ? KNOWLEDGE_SYSTEM_PROMPT : LESSON_SYSTEM_PROMPT },
         { role: "user", content: userPrompt },
     ];
     // Single bounded LLM call. The wrapper handles the gate-check, 30 s
@@ -190,14 +359,21 @@ export async function akmDistill(options) {
             throw new ConfigError("No LLM connection configured. Set `llm.endpoint` and `llm.model` in the akm config.", "LLM_NOT_CONFIGURED");
         }
         return chat(config.llm, messages);
-    }, null);
+    }, null, {
+        onFallback: (evt) => {
+            // Log the fallback reason; the caller (raw === null path) handles
+            // emitting the distill_invoked event so we don't double-emit here.
+            warnVerbose(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
+        },
+    });
     if (raw === null || raw.trim() === "") {
         appendEvent({
             eventType: "distill_invoked",
             ref: inputRef,
             metadata: {
                 outcome: "skipped",
-                lessonRef,
+                lessonRef: effectiveLessonRef,
+                proposalKind: effectiveProposalKind,
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
             },
         });
@@ -206,7 +382,9 @@ export async function akmDistill(options) {
             ok: true,
             outcome: "skipped",
             inputRef,
-            lessonRef,
+            lessonRef: effectiveLessonRef,
+            proposalRef: effectiveLessonRef,
+            proposalKind: effectiveProposalKind,
             message: "feedback distillation is disabled or the LLM call failed; no proposal created.",
             ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
         };
@@ -217,27 +395,40 @@ export async function akmDistill(options) {
     // canonical gate for required frontmatter (v1 spec §13). On failure we
     // surface a structured error and exit non-zero — but still emit
     // `distill_invoked` so the failure is observable.
-    const lintReport = lintLessonContent(content, `distill:${inputRef}`);
-    if (lintReport.findings.length > 0) {
+    const findings = effectiveProposalKind === "knowledge"
+        ? validateKnowledgeContent(content, inputRef)
+        : lintLessonContent(content, `distill:${inputRef}`).findings;
+    if (findings.length > 0) {
         appendEvent({
             eventType: "distill_invoked",
             ref: inputRef,
             metadata: {
                 outcome: "validation_failed",
-                lessonRef,
-                findingKinds: lintReport.findings.map((f) => f.kind),
+                lessonRef: effectiveLessonRef,
+                proposalKind: effectiveProposalKind,
+                findingKinds: findings.map((f) => f.kind),
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
             },
         });
-        const message = lintReport.findings.map((f) => f.message).join("\n");
-        throw new UsageError(`Distilled lesson failed validation:\n${message}`, "MISSING_REQUIRED_ARGUMENT", "Lessons require non-empty `description` and `when_to_use` frontmatter fields. See v1 spec §13.");
+        const message = findings.map((f) => f.message).join("\n");
+        throw new UsageError(`Distilled ${effectiveProposalKind} failed validation:\n${message}`, "MISSING_REQUIRED_ARGUMENT", effectiveProposalKind === "knowledge"
+            ? "Knowledge proposals require a non-empty markdown body."
+            : "Lessons require non-empty `description` and `when_to_use` frontmatter fields. See v1 spec §13.");
+    }
+    // LLM-as-judge quality gate (P2-B). Only active when the feature flag is
+    // explicitly enabled. Fail-open: judge failures always pass through.
+    if (isLlmFeatureEnabled(config, "lesson_quality_gate")) {
+        const judgeResult = await runLessonQualityJudge(config, content, assetContent ?? "", chat);
+        if (!judgeResult.pass) {
+            return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {});
+        }
     }
     // Round-trip the parsed frontmatter so the proposal carries it as a
     // structured payload alongside the raw content (matches the shape used by
     // other proposal sources).
     const parsed = parseFrontmatter(content);
     const proposal = createProposal(stash, {
-        ref: lessonRef,
+        ref: effectiveLessonRef,
         source: "distill",
         ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
         payload: {
@@ -250,7 +441,9 @@ export async function akmDistill(options) {
         ref: inputRef,
         metadata: {
             outcome: "queued",
-            lessonRef,
+            lessonRef: effectiveLessonRef,
+            proposalRef: effectiveLessonRef,
+            proposalKind: effectiveProposalKind,
             proposalId: proposal.id,
             ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
             ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
@@ -261,7 +454,9 @@ export async function akmDistill(options) {
         ok: true,
         outcome: "queued",
         inputRef,
-        lessonRef,
+        lessonRef: effectiveLessonRef,
+        proposalRef: effectiveLessonRef,
+        proposalKind: effectiveProposalKind,
         proposalId: proposal.id,
         proposal,
         ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
@@ -269,25 +464,5 @@ export async function akmDistill(options) {
 }
 // ── Helpers ─────────────────────────────────────────────────────────────────
 async function defaultLookup(ref) {
-    try {
-        const entry = await indexerLookup(parseAssetRef(ref));
-        return entry?.filePath ?? null;
-    }
-    catch {
-        return null;
-    }
-}
-/** Best-effort fence stripping. Keeps the body intact when no fence is present. */
-function stripMarkdownFences(raw) {
-    // Strip <think>…</think> reasoning blocks first — local LLMs (e.g. Qwen3)
-    // emit these before the content, which breaks YAML frontmatter detection.
-    const stripped = raw
-        .trim()
-        .replace(/<think>[\s\S]*?<\/think>/gi, "")
-        .trim();
-    // Only strip outer triple-fence pairs — leave inner code blocks alone.
-    const fence = stripped.match(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```\s*$/i);
-    if (fence)
-        return fence[1].trim();
-    return stripped;
+    return resolveAssetPath(ref, { mode: "index-only" });
 }

package/dist/commands/eval-cases.js

@@ -0,0 +1,40 @@
+import fs from "node:fs";
+import path from "node:path";
+import { writeFileAtomic } from "../core/common";
+export function writeEvalCase(stashDir, evalCase) {
+    const evalDir = path.join(stashDir, ".akm", "eval-cases");
+    fs.mkdirSync(evalDir, { recursive: true });
+    const fileName = `${evalCase.slug}.md`;
+    const filePath = path.join(evalDir, fileName);
+    const content = `---
+ref: ${evalCase.ref}
+failureReason: ${evalCase.failureReason}
+assetType: ${evalCase.assetType}
+rejectedAt: ${evalCase.rejectedAt}
+source: ${evalCase.source}
+---
+
+# Eval Case: ${evalCase.ref}
+
+**Failure reason:** ${evalCase.failureReason}
+**Source:** ${evalCase.source}
+**Asset type:** ${evalCase.assetType}
+
+This case was automatically captured when a distillation or proposal was rejected.
+Use it as a regression test: future improve runs on this ref should not produce
+output that would be rejected for the same reason.
+`;
+    writeFileAtomic(filePath, content);
+    return filePath;
+}
+export function countEvalCases(stashDir) {
+    const evalDir = path.join(stashDir, ".akm", "eval-cases");
+    if (!fs.existsSync(evalDir))
+        return 0;
+    try {
+        return fs.readdirSync(evalDir).filter((f) => f.endsWith(".md")).length;
+    }
+    catch {
+        return 0;
+    }
+}

package/dist/commands/events.js

@@ -10,6 +10,7 @@
 import { parseAssetRef } from "../core/asset-ref";
 import { UsageError } from "../core/errors";
 import { readEvents, tailEvents } from "../core/events";
+import { parseSinceToIso } from "../core/time";
 /**
  * Parse `--since` accepting either a byte-offset cursor (`@offset:<int>`) for
  * cross-process resumption, or a timestamp / epoch-ms (the existing form).
@@ -30,7 +31,7 @@ function parseSinceFlag(since) {
         }
         return { sinceOffset: value };
     }
-    return { since: normalizeSince(trimmed) };
+    return { since: parseSinceToIso(trimmed) };
 }
 function validateRef(ref) {
     if (ref === undefined)
@@ -42,28 +43,6 @@ function validateRef(ref) {
     parseAssetRef(trimmed);
     return trimmed;
 }
-function normalizeSince(since) {
-    if (since === undefined)
-        return undefined;
-    const trimmed = since.trim();
-    if (!trimmed) {
-        throw new UsageError("--since cannot be empty.", "INVALID_FLAG_VALUE");
-    }
-    // Accept ISO timestamp (preferred), epoch ms, or plain date.
-    if (/^\d+$/.test(trimmed)) {
-        const ms = Number.parseInt(trimmed, 10);
-        const d = new Date(ms);
-        if (Number.isNaN(d.getTime())) {
-            throw new UsageError(`Invalid --since value: ${since}`, "INVALID_FLAG_VALUE");
-        }
-        return d.toISOString();
-    }
-    const parsed = new Date(trimmed);
-    if (Number.isNaN(parsed.getTime())) {
-        throw new UsageError(`Invalid --since value: ${since}. Expected ISO timestamp (e.g. 2026-04-01T00:00:00Z) or epoch ms.`, "INVALID_FLAG_VALUE");
-    }
-    return parsed.toISOString();
-}
 export function akmEventsList(options = {}) {
     const ref = validateRef(options.ref);
     const parsed = parseSinceFlag(options.since);