akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/commands/improve/proactive-maintenance.js

@@ -1,32 +1,13 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { computeSalience } from "./salience.js";
 /** One day in milliseconds. */
 const DAY_MS = 86_400_000;
-/**
- * Importance multipliers by asset type. Higher = more worth maintaining. These
- * are the design defaults; callers may override any subset via config.
- */
-export const DEFAULT_IMPORTANCE_WEIGHTS = Object.freeze({
-    skill: 1.5,
-    agent: 1.5,
-    command: 1.3,
-    workflow: 1.3,
-    lesson: 1.2,
-    knowledge: 1.0,
-    script: 0.9,
-    memory: 0.7,
-});
 /** Default staleness gate: an asset is due when last reflected > this many days ago (or never). */
 export const DEFAULT_DUE_DAYS = 30;
 /** Default bound on how many assets the selector surfaces per run. */
 export const DEFAULT_MAX_PER_RUN = 25;
-/**
- * Half-life (days) for the recency-of-use decay term. An asset used today
- * contributes a full recency multiplier; one unused for one half-life
- * contributes half. Mirrors the validated prototype (21 days).
- */
-const RECENCY_HALFLIFE_DAYS = 21;
 /** Lower bound on size used in the cost denominator so tiny files don't divide by ~0. */
 const SIZE_FLOOR_BYTES = 200;
 /** Parse the bare asset type out of a `type:name` ref. Returns "" when unparseable. */
@@ -37,10 +18,9 @@ function refType(ref) {
 /**
  * Score and select due assets for proactive maintenance.
  *
- * Priority formula (mirrors the validated prototype):
- *
- *   priority = (importance × log(1 + retrievalFreq) × (0.1 + 0.5^(useAgeDays/21)))
- *              / log10(max(size, 200))
+ * Priority: delegates to `computeSalience(...).rankScore` (WS-1 unified salience
+ * vector). The ranking key is therefore identical to every other selector that
+ * uses salience — a single formula governs attention across the whole improve loop.
  *
  * DUE gate: an asset is eligible only if it was never reflected OR last
  * reflected/distilled more than `dueDays` ago. The same gate doubles as the
@@ -52,7 +32,6 @@ export function selectProactiveMaintenanceRefs(params) {
     const now = params.now ?? Date.now();
     const dueDays = params.dueDays ?? DEFAULT_DUE_DAYS;
     const maxPerRun = params.maxPerRun ?? DEFAULT_MAX_PER_RUN;
-    const weights = { ...DEFAULT_IMPORTANCE_WEIGHTS, ...(params.importanceWeights ?? {}) };
     const scored = [];
     for (const candidate of params.candidates) {
         const ref = candidate.ref;
@@ -70,28 +49,23 @@ export function selectProactiveMaintenanceRefs(params) {
         const staleDays = neverReflected ? Number.POSITIVE_INFINITY : (now - lastTouchMs) / DAY_MS;
         // DUE / rotation gate.
         const due = neverReflected || staleDays > dueDays;
-        // Retrieval frequency + recency decay.
+        // Retrieval frequency (for salience inputs).
         const retrievalFreq = params.retrievalCounts.get(ref) ?? 0;
         const lastUse = params.lastUseMs?.get(ref) ?? 0;
-        const useAgeDays = lastUse > 0 ? (now - lastUse) / DAY_MS : 9999;
-        const recencyDecay = 0.1 + 0.5 ** (useAgeDays / RECENCY_HALFLIFE_DAYS);
-        // Size proxy (cost): larger assets are slightly deprioritized, but only by
-        // log10 so a big-but-hot asset is never starved.
+        // Size proxy (cost): kept for salience input — computeSalience applies
+        // the log10 denominator internally.
         let sizeBytes = params.sizeBytesOf?.(candidate) ?? 0;
         if (!sizeBytes || sizeBytes < 0)
             sizeBytes = SIZE_FLOOR_BYTES;
-        const sizeProxy = Math.max(SIZE_FLOOR_BYTES, sizeBytes);
-        const importance = weights[type] ?? 1.0;
-        const priority = (importance * Math.log(1 + retrievalFreq) * recencyDecay) / Math.log10(sizeProxy);
+        // Unified priority via WS-1 salience vector (replaces the old inline formula).
+        const priority = computeSalience({ ref, type, retrievalFreq, lastUseMs: lastUse, sizeBytes, now }).rankScore;
         scored.push({
             ref: candidate,
             type,
             staleDays,
             neverReflected,
             retrievalFreq,
-            recencyDecay,
             sizeBytes,
-            importance,
             priority,
             due,
         });
@@ -111,3 +85,31 @@ export function selectProactiveMaintenanceRefs(params) {
     const selected = ranked.slice(0, Math.max(0, maxPerRun)).map((s) => s.ref);
     return { selected, dueTotal, neverReflected, scored };
 }
+/**
+ * Post-lock re-filter for proactive refs.
+ *
+ * Called INSIDE the reflect-distill.lock window with freshly-read timestamp
+ * maps so that any `reflect_invoked` events committed by a concurrent run
+ * while this run was waiting for the lock are visible. Refs that became
+ * non-due (staleDays ≤ dueDays) since planning time are dropped before
+ * execution, closing the SELECT-time cooldown race.
+ *
+ * The logic mirrors the DUE gate in `selectProactiveMaintenanceRefs` — an
+ * asset is due only when never touched OR last touched more than `dueDays`
+ * ago. The selector and this filter must always agree on the gate predicate.
+ */
+export function filterProactiveDue(selected, lastReflectTs, lastDistillTs, dueDays, now) {
+    return selected.filter((candidate) => {
+        const ref = candidate.ref;
+        const reflectIso = lastReflectTs.get(ref);
+        const distillIso = lastDistillTs.get(ref);
+        let lastTouchMs = 0;
+        if (reflectIso)
+            lastTouchMs = Math.max(lastTouchMs, Date.parse(reflectIso) || 0);
+        if (distillIso)
+            lastTouchMs = Math.max(lastTouchMs, Date.parse(distillIso) || 0);
+        const neverReflected = lastTouchMs === 0;
+        const staleDays = neverReflected ? Number.POSITIVE_INFINITY : (now - lastTouchMs) / DAY_MS;
+        return neverReflected || staleDays > dueDays;
+    });
+}

package/dist/commands/improve/procedural.js

@@ -0,0 +1,398 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * #615 — procedural-compilation pass.
+ *
+ * An OPT-IN post-loop improve stage (default disabled via
+ * `IMPROVE_PROCESS_DEFAULTS.procedural`). It reads assets that carry an
+ * `orderedActions` frontmatter list (captured by #619), detects RECURRING
+ * successful action sequences across sessions (the SAME normalized ordered step
+ * list appearing >= `minRecurrence` times with a non-failure `outcomeData`), and
+ * emits ONE normal `type: workflow` proposal per recurring sequence through the
+ * existing proposal queue + quality gate.
+ *
+ * The ordered step list — NOT the LLM — is the source of truth. The bounded LLM
+ * call only NAMES the workflow and per-step titles/instructions; the parser
+ * rejects any output whose step count / order drifts from the deterministic
+ * sequence, and the assembled workflow markdown is re-parsed locally before it
+ * is ever queued. A justified null (the LLM determines the sequence is not a
+ * coherent procedure) is an acceptable outcome and produces no proposal.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import proceduralSystemPrompt from "../../assets/prompts/procedural-system.md" with { type: "text" };
+import { parseFrontmatter } from "../../core/asset/frontmatter.js";
+import { resolveStashDir } from "../../core/common.js";
+import { loadConfig } from "../../core/config/config.js";
+import { appendEvent } from "../../core/events.js";
+import { parseEmbeddedJsonResponse } from "../../core/parse.js";
+import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
+import { closeDatabase, getAllEntries, openExistingDatabase } from "../../indexer/db/db.js";
+import { parseWorkflow } from "../../workflows/parser.js";
+import { createProposal, isProposalSkipped } from "../proposal/repository.js";
+import { validateProposalFrontmatter } from "../proposal/validators/proposal-quality-validators.js";
+import { resolveImproveLlmFn } from "./shared.js";
+const PROCEDURAL_SYSTEM_PROMPT = proceduralSystemPrompt;
+const DEFAULT_MIN_RECURRENCE = 3;
+const DEFAULT_MAX_PROPOSALS_PER_RUN = 3;
+/** Failure-signal heuristic: an outcome matching this is NOT counted as success. */
+const FAILURE_SIGNAL = /\b(fail|failed|failure|error|errored|abort|aborted|rollback|reverted)\b/i;
+// ── Normalization + recurrence model (deterministic, no LLM) ────────────────────
+/** Normalize a single action string: trim, lowercase, collapse whitespace, strip trailing punctuation. */
+function normalizeStep(s) {
+    return s
+        .trim()
+        .toLowerCase()
+        .replace(/\s+/g, " ")
+        .replace(/[.;,:]+$/, "")
+        .trim();
+}
+/**
+ * Normalize an ordered action list: each step token-normalized (case /
+ * whitespace / trailing-punctuation insensitive), with empties dropped. The
+ * result is ORDER-SENSITIVE — reordered sequences normalize distinctly.
+ */
+export function normalizeSequence(actions) {
+    return actions.map(normalizeStep).filter((s) => s.length > 0);
+}
+/** A member counts toward recurrence only when its outcome is present, non-empty, and not a failure signal. */
+function isSuccessfulOutcome(outcome) {
+    if (!outcome)
+        return false;
+    const trimmed = outcome.trim();
+    if (!trimmed)
+        return false;
+    return !FAILURE_SIGNAL.test(trimmed);
+}
+/** Recover the ordered-action sequence + outcome from an asset's frontmatter (NOT a DB column). */
+function readOrderedSequence(entry) {
+    let data;
+    try {
+        const raw = fs.readFileSync(entry.filePath, "utf8");
+        data = parseFrontmatter(raw).data;
+    }
+    catch {
+        return null;
+    }
+    const rawActions = data.orderedActions;
+    if (!Array.isArray(rawActions))
+        return null;
+    const actions = rawActions.filter((a) => typeof a === "string");
+    if (actions.length === 0)
+        return null;
+    const outcome = typeof data.outcomeData === "string" ? data.outcomeData : undefined;
+    const ref = `${entry.entry.type}:${entry.entry.name}`;
+    return { actions, ...(outcome !== undefined ? { outcome } : {}), ref, entryKey: entry.entryKey };
+}
+/**
+ * Group entries by their normalized ordered sequence. Only successful members
+ * (non-failure outcome) count toward recurrence. Keep groups whose successful
+ * member count >= `minRecurrence`, sort deterministically (member-count desc
+ * then groupKey asc), and cap to `maxProposalsPerRun`. Exported for unit tests.
+ */
+export function buildSequenceClusters(entries, opts) {
+    const groups = new Map();
+    for (const entry of entries) {
+        const seq = readOrderedSequence(entry);
+        if (!seq)
+            continue;
+        const normalized = normalizeSequence(seq.actions);
+        if (normalized.length === 0)
+            continue;
+        if (!isSuccessfulOutcome(seq.outcome))
+            continue;
+        const groupKey = JSON.stringify(normalized);
+        const existing = groups.get(groupKey);
+        const member = { ref: seq.ref, entryKey: seq.entryKey, outcome: seq.outcome ?? "" };
+        if (existing) {
+            if (!existing.members.some((m) => m.entryKey === member.entryKey))
+                existing.members.push(member);
+        }
+        else {
+            groups.set(groupKey, { groupKey, normalized, members: [member] });
+        }
+    }
+    const clusters = [...groups.values()].filter((c) => c.members.length >= opts.minRecurrence);
+    clusters.sort((a, b) => b.members.length - a.members.length || a.groupKey.localeCompare(b.groupKey));
+    return clusters.slice(0, Math.max(0, opts.maxProposalsPerRun));
+}
+/**
+ * Stable workflow ref for a sequence cluster. The hash of the sorted member keys
+ * + the normalized sequence keeps the ref deterministic across runs, so
+ * re-detection maps to the same ref and the content-hash dedup in createProposal
+ * suppresses queue churn.
+ */
+export function deriveProceduralWorkflowRef(cluster) {
+    const slug = cluster.normalized
+        .slice(0, 3)
+        .join("-")
+        .toLowerCase()
+        .replace(/[^a-z0-9-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "");
+    const memberKey = cluster.members
+        .map((m) => m.entryKey)
+        .sort()
+        .join("|");
+    const hash = createHash("sha256")
+        .update(`${memberKey} ${JSON.stringify(cluster.normalized)}`, "utf8")
+        .digest("hex")
+        .slice(0, 8);
+    return `workflow:compiled/${slug || "sequence"}-${hash}`;
+}
+// ── Prompt + parse ──────────────────────────────────────────────────────────────
+/** Assemble the per-sequence user prompt fed to the procedural LLM. */
+export function buildProceduralPrompt(cluster, standardsContext = "") {
+    const lines = [
+        `A recurring successful action sequence observed across ${cluster.members.length} sessions.`,
+        "",
+    ];
+    if (standardsContext.trim()) {
+        lines.push("Standards to follow (the rulebook for this target):");
+        lines.push(standardsContext.trim());
+        lines.push("");
+    }
+    lines.push("Ordered actions (turn EACH into exactly one step, in this order):");
+    cluster.normalized.forEach((step, i) => {
+        lines.push(`${i + 1}. ${step}`);
+    });
+    lines.push("", "Sample successful outcomes:");
+    for (const m of cluster.members.slice(0, 5)) {
+        if (m.outcome)
+            lines.push(`- ${m.outcome}`);
+    }
+    lines.push("", `Return strict JSON with EXACTLY ${cluster.normalized.length} steps in the same order, or an explicit null.`);
+    return lines.join("\n");
+}
+/** Parse the raw LLM output into a workflow, or `null` for the justified-null path. */
+function parseProceduralWorkflow(raw, expectedSteps) {
+    if (raw === null)
+        return null;
+    const trimmed = raw.trim();
+    if (!trimmed || trimmed.toLowerCase() === "null")
+        return null;
+    const parsed = parseEmbeddedJsonResponse(trimmed);
+    if (parsed === undefined || parsed === null || typeof parsed !== "object")
+        return null;
+    const obj = parsed;
+    const title = typeof obj.title === "string" ? obj.title.trim() : "";
+    const description = typeof obj.description === "string" ? obj.description : "";
+    if (!title && !description)
+        return null;
+    if (!Array.isArray(obj.steps) || obj.steps.length !== expectedSteps)
+        return null;
+    const steps = [];
+    for (const rawStep of obj.steps) {
+        if (typeof rawStep !== "object" || rawStep === null)
+            return null;
+        const s = rawStep;
+        const stepTitle = typeof s.title === "string" ? s.title.trim() : "";
+        const instructions = typeof s.instructions === "string" ? s.instructions.trim() : "";
+        if (!stepTitle || !instructions)
+            return null;
+        const completionCriteria = Array.isArray(s.completionCriteria)
+            ? s.completionCriteria.filter((c) => typeof c === "string" && c.trim().length > 0)
+            : undefined;
+        steps.push({
+            title: stepTitle,
+            instructions,
+            ...(completionCriteria && completionCriteria.length > 0 ? { completionCriteria } : {}),
+        });
+    }
+    return { title: title || "Compiled Workflow", description, steps };
+}
+// ── Workflow markdown assembly ────────────────────────────────────────────────
+/** Convert a step title into a kebab-case step id. */
+function kebab(s) {
+    return (s
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "") || "step");
+}
+/** Build the exact workflow markdown the parser accepts. */
+export function assembleWorkflowMarkdown(doc) {
+    const lines = [
+        "---",
+        `description: ${JSON.stringify(doc.description)}`,
+        "---",
+        "",
+        `# Workflow: ${doc.title}`,
+        "",
+    ];
+    const usedIds = new Set();
+    doc.steps.forEach((step, idx) => {
+        let id = kebab(step.title);
+        if (usedIds.has(id))
+            id = `${id}-${idx + 1}`;
+        usedIds.add(id);
+        lines.push(`## Step: ${step.title}`, `Step ID: ${id}`, "", "### Instructions", step.instructions, "");
+        if (step.completionCriteria && step.completionCriteria.length > 0) {
+            lines.push("### Completion Criteria");
+            for (const c of step.completionCriteria)
+                lines.push(`- ${c}`);
+            lines.push("");
+        }
+    });
+    return lines.join("\n");
+}
+// ── Production LLM seam ───────────────────────────────────────────────────────
+// ── Main entry point ───────────────────────────────────────────────────────────
+export async function akmProcedural(opts) {
+    const startMs = Date.now();
+    const config = opts.config ?? loadConfig();
+    const stashDir = opts.stashDir ?? resolveStashDir();
+    const sourceRun = opts.sourceRun ?? `procedural-${startMs}`;
+    const eligibilitySource = opts.eligibilitySource ?? "procedural";
+    const minRecurrence = opts.minRecurrence ?? DEFAULT_MIN_RECURRENCE;
+    const maxProposalsPerRun = opts.maxProposalsPerRun ?? DEFAULT_MAX_PROPOSALS_PER_RUN;
+    const warnings = [];
+    const finish = (over) => ({
+        schemaVersion: 1,
+        ok: true,
+        sequencesScanned: 0,
+        clustersFormed: 0,
+        proposalsEmitted: 0,
+        nullsReturned: 0,
+        durationMs: Date.now() - startMs,
+        warnings,
+        ...over,
+    });
+    // Budget guard: an already-aborted signal short-circuits before any LLM call.
+    if (opts.signal?.aborted) {
+        return finish({ ok: false, warnings: [...warnings, "aborted-before-start"] });
+    }
+    // Load all entries from the index (orderedActions can ride any asset type).
+    let entries = [];
+    let db;
+    try {
+        db = openExistingDatabase();
+        entries = getAllEntries(db);
+    }
+    catch (e) {
+        warnings.push(`procedural: failed to open index — ${String(e)}`);
+        return finish({ ok: false });
+    }
+    finally {
+        if (db)
+            closeDatabase(db);
+    }
+    const clusters = buildSequenceClusters(entries, { minRecurrence, maxProposalsPerRun });
+    const sequencesScanned = entries.length;
+    let clustersFormed = 0;
+    let proposalsEmitted = 0;
+    let nullsReturned = 0;
+    if (clusters.length === 0) {
+        return finish({ sequencesScanned, clustersFormed: 0 });
+    }
+    const llmFn = opts.proceduralLlmFn ??
+        resolveImproveLlmFn(config, {
+            processKey: "procedural",
+            systemPrompt: PROCEDURAL_SYSTEM_PROMPT,
+            tag: "[procedural]",
+            signal: opts.signal,
+            activeProfile: opts.improveProfile,
+        });
+    if (!llmFn) {
+        warnings.push("procedural: no LLM configured — skipping");
+        return finish({ sequencesScanned, clustersFormed: 0 });
+    }
+    // Procedural output is a workflow (non-wiki) → stash authoring standards.
+    // Resolved ONCE per run and passed to each sequence prompt.
+    const standardsContext = resolveStashStandards(stashDir);
+    for (const cluster of clusters) {
+        if (opts.signal?.aborted) {
+            warnings.push("aborted-mid-run");
+            break;
+        }
+        clustersFormed += 1;
+        const workflowRef = deriveProceduralWorkflowRef(cluster);
+        const prompt = buildProceduralPrompt(cluster, standardsContext);
+        const raw = await llmFn(prompt);
+        const doc = parseProceduralWorkflow(raw, cluster.normalized.length);
+        if (!doc) {
+            nullsReturned += 1;
+            appendEvent({
+                eventType: "procedural_compiled",
+                ref: workflowRef,
+                metadata: {
+                    groupKey: cluster.groupKey,
+                    memberCount: cluster.members.length,
+                    outcome: "null_returned",
+                    sourceRun,
+                },
+            }, opts.ctx);
+            continue;
+        }
+        // Quality gate (always-run, never bypassed): the description must be present
+        // and non-truncated. Runs BEFORE createProposal.
+        const fmCheck = validateProposalFrontmatter({ description: doc.description });
+        if (!fmCheck.ok) {
+            appendEvent({
+                eventType: "procedural_compiled",
+                ref: workflowRef,
+                metadata: {
+                    groupKey: cluster.groupKey,
+                    memberCount: cluster.members.length,
+                    outcome: "quality_rejected",
+                    reason: fmCheck.reason,
+                    sourceRun,
+                },
+            }, opts.ctx);
+            continue;
+        }
+        // Assemble + locally validate the workflow markdown. Never queue an
+        // unparseable workflow.
+        const content = assembleWorkflowMarkdown(doc);
+        const parsed = parseWorkflow(content, { path: workflowRef });
+        if (!parsed.ok) {
+            appendEvent({
+                eventType: "procedural_compiled",
+                ref: workflowRef,
+                metadata: {
+                    groupKey: cluster.groupKey,
+                    memberCount: cluster.members.length,
+                    outcome: "invalid_workflow",
+                    reason: parsed.errors[0]?.message,
+                    sourceRun,
+                },
+            }, opts.ctx);
+            continue;
+        }
+        const proposalResult = createProposal(stashDir, {
+            ref: workflowRef,
+            source: "procedural",
+            sourceRun,
+            payload: { content, frontmatter: { description: doc.description } },
+            eligibilitySource,
+        }, opts.ctx);
+        if (isProposalSkipped(proposalResult)) {
+            appendEvent({
+                eventType: "procedural_compiled",
+                ref: workflowRef,
+                metadata: {
+                    groupKey: cluster.groupKey,
+                    memberCount: cluster.members.length,
+                    outcome: "skipped",
+                    skipReason: proposalResult.reason,
+                    sourceRun,
+                },
+            }, opts.ctx);
+            continue;
+        }
+        proposalsEmitted += 1;
+        appendEvent({
+            eventType: "procedural_compiled",
+            ref: workflowRef,
+            metadata: {
+                groupKey: cluster.groupKey,
+                memberCount: cluster.members.length,
+                outcome: "queued",
+                proposalId: proposalResult.id,
+                sourceRun,
+            },
+        }, opts.ctx);
+    }
+    return finish({ sequencesScanned, clustersFormed, proposalsEmitted, nullsReturned });
+}