akm-cli 0.9.0-beta.5 → 0.9.0-beta.50

package/dist/commands/improve/improve-auto-accept.js

@@ -3,8 +3,26 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { loadConfig } from "../../core/config/config.js";
 import { appendEvent } from "../../core/events.js";
+import { getPhaseThreshold, withStateDb } from "../../core/state-db.js";
 import { info, warn } from "../../core/warn.js";
 import { promoteProposal, recordGateDecision } from "../proposal/validators/proposals.js";
+/**
+ * Derive a stable, low-cardinality reason bucket from an auto-accept promotion
+ * error. `promoteProposal` throws a `validateProposal` report formatted as
+ * `[kind] message` lines; we extract the first finding kind. Non-validation
+ * throws collapse to `promote-error`.
+ */
+function classifyPromoteFailure(err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const finding = /\[([a-z][a-z0-9-]*)\]/i.exec(message);
+    if (finding)
+        return `validation:${finding[1]}`;
+    if (/not pending/i.test(message))
+        return "not-pending";
+    if (/unknown asset type/i.test(message))
+        return "unknown-type";
+    return "promote-error";
+}
 // ---------------------------------------------------------------------------
 // Gate implementation
 // ---------------------------------------------------------------------------
@@ -18,13 +36,25 @@ import { promoteProposal, recordGateDecision } from "../proposal/validators/prop
  * @param promoteFn   Injectable override for `promoteProposal` (test seam).
  */
 export async function runAutoAcceptGate(candidates, cfg, promoteFn = promoteProposal) {
-    const result = { promoted: [], skipped: [], failed: [] };
+    const result = { promoted: [], skipped: [], failed: [], failedByReason: {} };
     // --- Guard: gate is disabled or context is incomplete ---
     if (cfg.dryRun || cfg.globalThreshold === undefined || !cfg.stashDir) {
         result.skipped = candidates.map((c) => c.proposalId);
         return result;
     }
-    const effectiveThreshold = Math.max(cfg.globalThreshold, cfg.minimumThreshold ?? 0) / 100;
+    // WS-4: per-phase threshold from state.db overrides the global threshold.
+    // The per-phase value is populated by makeGateConfig when a stateDbPath is
+    // available; callers that don't pass it get the global threshold unchanged.
+    const resolvedThreshold = cfg.phaseThreshold ?? cfg.globalThreshold;
+    const effectiveThreshold = Math.max(resolvedThreshold, cfg.minimumThreshold ?? 0) / 100;
+    // WS-4: Exploration budget — promote at most N candidates regardless of
+    // confidence to prevent the gate converging to pure exploitation.
+    // Exploration candidates are chosen from the LOWEST-confidence eligible set
+    // (i.e. those that would be deferred) so the budget truly samples the low-
+    // confidence tail and is meaningfully distinct from normal auto-accept.
+    // Promoted exploration proposals are logged with eligibilitySource="exploration".
+    const explorationBudget = cfg.explorationBudgetCount ?? 0;
+    let explorationRemaining = explorationBudget;
     const resolvedConfig = typeof cfg.config === "function" ? cfg.config() : cfg.config;
     const gateLabel = `improve:${cfg.phase}`;
     // #577: stamp the gate's verdict onto each proposal so `akm proposal show`
@@ -40,7 +70,12 @@ export async function runAutoAcceptGate(candidates, cfg, promoteFn = promoteProp
     };
     for (const candidate of candidates) {
         const { proposalId, confidence } = candidate;
-        if (confidence === undefined || confidence < effectiveThreshold) {
+        // Determine if this candidate is exploration-eligible: below-threshold
+        // (would normally be deferred) but with a valid confidence score and budget
+        // remaining. No-confidence candidates are never exploration-promoted.
+        const belowThreshold = confidence === undefined || confidence < effectiveThreshold;
+        const isExploration = belowThreshold && confidence !== undefined && explorationRemaining > 0;
+        if (belowThreshold && !isExploration) {
             stamp(proposalId, {
                 outcome: "deferred",
                 reason: confidence === undefined ? "no-confidence" : "below-threshold",
@@ -51,15 +86,25 @@ export async function runAutoAcceptGate(candidates, cfg, promoteFn = promoteProp
             result.skipped.push(proposalId);
             continue;
         }
+        // Either above-threshold (normal auto-accept) or exploration-budget promoted.
+        if (isExploration)
+            explorationRemaining -= 1;
+        const promoteReason = isExploration ? "exploration-budget" : "above-threshold";
         try {
             const promotion = await promoteFn(cfg.stashDir, resolvedConfig, proposalId, {}, undefined);
             stamp(promotion.proposal.id, {
                 outcome: "auto-accepted",
-                reason: "above-threshold",
+                reason: promoteReason,
                 confidence,
                 thresholds: { autoAccept: effectiveThreshold },
                 gate: gateLabel,
             });
+            // Resolve the eligibilitySource: exploration-promoted proposals get
+            // eligibilitySource="exploration" (WS-4); normal auto-accepts carry
+            // whatever the proposal was tagged with at selection time.
+            const resolvedEligibilitySource = isExploration
+                ? "exploration"
+                : promotion.proposal.eligibilitySource;
             appendEvent({
                 eventType: "promoted",
                 ref: promotion.ref,
@@ -72,14 +117,41 @@ export async function runAutoAcceptGate(candidates, cfg, promoteFn = promoteProp
                     confidence,
                     threshold: effectiveThreshold,
                     phase: cfg.phase,
+                    // Attribution tagging: carry the eligibility lane from the proposal
+                    // record onto the auto-accept promoted event so the lane survives to
+                    // accept time even when promotion happens in a later run.
+                    ...(resolvedEligibilitySource !== undefined ? { eligibilitySource: resolvedEligibilitySource } : {}),
+                    // WS-4: mark exploration promotions so health/telemetry can
+                    // distinguish them from calibration-signal promotions.
+                    ...(isExploration ? { explorationBudget: true } : {}),
                 },
             }, cfg.eventsCtx ?? {});
-            info(`[improve] auto-accepted ${promotion.ref} (${cfg.phase}; confidence=${confidence.toFixed(2)} >= threshold=${effectiveThreshold.toFixed(2)})`);
+            if (isExploration) {
+                info(`[improve] exploration-accepted ${promotion.ref} (${cfg.phase}; confidence=${confidence.toFixed(2)}; budgetRemaining=${explorationRemaining})`);
+            }
+            else {
+                info(`[improve] auto-accepted ${promotion.ref} (${cfg.phase}; confidence=${confidence.toFixed(2)} >= threshold=${effectiveThreshold.toFixed(2)})`);
+            }
             result.promoted.push(proposalId);
         }
         catch (err) {
-            warn(`[improve] ${cfg.phase} auto-accept failed for ${proposalId}: ${err instanceof Error ? err.message : String(err)}`);
+            const reason = classifyPromoteFailure(err);
+            warn(`[improve] ${cfg.phase} auto-accept failed for ${proposalId} (${reason}): ${err instanceof Error ? err.message : String(err)}`);
             result.failed.push(proposalId);
+            result.failedByReason[reason] = (result.failedByReason[reason] ?? 0) + 1;
+            // Record WHY on the proposal so `akm proposal show` explains the rejection
+            // and the leak is no longer blind. Best-effort.
+            stamp(proposalId, {
+                outcome: "auto-rejected",
+                reason,
+                confidence,
+                thresholds: { autoAccept: effectiveThreshold },
+                gate: gateLabel,
+            });
+            // If exploration budget was consumed but promotion failed, restore the slot
+            // so the budget isn't exhausted on errors.
+            if (isExploration)
+                explorationRemaining += 1;
         }
     }
     return result;
@@ -108,15 +180,46 @@ export function resolveExtractConfidence(proposal) {
 /**
  * Build a gate config for a phase, inheriting global settings from the
  * improve options. Callers supply only the phase-specific overrides.
+ *
+ * WS-4 additions:
+ * - When `shared.stateDbPath` is provided, reads the persisted per-phase
+ *   threshold from `improve_gate_thresholds` (Migration 012). The phase
+ *   value overrides `globalThreshold` but is still floored by
+ *   `minimumThreshold`. Falls back to `globalThreshold` when no row exists.
+ * - Computes `explorationBudgetCount` from
+ *   `config.improve.exploration.budgetFraction × candidateCount` when the
+ *   exploration budget is enabled. Defaults to 0 (no exploration).
  */
 export function makeGateConfig(phase, shared, overrides = {}) {
+    // WS-4: read per-phase threshold from state.db when available.
+    let phaseThreshold;
+    if (shared.stateDbPath && shared.globalThreshold !== undefined) {
+        try {
+            phaseThreshold = withStateDb((db) => getPhaseThreshold(db, phase) ?? undefined, {
+                path: shared.stateDbPath,
+            });
+        }
+        catch {
+            // DB unavailable — fall back to globalThreshold silently.
+        }
+    }
+    // WS-4: compute exploration budget count from config fraction × candidateCount.
+    let explorationBudgetCount;
+    const resolvedConfig = typeof shared.config === "function" ? shared.config() : shared.config;
+    const explorationCfg = resolvedConfig.improve?.exploration;
+    if (explorationCfg?.enabled && shared.candidateCount !== undefined && shared.candidateCount > 0) {
+        const fraction = Math.min(1, Math.max(0, explorationCfg.budgetFraction ?? 0.05));
+        explorationBudgetCount = Math.max(0, Math.floor(fraction * shared.candidateCount));
+    }
     return {
         phase,
         globalThreshold: shared.globalThreshold,
+        ...(phaseThreshold !== undefined ? { phaseThreshold } : {}),
         dryRun: shared.dryRun,
         stashDir: shared.stashDir,
         config: shared.config,
         eventsCtx: shared.eventsCtx,
+        ...(explorationBudgetCount !== undefined && explorationBudgetCount > 0 ? { explorationBudgetCount } : {}),
         ...overrides,
     };
 }

package/dist/commands/improve/improve-cli.js

@@ -13,6 +13,7 @@ import { resolveSourceEntries } from "../../indexer/search/search-source.js";
 import { getHyphenatedArg, getHyphenatedBoolean, parseFlagValue } from "../../output/context.js";
 import { akmImprove } from "./improve.js";
 import { buildImproveRunId, recordTerminatedImproveRun, relativeImproveResultPath, writeImproveResultFile, } from "./improve-result-file.js";
+import { runImproveSession } from "./improve-session.js";
 export const improveCommand = defineCommand({
     meta: {
         name: "improve",
@@ -144,67 +145,44 @@ export const improveCommand = defineCommand({
                     process.stderr.write(`warning: failed to persist terminated improve run ${runId}: ${err instanceof Error ? err.message : String(err)}\n`);
                 }
             };
-            // M5 (code-health round 2): signal -> {exit code, reason, ack message}
-            // as an explicit table instead of three near-identical handlers. The
-            // persist of the terminated-run row MUST complete before process.exit so
-            // a SIGTERM'd run (e.g. cron timeout) always leaves a row in
-            // improve_runs. recordTerminatedImproveRun is fully synchronous
-            // (bun:sqlite writes are sync), so the in-line call below blocks until
-            // the row is flushed before we exit.
-            const SIGNAL_TABLE = {
-                SIGTERM: { code: 143, reason: "SIGTERM", ack: true },
-                SIGINT: { code: 130, reason: "SIGINT", ack: true },
-                SIGHUP: { code: 129, reason: "SIGHUP", ack: false },
-            };
-            const makeSignalHandler = (sig) => () => {
-                const { code, reason, ack } = SIGNAL_TABLE[sig];
-                // Hard-exit fallback: if the synchronous persist ever hangs (e.g. a
-                // stuck sqlite lock under contention), the watchdog still exits with
-                // the correct code instead of leaving a zombie process. .unref() keeps
-                // the timer from holding the loop open on the normal (fast) path.
-                const watchdog = setTimeout(() => process.exit(code), 2000);
-                if (typeof watchdog.unref === "function")
-                    watchdog.unref();
-                try {
-                    persistTerminated(reason);
-                }
-                finally {
-                    clearTimeout(watchdog);
-                }
-                if (ack) {
-                    process.stderr.write(`[improve] received ${sig}; recorded terminated run ${runId}\n`);
-                }
-                process.exit(code);
-            };
-            const sigtermHandler = makeSignalHandler("SIGTERM");
-            const sigintHandler = makeSignalHandler("SIGINT");
-            const sighupHandler = makeSignalHandler("SIGHUP");
-            process.once("SIGTERM", sigtermHandler);
-            process.once("SIGINT", sigintHandler);
-            process.once("SIGHUP", sighupHandler);
+            // R8: the signal table / handlers / watchdog / persist-before-exit
+            // choreography lives in `runImproveSession`. It registers the
+            // SIGTERM/SIGINT/SIGHUP handlers (each persists the terminated-run row
+            // BEFORE process.exit so a SIGTERM'd run — e.g. cron timeout — always
+            // leaves a row in improve_runs), awaits the work, then removes the
+            // handlers on the way out. `onTerminate` persists synchronously
+            // (recordTerminatedImproveRun -> bun:sqlite writes are sync), and the
+            // 2000ms watchdog inside the session force-exits if that ever hangs.
             let improveResult;
             try {
-                improveResult = await akmImprove({
-                    scope: scopeArg,
-                    task: taskArg,
-                    dryRun,
-                    target: targetArg,
-                    autoAccept,
-                    ...(runId !== undefined ? { runId } : {}),
-                    ...(limitRaw !== undefined ? { limit: limitRaw } : {}),
-                    ...(timeoutMs !== undefined ? { timeoutMs } : {}),
-                    ...(minRetrievalCount !== undefined ? { minRetrievalCount } : {}),
-                    ...(requireFeedbackSignal ? { requireFeedbackSignal } : {}),
-                    ...(skipIfLocked ? { skipIfLocked } : {}),
-                    ...(profileArg !== undefined ? { profile: profileArg } : {}),
-                    ...(Object.keys(syncOverride).length > 0 ? { sync: syncOverride } : {}),
-                    consolidateOptions: {
-                        target: targetArg,
+                improveResult = await runImproveSession({
+                    runWork: () => akmImprove({
+                        scope: scopeArg,
+                        task: taskArg,
                         dryRun,
+                        target: targetArg,
                         autoAccept,
-                        task: taskArg,
-                        ...(consolidateRecovery !== undefined ? { recoveryMode: consolidateRecovery } : {}),
-                    },
+                        ...(runId !== undefined ? { runId } : {}),
+                        ...(limitRaw !== undefined ? { limit: limitRaw } : {}),
+                        ...(timeoutMs !== undefined ? { timeoutMs } : {}),
+                        ...(minRetrievalCount !== undefined ? { minRetrievalCount } : {}),
+                        ...(requireFeedbackSignal ? { requireFeedbackSignal } : {}),
+                        ...(skipIfLocked ? { skipIfLocked } : {}),
+                        ...(profileArg !== undefined ? { profile: profileArg } : {}),
+                        ...(Object.keys(syncOverride).length > 0 ? { sync: syncOverride } : {}),
+                        consolidateOptions: {
+                            target: targetArg,
+                            dryRun,
+                            autoAccept,
+                            task: taskArg,
+                            ...(consolidateRecovery !== undefined ? { recoveryMode: consolidateRecovery } : {}),
+                        },
+                    }),
+                }, {
+                    signalSource: process,
+                    exit: process.exit,
+                    onTerminate: (reason) => persistTerminated(reason),
+                    ack: (message) => process.stderr.write(`[improve] ${message}; recorded terminated run ${runId}\n`),
                 });
             }
             catch (err) {
@@ -216,9 +194,6 @@ export const improveCommand = defineCommand({
                 throw err;
             }
             finally {
-                process.removeListener("SIGTERM", sigtermHandler);
-                process.removeListener("SIGINT", sigintHandler);
-                process.removeListener("SIGHUP", sighupHandler);
                 clearLogFile();
             }
             const durationMs = Date.now() - startedAtMs;

package/dist/commands/improve/improve-profiles.js

@@ -8,6 +8,7 @@ import profileFrequent from "../../assets/profiles/frequent.json" with { type: "
 import profileGraphRefresh from "../../assets/profiles/graph-refresh.json" with { type: "json" };
 import profileMemoryFocus from "../../assets/profiles/memory-focus.json" with { type: "json" };
 import profileQuick from "../../assets/profiles/quick.json" with { type: "json" };
+import profileSynthesize from "../../assets/profiles/synthesize.json" with { type: "json" };
 import profileThorough from "../../assets/profiles/thorough.json" with { type: "json" };
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
 import { warn } from "../../core/warn.js";
@@ -31,6 +32,7 @@ const BUILTIN_PROFILES = {
     frequent: profileFrequent,
     consolidate: profileConsolidate,
     catchup: profileCatchup,
+    synthesize: profileSynthesize,
 };
 /**
  * Default enabled-state for known improve processes when neither the user
@@ -54,6 +56,18 @@ const IMPROVE_PROCESS_DEFAULTS = {
     // proposal-queue triage drains the standing backlog. Opt-in (default off),
     // like `validation` — needs an explicit `enabled: true`.
     triage: false,
+    // Layer 2 proactive-maintenance selector. Opt-in (default off) — surfaces
+    // stale high-value assets on a schedule. Enable per-profile with an explicit
+    // `processes.proactiveMaintenance.enabled: true`.
+    proactiveMaintenance: false,
+    // #609 recombine / synthesize pass — whole-corpus cross-episodic
+    // generalization. Opt-in (default off); enable per-profile with an explicit
+    // `processes.recombine.enabled: true`.
+    recombine: false,
+    // #615 procedural-compilation pass — detects recurring successful ordered
+    // action sequences and compiles them into workflow proposals. Opt-in (default
+    // off); enable per-profile with an explicit `processes.procedural.enabled: true`.
+    procedural: false,
 };
 /**
  * Compute the effective enabled-state for a named improve process.

package/dist/commands/improve/improve-result-file.js

@@ -27,7 +27,7 @@
  */
 import crypto from "node:crypto";
 import path from "node:path";
-import { openStateDatabase, recordImproveRun } from "../../core/state-db.js";
+import { recordImproveRun, withStateDb } from "../../core/state-db.js";
 /**
  * Build a stable run-id for a single improve invocation.
  *
@@ -74,8 +74,7 @@ export function relativeImproveResultPath(runId) {
  * `feedback_akm_dryrun_artifact_trap`).
  */
 export function writeImproveResultFile(stashDir, runId, result, startedAt) {
-    const db = openStateDatabase();
-    try {
+    withStateDb((db) => {
         const completedAt = new Date().toISOString();
         // startedAt is the ISO timestamp captured at process launch (passed from the
         // CLI entry point). If omitted, fall back to the run-id's embedded timestamp
@@ -95,15 +94,7 @@ export function writeImproveResultFile(stashDir, runId, result, startedAt) {
             ok: Boolean(result.ok),
             result,
         });
-    }
-    finally {
-        try {
-            db.close();
-        }
-        catch {
-            // best-effort
-        }
-    }
+    });
     return relativeImproveResultPath(runId);
 }
 /**
@@ -138,8 +129,7 @@ export function recordTerminatedImproveRun(stashDir, runId, startedAt, reason, c
             ...(ctx?.errorMessage ? { errorMessage: ctx.errorMessage } : {}),
         },
     };
-    const db = openStateDatabase();
-    try {
+    withStateDb((db) => {
         recordImproveRun(db, {
             id: runId,
             startedAt,
@@ -160,13 +150,5 @@ export function recordTerminatedImproveRun(stashDir, runId, startedAt, reason, c
                 },
             },
         });
-    }
-    finally {
-        try {
-            db.close();
-        }
-        catch {
-            // best-effort
-        }
-    }
+    });
 }

package/dist/commands/improve/improve-session.js

@@ -0,0 +1,58 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/** Signal -> {exit code, reason, ack message}. */
+export const SIGNAL_TABLE = {
+    SIGTERM: { code: 143, reason: "SIGTERM", ack: true },
+    SIGINT: { code: 130, reason: "SIGINT", ack: true },
+    SIGHUP: { code: 129, reason: "SIGHUP", ack: false },
+};
+/**
+ * Drive the improve-run lifecycle: register signal handlers, await `runWork`,
+ * and on clean completion remove the handlers and resolve with the result. The
+ * caller (improve-cli.ts) owns success-path result persistence + the final
+ * `exit(0)`; this function returns the work result so that choreography stays
+ * in the CLI exactly as before.
+ *
+ * Abnormal paths:
+ *   - A registered signal fires -> persist terminated row (watchdog-guarded) ->
+ *     optional ack -> `exit(code)`.
+ *   - `runWork` rejects -> handlers are removed and the rejection propagates so
+ *     the CLI's existing catch can persist `"exception"` and rethrow.
+ */
+export async function runImproveSession(opts, deps) {
+    const { signalSource, exit, onTerminate, ack } = deps;
+    const makeSignalHandler = (sig) => () => {
+        const { code, reason, ack: shouldAck } = SIGNAL_TABLE[sig];
+        // Hard-exit fallback: if the synchronous persist ever hangs (e.g. a stuck
+        // sqlite lock under contention), the watchdog still exits with the correct
+        // code instead of leaving a zombie process. .unref() keeps the timer from
+        // holding the loop open on the normal (fast) path.
+        const watchdog = setTimeout(() => exit(code), 2000);
+        if (typeof watchdog.unref === "function")
+            watchdog.unref();
+        try {
+            onTerminate(reason);
+        }
+        finally {
+            clearTimeout(watchdog);
+        }
+        if (shouldAck)
+            ack?.(`received ${sig}`);
+        exit(code);
+    };
+    const sigtermHandler = makeSignalHandler("SIGTERM");
+    const sigintHandler = makeSignalHandler("SIGINT");
+    const sighupHandler = makeSignalHandler("SIGHUP");
+    signalSource.once("SIGTERM", sigtermHandler);
+    signalSource.once("SIGINT", sigintHandler);
+    signalSource.once("SIGHUP", sighupHandler);
+    try {
+        return await opts.runWork();
+    }
+    finally {
+        signalSource.removeListener("SIGTERM", sigtermHandler);
+        signalSource.removeListener("SIGINT", sigintHandler);
+        signalSource.removeListener("SIGHUP", sighupHandler);
+    }
+}