akm-cli 0.9.23 → 0.9.25-alpha.1

package/dist/commands/improve/improve-cli.js

@@ -15,6 +15,7 @@ import { redactSensitiveText } from "../../core/redaction.js";
 import { clearLogFile, setLogFile, warn } from "../../core/warn.js";
 import { resolveWriteTarget } from "../../core/write-source.js";
 import { DEFAULT_LLM_TIMEOUT_MS } from "../../integrations/agent/config.js";
+import { defaultWhich } from "../../integrations/agent/detect.js";
 import { collectEngineCredentialValues } from "../../integrations/agent/engine-resolution.js";
 import { probeLlmReachable } from "../../llm/client.js";
 import { getOutputMode } from "../../output/context.js";
@@ -73,26 +74,19 @@ function assertRequiredEnginesAvailable(plan) {
     const lines = plan.engineUnavailable.map((item) => `  - ${item.process} (${item.configKey}): ${item.reason}`);
     throw new ConfigError(`--require-engines: ${plan.engineUnavailable.length} improve process${plan.engineUnavailable.length === 1 ? "" : "es"} cannot run because ${plan.engineUnavailable.length === 1 ? "its" : "their"} engine is unavailable:\n${lines.join("\n")}`, "LLM_NOT_CONFIGURED");
 }
-/** Every LLM connection the plan would dispatch to, triage's judgment engine included. */
+/** Every engine the plan would dispatch to, triage's judgment engine included. */
 function collectRequiredEngineTargets(plan) {
-    const targets = [];
-    for (const [processName, process] of Object.entries(plan.processes)) {
-        if (process.runner) {
-            targets.push({
-                process: processName,
-                engine: process.runner.engine,
-                connection: probeConnection(process.runner),
-            });
-        }
-    }
-    if (plan.triageJudgment?.kind === "llm") {
-        targets.push({
-            process: "triage.judgment",
-            engine: plan.triageJudgment.engine,
-            connection: probeConnection(plan.triageJudgment),
-        });
-    }
-    return targets;
+    const runners = Object.entries(plan.processes)
+        .flatMap(([processName, process]) => (process.runner ? [[processName, process.runner]] : []))
+        .concat(plan.triageJudgment ? [["triage.judgment", plan.triageJudgment]] : []);
+    return runners.map(([processName, runner]) => ({
+        process: processName,
+        engine: runner.engine,
+        ...(runner.kind === "llm" ? { connection: probeConnection(runner) } : { bin: runner.profile.bin }),
+        ...(runner.kind === "sdk" && runner.fallbackConnection
+            ? { connection: probeConnection({ connection: runner.fallbackConnection, timeoutMs: runner.fallbackTimeoutMs }) }
+            : {}),
+    }));
 }
 /** The resolved engine keeps its request timeout beside the connection (the runtime merges it in); the probe needs it on the connection. */
 function probeConnection(runner) {
@@ -112,37 +106,42 @@ const REQUIRED_ENGINE_PROBE_MAX_MS = 120_000;
 /**
  * `--require-engines`, live: probe each connection's real completion path
  * (a gateway can list a model whose completion route is dead, #980), once per
- * endpoint + model, within {@link requiredEngineProbeTimeoutMs}. Returns each
- * target's latency for the run result (R17); an unreachable one fails the run.
+ * endpoint + model, within {@link requiredEngineProbeTimeoutMs}, and look each
+ * agent harness's binary up on PATH. Returns each target's latency for the run
+ * result (R17); an unreachable one fails the run.
  */
-export async function assertRequiredEnginesReachable(plan, probeReachable = (connection) => probeLlmReachable(connection, requiredEngineProbeTimeoutMs(connection))) {
+export async function assertRequiredEnginesReachable(plan, probeReachable = (connection) => probeLlmReachable(connection, requiredEngineProbeTimeoutMs(connection)), which = defaultWhich) {
     const targets = collectRequiredEngineTargets(plan);
     if (targets.length === 0)
         return [];
     const probesByConnection = new Map();
-    const probed = await Promise.all(targets.map(async (target) => {
-        const key = `${target.connection.endpoint.replace(/\/+$/, "")}|${target.connection.model}`;
+    const probeConnectionOnce = (connection) => {
+        const key = `${connection.endpoint.replace(/\/+$/, "")}|${connection.model}`;
         let pending = probesByConnection.get(key);
         if (!pending) {
             const probeStartedAt = Date.now();
-            pending = probeReachable(target.connection).then((reach) => ({
-                reach,
-                latencyMs: Date.now() - probeStartedAt,
-            }));
+            pending = probeReachable(connection).then((reach) => ({ reach, latencyMs: Date.now() - probeStartedAt }));
             probesByConnection.set(key, pending);
         }
-        const { reach, latencyMs } = await pending;
-        return { ...target, reach, latencyMs };
+        return pending;
+    };
+    const probed = await Promise.all(targets.map(async (target) => {
+        if (target.bin !== undefined && which(target.bin) === undefined) {
+            return { ...target, reach: { reachable: false, error: `${target.bin} is not on PATH` }, latencyMs: 0 };
+        }
+        if (!target.connection)
+            return { ...target, reach: { reachable: true }, latencyMs: 0 };
+        return { ...target, ...(await probeConnectionOnce(target.connection)) };
     }));
     const unreachable = probed.filter((item) => !item.reach.reachable);
     if (unreachable.length > 0) {
-        const lines = unreachable.map((item) => `  - ${item.process} (engine "${item.engine}", ${item.connection.endpoint}): ${item.reach.error ?? "did not respond"}`);
-        throw new ConfigError(`--require-engines: ${unreachable.length} improve process${unreachable.length === 1 ? "" : "es"} cannot run because ${unreachable.length === 1 ? "its" : "their"} engine completion path is not reachable:\n${lines.join("\n")}`, "LLM_NOT_CONFIGURED", "Check that each listed endpoint is up and serves its model. The probe is one short completion, bounded by the engine's timeoutMs (at most two minutes).");
+        const lines = unreachable.map((item) => `  - ${item.process} (engine "${item.engine}", ${item.connection?.endpoint ?? item.bin}): ${item.reach.error ?? "did not respond"}`);
+        throw new ConfigError(`--require-engines: ${unreachable.length} improve process${unreachable.length === 1 ? "" : "es"} cannot run because ${unreachable.length === 1 ? "its" : "their"} engine completion path is not reachable:\n${lines.join("\n")}`, "LLM_NOT_CONFIGURED", "Check that each listed endpoint is up and serves its model, and that each listed agent binary is installed. The endpoint probe is one short completion, bounded by the engine's timeoutMs (at most two minutes).");
     }
     return probed.map((item) => ({
         process: item.process,
         engine: item.engine,
-        endpoint: item.connection.endpoint,
+        endpoint: item.connection?.endpoint ?? item.bin,
         reachable: item.reach.reachable,
         latencyMs: item.latencyMs,
     }));

package/dist/commands/improve/improve-strategies.js

@@ -11,11 +11,12 @@ import thorough from "../../assets/improve-strategies/thorough.json" with { type
 import { conceptIdFromTypeName, parseRefInput } from "../../core/asset/resolve-ref.js";
 import { ImproveProfileConfigSchema } from "../../core/config/config-schema.js";
 import { deepMergeConfig } from "../../core/config/deep-merge.js";
-import { BUILTIN_IMPROVE_STRATEGY_NAMES, IMPROVE_PROCESS_ENGINE_CAPABILITIES, } from "../../core/config/engine-semantics.js";
+import { BUILTIN_IMPROVE_STRATEGY_NAMES, IMPROVE_ENGINE_PROCESSES, IMPROVE_PROCESS_NAMES, } from "../../core/config/engine-semantics.js";
 import { ConfigError } from "../../core/errors.js";
-import { describeLlmCredentialAvailability } from "../../integrations/agent/engine-resolution.js";
+import { describeLlmCredentialAvailability, } from "../../integrations/agent/engine-resolution.js";
+import { runnerLlmConnection } from "../../integrations/agent/runner.js";
 import { applyAutonomyGate } from "./autonomy-gate.js";
-import { resolveImproveExecution, resolveImproveLlmExecution } from "./execution.js";
+import { resolveImproveExecution } from "./execution.js";
 import { stripBundle } from "./ledger.js";
 export const DEFAULT_ALLOWED_TYPES = {
     reflect: ["agent", "command", "knowledge", "lesson", "memory", "skill", "workflow"],
@@ -113,9 +114,14 @@ export function resolveImproveStrategy(name, config) {
     const resolved = deepMergeConfig(baseStrategy, (userStrategies[selectedName] ?? {}));
     return { name: selectedName, config: ImproveProfileConfigSchema.parse(resolved) };
 }
+/**
+ * The processes that make their own model calls, so the plan resolves a runner
+ * for each: every engine process but triage, whose engine is its judgment's.
+ */
+export const MODEL_CALLING_PROCESSES = new Set(IMPROVE_ENGINE_PROCESSES.filter((name) => name !== "triage"));
 /**
  * Project a resolved improve plan into one routing row per
- * {@link IMPROVE_PROCESS_ENGINE_CAPABILITIES} name, plus a `"triage.judgment"`
+ * {@link IMPROVE_PROCESS_NAMES} name, plus a `"triage.judgment"`
  * pseudo-row when the strategy configures a judgment engine. Pure — no I/O,
  * no re-resolution. Shared by `improve --dry-run`'s `plan.processes` (#947)
  * and `akm health`'s post-run reporting (#944); health's own
@@ -124,14 +130,18 @@ export function resolveImproveStrategy(name, config) {
 export function projectResolvedProcessRouting(plan) {
     const unavailableByProcess = new Map(plan.engineUnavailable.map((item) => [item.process, item]));
     const rows = [];
-    for (const processName of Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES)) {
+    for (const processName of IMPROVE_PROCESS_NAMES) {
         const process = plan.processes[processName];
         const unavailable = unavailableByProcess.get(processName);
         rows.push({
             process: processName,
             enabled: process.enabled,
             ...(process.runner
-                ? { engine: process.runner.engine, model: process.runner.connection.model, engineKind: process.runner.kind }
+                ? {
+                    engine: process.runner.engine,
+                    model: runnerLlmConnection(process.runner)?.model,
+                    engineKind: process.runner.kind,
+                }
                 : // #800/#957 round 3 — a credential-unavailable process never carries a
                     // runner, but its structurally resolved engine/model is still worth
                     // showing in the routing table (dry-run preview, health probe).
@@ -192,6 +202,22 @@ export function resolveImprovePlan(name, config, options = {}) {
 function credentialUnavailableReason(engineName, status) {
     return `requires a credential that is not available in this process's environment (engine "${engineName}": ${status.reason})`;
 }
+/**
+ * Whether the credential akm reads for `runner` is reachable in `env`: an LLM
+ * engine's own, or an SDK engine's LLM fallback. An agent CLI reads its own.
+ */
+function runnerCredentialStatus(runner, env) {
+    if (runner.kind === "llm")
+        return describeLlmCredentialAvailability(runner, env);
+    if (runner.kind === "sdk") {
+        return describeLlmCredentialAvailability({
+            credential: runner.fallbackCredential,
+            apiKeyFile: runner.fallbackApiKeyFile,
+            apiKeySecretRef: runner.fallbackApiKeySecretRef,
+        }, env);
+    }
+    return { available: true };
+}
 function buildImprovePlan(strategy, config, options) {
     const env = options.env ?? process.env;
     const processes = {};
@@ -205,12 +231,12 @@ function buildImprovePlan(strategy, config, options) {
     // install is not the credential-in-the-wrong-environment case this option
     // exists for.
     let anyEngineNotConfigured = false;
-    for (const processName of Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES)) {
+    for (const processName of IMPROVE_PROCESS_NAMES) {
         const sourceProcessConfig = strategy.config.processes?.[processName] ?? {};
         const enabled = sourceProcessConfig.enabled === true;
         let runner = null;
         let notices = [];
-        if (IMPROVE_PROCESS_ENGINE_CAPABILITIES[processName] !== "llm" || !enabled) {
+        if (!MODEL_CALLING_PROCESSES.has(processName) || !enabled) {
             processes[processName] = Object.freeze({ enabled, config: cloneAndFreeze(sourceProcessConfig), runner });
             continue;
         }
@@ -227,7 +253,7 @@ function buildImprovePlan(strategy, config, options) {
         // without the runner ever carrying a credential-less connection forward.
         let credentialUnavailableRouting;
         if (!skipsRepairEngine) {
-            const resolved = resolveImproveLlmExecution({
+            const resolved = resolveImproveExecution({
                 config,
                 profile: strategy.config,
                 process: sourceProcessConfig,
@@ -236,15 +262,14 @@ function buildImprovePlan(strategy, config, options) {
             runner = resolved?.runner ?? null;
             notices = resolved?.notices ?? [];
             if (runner) {
-                const credentialStatus = describeLlmCredentialAvailability(runner, env);
+                const credentialStatus = runnerCredentialStatus(runner, env);
                 if (!credentialStatus.available) {
+                    const connection = runnerLlmConnection(runner);
                     credentialUnavailableMessage = credentialUnavailableReason(runner.engine, credentialStatus);
                     credentialUnavailableRouting = {
                         engine: runner.engine,
-                        ...(runner.connection.model !== undefined ? { model: runner.connection.model } : {}),
-                        ...(runner.connection.contextLength !== undefined
-                            ? { contextLength: runner.connection.contextLength }
-                            : {}),
+                        ...(connection?.model !== undefined ? { model: connection.model } : {}),
+                        ...(connection?.contextLength !== undefined ? { contextLength: connection.contextLength } : {}),
                     };
                     runner = null;
                     notices = [];
@@ -259,7 +284,7 @@ function buildImprovePlan(strategy, config, options) {
                 process: processName,
                 configKey,
                 reason: credentialUnavailableMessage ??
-                    `requires an LLM engine that is not configured. Set defaults.llmEngine or ${configKey}`,
+                    `requires an engine that is not configured. Set defaults.llmEngine or ${configKey}`,
                 ...credentialUnavailableRouting,
             });
             processes[processName] = Object.freeze({
@@ -282,7 +307,7 @@ function buildImprovePlan(strategy, config, options) {
         !Object.values(processes).some((process) => process.enabled) &&
         (!options.allowAllDisabled || anyEngineNotConfigured)) {
         const names = engineUnavailable.map((item) => `"${item.process}"`).join(", ");
-        throw new ConfigError(`No improve process can run: ${names} ${engineUnavailable.length === 1 ? "requires" : "require"} an LLM engine that is not configured. Set defaults.llmEngine, or the per-process engine key named for each.`, "LLM_NOT_CONFIGURED");
+        throw new ConfigError(`No improve process can run: ${names} ${engineUnavailable.length === 1 ? "requires" : "require"} an engine that is not configured. Set defaults.llmEngine, or the per-process engine key named for each.`, "LLM_NOT_CONFIGURED");
     }
     const triage = strategy.config.processes?.triage;
     const judgmentEnabled = triage?.judgment?.enabled === true;
@@ -296,10 +321,6 @@ function buildImprovePlan(strategy, config, options) {
         })
         : null;
     let triageJudgment = triageJudgmentResolution?.runner ?? null;
-    const effectiveJudgmentLlm = triage?.judgment?.llm ?? triage?.llm ?? strategy.config.llm;
-    if (triageJudgment && triageJudgment.kind !== "llm" && effectiveJudgmentLlm) {
-        throw new ConfigError(`Triage judgment engine "${triageJudgment.engine ?? "unknown"}" is an agent engine and cannot receive llm overrides.`, "INVALID_CONFIG_FILE");
-    }
     if (processes.triage.enabled && judgmentEnabled && !triageJudgment) {
         throw new ConfigError(`Enabled improve triage judgment requires an engine. Set defaults.llmEngine or improve.strategies.${strategy.name}.processes.triage.judgment.engine.`, "LLM_NOT_CONFIGURED");
     }
@@ -310,29 +331,14 @@ function buildImprovePlan(strategy, config, options) {
     // `engineUnavailable` under the reserved `"triage.judgment"` process name
     // instead of silently reaching dispatch with a doomed credential.
     if (triageJudgment) {
-        const judgmentCredentialFields = triageJudgment.kind === "llm"
-            ? {
-                credential: triageJudgment.credential,
-                apiKeyFile: triageJudgment.apiKeyFile,
-                apiKeySecretRef: triageJudgment.apiKeySecretRef,
-            }
-            : triageJudgment.kind === "sdk"
-                ? {
-                    credential: triageJudgment.fallbackCredential,
-                    apiKeyFile: triageJudgment.fallbackApiKeyFile,
-                    apiKeySecretRef: triageJudgment.fallbackApiKeySecretRef,
-                }
-                : undefined;
-        if (judgmentCredentialFields) {
-            const credentialStatus = describeLlmCredentialAvailability(judgmentCredentialFields, env);
-            if (!credentialStatus.available) {
-                engineUnavailable.push({
-                    process: "triage.judgment",
-                    configKey: `improve.strategies.${strategy.name}.processes.triage.judgment.engine`,
-                    reason: credentialUnavailableReason(triageJudgment.engine, credentialStatus),
-                });
-                triageJudgment = null;
-            }
+        const credentialStatus = runnerCredentialStatus(triageJudgment, env);
+        if (!credentialStatus.available) {
+            engineUnavailable.push({
+                process: "triage.judgment",
+                configKey: `improve.strategies.${strategy.name}.processes.triage.judgment.engine`,
+                reason: credentialUnavailableReason(triageJudgment.engine, credentialStatus),
+            });
+            triageJudgment = null;
         }
     }
     const frozenProcesses = Object.freeze(processes);

package/dist/commands/improve/improve-usage-report.js

@@ -1,20 +1,8 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
-/**
- * The per-run LLM usage report (#944): the process × engine × model cross-tab
- * plus the enabled processes that made no call and why, rendered as one table
- * for the end-of-run stderr summary and `akm improve report`.
- */
-import { IMPROVE_PROCESS_ENGINE_CAPABILITIES } from "../../core/config/engine-semantics.js";
-import { eligibleRefCount, projectResolvedProcessRouting, } from "./improve-strategies.js";
-/**
- * Only processes that call an LLM themselves: triage (a runner, attributed to
- * its judgment engine) and proactive maintenance (no engine) would always read
- * as "zero calls".
- */
-const LLM_BACKED_PROCESSES = new Set(Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES).filter((name) => IMPROVE_PROCESS_ENGINE_CAPABILITIES[name] === "llm"));
-/** The autonomy lane gating each LLM-backed process, if any. */
+import { eligibleRefCount, MODEL_CALLING_PROCESSES, projectResolvedProcessRouting, } from "./improve-strategies.js";
+/** The autonomy lane gating each model-calling process, if any. */
 const AUTONOMY_LANE_BY_PROCESS = {
     memoryInference: "memoryInference",
 };
@@ -30,7 +18,7 @@ function dominantReason(counts) {
     return best;
 }
 /**
- * Why an LLM-backed process made no call, in priority order: its engine is
+ * Why a model-calling process made no call, in priority order: its engine is
  * unavailable; disabled — `autonomy_gated` when its lane was gated, else
  * nothing to report; every ref strategy-filtered; its dominant skip reason;
  * else `no_signal`. The reasons are the ones the events and results already
@@ -65,8 +53,11 @@ function countReflectSkipReasons(actions) {
 }
 /** This run's `usageReport`, or `undefined` when both halves are empty. */
 export function buildImproveUsageReport(args) {
-    // The "triage.judgment" row is dropped too: its calls are never attributed to it (#947).
-    const routing = projectResolvedProcessRouting(args.resolvedPlan).filter((row) => row.process !== "triage.judgment" && LLM_BACKED_PROCESSES.has(row.process));
+    // Only the processes that call a model themselves, on any engine kind: triage
+    // (attributed to its judgment) and proactive maintenance (no engine) would
+    // always read as "zero calls". The "triage.judgment" row is dropped too: its
+    // calls are never attributed to it (#947).
+    const routing = projectResolvedProcessRouting(args.resolvedPlan).filter((row) => row.process !== "triage.judgment" && MODEL_CALLING_PROCESSES.has(row.process));
     const calledProcesses = new Set(args.byProcessEngineModel.filter((row) => row.calls > 0).map((row) => row.process));
     const reflectSkipCounts = countReflectSkipReasons(args.persistedActions);
     const noCalls = [];

package/dist/commands/improve/preparation.js

@@ -24,6 +24,7 @@ import { appendEvent, readEvents } from "../../core/events.js";
 import { withStateDb } from "../../core/state-db.js";
 import { info, warn } from "../../core/warn.js";
 import { countUsageEventsByType, USAGE_EVENT_RETENTION_DAYS } from "../../indexer/usage/usage-events.js";
+import { runnerLlmConnection } from "../../integrations/agent/runner.js";
 import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
 import { getZeroResultSearches } from "../../storage/repositories/index-entries-repository.js";
 import { getRetrievalCounts } from "../../storage/repositories/index-utility-repository.js";
@@ -119,7 +120,8 @@ function planConsolidationPass(args) {
         : { poolSize: 0, candidatePoolSize: 0, judgedUnchanged: 0 };
     // A credential-unavailable engine still resolved its context length.
     const unavailable = resolvedPlan.engineUnavailable.find((item) => item.process === "consolidate");
-    const chunkSize = computeSafeChunkSize(resolvedPlan.processes.consolidate.runner?.connection.contextLength ??
+    const chunkSize = computeSafeChunkSize((resolvedPlan.processes.consolidate.runner &&
+        runnerLlmConnection(resolvedPlan.processes.consolidate.runner)?.contextLength) ??
         unavailable?.contextLength ??
         DEFAULT_CONTEXT_LENGTH_TOKENS, 500, processConfig?.maxChunkSize);
     const profilePassed = processConfig?.enabled !== false;