akm-cli 0.9.14 → 0.9.15-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +559 -0
- package/STABILITY.md +6 -3
- package/dist/akm +54 -1
- package/dist/akm-migrate +34 -1
- package/dist/assets/prompts/reflect-feedback-framing.md +1 -0
- package/dist/assets/prompts/reflect-llm-framed-contract.md +2 -0
- package/dist/assets/prompts/reflect-llm-schema-contract.md +2 -0
- package/dist/assets/tasks/core/improve.yml +1 -1
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +1 -1
- package/dist/cli/retired-commands.js +0 -1
- package/dist/cli/shared.js +9 -0
- package/dist/cli/unknown-flags.js +1 -0
- package/dist/cli.js +40 -3
- package/dist/commands/config-cli.js +85 -3
- package/dist/commands/env/env-cli.js +1 -42
- package/dist/commands/env/env.js +1 -1
- package/dist/commands/env/secret-cli.js +1 -2
- package/dist/commands/health/checks.js +357 -63
- package/dist/commands/health/engine-usage.js +45 -0
- package/dist/commands/health/improve-metrics.js +18 -0
- package/dist/commands/health/llm-usage.js +41 -1
- package/dist/commands/health/plugin-staleness.js +7 -3
- package/dist/commands/health/version-drift.js +93 -0
- package/dist/commands/health/windows.js +3 -1
- package/dist/commands/health.js +44 -9
- package/dist/commands/improve/consolidate/chunking.js +4 -2
- package/dist/commands/improve/improve-cli.js +99 -5
- package/dist/commands/improve/improve-report.js +154 -0
- package/dist/commands/improve/improve-result-file.js +45 -33
- package/dist/commands/improve/improve-strategies.js +133 -3
- package/dist/commands/improve/improve-usage-report.js +182 -0
- package/dist/commands/improve/improve.js +40 -3
- package/dist/commands/improve/locks.js +28 -78
- package/dist/commands/improve/planner.js +1 -0
- package/dist/commands/improve/preparation.js +9 -1
- package/dist/commands/improve/reflect.js +44 -4
- package/dist/commands/models-cli.js +50 -1
- package/dist/commands/proposal/repository.js +8 -3
- package/dist/commands/proposal/validators/proposal-quality-validators.js +41 -6
- package/dist/commands/proposal/validators/proposal-validators.js +24 -0
- package/dist/commands/read/search-cli.js +38 -2
- package/dist/commands/read/show.js +103 -4
- package/dist/commands/sources/info.js +5 -1
- package/dist/commands/sources/installed-stashes.js +58 -16
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/stash-cli.js +48 -0
- package/dist/commands/tasks/tasks-cli.js +49 -2
- package/dist/commands/workflow-cli.js +86 -12
- package/dist/core/asset/markdown-fragments.js +35 -0
- package/dist/core/config/config-schema.js +14 -0
- package/dist/core/config/config.js +302 -24
- package/dist/core/config/schema/embedding.js +41 -0
- package/dist/core/env-secret-ref.js +58 -5
- package/dist/core/errors.js +30 -0
- package/dist/core/file-lock.js +49 -15
- package/dist/core/improve-result.js +51 -0
- package/dist/core/loopback.js +17 -0
- package/dist/core/parent-watchdog.js +64 -0
- package/dist/core/paths.js +11 -0
- package/dist/core/run-lock.js +107 -0
- package/dist/core/sensitive-marker-path.js +19 -0
- package/dist/core/state-db.js +74 -14
- package/dist/indexer/index-rebuild-lock.js +73 -0
- package/dist/indexer/index-writer-lock.js +40 -1
- package/dist/indexer/index-written-assets.js +29 -1
- package/dist/indexer/indexer.js +93 -29
- package/dist/indexer/materialize-embeddings.js +564 -48
- package/dist/indexer/search/db-search.js +49 -2
- package/dist/indexer/search/search-source.js +23 -1
- package/dist/integrations/agent/engine-resolution.js +96 -6
- package/dist/integrations/agent/execution-definitions.js +6 -15
- package/dist/integrations/agent/execution-lowering.js +6 -1
- package/dist/integrations/agent/execution-preparation.js +1 -1
- package/dist/integrations/agent/model-map.js +123 -20
- package/dist/integrations/agent/prompts.js +40 -8
- package/dist/integrations/agent/runner-dispatch.js +9 -3
- package/dist/integrations/agent/runner.js +2 -0
- package/dist/llm/client.js +8 -3
- package/dist/llm/embedder.js +20 -8
- package/dist/llm/embedders/local.js +10 -2
- package/dist/llm/embedders/remote.js +497 -32
- package/dist/output/shapes/helpers.js +38 -2
- package/dist/output/shapes/models-list.js +16 -0
- package/dist/output/shapes/passthrough.js +2 -0
- package/dist/output/shapes.js +4 -0
- package/dist/output/text/command-format.js +29 -0
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/improve-report.js +27 -0
- package/dist/{commands/env/marker-path.js → output/text/models.js} +4 -3
- package/dist/output/text/show-format.js +4 -0
- package/dist/output/text.js +4 -0
- package/dist/scripts/akm-migrate-node.js +25146 -21759
- package/dist/scripts/akm-migrate.js +24271 -20885
- package/dist/storage/repositories/embedding-salvage-repository.js +184 -0
- package/dist/storage/repositories/improve-runs-repository.js +34 -0
- package/dist/storage/repositories/index-fts-repository.js +49 -6
- package/dist/storage/repositories/index-schema.js +16 -0
- package/dist/storage/repositories/index-vec-repository.js +30 -0
- package/dist/storage/repositories/workflow-runs-repository.js +55 -18
- package/dist/tasks/backends/cron.js +14 -7
- package/dist/tasks/run/run-native-task.js +23 -1
- package/dist/tasks/run/run-workflow-task.js +16 -0
- package/dist/workflows/exec/child-workflow.js +2 -2
- package/dist/workflows/exec/dispatch-redaction.js +21 -9
- package/dist/workflows/exec/run-workflow.js +6 -5
- package/dist/workflows/runtime/runs.js +33 -5
- package/docs/migration/release-notes/0.9.15.md +133 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/reference/cli.md +271 -30
- package/docs/reference/configuration.md +234 -21
- package/docs/reference/data-and-telemetry.md +8 -0
- package/docs/reference/tasks.md +16 -1
- package/docs/reference/workflow-schema.md +5 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +47 -0
|
@@ -3,15 +3,19 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { spawnSync } from "node:child_process";
|
|
5
5
|
import { loadConfig } from "../../core/config/config.js";
|
|
6
|
+
import { IMPROVE_PROCESS_ENGINE_CAPABILITIES } from "../../core/config/engine-semantics.js";
|
|
7
|
+
import { listEnvsRecursive } from "../../core/env-secret-ref.js";
|
|
6
8
|
import { ConfigError } from "../../core/errors.js";
|
|
7
9
|
import { EXTRACT_INFRASTRUCTURE_SKIP_REASONS } from "../../core/improve-types.js";
|
|
8
10
|
import { listPendingStateMigrations } from "../../core/state-db.js";
|
|
9
11
|
import { withEngineFallback } from "../../integrations/agent/engine-fallback.js";
|
|
10
|
-
import {
|
|
12
|
+
import { isLlmCredentialAvailable, resolveEngine, } from "../../integrations/agent/engine-resolution.js";
|
|
11
13
|
import { executionEngineDefinitionsFromConfig } from "../../integrations/agent/execution-definitions.js";
|
|
12
14
|
import { loadModelMap, mergeModelMapLayers, parseModelMapLayer, readInstalledModelMapText, resolveModelMapAlias, userModelMapPath, } from "../../integrations/agent/model-map.js";
|
|
15
|
+
import { listKeys } from "../env/env.js";
|
|
13
16
|
import { resolveImprovePlan } from "../improve/improve-strategies.js";
|
|
14
|
-
import {
|
|
17
|
+
import { ENGINE_LAST_USED_LOOKBACK_DAYS } from "./engine-usage.js";
|
|
18
|
+
import { ACTIVE_RUN_WARN_MS, TASK_FAIL_RATE_WARN, } from "./types.js";
|
|
15
19
|
/** Probe one connection's reachability, once per endpoint; `undefined` when no probe seam is supplied. */
|
|
16
20
|
function probeConnectionReachable(connection, deps, cache) {
|
|
17
21
|
if (!deps.probeReachable)
|
|
@@ -43,15 +47,64 @@ function escalateDefaultLlmEngineFailure(result) {
|
|
|
43
47
|
}
|
|
44
48
|
/** Rolling window the `session-extraction` check reads from the extract ledger (#914). */
|
|
45
49
|
export const SESSION_EXTRACTION_LEDGER_WINDOW_DAYS = 7;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
50
|
+
/**
|
|
51
|
+
* Delegates to {@link isLlmCredentialAvailable} (#953) so `akm health` and the
|
|
52
|
+
* improve-strategy probe (#957 builds on this same helper) share one
|
|
53
|
+
* non-throwing "is a credential present" check with the real dispatch
|
|
54
|
+
* boundary — env value, file-backed (#905), or secret-store-backed (#953).
|
|
55
|
+
*/
|
|
56
|
+
function credentialAvailable(credential, env, apiKeyFile, apiKeySecretRef) {
|
|
57
|
+
return isLlmCredentialAvailable({ credential, apiKeyFile, apiKeySecretRef }, env);
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* #950: when a required `$VAR`-style credential is missing from the shell,
|
|
61
|
+
* find the env asset (if any) whose key names include one of
|
|
62
|
+
* `credential.names` — so the warn can say "run under env/lab" instead of a
|
|
63
|
+
* bare "unavailable". Returns the asset's ref, or `null` when no credential,
|
|
64
|
+
* no candidate names, or no matching env asset. Never returns or logs the
|
|
65
|
+
* variable name itself (only ref/path/keys ever leave `listEnvsRecursive`,
|
|
66
|
+
* and only the ref is read here) — `tests/health-engine-probe.test.ts` pins
|
|
67
|
+
* that a credential's env-var name never appears in health's JSON output.
|
|
68
|
+
* Best-effort: any failure walking env assets (unreadable stash, bad config)
|
|
69
|
+
* degrades to `null`, never a crash. `deps.listEnvAssets` is expected to
|
|
70
|
+
* already be normalised (see {@link withMemoizedEnvAssets}) by the time this
|
|
71
|
+
* runs, so the walk happens at most once per probe run — every caller of
|
|
72
|
+
* this function passes deps that have already gone through
|
|
73
|
+
* {@link withMemoizedEnvAssets}, so there is no separate default to fall
|
|
74
|
+
* back to here.
|
|
75
|
+
*/
|
|
76
|
+
function findSuppliedByEnvAsset(credential, deps) {
|
|
77
|
+
if (!credential || credential.names.length === 0)
|
|
78
|
+
return null;
|
|
79
|
+
try {
|
|
80
|
+
const envs = deps.listEnvAssets?.();
|
|
81
|
+
if (!envs)
|
|
82
|
+
return null;
|
|
83
|
+
for (const envAsset of envs) {
|
|
84
|
+
if (credential.names.some((name) => envAsset.keys.includes(name)))
|
|
85
|
+
return envAsset.ref;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
// Best-effort — env-asset discovery must never break a health probe.
|
|
90
|
+
}
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* #950: normalise `deps.listEnvAssets` once per probe run so
|
|
95
|
+
* {@link findSuppliedByEnvAsset}'s env-asset walk happens at most once even
|
|
96
|
+
* when a run probes several engines with missing credentials — the walk is
|
|
97
|
+
* real IO (or config-injected IO in tests) with no reason to repeat per
|
|
98
|
+
* engine. Wraps whatever seam is already present (test-injected or the
|
|
99
|
+
* config-honouring default) in a memoising closure; called once by each of
|
|
100
|
+
* the four probe entry points that share {@link runConfiguredEngineProbe}
|
|
101
|
+
* (`runDefaultEngineProbe`, `runDefaultLlmEngineProbe`,
|
|
102
|
+
* `runConfiguredEnginesProbe`, `runHealthEngineProbes`).
|
|
103
|
+
*/
|
|
104
|
+
function withMemoizedEnvAssets(deps) {
|
|
105
|
+
const listEnvAssets = deps.listEnvAssets ?? (() => listEnvsRecursive(listKeys, (deps.loadConfig ?? loadConfig)()));
|
|
106
|
+
let cached;
|
|
107
|
+
return { ...deps, listEnvAssets: () => (cached ??= listEnvAssets()) };
|
|
55
108
|
}
|
|
56
109
|
async function runConfiguredEngineProbe(checkName, engineName, config, deps, reachabilityCache) {
|
|
57
110
|
if (!engineName) {
|
|
@@ -97,6 +150,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
|
|
|
97
150
|
let fallback;
|
|
98
151
|
let fallbackCredential;
|
|
99
152
|
let fallbackApiKeyFile;
|
|
153
|
+
let fallbackApiKeySecretRef;
|
|
100
154
|
let sdkRunner;
|
|
101
155
|
const resolve = deps.resolveEngine ?? resolveEngine;
|
|
102
156
|
try {
|
|
@@ -111,6 +165,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
|
|
|
111
165
|
fallback = { kind: "llm", engine: fallbackEngine, connection: sdkRunner.fallbackConnection };
|
|
112
166
|
fallbackCredential = sdkRunner.fallbackCredential;
|
|
113
167
|
fallbackApiKeyFile = sdkRunner.fallbackApiKeyFile;
|
|
168
|
+
fallbackApiKeySecretRef = sdkRunner.fallbackApiKeySecretRef;
|
|
114
169
|
}
|
|
115
170
|
else if (fallbackEngine) {
|
|
116
171
|
try {
|
|
@@ -119,6 +174,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
|
|
|
119
174
|
fallback = resolved;
|
|
120
175
|
fallbackCredential = resolved.credential;
|
|
121
176
|
fallbackApiKeyFile = resolved.apiKeyFile;
|
|
177
|
+
fallbackApiKeySecretRef = resolved.apiKeySecretRef;
|
|
122
178
|
}
|
|
123
179
|
}
|
|
124
180
|
catch {
|
|
@@ -127,12 +183,19 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
|
|
|
127
183
|
}
|
|
128
184
|
const configuredModel = configuredEngine.model;
|
|
129
185
|
const effectiveModel = sdkRunner?.profile.model ?? configuredModel ?? fallback?.connection.model;
|
|
130
|
-
const fallbackCredentialAvailable = credentialAvailable(fallbackCredential, env, fallbackApiKeyFile);
|
|
186
|
+
const fallbackCredentialAvailable = credentialAvailable(fallbackCredential, env, fallbackApiKeyFile, fallbackApiKeySecretRef);
|
|
187
|
+
const fallbackSuppliedByEnvAsset = !fallbackCredentialAvailable
|
|
188
|
+
? findSuppliedByEnvAsset(fallbackCredential, deps)
|
|
189
|
+
: null;
|
|
131
190
|
const missing = [
|
|
132
191
|
!packageAvailable ? "@opencode-ai/sdk package" : undefined,
|
|
133
192
|
!binaryAvailable ? `${binary} binary` : undefined,
|
|
134
193
|
fallbackEngine && !fallback ? "configured fallback LLM connection" : undefined,
|
|
135
|
-
!fallbackCredentialAvailable
|
|
194
|
+
!fallbackCredentialAvailable
|
|
195
|
+
? fallbackSuppliedByEnvAsset
|
|
196
|
+
? `required fallback credential (available via env asset ${fallbackSuppliedByEnvAsset}; run under it: akm env run ${fallbackSuppliedByEnvAsset} -- ...)`
|
|
197
|
+
: "required fallback credential"
|
|
198
|
+
: undefined,
|
|
136
199
|
].filter((value) => value !== undefined);
|
|
137
200
|
const sdkEvidence = {
|
|
138
201
|
engine: engineName,
|
|
@@ -149,6 +212,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
|
|
|
149
212
|
fallbackEndpoint: fallback?.connection.endpoint ?? null,
|
|
150
213
|
fallbackModel: fallback?.connection.model ?? null,
|
|
151
214
|
requiredCredentialAvailable: fallbackCredentialAvailable,
|
|
215
|
+
suppliedByEnvAsset: fallbackSuppliedByEnvAsset,
|
|
152
216
|
};
|
|
153
217
|
if (missing.length > 0) {
|
|
154
218
|
return {
|
|
@@ -181,7 +245,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
|
|
|
181
245
|
try {
|
|
182
246
|
const runner = (deps.resolveEngine ?? resolveEngine)(engineName, config);
|
|
183
247
|
if (runner.kind === "llm") {
|
|
184
|
-
const requiredCredentialAvailable = credentialAvailable(runner.credential, env, runner.apiKeyFile);
|
|
248
|
+
const requiredCredentialAvailable = credentialAvailable(runner.credential, env, runner.apiKeyFile, runner.apiKeySecretRef);
|
|
185
249
|
const llmEvidence = {
|
|
186
250
|
engine: engineName,
|
|
187
251
|
platform: null,
|
|
@@ -191,13 +255,20 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
|
|
|
191
255
|
requiredCredentialAvailable,
|
|
192
256
|
};
|
|
193
257
|
if (!requiredCredentialAvailable) {
|
|
258
|
+
// #950: a missing-in-shell credential is common when the operator's
|
|
259
|
+
// real workflow is `akm env run env/lab -- akm improve` — name the env
|
|
260
|
+
// asset that supplies it (never the variable name) so the warn is
|
|
261
|
+
// actionable instead of looking like a broken engine.
|
|
262
|
+
const suppliedByEnvAsset = findSuppliedByEnvAsset(runner.credential, deps);
|
|
194
263
|
return {
|
|
195
264
|
name: checkName,
|
|
196
265
|
kind: "deterministic",
|
|
197
266
|
status: "warn",
|
|
198
267
|
confidence: "high",
|
|
199
|
-
message:
|
|
200
|
-
|
|
268
|
+
message: suppliedByEnvAsset
|
|
269
|
+
? `LLM engine "${engineName}" is configured, but its required credential is not available in this shell; env asset ${suppliedByEnvAsset} supplies it — run under it (akm env run ${suppliedByEnvAsset} -- ...).`
|
|
270
|
+
: `LLM engine "${engineName}" is configured, but its required credential is unavailable.`,
|
|
271
|
+
evidence: { ...llmEvidence, suppliedByEnvAsset },
|
|
201
272
|
};
|
|
202
273
|
}
|
|
203
274
|
const reach = await probeConnectionReachable(runner.connection, deps, reachabilityCache);
|
|
@@ -284,6 +355,57 @@ function unconfiguredEngineProbe(name) {
|
|
|
284
355
|
message: name === "default-llm-engine" ? "No default LLM engine is configured." : "No default engine is configured.",
|
|
285
356
|
};
|
|
286
357
|
}
|
|
358
|
+
/**
|
|
359
|
+
* #949: `enableThinking: false` is a request an engine's endpoint (or a
|
|
360
|
+
* gateway in front of it) can silently ignore — `chatCompletionAttemptOnce`
|
|
361
|
+
* (src/llm/client.ts) already warns to stderr whenever a response reports
|
|
362
|
+
* reasoning tokens despite it. This makes that same signal visible in
|
|
363
|
+
* `akm health` as a structured finding, purely by re-reading the window's
|
|
364
|
+
* `llm_usage` aggregate already threaded into the context: no extra
|
|
365
|
+
* completion call, no live probe — a cold local model must not be woken just
|
|
366
|
+
* to run `akm health`.
|
|
367
|
+
*/
|
|
368
|
+
function projectThinkingControlCheck(thinkingOffEngines, llmUsage, since) {
|
|
369
|
+
if (thinkingOffEngines.length === 0) {
|
|
370
|
+
return {
|
|
371
|
+
name: "thinking-control",
|
|
372
|
+
kind: "deterministic",
|
|
373
|
+
status: "unknown",
|
|
374
|
+
confidence: "high",
|
|
375
|
+
message: "No configured engine sets enableThinking: false — nothing to verify.",
|
|
376
|
+
evidence: { engines: [] },
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
const engines = [...thinkingOffEngines].sort().map((engine) => {
|
|
380
|
+
const usage = llmUsage.byEngine[engine];
|
|
381
|
+
const calls = usage?.calls ?? 0;
|
|
382
|
+
const reasoningTokens = usage?.reasoningTokens ?? 0;
|
|
383
|
+
const status = calls === 0 ? "unknown" : reasoningTokens > 0 ? "warn" : "pass";
|
|
384
|
+
return { engine, status, calls, reasoningTokens };
|
|
385
|
+
});
|
|
386
|
+
const warning = engines.filter((e) => e.status === "warn");
|
|
387
|
+
const status = warning.length > 0 ? "warn" : engines.every((e) => e.status === "unknown") ? "unknown" : "pass";
|
|
388
|
+
let message;
|
|
389
|
+
if (warning.length > 0) {
|
|
390
|
+
message = warning
|
|
391
|
+
.map((e) => `LLM engine "${e.engine}" returned ${e.reasoningTokens} reasoning tokens since ${since} despite enableThinking: false — the endpoint (or a gateway in front of it) is not honouring the thinking-off control.`)
|
|
392
|
+
.join(" ");
|
|
393
|
+
}
|
|
394
|
+
else if (status === "unknown") {
|
|
395
|
+
message = `No calls were recorded for the engine(s) configured with enableThinking: false (${engines.map((e) => e.engine).join(", ")}) in the report window.`;
|
|
396
|
+
}
|
|
397
|
+
else {
|
|
398
|
+
message = `${engines.length} engine(s) configured with enableThinking: false (${engines.map((e) => e.engine).join(", ")}) returned no reasoning tokens in the report window.`;
|
|
399
|
+
}
|
|
400
|
+
return {
|
|
401
|
+
name: "thinking-control",
|
|
402
|
+
kind: "deterministic",
|
|
403
|
+
status,
|
|
404
|
+
confidence: "high",
|
|
405
|
+
message,
|
|
406
|
+
evidence: { engines },
|
|
407
|
+
};
|
|
408
|
+
}
|
|
287
409
|
function configuredEnginesProjection(engineNames, availability) {
|
|
288
410
|
if (engineNames.length === 0) {
|
|
289
411
|
return {
|
|
@@ -316,14 +438,14 @@ export async function runDefaultEngineProbe(deps = {}) {
|
|
|
316
438
|
// opencode binary DOES have a working default, and reporting otherwise would
|
|
317
439
|
// contradict what `workflow run` / `task run` actually do.
|
|
318
440
|
const { config } = withEngineFallback(deps.loadConfig?.() ?? loadConfig(), deps.which);
|
|
319
|
-
return runConfiguredEngineProbe("default-engine", config.defaults?.engine, config, deps, new Map());
|
|
441
|
+
return runConfiguredEngineProbe("default-engine", config.defaults?.engine, config, withMemoizedEnvAssets(deps), new Map());
|
|
320
442
|
}
|
|
321
443
|
export async function runDefaultLlmEngineProbe(deps = {}) {
|
|
322
444
|
const config = deps.loadConfig?.() ?? loadConfig();
|
|
323
445
|
const engineName = config.defaults?.llmEngine;
|
|
324
446
|
if (!engineName)
|
|
325
447
|
return unconfiguredEngineProbe("default-llm-engine");
|
|
326
|
-
const result = await runConfiguredEngineProbe("configured-engine", engineName, config, deps, new Map());
|
|
448
|
+
const result = await runConfiguredEngineProbe("configured-engine", engineName, config, withMemoizedEnvAssets(deps), new Map());
|
|
327
449
|
return projectSelectedEngineProbe(new Map([[engineName, result]]), engineName, "default-llm-engine");
|
|
328
450
|
}
|
|
329
451
|
/** Probe every explicitly configured engine without exposing connection or model material. */
|
|
@@ -331,7 +453,8 @@ export async function runConfiguredEnginesProbe(deps = {}) {
|
|
|
331
453
|
const config = deps.loadConfig?.() ?? loadConfig();
|
|
332
454
|
const engineNames = Object.keys(config.engines ?? {}).sort();
|
|
333
455
|
const cache = new Map();
|
|
334
|
-
const
|
|
456
|
+
const runDeps = withMemoizedEnvAssets(deps);
|
|
457
|
+
const availability = new Map(await Promise.all(engineNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, config, runDeps, cache)])));
|
|
335
458
|
return configuredEnginesProjection(engineNames, availability);
|
|
336
459
|
}
|
|
337
460
|
/**
|
|
@@ -350,7 +473,8 @@ export async function runHealthEngineProbes(deps = {}) {
|
|
|
350
473
|
.filter((name) => name !== undefined)
|
|
351
474
|
.sort();
|
|
352
475
|
const cache = new Map();
|
|
353
|
-
const
|
|
476
|
+
const runDeps = withMemoizedEnvAssets(deps);
|
|
477
|
+
const availability = new Map(await Promise.all(probeNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, effective, runDeps, cache)])));
|
|
354
478
|
return Object.freeze({
|
|
355
479
|
defaultEngine: defaultEngineName
|
|
356
480
|
? projectSelectedEngineProbe(availability, defaultEngineName, "default-engine")
|
|
@@ -361,32 +485,34 @@ export async function runHealthEngineProbes(deps = {}) {
|
|
|
361
485
|
configuredEngines: configuredEnginesProjection(explicitEngineNames, availability),
|
|
362
486
|
});
|
|
363
487
|
}
|
|
364
|
-
|
|
488
|
+
/**
|
|
489
|
+
* #950: probes the active improve strategy and returns both its
|
|
490
|
+
* `HealthCheckResult` and the typed process→engine map, computed once. See
|
|
491
|
+
* {@link runActiveImproveStrategyProbe} for the plain-`HealthCheckResult`
|
|
492
|
+
* wrapper existing callers/tests use.
|
|
493
|
+
*/
|
|
494
|
+
export function probeActiveImproveStrategy(deps = {}) {
|
|
365
495
|
const config = deps.loadConfig?.() ?? loadConfig();
|
|
366
496
|
const strategyName = config.defaults?.improveStrategy ?? "default";
|
|
497
|
+
const env = deps.env ?? process.env;
|
|
367
498
|
try {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
: undefined;
|
|
386
|
-
if (!credentialAvailable(judgmentCredential, env, judgmentApiKeyFile)) {
|
|
387
|
-
unavailableProcesses.push("triage.judgment");
|
|
388
|
-
}
|
|
389
|
-
}
|
|
499
|
+
// #957: credential availability (including the triage-judgment engine) is
|
|
500
|
+
// now derived once, inside `buildImprovePlan` itself — the same
|
|
501
|
+
// `plan.engineUnavailable` the real `improve` run reads to skip processes
|
|
502
|
+
// and populate `AkmImproveResult.skippedProcesses`. This check is a pure
|
|
503
|
+
// projection of that list rather than its own re-derivation.
|
|
504
|
+
//
|
|
505
|
+
// `allowAllDisabled: true` because a strategy left with every process
|
|
506
|
+
// disabled purely by credential unavailability must still come back as
|
|
507
|
+
// an inspectable plan here, not a thrown ConfigError — the
|
|
508
|
+
// `allRequiredUnavailable`/`fail` logic below needs `plan.engineUnavailable`
|
|
509
|
+
// to run this check's fail path instead of falling into the generic
|
|
510
|
+
// catch block's warn/unknown fallback (round 2 finding on #957). A
|
|
511
|
+
// strategy with no engine configured at all still throws (see
|
|
512
|
+
// `allowAllDisabled`'s doc comment) and keeps hitting that catch block,
|
|
513
|
+
// unchanged from before this fix.
|
|
514
|
+
const plan = resolveImprovePlan(strategyName, config, { env, allowAllDisabled: true });
|
|
515
|
+
const unavailableProcesses = plan.engineUnavailable.map((item) => item.process);
|
|
390
516
|
// #913: name the engine each process actually resolved to, so a
|
|
391
517
|
// strategy-level `engine` pin that shadows `defaults.llmEngine` is
|
|
392
518
|
// visible on `akm health` instead of requiring config archaeology.
|
|
@@ -401,37 +527,133 @@ export function runActiveImproveStrategyProbe(deps = {}) {
|
|
|
401
527
|
.sort(([a], [b]) => a.localeCompare(b))
|
|
402
528
|
.map(([process, engine]) => `${process}: "${engine}"`)
|
|
403
529
|
.join(", ");
|
|
530
|
+
// #957: fail only when the strategy's LLM-backed work would be a total
|
|
531
|
+
// no-op — every process the strategy actually enabled among the
|
|
532
|
+
// `capability: "llm"` set (see IMPROVE_PROCESS_ENGINE_CAPABILITIES) ended
|
|
533
|
+
// up unavailable. A partial failure (some processes still have a working
|
|
534
|
+
// engine) stays a `warn`, matching #914's "a credential warn stays a warn"
|
|
535
|
+
// policy for the general per-engine probes; this is the strategy-scoped
|
|
536
|
+
// "is the whole run a no-op" question instead.
|
|
537
|
+
const llmProcessNames = Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES).filter((name) => IMPROVE_PROCESS_ENGINE_CAPABILITIES[name] === "llm");
|
|
538
|
+
const requiredLlmProcessNames = llmProcessNames.filter((name) => plan.processes[name].enabled || plan.engineUnavailable.some((item) => item.process === name));
|
|
539
|
+
const availableLlmProcessNames = llmProcessNames.filter((name) => plan.processes[name].enabled);
|
|
540
|
+
const allRequiredUnavailable = requiredLlmProcessNames.length > 0 && availableLlmProcessNames.length === 0;
|
|
541
|
+
const status = unavailableProcesses.length === 0 ? "pass" : allRequiredUnavailable ? "fail" : "warn";
|
|
404
542
|
return {
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
543
|
+
check: {
|
|
544
|
+
name: "active-improve-strategy",
|
|
545
|
+
kind: "deterministic",
|
|
546
|
+
status,
|
|
547
|
+
confidence: "high",
|
|
548
|
+
message: unavailableProcesses.length === 0
|
|
549
|
+
? `Active improve strategy "${plan.strategy.name}" has available process engines${engineList ? ` (${engineList})` : ""}.`
|
|
550
|
+
: allRequiredUnavailable
|
|
551
|
+
? `Active improve strategy "${plan.strategy.name}" cannot run any LLM-backed process; the nightly run would be a no-op: ${unavailableProcesses.join(", ")}${engineList ? ` (engines: ${engineList})` : ""}.`
|
|
552
|
+
: `Active improve strategy "${plan.strategy.name}" has unavailable required credentials for: ${unavailableProcesses.join(", ")}${engineList ? ` (engines: ${engineList})` : ""}.`,
|
|
553
|
+
evidence: {
|
|
554
|
+
strategy: plan.strategy.name,
|
|
555
|
+
unavailableProcesses,
|
|
556
|
+
engines,
|
|
557
|
+
},
|
|
416
558
|
},
|
|
559
|
+
processEngines: Object.freeze({ ...engines }),
|
|
417
560
|
};
|
|
418
561
|
}
|
|
419
562
|
catch (error) {
|
|
420
563
|
const explicitlyConfigured = config.defaults?.improveStrategy !== undefined || Object.keys(config.improve?.strategies ?? {}).length > 0;
|
|
421
564
|
return {
|
|
422
|
-
|
|
565
|
+
check: {
|
|
566
|
+
name: "active-improve-strategy",
|
|
567
|
+
kind: "deterministic",
|
|
568
|
+
status: explicitlyConfigured ? "warn" : "unknown",
|
|
569
|
+
confidence: "high",
|
|
570
|
+
message: `Active improve strategy "${strategyName}" is unavailable: ${error instanceof Error ? error.message : String(error)}`,
|
|
571
|
+
evidence: { strategy: strategyName, unavailableProcesses: [] },
|
|
572
|
+
},
|
|
573
|
+
processEngines: {},
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
/** Plain-`HealthCheckResult` wrapper around {@link probeActiveImproveStrategy} for existing callers/tests. */
|
|
578
|
+
export function runActiveImproveStrategyProbe(deps = {}) {
|
|
579
|
+
return probeActiveImproveStrategy(deps).check;
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* #950: the `engine-last-used` advisory. Pure projection of context computed
|
|
583
|
+
* once in `health.ts` (`activeImproveStrategyEngines`, `engineLastUsed`,
|
|
584
|
+
* `improveRunsInLookbackWindow`) — no IO here, mirrors `thinking-control`.
|
|
585
|
+
*
|
|
586
|
+
* `unknown` (not a noisy `warn`) both when no engine is bound to an enabled
|
|
587
|
+
* process, AND when no improve run has been recorded (started) in the
|
|
588
|
+
* lookback window at all — a fresh install has never been given the chance
|
|
589
|
+
* to use its engines.
|
|
590
|
+
*/
|
|
591
|
+
function projectEngineLastUsedCheck(processEngineMap, lastUsed, improveRunsInLookbackWindow, lookbackDays) {
|
|
592
|
+
const engineProcesses = new Map();
|
|
593
|
+
for (const [process, engine] of Object.entries(processEngineMap)) {
|
|
594
|
+
const processes = engineProcesses.get(engine) ?? [];
|
|
595
|
+
processes.push(process);
|
|
596
|
+
engineProcesses.set(engine, processes);
|
|
597
|
+
}
|
|
598
|
+
const engines = [...engineProcesses.keys()].sort();
|
|
599
|
+
if (engines.length === 0) {
|
|
600
|
+
return {
|
|
601
|
+
name: "engine-last-used",
|
|
602
|
+
kind: "deterministic",
|
|
603
|
+
status: "unknown",
|
|
604
|
+
confidence: "high",
|
|
605
|
+
message: "No engine is bound to an enabled improve process.",
|
|
606
|
+
evidence: { engines: [] },
|
|
607
|
+
};
|
|
608
|
+
}
|
|
609
|
+
if (improveRunsInLookbackWindow === 0) {
|
|
610
|
+
return {
|
|
611
|
+
name: "engine-last-used",
|
|
423
612
|
kind: "deterministic",
|
|
424
|
-
status:
|
|
613
|
+
status: "unknown",
|
|
425
614
|
confidence: "high",
|
|
426
|
-
message: `
|
|
427
|
-
evidence: {
|
|
615
|
+
message: `No improve runs in the last ${lookbackDays} days.`,
|
|
616
|
+
evidence: {
|
|
617
|
+
engines: engines.map((engine) => ({ engine, processes: (engineProcesses.get(engine) ?? []).sort() })),
|
|
618
|
+
},
|
|
428
619
|
};
|
|
429
620
|
}
|
|
621
|
+
const entries = engines.map((engine) => ({
|
|
622
|
+
engine,
|
|
623
|
+
processes: (engineProcesses.get(engine) ?? []).sort(),
|
|
624
|
+
lastUsedAt: lastUsed.get(engine)?.lastUsedAt ?? null,
|
|
625
|
+
}));
|
|
626
|
+
const idle = entries.filter((entry) => entry.lastUsedAt === null);
|
|
627
|
+
const status = idle.length > 0 ? "warn" : "pass";
|
|
628
|
+
const message = idle.length > 0
|
|
629
|
+
? idle
|
|
630
|
+
.map((entry) => {
|
|
631
|
+
const quoted = entry.processes.map((process) => `"${process}"`).join(", ");
|
|
632
|
+
const noun = entry.processes.length === 1 ? "process" : "processes";
|
|
633
|
+
return `Engine "${entry.engine}" is bound to ${noun} ${quoted} but has not been used in the last ${lookbackDays} days.`;
|
|
634
|
+
})
|
|
635
|
+
.join(" ")
|
|
636
|
+
: entries
|
|
637
|
+
.map((entry) => {
|
|
638
|
+
const process = lastUsed.get(entry.engine)?.process ?? entry.processes.join(", ");
|
|
639
|
+
return `Engine "${entry.engine}" last used by "${process}" at ${entry.lastUsedAt}.`;
|
|
640
|
+
})
|
|
641
|
+
.join(" ");
|
|
642
|
+
return {
|
|
643
|
+
name: "engine-last-used",
|
|
644
|
+
kind: "deterministic",
|
|
645
|
+
status,
|
|
646
|
+
confidence: "high",
|
|
647
|
+
message,
|
|
648
|
+
evidence: { engines: entries },
|
|
649
|
+
};
|
|
430
650
|
}
|
|
431
651
|
/**
|
|
432
652
|
* Validate the immutable installed model map and optional user overlay.
|
|
433
653
|
* Installed corruption is a package defect (fail); a bad optional user file
|
|
434
|
-
*
|
|
654
|
+
* (including an `engine`-backed profile referencing a missing/broken engine,
|
|
655
|
+
* once resolved against real `config.engines` — #946) is operator-fixable
|
|
656
|
+
* configuration (warn); absence is the normal state.
|
|
435
657
|
*/
|
|
436
658
|
export function runModelMapProbe(options = {}) {
|
|
437
659
|
let installedText;
|
|
@@ -449,8 +671,20 @@ export function runModelMapProbe(options = {}) {
|
|
|
449
671
|
evidence: { installedSource: "package asset" },
|
|
450
672
|
};
|
|
451
673
|
}
|
|
674
|
+
// Load config OUTSIDE the models.json try/catch below: a broken
|
|
675
|
+
// config.json must surface as a config problem reported by the config
|
|
676
|
+
// checks, never get misreported as a models.json warning carrying the
|
|
677
|
+
// config error's own text. When config fails to load, validate the model
|
|
678
|
+
// map shape-only (no `engine` resolution), matching pre-#946 behavior.
|
|
679
|
+
let engines;
|
|
452
680
|
try {
|
|
453
|
-
|
|
681
|
+
engines = (options.loadConfig?.() ?? loadConfig()).engines;
|
|
682
|
+
}
|
|
683
|
+
catch {
|
|
684
|
+
engines = undefined;
|
|
685
|
+
}
|
|
686
|
+
try {
|
|
687
|
+
const loaded = loadModelMap({ ...options, installedText, engines });
|
|
454
688
|
return {
|
|
455
689
|
name: "model-map-files",
|
|
456
690
|
kind: "deterministic",
|
|
@@ -510,7 +744,11 @@ export function runSelectedModelAliasesProbe(deps = {}) {
|
|
|
510
744
|
}
|
|
511
745
|
let modelMap;
|
|
512
746
|
try {
|
|
513
|
-
modelMap = (deps.loadModelMap ?? loadModelMap)({
|
|
747
|
+
modelMap = (deps.loadModelMap ?? loadModelMap)({
|
|
748
|
+
env: deps.env,
|
|
749
|
+
installedText: deps.installedText,
|
|
750
|
+
engines: config.engines,
|
|
751
|
+
});
|
|
514
752
|
}
|
|
515
753
|
catch {
|
|
516
754
|
return {
|
|
@@ -583,6 +821,22 @@ export function runPendingStateMigrationsCheck(stateDbPath, deps = {}) {
|
|
|
583
821
|
evidence: { path: stateDbPath, pending },
|
|
584
822
|
};
|
|
585
823
|
}
|
|
824
|
+
/**
|
|
825
|
+
* #943: name the dominant `detail.reason` behind `task-fail-rate`'s warning
|
|
826
|
+
* when one reason accounts for at least half of the window's command-task
|
|
827
|
+
* failures — e.g. "(timeout-dominant: 9/12 command-task failures)" — so an
|
|
828
|
+
* operator sees the shape of the failure from data instead of grepping task
|
|
829
|
+
* logs. Ties break on reason name for determinism. `undefined` when there are
|
|
830
|
+
* no counted failures or no single reason reaches the 50% floor.
|
|
831
|
+
*/
|
|
832
|
+
export function dominantAgentFailureReason(counts) {
|
|
833
|
+
const entries = Object.entries(counts);
|
|
834
|
+
const total = entries.reduce((sum, [, count]) => sum + count, 0);
|
|
835
|
+
if (total === 0)
|
|
836
|
+
return undefined;
|
|
837
|
+
const [reason, count] = entries.reduce((best, entry) => entry[1] > best[1] || (entry[1] === best[1] && entry[0] < best[0]) ? entry : best);
|
|
838
|
+
return count / total >= 0.5 ? { reason, count, total } : undefined;
|
|
839
|
+
}
|
|
586
840
|
/**
|
|
587
841
|
* The ordered health-check registry. ORDER IS LOAD-BEARING: `akmHealth`
|
|
588
842
|
* iterates this array and appends to hardChecks/advisories in sequence, so the
|
|
@@ -699,7 +953,10 @@ export const HEALTH_CHECKS = [
|
|
|
699
953
|
{
|
|
700
954
|
name: "active-improve-strategy",
|
|
701
955
|
channel: "hard",
|
|
702
|
-
|
|
956
|
+
// #950: projects the context field instead of recomputing, so
|
|
957
|
+
// `engine-last-used` below can reuse the same process→engine map without
|
|
958
|
+
// a second `runActiveImproveStrategyProbe()` call.
|
|
959
|
+
run: (ctx) => ctx.activeImproveStrategy,
|
|
703
960
|
},
|
|
704
961
|
{
|
|
705
962
|
// C2 (13-bus-factor): the cron task-failure rate was computed and rendered
|
|
@@ -721,6 +978,14 @@ export const HEALTH_CHECKS = [
|
|
|
721
978
|
const worst = ctx.worstTaskFailRate;
|
|
722
979
|
const worstWarn = worst !== null && worst.rate >= TASK_FAIL_RATE_WARN;
|
|
723
980
|
const warn = aggregateWarn || worstWarn;
|
|
981
|
+
// #943: name the dominant command-task failure reason (timeout,
|
|
982
|
+
// non_zero_exit, spawn_failed, …) when the check is already warning, so
|
|
983
|
+
// "timeout-dominant" is visible without grepping task logs.
|
|
984
|
+
const agentFailureReasonCounts = ctx.agentFailureReasonCounts ?? {};
|
|
985
|
+
const dominant = warn ? dominantAgentFailureReason(agentFailureReasonCounts) : undefined;
|
|
986
|
+
const dominantSuffix = dominant
|
|
987
|
+
? ` (${dominant.reason}-dominant: ${dominant.count}/${dominant.total} command-task failures)`
|
|
988
|
+
: "";
|
|
724
989
|
let message;
|
|
725
990
|
if (ctx.taskRowCount === 0) {
|
|
726
991
|
message = `No cron tasks ran since ${ctx.since} — no task-fail-rate signal.`;
|
|
@@ -734,7 +999,7 @@ export const HEALTH_CHECKS = [
|
|
|
734
999
|
const worstPctStr = `${(worst.rate * 100).toFixed(1)}%`;
|
|
735
1000
|
parts.push(`task "${worst.taskId}" fails ${worstPctStr} of its ${worst.rows} run(s) ≥ ${thresholdPct}`);
|
|
736
1001
|
}
|
|
737
|
-
message = `Cron task fail rate warning: ${parts.join("; ")} — inspect failed runs (ok=false) for early-exit/harness errors.`;
|
|
1002
|
+
message = `Cron task fail rate warning: ${parts.join("; ")}${dominantSuffix} — inspect failed runs (ok=false) for early-exit/harness errors.`;
|
|
738
1003
|
}
|
|
739
1004
|
else {
|
|
740
1005
|
message = `Cron task fail rate ${pctStr} across ${ctx.taskRowCount} task(s) since ${ctx.since} (below ${thresholdPct} threshold).`;
|
|
@@ -750,6 +1015,9 @@ export const HEALTH_CHECKS = [
|
|
|
750
1015
|
taskRowCount: ctx.taskRowCount,
|
|
751
1016
|
threshold: TASK_FAIL_RATE_WARN,
|
|
752
1017
|
worstTaskFailRate: worst,
|
|
1018
|
+
// #943: always present (even pass/empty) so scripts reading evidence
|
|
1019
|
+
// never need to branch on check status to find the breakdown.
|
|
1020
|
+
agentFailureReasonCounts,
|
|
753
1021
|
},
|
|
754
1022
|
};
|
|
755
1023
|
},
|
|
@@ -877,4 +1145,30 @@ export const HEALTH_CHECKS = [
|
|
|
877
1145
|
};
|
|
878
1146
|
},
|
|
879
1147
|
},
|
|
1148
|
+
{
|
|
1149
|
+
// #949: advisory channel, but `kind: "deterministic"` (same as the
|
|
1150
|
+
// pre-existing task-fail-rate advisory) — a warn here DOES flip
|
|
1151
|
+
// AkmHealthResult.status to "warn" and akm health's exit code to
|
|
1152
|
+
// EXIT_HEALTH_WARN, because the overall-status computation ORs
|
|
1153
|
+
// deterministic warns across both hardChecks and advisories.
|
|
1154
|
+
name: "thinking-control",
|
|
1155
|
+
channel: "advisory",
|
|
1156
|
+
run: (ctx) => projectThinkingControlCheck(ctx.thinkingOffEngines, ctx.llmUsage, ctx.since),
|
|
1157
|
+
},
|
|
1158
|
+
{
|
|
1159
|
+
// #950: best-effort "installed vs latest release" advisory, gated behind
|
|
1160
|
+
// the same --probe/--no-probe flag as engine reachability. Computed once
|
|
1161
|
+
// in health.ts (network IO), projected here like engineProbes.
|
|
1162
|
+
name: "cli-version",
|
|
1163
|
+
channel: "advisory",
|
|
1164
|
+
run: (ctx) => ctx.versionDrift,
|
|
1165
|
+
},
|
|
1166
|
+
{
|
|
1167
|
+
// #950: registered last — order is load-bearing (see the HEALTH_CHECKS
|
|
1168
|
+
// doc comment above). Advisory channel, `kind: "deterministic"` — same
|
|
1169
|
+
// exit-code-gating rationale as thinking-control above.
|
|
1170
|
+
name: "engine-last-used",
|
|
1171
|
+
channel: "advisory",
|
|
1172
|
+
run: (ctx) => projectEngineLastUsedCheck(ctx.activeImproveStrategyEngines, ctx.engineLastUsed, ctx.improveRunsInLookbackWindow, ENGINE_LAST_USED_LOOKBACK_DAYS),
|
|
1173
|
+
},
|
|
880
1174
|
];
|