akm-cli 0.9.14 → 0.9.15-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/CHANGELOG.md +397 -0
  2. package/STABILITY.md +6 -3
  3. package/dist/assets/prompts/reflect-feedback-framing.md +1 -0
  4. package/dist/assets/prompts/reflect-llm-framed-contract.md +2 -0
  5. package/dist/assets/prompts/reflect-llm-schema-contract.md +2 -0
  6. package/dist/assets/tasks/core/improve.yml +1 -1
  7. package/dist/assets/tasks/core/index-refresh.yml +1 -1
  8. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +1 -1
  9. package/dist/assets/tasks/improve/akm-improve-catchup.yml +1 -1
  10. package/dist/assets/tasks/improve/akm-improve-consolidate.yml +1 -1
  11. package/dist/assets/tasks/improve/akm-improve-frequent.yml +1 -1
  12. package/dist/assets/tasks/improve/akm-improve-nightly.yml +1 -1
  13. package/dist/cli/retired-commands.js +0 -1
  14. package/dist/cli/shared.js +9 -0
  15. package/dist/cli/unknown-flags.js +1 -0
  16. package/dist/cli.js +3 -2
  17. package/dist/commands/config-cli.js +85 -3
  18. package/dist/commands/env/env-cli.js +1 -42
  19. package/dist/commands/env/env.js +1 -1
  20. package/dist/commands/env/secret-cli.js +1 -2
  21. package/dist/commands/health/checks.js +357 -63
  22. package/dist/commands/health/engine-usage.js +45 -0
  23. package/dist/commands/health/improve-metrics.js +18 -0
  24. package/dist/commands/health/llm-usage.js +41 -1
  25. package/dist/commands/health/plugin-staleness.js +7 -3
  26. package/dist/commands/health/version-drift.js +93 -0
  27. package/dist/commands/health/windows.js +3 -1
  28. package/dist/commands/health.js +44 -9
  29. package/dist/commands/improve/consolidate/chunking.js +4 -2
  30. package/dist/commands/improve/improve-cli.js +99 -5
  31. package/dist/commands/improve/improve-report.js +154 -0
  32. package/dist/commands/improve/improve-result-file.js +45 -33
  33. package/dist/commands/improve/improve-strategies.js +133 -3
  34. package/dist/commands/improve/improve-usage-report.js +182 -0
  35. package/dist/commands/improve/improve.js +40 -3
  36. package/dist/commands/improve/locks.js +27 -78
  37. package/dist/commands/improve/planner.js +1 -0
  38. package/dist/commands/improve/preparation.js +9 -1
  39. package/dist/commands/improve/reflect.js +44 -4
  40. package/dist/commands/models-cli.js +50 -1
  41. package/dist/commands/proposal/repository.js +8 -3
  42. package/dist/commands/proposal/validators/proposal-quality-validators.js +41 -6
  43. package/dist/commands/proposal/validators/proposal-validators.js +24 -0
  44. package/dist/commands/read/search-cli.js +38 -2
  45. package/dist/commands/read/show.js +103 -4
  46. package/dist/commands/sources/info.js +5 -1
  47. package/dist/commands/sources/self-update.js +2 -2
  48. package/dist/commands/sources/stash-cli.js +31 -0
  49. package/dist/commands/tasks/tasks-cli.js +49 -2
  50. package/dist/commands/workflow-cli.js +86 -12
  51. package/dist/core/asset/markdown-fragments.js +35 -0
  52. package/dist/core/config/config-schema.js +14 -0
  53. package/dist/core/config/config.js +302 -24
  54. package/dist/core/env-secret-ref.js +58 -5
  55. package/dist/core/errors.js +30 -0
  56. package/dist/core/improve-result.js +51 -0
  57. package/dist/core/loopback.js +17 -0
  58. package/dist/core/paths.js +11 -0
  59. package/dist/core/run-lock.js +96 -0
  60. package/dist/core/sensitive-marker-path.js +19 -0
  61. package/dist/core/state-db.js +74 -14
  62. package/dist/indexer/index-rebuild-lock.js +73 -0
  63. package/dist/indexer/index-writer-lock.js +40 -1
  64. package/dist/indexer/index-written-assets.js +21 -1
  65. package/dist/indexer/indexer.js +18 -17
  66. package/dist/indexer/materialize-embeddings.js +282 -32
  67. package/dist/indexer/search/db-search.js +49 -2
  68. package/dist/integrations/agent/engine-resolution.js +96 -6
  69. package/dist/integrations/agent/execution-definitions.js +6 -15
  70. package/dist/integrations/agent/execution-lowering.js +6 -1
  71. package/dist/integrations/agent/execution-preparation.js +1 -1
  72. package/dist/integrations/agent/model-map.js +123 -20
  73. package/dist/integrations/agent/prompts.js +40 -8
  74. package/dist/integrations/agent/runner-dispatch.js +9 -3
  75. package/dist/integrations/agent/runner.js +2 -0
  76. package/dist/llm/client.js +8 -3
  77. package/dist/llm/embedder.js +20 -8
  78. package/dist/llm/embedders/local.js +10 -2
  79. package/dist/llm/embedders/remote.js +188 -21
  80. package/dist/output/shapes/helpers.js +38 -2
  81. package/dist/output/shapes/models-list.js +16 -0
  82. package/dist/output/shapes/passthrough.js +2 -0
  83. package/dist/output/shapes.js +4 -0
  84. package/dist/output/text/command-format.js +29 -0
  85. package/dist/output/text/helpers.js +1 -1
  86. package/dist/output/text/improve-report.js +27 -0
  87. package/dist/{commands/env/marker-path.js → output/text/models.js} +4 -3
  88. package/dist/output/text/show-format.js +4 -0
  89. package/dist/output/text.js +4 -0
  90. package/dist/scripts/akm-migrate-node.js +24798 -21732
  91. package/dist/scripts/akm-migrate.js +23408 -20343
  92. package/dist/storage/repositories/improve-runs-repository.js +34 -0
  93. package/dist/storage/repositories/index-fts-repository.js +49 -6
  94. package/dist/storage/repositories/index-vec-repository.js +30 -0
  95. package/dist/storage/repositories/workflow-runs-repository.js +55 -18
  96. package/dist/tasks/backends/cron.js +14 -7
  97. package/dist/tasks/run/run-workflow-task.js +16 -0
  98. package/dist/workflows/exec/child-workflow.js +2 -2
  99. package/dist/workflows/exec/dispatch-redaction.js +21 -9
  100. package/dist/workflows/exec/run-workflow.js +6 -5
  101. package/dist/workflows/runtime/runs.js +33 -5
  102. package/docs/migration/release-notes/0.9.15.md +52 -0
  103. package/docs/migration/release-notes/README.md +4 -0
  104. package/docs/reference/cli.md +245 -29
  105. package/docs/reference/configuration.md +180 -19
  106. package/docs/reference/data-and-telemetry.md +8 -0
  107. package/docs/reference/tasks.md +16 -1
  108. package/docs/reference/workflow-schema.md +5 -1
  109. package/package.json +1 -1
  110. package/schemas/akm-config.json +8 -0
@@ -3,15 +3,19 @@
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { spawnSync } from "node:child_process";
5
5
  import { loadConfig } from "../../core/config/config.js";
6
+ import { IMPROVE_PROCESS_ENGINE_CAPABILITIES } from "../../core/config/engine-semantics.js";
7
+ import { listEnvsRecursive } from "../../core/env-secret-ref.js";
6
8
  import { ConfigError } from "../../core/errors.js";
7
9
  import { EXTRACT_INFRASTRUCTURE_SKIP_REASONS } from "../../core/improve-types.js";
8
10
  import { listPendingStateMigrations } from "../../core/state-db.js";
9
11
  import { withEngineFallback } from "../../integrations/agent/engine-fallback.js";
10
- import { lookupApiKeyFileValue, resolveEngine } from "../../integrations/agent/engine-resolution.js";
12
+ import { isLlmCredentialAvailable, resolveEngine, } from "../../integrations/agent/engine-resolution.js";
11
13
  import { executionEngineDefinitionsFromConfig } from "../../integrations/agent/execution-definitions.js";
12
14
  import { loadModelMap, mergeModelMapLayers, parseModelMapLayer, readInstalledModelMapText, resolveModelMapAlias, userModelMapPath, } from "../../integrations/agent/model-map.js";
15
+ import { listKeys } from "../env/env.js";
13
16
  import { resolveImprovePlan } from "../improve/improve-strategies.js";
14
- import { ACTIVE_RUN_WARN_MS, TASK_FAIL_RATE_WARN } from "./types.js";
17
+ import { ENGINE_LAST_USED_LOOKBACK_DAYS } from "./engine-usage.js";
18
+ import { ACTIVE_RUN_WARN_MS, TASK_FAIL_RATE_WARN, } from "./types.js";
15
19
  /** Probe one connection's reachability, once per endpoint; `undefined` when no probe seam is supplied. */
16
20
  function probeConnectionReachable(connection, deps, cache) {
17
21
  if (!deps.probeReachable)
@@ -43,15 +47,64 @@ function escalateDefaultLlmEngineFailure(result) {
43
47
  }
44
48
  /** Rolling window the `session-extraction` check reads from the extract ledger (#914). */
45
49
  export const SESSION_EXTRACTION_LEDGER_WINDOW_DAYS = 7;
46
- function credentialAvailable(credential, env, apiKeyFile) {
47
- if (credential?.required)
48
- return credential.names.some((name) => Boolean(env[name]?.trim()));
49
- // #905: an engine with no env descriptor may still require a file-backed
50
- // credentialprobe it too, rather than reporting an unreadable/empty
51
- // apiKeyFile as available just because it carries no env var names.
52
- if (apiKeyFile !== undefined)
53
- return lookupApiKeyFileValue(apiKeyFile) !== undefined;
54
- return true;
50
+ /**
51
+ * Delegates to {@link isLlmCredentialAvailable} (#953) so `akm health` and the
52
+ * improve-strategy probe (#957 builds on this same helper) share one
53
+ * non-throwing "is a credential present" check with the real dispatch
54
+ * boundary env value, file-backed (#905), or secret-store-backed (#953).
55
+ */
56
+ function credentialAvailable(credential, env, apiKeyFile, apiKeySecretRef) {
57
+ return isLlmCredentialAvailable({ credential, apiKeyFile, apiKeySecretRef }, env);
58
+ }
59
+ /**
60
+ * #950: when a required `$VAR`-style credential is missing from the shell,
61
+ * find the env asset (if any) whose key names include one of
62
+ * `credential.names` — so the warn can say "run under env/lab" instead of a
63
+ * bare "unavailable". Returns the asset's ref, or `null` when no credential,
64
+ * no candidate names, or no matching env asset. Never returns or logs the
65
+ * variable name itself (only ref/path/keys ever leave `listEnvsRecursive`,
66
+ * and only the ref is read here) — `tests/health-engine-probe.test.ts` pins
67
+ * that a credential's env-var name never appears in health's JSON output.
68
+ * Best-effort: any failure walking env assets (unreadable stash, bad config)
69
+ * degrades to `null`, never a crash. `deps.listEnvAssets` is expected to
70
+ * already be normalised (see {@link withMemoizedEnvAssets}) by the time this
71
+ * runs, so the walk happens at most once per probe run — every caller of
72
+ * this function passes deps that have already gone through
73
+ * {@link withMemoizedEnvAssets}, so there is no separate default to fall
74
+ * back to here.
75
+ */
76
+ function findSuppliedByEnvAsset(credential, deps) {
77
+ if (!credential || credential.names.length === 0)
78
+ return null;
79
+ try {
80
+ const envs = deps.listEnvAssets?.();
81
+ if (!envs)
82
+ return null;
83
+ for (const envAsset of envs) {
84
+ if (credential.names.some((name) => envAsset.keys.includes(name)))
85
+ return envAsset.ref;
86
+ }
87
+ }
88
+ catch {
89
+ // Best-effort — env-asset discovery must never break a health probe.
90
+ }
91
+ return null;
92
+ }
93
+ /**
94
+ * #950: normalise `deps.listEnvAssets` once per probe run so
95
+ * {@link findSuppliedByEnvAsset}'s env-asset walk happens at most once even
96
+ * when a run probes several engines with missing credentials — the walk is
97
+ * real IO (or config-injected IO in tests) with no reason to repeat per
98
+ * engine. Wraps whatever seam is already present (test-injected or the
99
+ * config-honouring default) in a memoising closure; called once by each of
100
+ * the four probe entry points that share {@link runConfiguredEngineProbe}
101
+ * (`runDefaultEngineProbe`, `runDefaultLlmEngineProbe`,
102
+ * `runConfiguredEnginesProbe`, `runHealthEngineProbes`).
103
+ */
104
+ function withMemoizedEnvAssets(deps) {
105
+ const listEnvAssets = deps.listEnvAssets ?? (() => listEnvsRecursive(listKeys, (deps.loadConfig ?? loadConfig)()));
106
+ let cached;
107
+ return { ...deps, listEnvAssets: () => (cached ??= listEnvAssets()) };
55
108
  }
56
109
  async function runConfiguredEngineProbe(checkName, engineName, config, deps, reachabilityCache) {
57
110
  if (!engineName) {
@@ -97,6 +150,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
97
150
  let fallback;
98
151
  let fallbackCredential;
99
152
  let fallbackApiKeyFile;
153
+ let fallbackApiKeySecretRef;
100
154
  let sdkRunner;
101
155
  const resolve = deps.resolveEngine ?? resolveEngine;
102
156
  try {
@@ -111,6 +165,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
111
165
  fallback = { kind: "llm", engine: fallbackEngine, connection: sdkRunner.fallbackConnection };
112
166
  fallbackCredential = sdkRunner.fallbackCredential;
113
167
  fallbackApiKeyFile = sdkRunner.fallbackApiKeyFile;
168
+ fallbackApiKeySecretRef = sdkRunner.fallbackApiKeySecretRef;
114
169
  }
115
170
  else if (fallbackEngine) {
116
171
  try {
@@ -119,6 +174,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
119
174
  fallback = resolved;
120
175
  fallbackCredential = resolved.credential;
121
176
  fallbackApiKeyFile = resolved.apiKeyFile;
177
+ fallbackApiKeySecretRef = resolved.apiKeySecretRef;
122
178
  }
123
179
  }
124
180
  catch {
@@ -127,12 +183,19 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
127
183
  }
128
184
  const configuredModel = configuredEngine.model;
129
185
  const effectiveModel = sdkRunner?.profile.model ?? configuredModel ?? fallback?.connection.model;
130
- const fallbackCredentialAvailable = credentialAvailable(fallbackCredential, env, fallbackApiKeyFile);
186
+ const fallbackCredentialAvailable = credentialAvailable(fallbackCredential, env, fallbackApiKeyFile, fallbackApiKeySecretRef);
187
+ const fallbackSuppliedByEnvAsset = !fallbackCredentialAvailable
188
+ ? findSuppliedByEnvAsset(fallbackCredential, deps)
189
+ : null;
131
190
  const missing = [
132
191
  !packageAvailable ? "@opencode-ai/sdk package" : undefined,
133
192
  !binaryAvailable ? `${binary} binary` : undefined,
134
193
  fallbackEngine && !fallback ? "configured fallback LLM connection" : undefined,
135
- !fallbackCredentialAvailable ? "required fallback credential" : undefined,
194
+ !fallbackCredentialAvailable
195
+ ? fallbackSuppliedByEnvAsset
196
+ ? `required fallback credential (available via env asset ${fallbackSuppliedByEnvAsset}; run under it: akm env run ${fallbackSuppliedByEnvAsset} -- ...)`
197
+ : "required fallback credential"
198
+ : undefined,
136
199
  ].filter((value) => value !== undefined);
137
200
  const sdkEvidence = {
138
201
  engine: engineName,
@@ -149,6 +212,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
149
212
  fallbackEndpoint: fallback?.connection.endpoint ?? null,
150
213
  fallbackModel: fallback?.connection.model ?? null,
151
214
  requiredCredentialAvailable: fallbackCredentialAvailable,
215
+ suppliedByEnvAsset: fallbackSuppliedByEnvAsset,
152
216
  };
153
217
  if (missing.length > 0) {
154
218
  return {
@@ -181,7 +245,7 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
181
245
  try {
182
246
  const runner = (deps.resolveEngine ?? resolveEngine)(engineName, config);
183
247
  if (runner.kind === "llm") {
184
- const requiredCredentialAvailable = credentialAvailable(runner.credential, env, runner.apiKeyFile);
248
+ const requiredCredentialAvailable = credentialAvailable(runner.credential, env, runner.apiKeyFile, runner.apiKeySecretRef);
185
249
  const llmEvidence = {
186
250
  engine: engineName,
187
251
  platform: null,
@@ -191,13 +255,20 @@ async function runConfiguredEngineProbe(checkName, engineName, config, deps, rea
191
255
  requiredCredentialAvailable,
192
256
  };
193
257
  if (!requiredCredentialAvailable) {
258
+ // #950: a missing-in-shell credential is common when the operator's
259
+ // real workflow is `akm env run env/lab -- akm improve` — name the env
260
+ // asset that supplies it (never the variable name) so the warn is
261
+ // actionable instead of looking like a broken engine.
262
+ const suppliedByEnvAsset = findSuppliedByEnvAsset(runner.credential, deps);
194
263
  return {
195
264
  name: checkName,
196
265
  kind: "deterministic",
197
266
  status: "warn",
198
267
  confidence: "high",
199
- message: `LLM engine "${engineName}" is configured, but its required credential is unavailable.`,
200
- evidence: llmEvidence,
268
+ message: suppliedByEnvAsset
269
+ ? `LLM engine "${engineName}" is configured, but its required credential is not available in this shell; env asset ${suppliedByEnvAsset} supplies it — run under it (akm env run ${suppliedByEnvAsset} -- ...).`
270
+ : `LLM engine "${engineName}" is configured, but its required credential is unavailable.`,
271
+ evidence: { ...llmEvidence, suppliedByEnvAsset },
201
272
  };
202
273
  }
203
274
  const reach = await probeConnectionReachable(runner.connection, deps, reachabilityCache);
@@ -284,6 +355,57 @@ function unconfiguredEngineProbe(name) {
284
355
  message: name === "default-llm-engine" ? "No default LLM engine is configured." : "No default engine is configured.",
285
356
  };
286
357
  }
358
+ /**
359
+ * #949: `enableThinking: false` is a request an engine's endpoint (or a
360
+ * gateway in front of it) can silently ignore — `chatCompletionAttemptOnce`
361
+ * (src/llm/client.ts) already warns to stderr whenever a response reports
362
+ * reasoning tokens despite it. This makes that same signal visible in
363
+ * `akm health` as a structured finding, purely by re-reading the window's
364
+ * `llm_usage` aggregate already threaded into the context: no extra
365
+ * completion call, no live probe — a cold local model must not be woken just
366
+ * to run `akm health`.
367
+ */
368
+ function projectThinkingControlCheck(thinkingOffEngines, llmUsage, since) {
369
+ if (thinkingOffEngines.length === 0) {
370
+ return {
371
+ name: "thinking-control",
372
+ kind: "deterministic",
373
+ status: "unknown",
374
+ confidence: "high",
375
+ message: "No configured engine sets enableThinking: false — nothing to verify.",
376
+ evidence: { engines: [] },
377
+ };
378
+ }
379
+ const engines = [...thinkingOffEngines].sort().map((engine) => {
380
+ const usage = llmUsage.byEngine[engine];
381
+ const calls = usage?.calls ?? 0;
382
+ const reasoningTokens = usage?.reasoningTokens ?? 0;
383
+ const status = calls === 0 ? "unknown" : reasoningTokens > 0 ? "warn" : "pass";
384
+ return { engine, status, calls, reasoningTokens };
385
+ });
386
+ const warning = engines.filter((e) => e.status === "warn");
387
+ const status = warning.length > 0 ? "warn" : engines.every((e) => e.status === "unknown") ? "unknown" : "pass";
388
+ let message;
389
+ if (warning.length > 0) {
390
+ message = warning
391
+ .map((e) => `LLM engine "${e.engine}" returned ${e.reasoningTokens} reasoning tokens since ${since} despite enableThinking: false — the endpoint (or a gateway in front of it) is not honouring the thinking-off control.`)
392
+ .join(" ");
393
+ }
394
+ else if (status === "unknown") {
395
+ message = `No calls were recorded for the engine(s) configured with enableThinking: false (${engines.map((e) => e.engine).join(", ")}) in the report window.`;
396
+ }
397
+ else {
398
+ message = `${engines.length} engine(s) configured with enableThinking: false (${engines.map((e) => e.engine).join(", ")}) returned no reasoning tokens in the report window.`;
399
+ }
400
+ return {
401
+ name: "thinking-control",
402
+ kind: "deterministic",
403
+ status,
404
+ confidence: "high",
405
+ message,
406
+ evidence: { engines },
407
+ };
408
+ }
287
409
  function configuredEnginesProjection(engineNames, availability) {
288
410
  if (engineNames.length === 0) {
289
411
  return {
@@ -316,14 +438,14 @@ export async function runDefaultEngineProbe(deps = {}) {
316
438
  // opencode binary DOES have a working default, and reporting otherwise would
317
439
  // contradict what `workflow run` / `task run` actually do.
318
440
  const { config } = withEngineFallback(deps.loadConfig?.() ?? loadConfig(), deps.which);
319
- return runConfiguredEngineProbe("default-engine", config.defaults?.engine, config, deps, new Map());
441
+ return runConfiguredEngineProbe("default-engine", config.defaults?.engine, config, withMemoizedEnvAssets(deps), new Map());
320
442
  }
321
443
  export async function runDefaultLlmEngineProbe(deps = {}) {
322
444
  const config = deps.loadConfig?.() ?? loadConfig();
323
445
  const engineName = config.defaults?.llmEngine;
324
446
  if (!engineName)
325
447
  return unconfiguredEngineProbe("default-llm-engine");
326
- const result = await runConfiguredEngineProbe("configured-engine", engineName, config, deps, new Map());
448
+ const result = await runConfiguredEngineProbe("configured-engine", engineName, config, withMemoizedEnvAssets(deps), new Map());
327
449
  return projectSelectedEngineProbe(new Map([[engineName, result]]), engineName, "default-llm-engine");
328
450
  }
329
451
  /** Probe every explicitly configured engine without exposing connection or model material. */
@@ -331,7 +453,8 @@ export async function runConfiguredEnginesProbe(deps = {}) {
331
453
  const config = deps.loadConfig?.() ?? loadConfig();
332
454
  const engineNames = Object.keys(config.engines ?? {}).sort();
333
455
  const cache = new Map();
334
- const availability = new Map(await Promise.all(engineNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, config, deps, cache)])));
456
+ const runDeps = withMemoizedEnvAssets(deps);
457
+ const availability = new Map(await Promise.all(engineNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, config, runDeps, cache)])));
335
458
  return configuredEnginesProjection(engineNames, availability);
336
459
  }
337
460
  /**
@@ -350,7 +473,8 @@ export async function runHealthEngineProbes(deps = {}) {
350
473
  .filter((name) => name !== undefined)
351
474
  .sort();
352
475
  const cache = new Map();
353
- const availability = new Map(await Promise.all(probeNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, effective, deps, cache)])));
476
+ const runDeps = withMemoizedEnvAssets(deps);
477
+ const availability = new Map(await Promise.all(probeNames.map(async (engine) => [engine, await runConfiguredEngineProbe("configured-engine", engine, effective, runDeps, cache)])));
354
478
  return Object.freeze({
355
479
  defaultEngine: defaultEngineName
356
480
  ? projectSelectedEngineProbe(availability, defaultEngineName, "default-engine")
@@ -361,32 +485,34 @@ export async function runHealthEngineProbes(deps = {}) {
361
485
  configuredEngines: configuredEnginesProjection(explicitEngineNames, availability),
362
486
  });
363
487
  }
364
- export function runActiveImproveStrategyProbe(deps = {}) {
488
+ /**
489
+ * #950: probes the active improve strategy and returns both its
490
+ * `HealthCheckResult` and the typed process→engine map, computed once. See
491
+ * {@link runActiveImproveStrategyProbe} for the plain-`HealthCheckResult`
492
+ * wrapper existing callers/tests use.
493
+ */
494
+ export function probeActiveImproveStrategy(deps = {}) {
365
495
  const config = deps.loadConfig?.() ?? loadConfig();
366
496
  const strategyName = config.defaults?.improveStrategy ?? "default";
497
+ const env = deps.env ?? process.env;
367
498
  try {
368
- const plan = resolveImprovePlan(strategyName, config);
369
- const env = deps.env ?? process.env;
370
- const unavailableProcesses = Object.entries(plan.processes).flatMap(([name, process]) => {
371
- if (!process.enabled || !process.runner)
372
- return [];
373
- return credentialAvailable(process.runner.credential, env, process.runner.apiKeyFile) ? [] : [name];
374
- });
375
- if (plan.triageJudgment) {
376
- const judgmentCredential = plan.triageJudgment.kind === "llm"
377
- ? plan.triageJudgment.credential
378
- : plan.triageJudgment.kind === "sdk"
379
- ? plan.triageJudgment.fallbackCredential
380
- : undefined;
381
- const judgmentApiKeyFile = plan.triageJudgment.kind === "llm"
382
- ? plan.triageJudgment.apiKeyFile
383
- : plan.triageJudgment.kind === "sdk"
384
- ? plan.triageJudgment.fallbackApiKeyFile
385
- : undefined;
386
- if (!credentialAvailable(judgmentCredential, env, judgmentApiKeyFile)) {
387
- unavailableProcesses.push("triage.judgment");
388
- }
389
- }
499
+ // #957: credential availability (including the triage-judgment engine) is
500
+ // now derived once, inside `buildImprovePlan` itself — the same
501
+ // `plan.engineUnavailable` the real `improve` run reads to skip processes
502
+ // and populate `AkmImproveResult.skippedProcesses`. This check is a pure
503
+ // projection of that list rather than its own re-derivation.
504
+ //
505
+ // `allowAllDisabled: true` because a strategy left with every process
506
+ // disabled purely by credential unavailability must still come back as
507
+ // an inspectable plan here, not a thrown ConfigError — the
508
+ // `allRequiredUnavailable`/`fail` logic below needs `plan.engineUnavailable`
509
+ // to run this check's fail path instead of falling into the generic
510
+ // catch block's warn/unknown fallback (round 2 finding on #957). A
511
+ // strategy with no engine configured at all still throws (see
512
+ // `allowAllDisabled`'s doc comment) and keeps hitting that catch block,
513
+ // unchanged from before this fix.
514
+ const plan = resolveImprovePlan(strategyName, config, { env, allowAllDisabled: true });
515
+ const unavailableProcesses = plan.engineUnavailable.map((item) => item.process);
390
516
  // #913: name the engine each process actually resolved to, so a
391
517
  // strategy-level `engine` pin that shadows `defaults.llmEngine` is
392
518
  // visible on `akm health` instead of requiring config archaeology.
@@ -401,37 +527,133 @@ export function runActiveImproveStrategyProbe(deps = {}) {
401
527
  .sort(([a], [b]) => a.localeCompare(b))
402
528
  .map(([process, engine]) => `${process}: "${engine}"`)
403
529
  .join(", ");
530
+ // #957: fail only when the strategy's LLM-backed work would be a total
531
+ // no-op — every process the strategy actually enabled among the
532
+ // `capability: "llm"` set (see IMPROVE_PROCESS_ENGINE_CAPABILITIES) ended
533
+ // up unavailable. A partial failure (some processes still have a working
534
+ // engine) stays a `warn`, matching #914's "a credential warn stays a warn"
535
+ // policy for the general per-engine probes; this is the strategy-scoped
536
+ // "is the whole run a no-op" question instead.
537
+ const llmProcessNames = Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES).filter((name) => IMPROVE_PROCESS_ENGINE_CAPABILITIES[name] === "llm");
538
+ const requiredLlmProcessNames = llmProcessNames.filter((name) => plan.processes[name].enabled || plan.engineUnavailable.some((item) => item.process === name));
539
+ const availableLlmProcessNames = llmProcessNames.filter((name) => plan.processes[name].enabled);
540
+ const allRequiredUnavailable = requiredLlmProcessNames.length > 0 && availableLlmProcessNames.length === 0;
541
+ const status = unavailableProcesses.length === 0 ? "pass" : allRequiredUnavailable ? "fail" : "warn";
404
542
  return {
405
- name: "active-improve-strategy",
406
- kind: "deterministic",
407
- status: unavailableProcesses.length === 0 ? "pass" : "warn",
408
- confidence: "high",
409
- message: unavailableProcesses.length === 0
410
- ? `Active improve strategy "${plan.strategy.name}" has available process engines${engineList ? ` (${engineList})` : ""}.`
411
- : `Active improve strategy "${plan.strategy.name}" has unavailable required credentials for: ${unavailableProcesses.join(", ")}${engineList ? ` (engines: ${engineList})` : ""}.`,
412
- evidence: {
413
- strategy: plan.strategy.name,
414
- unavailableProcesses,
415
- engines,
543
+ check: {
544
+ name: "active-improve-strategy",
545
+ kind: "deterministic",
546
+ status,
547
+ confidence: "high",
548
+ message: unavailableProcesses.length === 0
549
+ ? `Active improve strategy "${plan.strategy.name}" has available process engines${engineList ? ` (${engineList})` : ""}.`
550
+ : allRequiredUnavailable
551
+ ? `Active improve strategy "${plan.strategy.name}" cannot run any LLM-backed process; the nightly run would be a no-op: ${unavailableProcesses.join(", ")}${engineList ? ` (engines: ${engineList})` : ""}.`
552
+ : `Active improve strategy "${plan.strategy.name}" has unavailable required credentials for: ${unavailableProcesses.join(", ")}${engineList ? ` (engines: ${engineList})` : ""}.`,
553
+ evidence: {
554
+ strategy: plan.strategy.name,
555
+ unavailableProcesses,
556
+ engines,
557
+ },
416
558
  },
559
+ processEngines: Object.freeze({ ...engines }),
417
560
  };
418
561
  }
419
562
  catch (error) {
420
563
  const explicitlyConfigured = config.defaults?.improveStrategy !== undefined || Object.keys(config.improve?.strategies ?? {}).length > 0;
421
564
  return {
422
- name: "active-improve-strategy",
565
+ check: {
566
+ name: "active-improve-strategy",
567
+ kind: "deterministic",
568
+ status: explicitlyConfigured ? "warn" : "unknown",
569
+ confidence: "high",
570
+ message: `Active improve strategy "${strategyName}" is unavailable: ${error instanceof Error ? error.message : String(error)}`,
571
+ evidence: { strategy: strategyName, unavailableProcesses: [] },
572
+ },
573
+ processEngines: {},
574
+ };
575
+ }
576
+ }
577
+ /** Plain-`HealthCheckResult` wrapper around {@link probeActiveImproveStrategy} for existing callers/tests. */
578
+ export function runActiveImproveStrategyProbe(deps = {}) {
579
+ return probeActiveImproveStrategy(deps).check;
580
+ }
581
+ /**
582
+ * #950: the `engine-last-used` advisory. Pure projection of context computed
583
+ * once in `health.ts` (`activeImproveStrategyEngines`, `engineLastUsed`,
584
+ * `improveRunsInLookbackWindow`) — no IO here, mirrors `thinking-control`.
585
+ *
586
+ * `unknown` (not a noisy `warn`) both when no engine is bound to an enabled
587
+ * process, AND when no improve run has been recorded (started) in the
588
+ * lookback window at all — a fresh install has never been given the chance
589
+ * to use its engines.
590
+ */
591
+ function projectEngineLastUsedCheck(processEngineMap, lastUsed, improveRunsInLookbackWindow, lookbackDays) {
592
+ const engineProcesses = new Map();
593
+ for (const [process, engine] of Object.entries(processEngineMap)) {
594
+ const processes = engineProcesses.get(engine) ?? [];
595
+ processes.push(process);
596
+ engineProcesses.set(engine, processes);
597
+ }
598
+ const engines = [...engineProcesses.keys()].sort();
599
+ if (engines.length === 0) {
600
+ return {
601
+ name: "engine-last-used",
602
+ kind: "deterministic",
603
+ status: "unknown",
604
+ confidence: "high",
605
+ message: "No engine is bound to an enabled improve process.",
606
+ evidence: { engines: [] },
607
+ };
608
+ }
609
+ if (improveRunsInLookbackWindow === 0) {
610
+ return {
611
+ name: "engine-last-used",
423
612
  kind: "deterministic",
424
- status: explicitlyConfigured ? "warn" : "unknown",
613
+ status: "unknown",
425
614
  confidence: "high",
426
- message: `Active improve strategy "${strategyName}" is unavailable: ${error instanceof Error ? error.message : String(error)}`,
427
- evidence: { strategy: strategyName, unavailableProcesses: [] },
615
+ message: `No improve runs in the last ${lookbackDays} days.`,
616
+ evidence: {
617
+ engines: engines.map((engine) => ({ engine, processes: (engineProcesses.get(engine) ?? []).sort() })),
618
+ },
428
619
  };
429
620
  }
621
+ const entries = engines.map((engine) => ({
622
+ engine,
623
+ processes: (engineProcesses.get(engine) ?? []).sort(),
624
+ lastUsedAt: lastUsed.get(engine)?.lastUsedAt ?? null,
625
+ }));
626
+ const idle = entries.filter((entry) => entry.lastUsedAt === null);
627
+ const status = idle.length > 0 ? "warn" : "pass";
628
+ const message = idle.length > 0
629
+ ? idle
630
+ .map((entry) => {
631
+ const quoted = entry.processes.map((process) => `"${process}"`).join(", ");
632
+ const noun = entry.processes.length === 1 ? "process" : "processes";
633
+ return `Engine "${entry.engine}" is bound to ${noun} ${quoted} but has not been used in the last ${lookbackDays} days.`;
634
+ })
635
+ .join(" ")
636
+ : entries
637
+ .map((entry) => {
638
+ const process = lastUsed.get(entry.engine)?.process ?? entry.processes.join(", ");
639
+ return `Engine "${entry.engine}" last used by "${process}" at ${entry.lastUsedAt}.`;
640
+ })
641
+ .join(" ");
642
+ return {
643
+ name: "engine-last-used",
644
+ kind: "deterministic",
645
+ status,
646
+ confidence: "high",
647
+ message,
648
+ evidence: { engines: entries },
649
+ };
430
650
  }
431
651
  /**
432
652
  * Validate the immutable installed model map and optional user overlay.
433
653
  * Installed corruption is a package defect (fail); a bad optional user file
434
- * is operator-fixable configuration (warn); absence is the normal state.
654
+ * (including an `engine`-backed profile referencing a missing/broken engine,
655
+ * once resolved against real `config.engines` — #946) is operator-fixable
656
+ * configuration (warn); absence is the normal state.
435
657
  */
436
658
  export function runModelMapProbe(options = {}) {
437
659
  let installedText;
@@ -449,8 +671,20 @@ export function runModelMapProbe(options = {}) {
449
671
  evidence: { installedSource: "package asset" },
450
672
  };
451
673
  }
674
+ // Load config OUTSIDE the models.json try/catch below: a broken
675
+ // config.json must surface as a config problem reported by the config
676
+ // checks, never get misreported as a models.json warning carrying the
677
+ // config error's own text. When config fails to load, validate the model
678
+ // map shape-only (no `engine` resolution), matching pre-#946 behavior.
679
+ let engines;
452
680
  try {
453
- const loaded = loadModelMap({ ...options, installedText });
681
+ engines = (options.loadConfig?.() ?? loadConfig()).engines;
682
+ }
683
+ catch {
684
+ engines = undefined;
685
+ }
686
+ try {
687
+ const loaded = loadModelMap({ ...options, installedText, engines });
454
688
  return {
455
689
  name: "model-map-files",
456
690
  kind: "deterministic",
@@ -510,7 +744,11 @@ export function runSelectedModelAliasesProbe(deps = {}) {
510
744
  }
511
745
  let modelMap;
512
746
  try {
513
- modelMap = (deps.loadModelMap ?? loadModelMap)({ env: deps.env, installedText: deps.installedText });
747
+ modelMap = (deps.loadModelMap ?? loadModelMap)({
748
+ env: deps.env,
749
+ installedText: deps.installedText,
750
+ engines: config.engines,
751
+ });
514
752
  }
515
753
  catch {
516
754
  return {
@@ -583,6 +821,22 @@ export function runPendingStateMigrationsCheck(stateDbPath, deps = {}) {
583
821
  evidence: { path: stateDbPath, pending },
584
822
  };
585
823
  }
824
+ /**
825
+ * #943: name the dominant `detail.reason` behind `task-fail-rate`'s warning
826
+ * when one reason accounts for at least half of the window's command-task
827
+ * failures — e.g. "(timeout-dominant: 9/12 command-task failures)" — so an
828
+ * operator sees the shape of the failure from data instead of grepping task
829
+ * logs. Ties break on reason name for determinism. `undefined` when there are
830
+ * no counted failures or no single reason reaches the 50% floor.
831
+ */
832
+ export function dominantAgentFailureReason(counts) {
833
+ const entries = Object.entries(counts);
834
+ const total = entries.reduce((sum, [, count]) => sum + count, 0);
835
+ if (total === 0)
836
+ return undefined;
837
+ const [reason, count] = entries.reduce((best, entry) => entry[1] > best[1] || (entry[1] === best[1] && entry[0] < best[0]) ? entry : best);
838
+ return count / total >= 0.5 ? { reason, count, total } : undefined;
839
+ }
586
840
  /**
587
841
  * The ordered health-check registry. ORDER IS LOAD-BEARING: `akmHealth`
588
842
  * iterates this array and appends to hardChecks/advisories in sequence, so the
@@ -699,7 +953,10 @@ export const HEALTH_CHECKS = [
699
953
  {
700
954
  name: "active-improve-strategy",
701
955
  channel: "hard",
702
- run: () => runActiveImproveStrategyProbe(),
956
+ // #950: projects the context field instead of recomputing, so
957
+ // `engine-last-used` below can reuse the same process→engine map without
958
+ // a second `runActiveImproveStrategyProbe()` call.
959
+ run: (ctx) => ctx.activeImproveStrategy,
703
960
  },
704
961
  {
705
962
  // C2 (13-bus-factor): the cron task-failure rate was computed and rendered
@@ -721,6 +978,14 @@ export const HEALTH_CHECKS = [
721
978
  const worst = ctx.worstTaskFailRate;
722
979
  const worstWarn = worst !== null && worst.rate >= TASK_FAIL_RATE_WARN;
723
980
  const warn = aggregateWarn || worstWarn;
981
+ // #943: name the dominant command-task failure reason (timeout,
982
+ // non_zero_exit, spawn_failed, …) when the check is already warning, so
983
+ // "timeout-dominant" is visible without grepping task logs.
984
+ const agentFailureReasonCounts = ctx.agentFailureReasonCounts ?? {};
985
+ const dominant = warn ? dominantAgentFailureReason(agentFailureReasonCounts) : undefined;
986
+ const dominantSuffix = dominant
987
+ ? ` (${dominant.reason}-dominant: ${dominant.count}/${dominant.total} command-task failures)`
988
+ : "";
724
989
  let message;
725
990
  if (ctx.taskRowCount === 0) {
726
991
  message = `No cron tasks ran since ${ctx.since} — no task-fail-rate signal.`;
@@ -734,7 +999,7 @@ export const HEALTH_CHECKS = [
734
999
  const worstPctStr = `${(worst.rate * 100).toFixed(1)}%`;
735
1000
  parts.push(`task "${worst.taskId}" fails ${worstPctStr} of its ${worst.rows} run(s) ≥ ${thresholdPct}`);
736
1001
  }
737
- message = `Cron task fail rate warning: ${parts.join("; ")} — inspect failed runs (ok=false) for early-exit/harness errors.`;
1002
+ message = `Cron task fail rate warning: ${parts.join("; ")}${dominantSuffix} — inspect failed runs (ok=false) for early-exit/harness errors.`;
738
1003
  }
739
1004
  else {
740
1005
  message = `Cron task fail rate ${pctStr} across ${ctx.taskRowCount} task(s) since ${ctx.since} (below ${thresholdPct} threshold).`;
@@ -750,6 +1015,9 @@ export const HEALTH_CHECKS = [
750
1015
  taskRowCount: ctx.taskRowCount,
751
1016
  threshold: TASK_FAIL_RATE_WARN,
752
1017
  worstTaskFailRate: worst,
1018
+ // #943: always present (even pass/empty) so scripts reading evidence
1019
+ // never need to branch on check status to find the breakdown.
1020
+ agentFailureReasonCounts,
753
1021
  },
754
1022
  };
755
1023
  },
@@ -877,4 +1145,30 @@ export const HEALTH_CHECKS = [
877
1145
  };
878
1146
  },
879
1147
  },
1148
+ {
1149
+ // #949: advisory channel, but `kind: "deterministic"` (same as the
1150
+ // pre-existing task-fail-rate advisory) — a warn here DOES flip
1151
+ // AkmHealthResult.status to "warn" and akm health's exit code to
1152
+ // EXIT_HEALTH_WARN, because the overall-status computation ORs
1153
+ // deterministic warns across both hardChecks and advisories.
1154
+ name: "thinking-control",
1155
+ channel: "advisory",
1156
+ run: (ctx) => projectThinkingControlCheck(ctx.thinkingOffEngines, ctx.llmUsage, ctx.since),
1157
+ },
1158
+ {
1159
+ // #950: best-effort "installed vs latest release" advisory, gated behind
1160
+ // the same --probe/--no-probe flag as engine reachability. Computed once
1161
+ // in health.ts (network IO), projected here like engineProbes.
1162
+ name: "cli-version",
1163
+ channel: "advisory",
1164
+ run: (ctx) => ctx.versionDrift,
1165
+ },
1166
+ {
1167
+ // #950: registered last — order is load-bearing (see the HEALTH_CHECKS
1168
+ // doc comment above). Advisory channel, `kind: "deterministic"` — same
1169
+ // exit-code-gating rationale as thinking-control above.
1170
+ name: "engine-last-used",
1171
+ channel: "advisory",
1172
+ run: (ctx) => projectEngineLastUsedCheck(ctx.activeImproveStrategyEngines, ctx.engineLastUsed, ctx.improveRunsInLookbackWindow, ENGINE_LAST_USED_LOOKBACK_DAYS),
1173
+ },
880
1174
  ];