akm-cli 0.9.14 → 0.9.15-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/CHANGELOG.md +397 -0
  2. package/STABILITY.md +6 -3
  3. package/dist/assets/prompts/reflect-feedback-framing.md +1 -0
  4. package/dist/assets/prompts/reflect-llm-framed-contract.md +2 -0
  5. package/dist/assets/prompts/reflect-llm-schema-contract.md +2 -0
  6. package/dist/assets/tasks/core/improve.yml +1 -1
  7. package/dist/assets/tasks/core/index-refresh.yml +1 -1
  8. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +1 -1
  9. package/dist/assets/tasks/improve/akm-improve-catchup.yml +1 -1
  10. package/dist/assets/tasks/improve/akm-improve-consolidate.yml +1 -1
  11. package/dist/assets/tasks/improve/akm-improve-frequent.yml +1 -1
  12. package/dist/assets/tasks/improve/akm-improve-nightly.yml +1 -1
  13. package/dist/cli/retired-commands.js +0 -1
  14. package/dist/cli/shared.js +9 -0
  15. package/dist/cli/unknown-flags.js +1 -0
  16. package/dist/cli.js +3 -2
  17. package/dist/commands/config-cli.js +85 -3
  18. package/dist/commands/env/env-cli.js +1 -42
  19. package/dist/commands/env/env.js +1 -1
  20. package/dist/commands/env/secret-cli.js +1 -2
  21. package/dist/commands/health/checks.js +357 -63
  22. package/dist/commands/health/engine-usage.js +45 -0
  23. package/dist/commands/health/improve-metrics.js +18 -0
  24. package/dist/commands/health/llm-usage.js +41 -1
  25. package/dist/commands/health/plugin-staleness.js +7 -3
  26. package/dist/commands/health/version-drift.js +93 -0
  27. package/dist/commands/health/windows.js +3 -1
  28. package/dist/commands/health.js +44 -9
  29. package/dist/commands/improve/consolidate/chunking.js +4 -2
  30. package/dist/commands/improve/improve-cli.js +99 -5
  31. package/dist/commands/improve/improve-report.js +154 -0
  32. package/dist/commands/improve/improve-result-file.js +45 -33
  33. package/dist/commands/improve/improve-strategies.js +133 -3
  34. package/dist/commands/improve/improve-usage-report.js +182 -0
  35. package/dist/commands/improve/improve.js +40 -3
  36. package/dist/commands/improve/locks.js +27 -78
  37. package/dist/commands/improve/planner.js +1 -0
  38. package/dist/commands/improve/preparation.js +9 -1
  39. package/dist/commands/improve/reflect.js +44 -4
  40. package/dist/commands/models-cli.js +50 -1
  41. package/dist/commands/proposal/repository.js +8 -3
  42. package/dist/commands/proposal/validators/proposal-quality-validators.js +41 -6
  43. package/dist/commands/proposal/validators/proposal-validators.js +24 -0
  44. package/dist/commands/read/search-cli.js +38 -2
  45. package/dist/commands/read/show.js +103 -4
  46. package/dist/commands/sources/info.js +5 -1
  47. package/dist/commands/sources/self-update.js +2 -2
  48. package/dist/commands/sources/stash-cli.js +31 -0
  49. package/dist/commands/tasks/tasks-cli.js +49 -2
  50. package/dist/commands/workflow-cli.js +86 -12
  51. package/dist/core/asset/markdown-fragments.js +35 -0
  52. package/dist/core/config/config-schema.js +14 -0
  53. package/dist/core/config/config.js +302 -24
  54. package/dist/core/env-secret-ref.js +58 -5
  55. package/dist/core/errors.js +30 -0
  56. package/dist/core/improve-result.js +51 -0
  57. package/dist/core/loopback.js +17 -0
  58. package/dist/core/paths.js +11 -0
  59. package/dist/core/run-lock.js +96 -0
  60. package/dist/core/sensitive-marker-path.js +19 -0
  61. package/dist/core/state-db.js +74 -14
  62. package/dist/indexer/index-rebuild-lock.js +73 -0
  63. package/dist/indexer/index-writer-lock.js +40 -1
  64. package/dist/indexer/index-written-assets.js +21 -1
  65. package/dist/indexer/indexer.js +18 -17
  66. package/dist/indexer/materialize-embeddings.js +282 -32
  67. package/dist/indexer/search/db-search.js +49 -2
  68. package/dist/integrations/agent/engine-resolution.js +96 -6
  69. package/dist/integrations/agent/execution-definitions.js +6 -15
  70. package/dist/integrations/agent/execution-lowering.js +6 -1
  71. package/dist/integrations/agent/execution-preparation.js +1 -1
  72. package/dist/integrations/agent/model-map.js +123 -20
  73. package/dist/integrations/agent/prompts.js +40 -8
  74. package/dist/integrations/agent/runner-dispatch.js +9 -3
  75. package/dist/integrations/agent/runner.js +2 -0
  76. package/dist/llm/client.js +8 -3
  77. package/dist/llm/embedder.js +20 -8
  78. package/dist/llm/embedders/local.js +10 -2
  79. package/dist/llm/embedders/remote.js +188 -21
  80. package/dist/output/shapes/helpers.js +38 -2
  81. package/dist/output/shapes/models-list.js +16 -0
  82. package/dist/output/shapes/passthrough.js +2 -0
  83. package/dist/output/shapes.js +4 -0
  84. package/dist/output/text/command-format.js +29 -0
  85. package/dist/output/text/helpers.js +1 -1
  86. package/dist/output/text/improve-report.js +27 -0
  87. package/dist/{commands/env/marker-path.js → output/text/models.js} +4 -3
  88. package/dist/output/text/show-format.js +4 -0
  89. package/dist/output/text.js +4 -0
  90. package/dist/scripts/akm-migrate-node.js +24798 -21732
  91. package/dist/scripts/akm-migrate.js +23408 -20343
  92. package/dist/storage/repositories/improve-runs-repository.js +34 -0
  93. package/dist/storage/repositories/index-fts-repository.js +49 -6
  94. package/dist/storage/repositories/index-vec-repository.js +30 -0
  95. package/dist/storage/repositories/workflow-runs-repository.js +55 -18
  96. package/dist/tasks/backends/cron.js +14 -7
  97. package/dist/tasks/run/run-workflow-task.js +16 -0
  98. package/dist/workflows/exec/child-workflow.js +2 -2
  99. package/dist/workflows/exec/dispatch-redaction.js +21 -9
  100. package/dist/workflows/exec/run-workflow.js +6 -5
  101. package/dist/workflows/runtime/runs.js +33 -5
  102. package/docs/migration/release-notes/0.9.15.md +52 -0
  103. package/docs/migration/release-notes/README.md +4 -0
  104. package/docs/reference/cli.md +245 -29
  105. package/docs/reference/configuration.md +180 -19
  106. package/docs/reference/data-and-telemetry.md +8 -0
  107. package/docs/reference/tasks.md +16 -1
  108. package/docs/reference/workflow-schema.md +5 -1
  109. package/package.json +1 -1
  110. package/schemas/akm-config.json +8 -0
@@ -0,0 +1,154 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * #944 Layer 2 — `akm improve report`. Reads one (or several, `--since`)
6
+ * `improve_runs` rows and returns the same `usageReport` shape
7
+ * `finalizeImproveResult` persists on a live run (`improve-usage-report.ts`),
8
+ * so the command and the end-of-run stderr table stay byte-identical in
9
+ * structure. A row with no persisted `usageReport` degrades rather than
10
+ * errors, per AGENTS.md's "reader must tolerate data older releases wrote":
11
+ * its cross-tab is recomputed from the run's own `llm_usage` events and a
12
+ * `notes` entry explains why — either the row predates #944, or (per
13
+ * `recordTerminatedImproveRun` in improve-result-file.ts) the run was
14
+ * terminated before it ever reached `finalizeImproveResult`. The `noCalls`
15
+ * half is never fabricated for either case.
16
+ */
17
+ import { NotFoundError, UsageError } from "../../core/errors.js";
18
+ import { readEvents } from "../../core/events.js";
19
+ import { decodeImproveResult } from "../../core/improve-result.js";
20
+ import { withStateDb } from "../../core/state-db.js";
21
+ import { LLM_USAGE_EVENT } from "../../llm/usage-persist.js";
22
+ import { getImproveRunById, getLatestImproveRun, queryImproveRuns, } from "../../storage/repositories/improve-runs-repository.js";
23
+ import { parseHealthSince } from "../health.js";
24
+ import { summarizeLlmUsageCrossTab } from "../health/llm-usage.js";
25
+ const PRE_USAGE_REPORT_NOTE = "eligibility reasons unavailable for runs recorded before 0.9.15";
26
+ const TERMINATED_RUN_NOTE = "run terminated before completion; no usage report was recorded";
27
+ /**
28
+ * Recompute the cross-tab from this run's own `llm_usage` events (a pre-#944
29
+ * row has no persisted one). `eventsCtx` threads the state.db connection
30
+ * `runImproveReportQuery` already holds via `withStateDb`, so a `--since`
31
+ * window with several pre-0.9.15 or undecodable rows reads through that one
32
+ * open connection instead of each row opening and closing its own.
33
+ */
34
+ function crossTabFromEvents(startedAt, completedAt, eventsCtx) {
35
+ const until = completedAt ?? new Date().toISOString();
36
+ const events = readEvents({ since: startedAt, type: LLM_USAGE_EVENT }, eventsCtx).events.filter((event) => new Date(event.ts ?? startedAt).getTime() < new Date(until).getTime());
37
+ return summarizeLlmUsageCrossTab(events);
38
+ }
39
+ /** One row's usage report — persisted (common case) or recomputed (pre-#944 row / terminated run / undecodable row). */
40
+ function usageReportForRow(row, eventsCtx) {
41
+ try {
42
+ const decoded = decodeImproveResult(row.result_json);
43
+ if (decoded.envelope.usageReport) {
44
+ return {
45
+ usageReport: {
46
+ byProcessEngineModel: [...decoded.envelope.usageReport.byProcessEngineModel],
47
+ noCalls: [...decoded.envelope.usageReport.noCalls],
48
+ },
49
+ strategy: decoded.strategy,
50
+ };
51
+ }
52
+ return {
53
+ usageReport: {
54
+ byProcessEngineModel: crossTabFromEvents(row.started_at, row.completed_at, eventsCtx),
55
+ noCalls: [],
56
+ },
57
+ strategy: decoded.strategy,
58
+ note: decoded.envelope.terminated ? TERMINATED_RUN_NOTE : PRE_USAGE_REPORT_NOTE,
59
+ };
60
+ }
61
+ catch {
62
+ // A row whose result_json this build cannot decode still has real
63
+ // llm_usage events on state.db — degrade to the recomputed cross-tab
64
+ // rather than excluding the run entirely.
65
+ return {
66
+ usageReport: {
67
+ byProcessEngineModel: crossTabFromEvents(row.started_at, row.completed_at, eventsCtx),
68
+ noCalls: [],
69
+ },
70
+ note: PRE_USAGE_REPORT_NOTE,
71
+ };
72
+ }
73
+ }
74
+ function mergeCrossTabRow(merged, row) {
75
+ const key = `${row.process}:${row.engine}:${row.model}`;
76
+ const acc = merged.get(key);
77
+ if (!acc) {
78
+ merged.set(key, { ...row });
79
+ return;
80
+ }
81
+ acc.calls += row.calls;
82
+ acc.failures += row.failures;
83
+ acc.promptTokens += row.promptTokens;
84
+ acc.completionTokens += row.completionTokens;
85
+ acc.totalTokens += row.totalTokens;
86
+ acc.reasoningTokens += row.reasoningTokens;
87
+ acc.totalDurationMs += row.totalDurationMs;
88
+ }
89
+ /**
90
+ * Resolve `akm improve report`'s target run(s) and build its `usageReport`.
91
+ * `runId` and `since` are mutually exclusive; neither given selects the most
92
+ * recent non-dry-run.
93
+ */
94
+ export function runImproveReportQuery(options) {
95
+ if (options.runId !== undefined && options.since !== undefined) {
96
+ throw new UsageError("akm improve report: --run and --since are mutually exclusive.", "INVALID_FLAG_VALUE");
97
+ }
98
+ return withStateDb((db) => {
99
+ // Thread this already-open connection into every reader below, so a
100
+ // `--since` window with several pre-0.9.15 or undecodable rows reads
101
+ // through it instead of each row opening and closing its own.
102
+ const eventsCtx = { db };
103
+ if (options.since !== undefined) {
104
+ const sinceIso = parseHealthSince(options.since);
105
+ const rows = queryImproveRuns(db, sinceIso);
106
+ const merged = new Map();
107
+ const calledEverByProcess = new Set();
108
+ const lastReasonByProcess = new Map();
109
+ const notes = new Set();
110
+ for (const row of rows) {
111
+ const { usageReport, note } = usageReportForRow(row, eventsCtx);
112
+ for (const crossTabRow of usageReport.byProcessEngineModel) {
113
+ mergeCrossTabRow(merged, crossTabRow);
114
+ if (crossTabRow.calls > 0)
115
+ calledEverByProcess.add(crossTabRow.process);
116
+ }
117
+ for (const noCallRow of usageReport.noCalls) {
118
+ // `rows` is newest-first (ORDER BY started_at DESC): keep only the
119
+ // first (most recent) reason seen per process, never overwrite it
120
+ // with an older run's reason.
121
+ if (!lastReasonByProcess.has(noCallRow.process)) {
122
+ lastReasonByProcess.set(noCallRow.process, { engine: noCallRow.engine, reason: noCallRow.reason });
123
+ }
124
+ }
125
+ if (note)
126
+ notes.add(note);
127
+ }
128
+ const noCalls = [...lastReasonByProcess.entries()]
129
+ .filter(([process]) => !calledEverByProcess.has(process))
130
+ .map(([process, { engine, reason }]) => ({ process, ...(engine ? { engine } : {}), reason }));
131
+ return {
132
+ mode: "since",
133
+ since: sinceIso,
134
+ runIds: rows.map((r) => r.id),
135
+ usageReport: { byProcessEngineModel: [...merged.values()], noCalls },
136
+ ...(notes.size > 0 ? { notes: [...notes] } : {}),
137
+ };
138
+ }
139
+ const row = options.runId !== undefined ? getImproveRunById(db, options.runId) : getLatestImproveRun(db);
140
+ if (!row) {
141
+ throw new NotFoundError(options.runId !== undefined
142
+ ? `akm improve report: no improve run found with id "${options.runId}".`
143
+ : "akm improve report: no improve runs recorded yet.", "IMPROVE_RUN_NOT_FOUND");
144
+ }
145
+ const { usageReport, strategy, note } = usageReportForRow(row, eventsCtx);
146
+ return {
147
+ mode: "run",
148
+ runId: row.id,
149
+ ...(strategy !== undefined ? { strategy } : {}),
150
+ usageReport,
151
+ ...(note ? { notes: [note] } : {}),
152
+ };
153
+ });
154
+ }
@@ -27,7 +27,7 @@
27
27
  import crypto from "node:crypto";
28
28
  import { decodeImproveResult } from "../../core/improve-result.js";
29
29
  import { redactSensitiveValue } from "../../core/redaction.js";
30
- import { withStateDb } from "../../core/state-db.js";
30
+ import { withImmediateTransaction, withStateDb } from "../../core/state-db.js";
31
31
  import { recordImproveRun } from "../../storage/repositories/improve-runs-repository.js";
32
32
  /**
33
33
  * Build a stable run-id for a single improve invocation.
@@ -63,18 +63,25 @@ export function recordImproveRunResult(stashDir, runId, result, startedAt, sensi
63
63
  // so started_at != completed_at even on older call sites.
64
64
  const resolvedStartedAt = startedAt ??
65
65
  runId.slice(0, 24).replace(/^(\d{4}-\d{2}-\d{2}T)(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, "$1$2:$3:$4.$5Z");
66
- recordImproveRun(db, {
67
- id: runId,
68
- startedAt: resolvedStartedAt,
69
- completedAt,
70
- stashDir,
71
- dryRun: Boolean(result.dryRun),
72
- strategy: redactSensitiveValue(decoded.strategy, sensitiveValues),
73
- scopeMode: result.scope?.mode ?? "all",
74
- scopeValue: persistedResult.scope?.value ?? null,
75
- guidance: persistedResult.guidance ?? null,
76
- ok: Boolean(result.ok),
77
- result: persistedResult,
66
+ // #948: route through the shared BEGIN IMMEDIATE retry/reclassify helper
67
+ // instead of a bare write — this INSERT used to rely solely on the
68
+ // connection's 30s busy_timeout, with no retry and no friendly
69
+ // reclassification on exhaustion, so a raw "database is locked" from
70
+ // improve's own ledger write could reach the CLI as exit 70.
71
+ withImmediateTransaction(db, () => {
72
+ recordImproveRun(db, {
73
+ id: runId,
74
+ startedAt: resolvedStartedAt,
75
+ completedAt,
76
+ stashDir,
77
+ dryRun: Boolean(result.dryRun),
78
+ strategy: redactSensitiveValue(decoded.strategy, sensitiveValues),
79
+ scopeMode: result.scope?.mode ?? "all",
80
+ scopeValue: persistedResult.scope?.value ?? null,
81
+ guidance: persistedResult.guidance ?? null,
82
+ ok: Boolean(result.ok),
83
+ result: persistedResult,
84
+ });
78
85
  });
79
86
  });
80
87
  }
@@ -116,27 +123,32 @@ export function recordTerminatedImproveRun(stashDir, runId, startedAt, reason, c
116
123
  },
117
124
  }, ctx.sensitiveValues ?? []);
118
125
  withStateDb((db) => {
119
- recordImproveRun(db, {
120
- id: runId,
121
- startedAt,
122
- completedAt,
123
- stashDir,
124
- dryRun: Boolean(ctx.dryRun),
125
- strategy: persistedStrategy,
126
- scopeMode: ctx.scopeMode ?? "all",
127
- scopeValue: persistedScopeValue ?? null,
128
- guidance: null,
129
- ok: false,
130
- result: minimalResult,
131
- metadata: {
132
- terminated: {
133
- reason: persistedReason,
134
- at: completedAt,
135
- ...(ctx.errorMessage
136
- ? { errorMessage: redactSensitiveValue(ctx.errorMessage, ctx.sensitiveValues ?? []) }
137
- : {}),
126
+ // #948: same rationale as recordImproveRunResult above — this is the
127
+ // signal-handler/terminated-run write path, which must not itself raise
128
+ // a raw "database is locked" while trying to record why the run ended.
129
+ withImmediateTransaction(db, () => {
130
+ recordImproveRun(db, {
131
+ id: runId,
132
+ startedAt,
133
+ completedAt,
134
+ stashDir,
135
+ dryRun: Boolean(ctx.dryRun),
136
+ strategy: persistedStrategy,
137
+ scopeMode: ctx.scopeMode ?? "all",
138
+ scopeValue: persistedScopeValue ?? null,
139
+ guidance: null,
140
+ ok: false,
141
+ result: minimalResult,
142
+ metadata: {
143
+ terminated: {
144
+ reason: persistedReason,
145
+ at: completedAt,
146
+ ...(ctx.errorMessage
147
+ ? { errorMessage: redactSensitiveValue(ctx.errorMessage, ctx.sensitiveValues ?? []) }
148
+ : {}),
149
+ },
138
150
  },
139
- },
151
+ });
140
152
  });
141
153
  });
142
154
  }
@@ -14,6 +14,7 @@ import { ImproveProfileConfigSchema } from "../../core/config/config-schema.js";
14
14
  import { deepMergeConfig } from "../../core/config/deep-merge.js";
15
15
  import { BUILTIN_IMPROVE_STRATEGY_NAMES, IMPROVE_PROCESS_ENGINE_CAPABILITIES, } from "../../core/config/engine-semantics.js";
16
16
  import { ConfigError } from "../../core/errors.js";
17
+ import { describeLlmCredentialAvailability } from "../../integrations/agent/engine-resolution.js";
17
18
  import { applyAutonomyGate } from "./autonomy-gate.js";
18
19
  import { resolveImproveExecution, resolveImproveLlmExecution } from "./execution.js";
19
20
  export const DEFAULT_ALLOWED_TYPES = {
@@ -64,6 +65,59 @@ export function resolveImproveStrategy(name, config) {
64
65
  const resolved = deepMergeConfig(baseStrategy, (userStrategies[selectedName] ?? {}));
65
66
  return { name: selectedName, config: ImproveProfileConfigSchema.parse(resolved) };
66
67
  }
68
+ /**
69
+ * Project a resolved improve plan into one routing row per
70
+ * {@link IMPROVE_PROCESS_ENGINE_CAPABILITIES} name, plus a `"triage.judgment"`
71
+ * pseudo-row when the strategy configures a judgment engine. Pure — no I/O,
72
+ * no re-resolution. Shared by `improve --dry-run`'s `plan.processes` (#947)
73
+ * and `akm health`'s post-run reporting (#944); health's own
74
+ * `active-improve-strategy` check keeps its existing `engines` shape.
75
+ */
76
+ export function projectResolvedProcessRouting(plan) {
77
+ const unavailableByProcess = new Map(plan.engineUnavailable.map((item) => [item.process, item]));
78
+ const rows = [];
79
+ for (const processName of Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES)) {
80
+ const process = plan.processes[processName];
81
+ const unavailable = unavailableByProcess.get(processName);
82
+ rows.push({
83
+ process: processName,
84
+ enabled: process.enabled,
85
+ ...(process.runner
86
+ ? { engine: process.runner.engine, model: process.runner.connection.model, engineKind: process.runner.kind }
87
+ : // #800/#957 round 3 — a credential-unavailable process never carries a
88
+ // runner, but its structurally resolved engine/model is still worth
89
+ // showing in the routing table (dry-run preview, health probe).
90
+ unavailable?.engine
91
+ ? {
92
+ engine: unavailable.engine,
93
+ ...(unavailable.model ? { model: unavailable.model } : {}),
94
+ engineKind: "llm",
95
+ }
96
+ : {}),
97
+ notices: process.notices ?? [],
98
+ ...(unavailable ? { unavailable: { configKey: unavailable.configKey, reason: unavailable.reason } } : {}),
99
+ });
100
+ }
101
+ if (plan.strategy.config.processes?.triage?.judgment?.enabled === true) {
102
+ const judgmentUnavailable = unavailableByProcess.get("triage.judgment");
103
+ rows.push({
104
+ process: "triage.judgment",
105
+ enabled: plan.triageJudgment !== null,
106
+ ...(plan.triageJudgment
107
+ ? {
108
+ engine: plan.triageJudgment.engine,
109
+ engineKind: plan.triageJudgment.kind,
110
+ ...(plan.triageJudgment.kind === "llm" ? { model: plan.triageJudgment.connection.model } : {}),
111
+ }
112
+ : {}),
113
+ notices: plan.triageJudgmentNotices ?? [],
114
+ ...(judgmentUnavailable
115
+ ? { unavailable: { configKey: judgmentUnavailable.configKey, reason: judgmentUnavailable.reason } }
116
+ : {}),
117
+ });
118
+ }
119
+ return rows;
120
+ }
67
121
  function cloneAndFreeze(value) {
68
122
  const clone = structuredClone(value);
69
123
  const freeze = (item) => {
@@ -86,9 +140,23 @@ export function resolveImprovePlan(name, config, options = {}) {
86
140
  const strategy = { name: selected.name, config: gatedStrategyConfig };
87
141
  return { ...buildImprovePlan(strategy, config, options), autonomyGated: gated };
88
142
  }
143
+ /** Describe why a resolved-but-uncredentialed runner belongs in `engineUnavailable` (#957). */
144
+ function credentialUnavailableReason(engineName, status) {
145
+ return `requires a credential that is not available in this process's environment (engine "${engineName}": ${status.reason})`;
146
+ }
89
147
  function buildImprovePlan(strategy, config, options) {
148
+ const env = options.env ?? process.env;
90
149
  const processes = {};
91
150
  const engineUnavailable = [];
151
+ // #957 round 2 — distinguishes "credential unavailable" (a working engine
152
+ // whose secret isn't in this environment) from "no engine selected at all"
153
+ // among the pushes below, so `allowAllDisabled` can bypass the ConfigError
154
+ // only for the former. The latter is the guard's original, pre-#957
155
+ // condition and must keep hard-aborting for every caller, including a
156
+ // health probe with `allowAllDisabled` set — a totally unconfigured
157
+ // install is not the credential-in-the-wrong-environment case this option
158
+ // exists for.
159
+ let anyEngineNotConfigured = false;
92
160
  for (const processName of Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES)) {
93
161
  const sourceProcessConfig = strategy.config.processes?.[processName] ?? {};
94
162
  const enabled = sourceProcessConfig.enabled === true;
@@ -101,6 +169,15 @@ function buildImprovePlan(strategy, config, options) {
101
169
  // Validation itself is structural and always runs. Only its optional repair
102
170
  // step needs a model, so disabling repair must not create an LLM preflight.
103
171
  const skipsRepairEngine = processName === "validation" && options.repairValidationFailures === false;
172
+ // #957: a resolved-but-uncredentialed runner is folded into the same
173
+ // "no engine" branch below — set here so the shared push/continue block
174
+ // can tell the two apart in its reason text.
175
+ let credentialUnavailableMessage;
176
+ // #800/#957 round 3 — the structurally resolved engine/model/contextLength
177
+ // for the credential-unavailable case only, so a dry-run preview can read
178
+ // what resolution decided (e.g. the consolidation chunk-size estimate)
179
+ // without the runner ever carrying a credential-less connection forward.
180
+ let credentialUnavailableRouting;
104
181
  if (!skipsRepairEngine) {
105
182
  const resolved = resolveImproveLlmExecution({
106
183
  config,
@@ -110,13 +187,32 @@ function buildImprovePlan(strategy, config, options) {
110
187
  });
111
188
  runner = resolved?.runner ?? null;
112
189
  notices = resolved?.notices ?? [];
190
+ if (runner) {
191
+ const credentialStatus = describeLlmCredentialAvailability(runner, env);
192
+ if (!credentialStatus.available) {
193
+ credentialUnavailableMessage = credentialUnavailableReason(runner.engine, credentialStatus);
194
+ credentialUnavailableRouting = {
195
+ engine: runner.engine,
196
+ ...(runner.connection.model !== undefined ? { model: runner.connection.model } : {}),
197
+ ...(runner.connection.contextLength !== undefined
198
+ ? { contextLength: runner.connection.contextLength }
199
+ : {}),
200
+ };
201
+ runner = null;
202
+ notices = [];
203
+ }
204
+ }
113
205
  }
114
206
  if (!runner && !skipsRepairEngine) {
115
207
  const configKey = `improve.strategies.${strategy.name}.processes.${processName}.engine`;
208
+ if (!credentialUnavailableMessage)
209
+ anyEngineNotConfigured = true;
116
210
  engineUnavailable.push({
117
211
  process: processName,
118
212
  configKey,
119
- reason: `requires an LLM engine that is not configured. Set defaults.llmEngine or ${configKey}`,
213
+ reason: credentialUnavailableMessage ??
214
+ `requires an LLM engine that is not configured. Set defaults.llmEngine or ${configKey}`,
215
+ ...credentialUnavailableRouting,
120
216
  });
121
217
  processes[processName] = Object.freeze({
122
218
  enabled: false,
@@ -134,7 +230,9 @@ function buildImprovePlan(strategy, config, options) {
134
230
  ...(notices.length > 0 ? { notices: cloneAndFreeze(notices) } : {}),
135
231
  });
136
232
  }
137
- if (engineUnavailable.length > 0 && !Object.values(processes).some((process) => process.enabled)) {
233
+ if (engineUnavailable.length > 0 &&
234
+ !Object.values(processes).some((process) => process.enabled) &&
235
+ (!options.allowAllDisabled || anyEngineNotConfigured)) {
138
236
  const names = engineUnavailable.map((item) => `"${item.process}"`).join(", ");
139
237
  throw new ConfigError(`No improve process can run: ${names} ${engineUnavailable.length === 1 ? "requires" : "require"} an LLM engine that is not configured. Set defaults.llmEngine, or the per-process engine key named for each.`, "LLM_NOT_CONFIGURED");
140
238
  }
@@ -149,7 +247,7 @@ function buildImprovePlan(strategy, config, options) {
149
247
  processName: "triage-judgment",
150
248
  })
151
249
  : null;
152
- const triageJudgment = triageJudgmentResolution?.runner ?? null;
250
+ let triageJudgment = triageJudgmentResolution?.runner ?? null;
153
251
  const effectiveJudgmentLlm = triage?.judgment?.llm ?? triage?.llm ?? strategy.config.llm;
154
252
  if (triageJudgment && triageJudgment.kind !== "llm" && effectiveJudgmentLlm) {
155
253
  throw new ConfigError(`Triage judgment engine "${triageJudgment.engine ?? "unknown"}" is an agent engine and cannot receive llm overrides.`, "INVALID_CONFIG_FILE");
@@ -157,6 +255,38 @@ function buildImprovePlan(strategy, config, options) {
157
255
  if (processes.triage.enabled && judgmentEnabled && !triageJudgment) {
158
256
  throw new ConfigError(`Enabled improve triage judgment requires an engine. Set defaults.llmEngine or improve.strategies.${strategy.name}.processes.triage.judgment.engine.`, "LLM_NOT_CONFIGURED");
159
257
  }
258
+ // #957: same credential-unavailable treatment as the main per-process loop
259
+ // above — a triage judgment engine that resolves structurally but whose
260
+ // credential (or, for an SDK judgment, its LLM fallback credential) is
261
+ // unavailable in this process's environment is folded into
262
+ // `engineUnavailable` under the reserved `"triage.judgment"` process name
263
+ // instead of silently reaching dispatch with a doomed credential.
264
+ if (triageJudgment) {
265
+ const judgmentCredentialFields = triageJudgment.kind === "llm"
266
+ ? {
267
+ credential: triageJudgment.credential,
268
+ apiKeyFile: triageJudgment.apiKeyFile,
269
+ apiKeySecretRef: triageJudgment.apiKeySecretRef,
270
+ }
271
+ : triageJudgment.kind === "sdk"
272
+ ? {
273
+ credential: triageJudgment.fallbackCredential,
274
+ apiKeyFile: triageJudgment.fallbackApiKeyFile,
275
+ apiKeySecretRef: triageJudgment.fallbackApiKeySecretRef,
276
+ }
277
+ : undefined;
278
+ if (judgmentCredentialFields) {
279
+ const credentialStatus = describeLlmCredentialAvailability(judgmentCredentialFields, env);
280
+ if (!credentialStatus.available) {
281
+ engineUnavailable.push({
282
+ process: "triage.judgment",
283
+ configKey: `improve.strategies.${strategy.name}.processes.triage.judgment.engine`,
284
+ reason: credentialUnavailableReason(triageJudgment.engine, credentialStatus),
285
+ });
286
+ triageJudgment = null;
287
+ }
288
+ }
289
+ }
160
290
  const frozenProcesses = Object.freeze(processes);
161
291
  const frozenStrategy = Object.freeze({
162
292
  name: strategy.name,
@@ -0,0 +1,182 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * #944 — per-run LLM usage reporting: fold the process x engine x model
6
+ * cross-tab (`summarizeLlmUsageCrossTab`, `health/llm-usage.ts`) together with
7
+ * the resolved process routing table (`projectResolvedProcessRouting`, #947)
8
+ * into the `usageReport` field `finalizeImproveResult` persists, and render
9
+ * both halves as one fixed-width table shared by the end-of-run stderr
10
+ * summary (`improve-cli.ts`) and `akm improve report`'s text output
11
+ * (`src/output/text/improve-report.ts`).
12
+ */
13
+ import { IMPROVE_PROCESS_ENGINE_CAPABILITIES } from "../../core/config/engine-semantics.js";
14
+ import { projectResolvedProcessRouting, shouldSkipRef, } from "./improve-strategies.js";
15
+ /**
16
+ * The processes this report covers — only the ones {@link IMPROVE_PROCESS_ENGINE_CAPABILITIES}
17
+ * marks `"llm"`. `triage` ("runner" kind — its LLM cost, if any, is
18
+ * attributed to the separate `"triage.judgment"` pseudo-row, itself excluded
19
+ * below) and `proactiveMaintenance` (no engine at all) never make an
20
+ * attributable LLM call, so listing them as "zero calls" would always be a
21
+ * false positive, not a real finding.
22
+ */
23
+ const LLM_BACKED_PROCESSES = new Set(Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES).filter((name) => IMPROVE_PROCESS_ENGINE_CAPABILITIES[name] === "llm"));
24
+ /** Ref-scoped processes `shouldSkipRef` understands — the only ones an eligible-ref count is meaningful for. */
25
+ const REF_SCOPED_PROCESSES = new Set(["reflect", "distill", "consolidate"]);
26
+ /** The autonomy lane (if any) gating each LLM-backed process, per `autonomy-gate.ts`'s `AUTONOMY_LANES`. */
27
+ const AUTONOMY_LANE_BY_PROCESS = {
28
+ memoryInference: "memoryInference",
29
+ };
30
+ /** `{reason -> count}` for the dominant-reason lookup in {@link deriveNoCallReason}. */
31
+ function dominantReason(counts) {
32
+ let best;
33
+ let bestCount = 0;
34
+ for (const [reason, count] of Object.entries(counts)) {
35
+ if (count > bestCount) {
36
+ best = reason;
37
+ bestCount = count;
38
+ }
39
+ }
40
+ return best;
41
+ }
42
+ /**
43
+ * The single decision point for why an LLM-backed process made zero LLM
44
+ * calls this run — every case `buildImproveUsageReport` needs to report,
45
+ * decided in one place instead of split between this function and its
46
+ * caller. Priority order: `"engine_unavailable"` (the row's engine or
47
+ * credential could not be resolved) beats everything else, including a
48
+ * process that also happens to be autonomy-gated; then, for a disabled row
49
+ * that IS resolvable, `"autonomy_gated"` when its lane was gated, else
50
+ * `undefined` (a `false`-in-config process never reached the routing table
51
+ * in the first place, so a disabled-and-not-gated row has no other
52
+ * explanation to report); for an enabled row, `"strategy_filtered_all_passes"`
53
+ * when every ref got filtered out, then the process's own dominant skip
54
+ * reason, else the `"no_signal"` fallback.
55
+ *
56
+ * Reuses the SAME reason strings already emitted elsewhere
57
+ * (`improve_skipped` events, reflect's `AkmReflectFailure.reason`, distill's
58
+ * `distillSkipped.byReason` keys) rather than inventing a translation layer —
59
+ * per the brief, never a fabricated category like `"limit_reached"`.
60
+ */
61
+ export function deriveNoCallReason(args) {
62
+ const { row } = args;
63
+ if (row.unavailable)
64
+ return "engine_unavailable";
65
+ const lane = AUTONOMY_LANE_BY_PROCESS[row.process];
66
+ const laneGated = lane !== undefined && args.autonomyGated.some((gated) => gated.lane === lane);
67
+ if (!row.enabled)
68
+ return laneGated ? "autonomy_gated" : undefined;
69
+ if (args.eligibleRefs === 0 && args.strategyFilteredRefsCount > 0)
70
+ return "strategy_filtered_all_passes";
71
+ const dominant = args.skipReasonCounts ? dominantReason(args.skipReasonCounts) : undefined;
72
+ if (dominant)
73
+ return dominant;
74
+ return "no_signal";
75
+ }
76
+ /** Reflect's per-ref skip reason, read off `AkmReflectFailure.reason` for both cooldown and skipped actions. */
77
+ function countReflectSkipReasons(actions) {
78
+ const counts = {};
79
+ for (const action of actions) {
80
+ if (action.mode !== "reflect-cooldown" && action.mode !== "reflect-skipped")
81
+ continue;
82
+ const result = action.result;
83
+ const reason = typeof result?.reason === "string" && result.reason.trim() ? result.reason : "unknown";
84
+ counts[reason] = (counts[reason] ?? 0) + 1;
85
+ }
86
+ return counts;
87
+ }
88
+ /**
89
+ * Assemble this run's `usageReport` (#944): the process x engine x model
90
+ * cross-tab plus which enabled processes made zero calls and why. Pure — no
91
+ * I/O; the caller (`finalizeImproveResult`) supplies the cross-tab (already
92
+ * computed from this run's `llm_usage` events) and every other input from
93
+ * data it already has in scope.
94
+ *
95
+ * Returns `undefined` when both halves would be empty (e.g. a run whose
96
+ * active strategy enables no LLM-backed process), matching the envelope's
97
+ * existing convention of omitting empty optional sections.
98
+ */
99
+ export function buildImproveUsageReport(args) {
100
+ // Only the LLM-backed processes (see LLM_BACKED_PROCESSES) — this also
101
+ // drops the "triage.judgment" pseudo-row (#947): judgment dispatch does
102
+ // not route through `withLlmStage`, so its calls (if any) are never
103
+ // attributable to a "triage.judgment" process in the cross-tab, and
104
+ // reporting it here would always read as a false "zero calls".
105
+ const routing = projectResolvedProcessRouting(args.resolvedPlan).filter((row) => row.process !== "triage.judgment" && LLM_BACKED_PROCESSES.has(row.process));
106
+ const calledProcesses = new Set(args.byProcessEngineModel.filter((row) => row.calls > 0).map((row) => row.process));
107
+ const reflectSkipCounts = countReflectSkipReasons(args.persistedActions);
108
+ const noCalls = [];
109
+ for (const row of routing) {
110
+ if (calledProcesses.has(row.process))
111
+ continue;
112
+ const eligibleRefs = REF_SCOPED_PROCESSES.has(row.process)
113
+ ? args.loopRefs.filter((entry) => !shouldSkipRef(entry.ref, row.process, args.resolvedPlan.strategy.config).skip).length
114
+ : undefined;
115
+ const reason = deriveNoCallReason({
116
+ row,
117
+ autonomyGated: args.resolvedPlan.autonomyGated,
118
+ strategyFilteredRefsCount: args.strategyFilteredRefsCount,
119
+ eligibleRefs,
120
+ skipReasonCounts: row.process === "reflect"
121
+ ? reflectSkipCounts
122
+ : row.process === "distill"
123
+ ? args.distillSkippedAggregate?.byReason
124
+ : undefined,
125
+ });
126
+ if (reason === undefined)
127
+ continue;
128
+ noCalls.push({ process: row.process, ...(row.engine ? { engine: row.engine } : {}), reason });
129
+ }
130
+ if (args.byProcessEngineModel.length === 0 && noCalls.length === 0)
131
+ return undefined;
132
+ return { byProcessEngineModel: args.byProcessEngineModel, noCalls };
133
+ }
134
+ // ── Shared fixed-width text rendering ────────────────────────────────────────
135
+ function renderFixedWidthTable(headers, rows) {
136
+ const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)));
137
+ const renderRow = (cells) => cells
138
+ .map((cell, index) => cell.padEnd(widths[index] ?? 0))
139
+ .join(" ")
140
+ .trimEnd();
141
+ return [renderRow(headers), ...rows.map(renderRow)];
142
+ }
143
+ /**
144
+ * Render a `usageReport` as fixed-width plain text — the ONE formatter shared
145
+ * by the end-of-run `[improve] ...` stderr table (`improve-cli.ts`) and `akm
146
+ * improve report`'s `--format text` output
147
+ * (`src/output/text/improve-report.ts`), per the brief. `notes` surfaces
148
+ * degraded-precision caveats (e.g. a pre-0.9.15 run recomputed from raw
149
+ * events, per `improve-report.ts`).
150
+ */
151
+ export function formatUsageReportTable(usageReport, notes) {
152
+ const lines = ["[improve] usage report (process x engine x model):"];
153
+ if (usageReport.byProcessEngineModel.length === 0) {
154
+ lines.push(" (no LLM calls recorded)");
155
+ }
156
+ else {
157
+ const headers = ["process", "engine", "model", "calls", "failures", "promptTok", "complTok", "totalTok", "ms"];
158
+ const rows = usageReport.byProcessEngineModel.map((row) => [
159
+ row.process,
160
+ row.engine,
161
+ row.model,
162
+ String(row.calls),
163
+ String(row.failures),
164
+ String(row.promptTokens),
165
+ String(row.completionTokens),
166
+ String(row.totalTokens),
167
+ String(row.totalDurationMs),
168
+ ]);
169
+ for (const line of renderFixedWidthTable(headers, rows))
170
+ lines.push(` ${line}`);
171
+ }
172
+ if (usageReport.noCalls.length > 0) {
173
+ lines.push("[improve] enabled processes with zero calls:");
174
+ const headers = ["process", "engine", "reason"];
175
+ const rows = usageReport.noCalls.map((row) => [row.process, row.engine ?? "-", row.reason]);
176
+ for (const line of renderFixedWidthTable(headers, rows))
177
+ lines.push(` ${line}`);
178
+ }
179
+ for (const note of notes ?? [])
180
+ lines.push(` note: ${note}`);
181
+ return lines.join("\n");
182
+ }