akm-cli 0.9.14 → 0.9.15-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +397 -0
- package/STABILITY.md +6 -3
- package/dist/assets/prompts/reflect-feedback-framing.md +1 -0
- package/dist/assets/prompts/reflect-llm-framed-contract.md +2 -0
- package/dist/assets/prompts/reflect-llm-schema-contract.md +2 -0
- package/dist/assets/tasks/core/improve.yml +1 -1
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-catchup.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-consolidate.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-frequent.yml +1 -1
- package/dist/assets/tasks/improve/akm-improve-nightly.yml +1 -1
- package/dist/cli/retired-commands.js +0 -1
- package/dist/cli/shared.js +9 -0
- package/dist/cli/unknown-flags.js +1 -0
- package/dist/cli.js +3 -2
- package/dist/commands/config-cli.js +85 -3
- package/dist/commands/env/env-cli.js +1 -42
- package/dist/commands/env/env.js +1 -1
- package/dist/commands/env/secret-cli.js +1 -2
- package/dist/commands/health/checks.js +357 -63
- package/dist/commands/health/engine-usage.js +45 -0
- package/dist/commands/health/improve-metrics.js +18 -0
- package/dist/commands/health/llm-usage.js +41 -1
- package/dist/commands/health/plugin-staleness.js +7 -3
- package/dist/commands/health/version-drift.js +93 -0
- package/dist/commands/health/windows.js +3 -1
- package/dist/commands/health.js +44 -9
- package/dist/commands/improve/consolidate/chunking.js +4 -2
- package/dist/commands/improve/improve-cli.js +99 -5
- package/dist/commands/improve/improve-report.js +154 -0
- package/dist/commands/improve/improve-result-file.js +45 -33
- package/dist/commands/improve/improve-strategies.js +133 -3
- package/dist/commands/improve/improve-usage-report.js +182 -0
- package/dist/commands/improve/improve.js +40 -3
- package/dist/commands/improve/locks.js +27 -78
- package/dist/commands/improve/planner.js +1 -0
- package/dist/commands/improve/preparation.js +9 -1
- package/dist/commands/improve/reflect.js +44 -4
- package/dist/commands/models-cli.js +50 -1
- package/dist/commands/proposal/repository.js +8 -3
- package/dist/commands/proposal/validators/proposal-quality-validators.js +41 -6
- package/dist/commands/proposal/validators/proposal-validators.js +24 -0
- package/dist/commands/read/search-cli.js +38 -2
- package/dist/commands/read/show.js +103 -4
- package/dist/commands/sources/info.js +5 -1
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/stash-cli.js +31 -0
- package/dist/commands/tasks/tasks-cli.js +49 -2
- package/dist/commands/workflow-cli.js +86 -12
- package/dist/core/asset/markdown-fragments.js +35 -0
- package/dist/core/config/config-schema.js +14 -0
- package/dist/core/config/config.js +302 -24
- package/dist/core/env-secret-ref.js +58 -5
- package/dist/core/errors.js +30 -0
- package/dist/core/improve-result.js +51 -0
- package/dist/core/loopback.js +17 -0
- package/dist/core/paths.js +11 -0
- package/dist/core/run-lock.js +96 -0
- package/dist/core/sensitive-marker-path.js +19 -0
- package/dist/core/state-db.js +74 -14
- package/dist/indexer/index-rebuild-lock.js +73 -0
- package/dist/indexer/index-writer-lock.js +40 -1
- package/dist/indexer/index-written-assets.js +21 -1
- package/dist/indexer/indexer.js +18 -17
- package/dist/indexer/materialize-embeddings.js +282 -32
- package/dist/indexer/search/db-search.js +49 -2
- package/dist/integrations/agent/engine-resolution.js +96 -6
- package/dist/integrations/agent/execution-definitions.js +6 -15
- package/dist/integrations/agent/execution-lowering.js +6 -1
- package/dist/integrations/agent/execution-preparation.js +1 -1
- package/dist/integrations/agent/model-map.js +123 -20
- package/dist/integrations/agent/prompts.js +40 -8
- package/dist/integrations/agent/runner-dispatch.js +9 -3
- package/dist/integrations/agent/runner.js +2 -0
- package/dist/llm/client.js +8 -3
- package/dist/llm/embedder.js +20 -8
- package/dist/llm/embedders/local.js +10 -2
- package/dist/llm/embedders/remote.js +188 -21
- package/dist/output/shapes/helpers.js +38 -2
- package/dist/output/shapes/models-list.js +16 -0
- package/dist/output/shapes/passthrough.js +2 -0
- package/dist/output/shapes.js +4 -0
- package/dist/output/text/command-format.js +29 -0
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/improve-report.js +27 -0
- package/dist/{commands/env/marker-path.js → output/text/models.js} +4 -3
- package/dist/output/text/show-format.js +4 -0
- package/dist/output/text.js +4 -0
- package/dist/scripts/akm-migrate-node.js +24798 -21732
- package/dist/scripts/akm-migrate.js +23408 -20343
- package/dist/storage/repositories/improve-runs-repository.js +34 -0
- package/dist/storage/repositories/index-fts-repository.js +49 -6
- package/dist/storage/repositories/index-vec-repository.js +30 -0
- package/dist/storage/repositories/workflow-runs-repository.js +55 -18
- package/dist/tasks/backends/cron.js +14 -7
- package/dist/tasks/run/run-workflow-task.js +16 -0
- package/dist/workflows/exec/child-workflow.js +2 -2
- package/dist/workflows/exec/dispatch-redaction.js +21 -9
- package/dist/workflows/exec/run-workflow.js +6 -5
- package/dist/workflows/runtime/runs.js +33 -5
- package/docs/migration/release-notes/0.9.15.md +52 -0
- package/docs/migration/release-notes/README.md +4 -0
- package/docs/reference/cli.md +245 -29
- package/docs/reference/configuration.md +180 -19
- package/docs/reference/data-and-telemetry.md +8 -0
- package/docs/reference/tasks.md +16 -1
- package/docs/reference/workflow-schema.md +5 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +8 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* #944 Layer 2 — `akm improve report`. Reads one (or several, `--since`)
|
|
6
|
+
* `improve_runs` rows and returns the same `usageReport` shape
|
|
7
|
+
* `finalizeImproveResult` persists on a live run (`improve-usage-report.ts`),
|
|
8
|
+
* so the command and the end-of-run stderr table stay byte-identical in
|
|
9
|
+
* structure. A row with no persisted `usageReport` degrades rather than
|
|
10
|
+
* errors, per AGENTS.md's "reader must tolerate data older releases wrote":
|
|
11
|
+
* its cross-tab is recomputed from the run's own `llm_usage` events and a
|
|
12
|
+
* `notes` entry explains why — either the row predates #944, or (per
|
|
13
|
+
* `recordTerminatedImproveRun` in improve-result-file.ts) the run was
|
|
14
|
+
* terminated before it ever reached `finalizeImproveResult`. The `noCalls`
|
|
15
|
+
* half is never fabricated for either case.
|
|
16
|
+
*/
|
|
17
|
+
import { NotFoundError, UsageError } from "../../core/errors.js";
|
|
18
|
+
import { readEvents } from "../../core/events.js";
|
|
19
|
+
import { decodeImproveResult } from "../../core/improve-result.js";
|
|
20
|
+
import { withStateDb } from "../../core/state-db.js";
|
|
21
|
+
import { LLM_USAGE_EVENT } from "../../llm/usage-persist.js";
|
|
22
|
+
import { getImproveRunById, getLatestImproveRun, queryImproveRuns, } from "../../storage/repositories/improve-runs-repository.js";
|
|
23
|
+
import { parseHealthSince } from "../health.js";
|
|
24
|
+
import { summarizeLlmUsageCrossTab } from "../health/llm-usage.js";
|
|
25
|
+
const PRE_USAGE_REPORT_NOTE = "eligibility reasons unavailable for runs recorded before 0.9.15";
|
|
26
|
+
const TERMINATED_RUN_NOTE = "run terminated before completion; no usage report was recorded";
|
|
27
|
+
/**
|
|
28
|
+
* Recompute the cross-tab from this run's own `llm_usage` events (a pre-#944
|
|
29
|
+
* row has no persisted one). `eventsCtx` threads the state.db connection
|
|
30
|
+
* `runImproveReportQuery` already holds via `withStateDb`, so a `--since`
|
|
31
|
+
* window with several pre-0.9.15 or undecodable rows reads through that one
|
|
32
|
+
* open connection instead of each row opening and closing its own.
|
|
33
|
+
*/
|
|
34
|
+
function crossTabFromEvents(startedAt, completedAt, eventsCtx) {
|
|
35
|
+
const until = completedAt ?? new Date().toISOString();
|
|
36
|
+
const events = readEvents({ since: startedAt, type: LLM_USAGE_EVENT }, eventsCtx).events.filter((event) => new Date(event.ts ?? startedAt).getTime() < new Date(until).getTime());
|
|
37
|
+
return summarizeLlmUsageCrossTab(events);
|
|
38
|
+
}
|
|
39
|
+
/** One row's usage report — persisted (common case) or recomputed (pre-#944 row / terminated run / undecodable row). */
|
|
40
|
+
function usageReportForRow(row, eventsCtx) {
|
|
41
|
+
try {
|
|
42
|
+
const decoded = decodeImproveResult(row.result_json);
|
|
43
|
+
if (decoded.envelope.usageReport) {
|
|
44
|
+
return {
|
|
45
|
+
usageReport: {
|
|
46
|
+
byProcessEngineModel: [...decoded.envelope.usageReport.byProcessEngineModel],
|
|
47
|
+
noCalls: [...decoded.envelope.usageReport.noCalls],
|
|
48
|
+
},
|
|
49
|
+
strategy: decoded.strategy,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
usageReport: {
|
|
54
|
+
byProcessEngineModel: crossTabFromEvents(row.started_at, row.completed_at, eventsCtx),
|
|
55
|
+
noCalls: [],
|
|
56
|
+
},
|
|
57
|
+
strategy: decoded.strategy,
|
|
58
|
+
note: decoded.envelope.terminated ? TERMINATED_RUN_NOTE : PRE_USAGE_REPORT_NOTE,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
// A row whose result_json this build cannot decode still has real
|
|
63
|
+
// llm_usage events on state.db — degrade to the recomputed cross-tab
|
|
64
|
+
// rather than excluding the run entirely.
|
|
65
|
+
return {
|
|
66
|
+
usageReport: {
|
|
67
|
+
byProcessEngineModel: crossTabFromEvents(row.started_at, row.completed_at, eventsCtx),
|
|
68
|
+
noCalls: [],
|
|
69
|
+
},
|
|
70
|
+
note: PRE_USAGE_REPORT_NOTE,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function mergeCrossTabRow(merged, row) {
|
|
75
|
+
const key = `${row.process}:${row.engine}:${row.model}`;
|
|
76
|
+
const acc = merged.get(key);
|
|
77
|
+
if (!acc) {
|
|
78
|
+
merged.set(key, { ...row });
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
acc.calls += row.calls;
|
|
82
|
+
acc.failures += row.failures;
|
|
83
|
+
acc.promptTokens += row.promptTokens;
|
|
84
|
+
acc.completionTokens += row.completionTokens;
|
|
85
|
+
acc.totalTokens += row.totalTokens;
|
|
86
|
+
acc.reasoningTokens += row.reasoningTokens;
|
|
87
|
+
acc.totalDurationMs += row.totalDurationMs;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Resolve `akm improve report`'s target run(s) and build its `usageReport`.
|
|
91
|
+
* `runId` and `since` are mutually exclusive; neither given selects the most
|
|
92
|
+
* recent non-dry-run.
|
|
93
|
+
*/
|
|
94
|
+
export function runImproveReportQuery(options) {
|
|
95
|
+
if (options.runId !== undefined && options.since !== undefined) {
|
|
96
|
+
throw new UsageError("akm improve report: --run and --since are mutually exclusive.", "INVALID_FLAG_VALUE");
|
|
97
|
+
}
|
|
98
|
+
return withStateDb((db) => {
|
|
99
|
+
// Thread this already-open connection into every reader below, so a
|
|
100
|
+
// `--since` window with several pre-0.9.15 or undecodable rows reads
|
|
101
|
+
// through it instead of each row opening and closing its own.
|
|
102
|
+
const eventsCtx = { db };
|
|
103
|
+
if (options.since !== undefined) {
|
|
104
|
+
const sinceIso = parseHealthSince(options.since);
|
|
105
|
+
const rows = queryImproveRuns(db, sinceIso);
|
|
106
|
+
const merged = new Map();
|
|
107
|
+
const calledEverByProcess = new Set();
|
|
108
|
+
const lastReasonByProcess = new Map();
|
|
109
|
+
const notes = new Set();
|
|
110
|
+
for (const row of rows) {
|
|
111
|
+
const { usageReport, note } = usageReportForRow(row, eventsCtx);
|
|
112
|
+
for (const crossTabRow of usageReport.byProcessEngineModel) {
|
|
113
|
+
mergeCrossTabRow(merged, crossTabRow);
|
|
114
|
+
if (crossTabRow.calls > 0)
|
|
115
|
+
calledEverByProcess.add(crossTabRow.process);
|
|
116
|
+
}
|
|
117
|
+
for (const noCallRow of usageReport.noCalls) {
|
|
118
|
+
// `rows` is newest-first (ORDER BY started_at DESC): keep only the
|
|
119
|
+
// first (most recent) reason seen per process, never overwrite it
|
|
120
|
+
// with an older run's reason.
|
|
121
|
+
if (!lastReasonByProcess.has(noCallRow.process)) {
|
|
122
|
+
lastReasonByProcess.set(noCallRow.process, { engine: noCallRow.engine, reason: noCallRow.reason });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (note)
|
|
126
|
+
notes.add(note);
|
|
127
|
+
}
|
|
128
|
+
const noCalls = [...lastReasonByProcess.entries()]
|
|
129
|
+
.filter(([process]) => !calledEverByProcess.has(process))
|
|
130
|
+
.map(([process, { engine, reason }]) => ({ process, ...(engine ? { engine } : {}), reason }));
|
|
131
|
+
return {
|
|
132
|
+
mode: "since",
|
|
133
|
+
since: sinceIso,
|
|
134
|
+
runIds: rows.map((r) => r.id),
|
|
135
|
+
usageReport: { byProcessEngineModel: [...merged.values()], noCalls },
|
|
136
|
+
...(notes.size > 0 ? { notes: [...notes] } : {}),
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
const row = options.runId !== undefined ? getImproveRunById(db, options.runId) : getLatestImproveRun(db);
|
|
140
|
+
if (!row) {
|
|
141
|
+
throw new NotFoundError(options.runId !== undefined
|
|
142
|
+
? `akm improve report: no improve run found with id "${options.runId}".`
|
|
143
|
+
: "akm improve report: no improve runs recorded yet.", "IMPROVE_RUN_NOT_FOUND");
|
|
144
|
+
}
|
|
145
|
+
const { usageReport, strategy, note } = usageReportForRow(row, eventsCtx);
|
|
146
|
+
return {
|
|
147
|
+
mode: "run",
|
|
148
|
+
runId: row.id,
|
|
149
|
+
...(strategy !== undefined ? { strategy } : {}),
|
|
150
|
+
usageReport,
|
|
151
|
+
...(note ? { notes: [note] } : {}),
|
|
152
|
+
};
|
|
153
|
+
});
|
|
154
|
+
}
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
import crypto from "node:crypto";
|
|
28
28
|
import { decodeImproveResult } from "../../core/improve-result.js";
|
|
29
29
|
import { redactSensitiveValue } from "../../core/redaction.js";
|
|
30
|
-
import { withStateDb } from "../../core/state-db.js";
|
|
30
|
+
import { withImmediateTransaction, withStateDb } from "../../core/state-db.js";
|
|
31
31
|
import { recordImproveRun } from "../../storage/repositories/improve-runs-repository.js";
|
|
32
32
|
/**
|
|
33
33
|
* Build a stable run-id for a single improve invocation.
|
|
@@ -63,18 +63,25 @@ export function recordImproveRunResult(stashDir, runId, result, startedAt, sensi
|
|
|
63
63
|
// so started_at != completed_at even on older call sites.
|
|
64
64
|
const resolvedStartedAt = startedAt ??
|
|
65
65
|
runId.slice(0, 24).replace(/^(\d{4}-\d{2}-\d{2}T)(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, "$1$2:$3:$4.$5Z");
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
66
|
+
// #948: route through the shared BEGIN IMMEDIATE retry/reclassify helper
|
|
67
|
+
// instead of a bare write — this INSERT used to rely solely on the
|
|
68
|
+
// connection's 30s busy_timeout, with no retry and no friendly
|
|
69
|
+
// reclassification on exhaustion, so a raw "database is locked" from
|
|
70
|
+
// improve's own ledger write could reach the CLI as exit 70.
|
|
71
|
+
withImmediateTransaction(db, () => {
|
|
72
|
+
recordImproveRun(db, {
|
|
73
|
+
id: runId,
|
|
74
|
+
startedAt: resolvedStartedAt,
|
|
75
|
+
completedAt,
|
|
76
|
+
stashDir,
|
|
77
|
+
dryRun: Boolean(result.dryRun),
|
|
78
|
+
strategy: redactSensitiveValue(decoded.strategy, sensitiveValues),
|
|
79
|
+
scopeMode: result.scope?.mode ?? "all",
|
|
80
|
+
scopeValue: persistedResult.scope?.value ?? null,
|
|
81
|
+
guidance: persistedResult.guidance ?? null,
|
|
82
|
+
ok: Boolean(result.ok),
|
|
83
|
+
result: persistedResult,
|
|
84
|
+
});
|
|
78
85
|
});
|
|
79
86
|
});
|
|
80
87
|
}
|
|
@@ -116,27 +123,32 @@ export function recordTerminatedImproveRun(stashDir, runId, startedAt, reason, c
|
|
|
116
123
|
},
|
|
117
124
|
}, ctx.sensitiveValues ?? []);
|
|
118
125
|
withStateDb((db) => {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
:
|
|
126
|
+
// #948: same rationale as recordImproveRunResult above — this is the
|
|
127
|
+
// signal-handler/terminated-run write path, which must not itself raise
|
|
128
|
+
// a raw "database is locked" while trying to record why the run ended.
|
|
129
|
+
withImmediateTransaction(db, () => {
|
|
130
|
+
recordImproveRun(db, {
|
|
131
|
+
id: runId,
|
|
132
|
+
startedAt,
|
|
133
|
+
completedAt,
|
|
134
|
+
stashDir,
|
|
135
|
+
dryRun: Boolean(ctx.dryRun),
|
|
136
|
+
strategy: persistedStrategy,
|
|
137
|
+
scopeMode: ctx.scopeMode ?? "all",
|
|
138
|
+
scopeValue: persistedScopeValue ?? null,
|
|
139
|
+
guidance: null,
|
|
140
|
+
ok: false,
|
|
141
|
+
result: minimalResult,
|
|
142
|
+
metadata: {
|
|
143
|
+
terminated: {
|
|
144
|
+
reason: persistedReason,
|
|
145
|
+
at: completedAt,
|
|
146
|
+
...(ctx.errorMessage
|
|
147
|
+
? { errorMessage: redactSensitiveValue(ctx.errorMessage, ctx.sensitiveValues ?? []) }
|
|
148
|
+
: {}),
|
|
149
|
+
},
|
|
138
150
|
},
|
|
139
|
-
}
|
|
151
|
+
});
|
|
140
152
|
});
|
|
141
153
|
});
|
|
142
154
|
}
|
|
@@ -14,6 +14,7 @@ import { ImproveProfileConfigSchema } from "../../core/config/config-schema.js";
|
|
|
14
14
|
import { deepMergeConfig } from "../../core/config/deep-merge.js";
|
|
15
15
|
import { BUILTIN_IMPROVE_STRATEGY_NAMES, IMPROVE_PROCESS_ENGINE_CAPABILITIES, } from "../../core/config/engine-semantics.js";
|
|
16
16
|
import { ConfigError } from "../../core/errors.js";
|
|
17
|
+
import { describeLlmCredentialAvailability } from "../../integrations/agent/engine-resolution.js";
|
|
17
18
|
import { applyAutonomyGate } from "./autonomy-gate.js";
|
|
18
19
|
import { resolveImproveExecution, resolveImproveLlmExecution } from "./execution.js";
|
|
19
20
|
export const DEFAULT_ALLOWED_TYPES = {
|
|
@@ -64,6 +65,59 @@ export function resolveImproveStrategy(name, config) {
|
|
|
64
65
|
const resolved = deepMergeConfig(baseStrategy, (userStrategies[selectedName] ?? {}));
|
|
65
66
|
return { name: selectedName, config: ImproveProfileConfigSchema.parse(resolved) };
|
|
66
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Project a resolved improve plan into one routing row per
|
|
70
|
+
* {@link IMPROVE_PROCESS_ENGINE_CAPABILITIES} name, plus a `"triage.judgment"`
|
|
71
|
+
* pseudo-row when the strategy configures a judgment engine. Pure — no I/O,
|
|
72
|
+
* no re-resolution. Shared by `improve --dry-run`'s `plan.processes` (#947)
|
|
73
|
+
* and `akm health`'s post-run reporting (#944); health's own
|
|
74
|
+
* `active-improve-strategy` check keeps its existing `engines` shape.
|
|
75
|
+
*/
|
|
76
|
+
export function projectResolvedProcessRouting(plan) {
|
|
77
|
+
const unavailableByProcess = new Map(plan.engineUnavailable.map((item) => [item.process, item]));
|
|
78
|
+
const rows = [];
|
|
79
|
+
for (const processName of Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES)) {
|
|
80
|
+
const process = plan.processes[processName];
|
|
81
|
+
const unavailable = unavailableByProcess.get(processName);
|
|
82
|
+
rows.push({
|
|
83
|
+
process: processName,
|
|
84
|
+
enabled: process.enabled,
|
|
85
|
+
...(process.runner
|
|
86
|
+
? { engine: process.runner.engine, model: process.runner.connection.model, engineKind: process.runner.kind }
|
|
87
|
+
: // #800/#957 round 3 — a credential-unavailable process never carries a
|
|
88
|
+
// runner, but its structurally resolved engine/model is still worth
|
|
89
|
+
// showing in the routing table (dry-run preview, health probe).
|
|
90
|
+
unavailable?.engine
|
|
91
|
+
? {
|
|
92
|
+
engine: unavailable.engine,
|
|
93
|
+
...(unavailable.model ? { model: unavailable.model } : {}),
|
|
94
|
+
engineKind: "llm",
|
|
95
|
+
}
|
|
96
|
+
: {}),
|
|
97
|
+
notices: process.notices ?? [],
|
|
98
|
+
...(unavailable ? { unavailable: { configKey: unavailable.configKey, reason: unavailable.reason } } : {}),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
if (plan.strategy.config.processes?.triage?.judgment?.enabled === true) {
|
|
102
|
+
const judgmentUnavailable = unavailableByProcess.get("triage.judgment");
|
|
103
|
+
rows.push({
|
|
104
|
+
process: "triage.judgment",
|
|
105
|
+
enabled: plan.triageJudgment !== null,
|
|
106
|
+
...(plan.triageJudgment
|
|
107
|
+
? {
|
|
108
|
+
engine: plan.triageJudgment.engine,
|
|
109
|
+
engineKind: plan.triageJudgment.kind,
|
|
110
|
+
...(plan.triageJudgment.kind === "llm" ? { model: plan.triageJudgment.connection.model } : {}),
|
|
111
|
+
}
|
|
112
|
+
: {}),
|
|
113
|
+
notices: plan.triageJudgmentNotices ?? [],
|
|
114
|
+
...(judgmentUnavailable
|
|
115
|
+
? { unavailable: { configKey: judgmentUnavailable.configKey, reason: judgmentUnavailable.reason } }
|
|
116
|
+
: {}),
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return rows;
|
|
120
|
+
}
|
|
67
121
|
function cloneAndFreeze(value) {
|
|
68
122
|
const clone = structuredClone(value);
|
|
69
123
|
const freeze = (item) => {
|
|
@@ -86,9 +140,23 @@ export function resolveImprovePlan(name, config, options = {}) {
|
|
|
86
140
|
const strategy = { name: selected.name, config: gatedStrategyConfig };
|
|
87
141
|
return { ...buildImprovePlan(strategy, config, options), autonomyGated: gated };
|
|
88
142
|
}
|
|
143
|
+
/** Describe why a resolved-but-uncredentialed runner belongs in `engineUnavailable` (#957). */
|
|
144
|
+
function credentialUnavailableReason(engineName, status) {
|
|
145
|
+
return `requires a credential that is not available in this process's environment (engine "${engineName}": ${status.reason})`;
|
|
146
|
+
}
|
|
89
147
|
function buildImprovePlan(strategy, config, options) {
|
|
148
|
+
const env = options.env ?? process.env;
|
|
90
149
|
const processes = {};
|
|
91
150
|
const engineUnavailable = [];
|
|
151
|
+
// #957 round 2 — distinguishes "credential unavailable" (a working engine
|
|
152
|
+
// whose secret isn't in this environment) from "no engine selected at all"
|
|
153
|
+
// among the pushes below, so `allowAllDisabled` can bypass the ConfigError
|
|
154
|
+
// only for the former. The latter is the guard's original, pre-#957
|
|
155
|
+
// condition and must keep hard-aborting for every caller, including a
|
|
156
|
+
// health probe with `allowAllDisabled` set — a totally unconfigured
|
|
157
|
+
// install is not the credential-in-the-wrong-environment case this option
|
|
158
|
+
// exists for.
|
|
159
|
+
let anyEngineNotConfigured = false;
|
|
92
160
|
for (const processName of Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES)) {
|
|
93
161
|
const sourceProcessConfig = strategy.config.processes?.[processName] ?? {};
|
|
94
162
|
const enabled = sourceProcessConfig.enabled === true;
|
|
@@ -101,6 +169,15 @@ function buildImprovePlan(strategy, config, options) {
|
|
|
101
169
|
// Validation itself is structural and always runs. Only its optional repair
|
|
102
170
|
// step needs a model, so disabling repair must not create an LLM preflight.
|
|
103
171
|
const skipsRepairEngine = processName === "validation" && options.repairValidationFailures === false;
|
|
172
|
+
// #957: a resolved-but-uncredentialed runner is folded into the same
|
|
173
|
+
// "no engine" branch below — set here so the shared push/continue block
|
|
174
|
+
// can tell the two apart in its reason text.
|
|
175
|
+
let credentialUnavailableMessage;
|
|
176
|
+
// #800/#957 round 3 — the structurally resolved engine/model/contextLength
|
|
177
|
+
// for the credential-unavailable case only, so a dry-run preview can read
|
|
178
|
+
// what resolution decided (e.g. the consolidation chunk-size estimate)
|
|
179
|
+
// without the runner ever carrying a credential-less connection forward.
|
|
180
|
+
let credentialUnavailableRouting;
|
|
104
181
|
if (!skipsRepairEngine) {
|
|
105
182
|
const resolved = resolveImproveLlmExecution({
|
|
106
183
|
config,
|
|
@@ -110,13 +187,32 @@ function buildImprovePlan(strategy, config, options) {
|
|
|
110
187
|
});
|
|
111
188
|
runner = resolved?.runner ?? null;
|
|
112
189
|
notices = resolved?.notices ?? [];
|
|
190
|
+
if (runner) {
|
|
191
|
+
const credentialStatus = describeLlmCredentialAvailability(runner, env);
|
|
192
|
+
if (!credentialStatus.available) {
|
|
193
|
+
credentialUnavailableMessage = credentialUnavailableReason(runner.engine, credentialStatus);
|
|
194
|
+
credentialUnavailableRouting = {
|
|
195
|
+
engine: runner.engine,
|
|
196
|
+
...(runner.connection.model !== undefined ? { model: runner.connection.model } : {}),
|
|
197
|
+
...(runner.connection.contextLength !== undefined
|
|
198
|
+
? { contextLength: runner.connection.contextLength }
|
|
199
|
+
: {}),
|
|
200
|
+
};
|
|
201
|
+
runner = null;
|
|
202
|
+
notices = [];
|
|
203
|
+
}
|
|
204
|
+
}
|
|
113
205
|
}
|
|
114
206
|
if (!runner && !skipsRepairEngine) {
|
|
115
207
|
const configKey = `improve.strategies.${strategy.name}.processes.${processName}.engine`;
|
|
208
|
+
if (!credentialUnavailableMessage)
|
|
209
|
+
anyEngineNotConfigured = true;
|
|
116
210
|
engineUnavailable.push({
|
|
117
211
|
process: processName,
|
|
118
212
|
configKey,
|
|
119
|
-
reason:
|
|
213
|
+
reason: credentialUnavailableMessage ??
|
|
214
|
+
`requires an LLM engine that is not configured. Set defaults.llmEngine or ${configKey}`,
|
|
215
|
+
...credentialUnavailableRouting,
|
|
120
216
|
});
|
|
121
217
|
processes[processName] = Object.freeze({
|
|
122
218
|
enabled: false,
|
|
@@ -134,7 +230,9 @@ function buildImprovePlan(strategy, config, options) {
|
|
|
134
230
|
...(notices.length > 0 ? { notices: cloneAndFreeze(notices) } : {}),
|
|
135
231
|
});
|
|
136
232
|
}
|
|
137
|
-
if (engineUnavailable.length > 0 &&
|
|
233
|
+
if (engineUnavailable.length > 0 &&
|
|
234
|
+
!Object.values(processes).some((process) => process.enabled) &&
|
|
235
|
+
(!options.allowAllDisabled || anyEngineNotConfigured)) {
|
|
138
236
|
const names = engineUnavailable.map((item) => `"${item.process}"`).join(", ");
|
|
139
237
|
throw new ConfigError(`No improve process can run: ${names} ${engineUnavailable.length === 1 ? "requires" : "require"} an LLM engine that is not configured. Set defaults.llmEngine, or the per-process engine key named for each.`, "LLM_NOT_CONFIGURED");
|
|
140
238
|
}
|
|
@@ -149,7 +247,7 @@ function buildImprovePlan(strategy, config, options) {
|
|
|
149
247
|
processName: "triage-judgment",
|
|
150
248
|
})
|
|
151
249
|
: null;
|
|
152
|
-
|
|
250
|
+
let triageJudgment = triageJudgmentResolution?.runner ?? null;
|
|
153
251
|
const effectiveJudgmentLlm = triage?.judgment?.llm ?? triage?.llm ?? strategy.config.llm;
|
|
154
252
|
if (triageJudgment && triageJudgment.kind !== "llm" && effectiveJudgmentLlm) {
|
|
155
253
|
throw new ConfigError(`Triage judgment engine "${triageJudgment.engine ?? "unknown"}" is an agent engine and cannot receive llm overrides.`, "INVALID_CONFIG_FILE");
|
|
@@ -157,6 +255,38 @@ function buildImprovePlan(strategy, config, options) {
|
|
|
157
255
|
if (processes.triage.enabled && judgmentEnabled && !triageJudgment) {
|
|
158
256
|
throw new ConfigError(`Enabled improve triage judgment requires an engine. Set defaults.llmEngine or improve.strategies.${strategy.name}.processes.triage.judgment.engine.`, "LLM_NOT_CONFIGURED");
|
|
159
257
|
}
|
|
258
|
+
// #957: same credential-unavailable treatment as the main per-process loop
|
|
259
|
+
// above — a triage judgment engine that resolves structurally but whose
|
|
260
|
+
// credential (or, for an SDK judgment, its LLM fallback credential) is
|
|
261
|
+
// unavailable in this process's environment is folded into
|
|
262
|
+
// `engineUnavailable` under the reserved `"triage.judgment"` process name
|
|
263
|
+
// instead of silently reaching dispatch with a doomed credential.
|
|
264
|
+
if (triageJudgment) {
|
|
265
|
+
const judgmentCredentialFields = triageJudgment.kind === "llm"
|
|
266
|
+
? {
|
|
267
|
+
credential: triageJudgment.credential,
|
|
268
|
+
apiKeyFile: triageJudgment.apiKeyFile,
|
|
269
|
+
apiKeySecretRef: triageJudgment.apiKeySecretRef,
|
|
270
|
+
}
|
|
271
|
+
: triageJudgment.kind === "sdk"
|
|
272
|
+
? {
|
|
273
|
+
credential: triageJudgment.fallbackCredential,
|
|
274
|
+
apiKeyFile: triageJudgment.fallbackApiKeyFile,
|
|
275
|
+
apiKeySecretRef: triageJudgment.fallbackApiKeySecretRef,
|
|
276
|
+
}
|
|
277
|
+
: undefined;
|
|
278
|
+
if (judgmentCredentialFields) {
|
|
279
|
+
const credentialStatus = describeLlmCredentialAvailability(judgmentCredentialFields, env);
|
|
280
|
+
if (!credentialStatus.available) {
|
|
281
|
+
engineUnavailable.push({
|
|
282
|
+
process: "triage.judgment",
|
|
283
|
+
configKey: `improve.strategies.${strategy.name}.processes.triage.judgment.engine`,
|
|
284
|
+
reason: credentialUnavailableReason(triageJudgment.engine, credentialStatus),
|
|
285
|
+
});
|
|
286
|
+
triageJudgment = null;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
160
290
|
const frozenProcesses = Object.freeze(processes);
|
|
161
291
|
const frozenStrategy = Object.freeze({
|
|
162
292
|
name: strategy.name,
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* #944 — per-run LLM usage reporting: fold the process x engine x model
|
|
6
|
+
* cross-tab (`summarizeLlmUsageCrossTab`, `health/llm-usage.ts`) together with
|
|
7
|
+
* the resolved process routing table (`projectResolvedProcessRouting`, #947)
|
|
8
|
+
* into the `usageReport` field `finalizeImproveResult` persists, and render
|
|
9
|
+
* both halves as one fixed-width table shared by the end-of-run stderr
|
|
10
|
+
* summary (`improve-cli.ts`) and `akm improve report`'s text output
|
|
11
|
+
* (`src/output/text/improve-report.ts`).
|
|
12
|
+
*/
|
|
13
|
+
import { IMPROVE_PROCESS_ENGINE_CAPABILITIES } from "../../core/config/engine-semantics.js";
|
|
14
|
+
import { projectResolvedProcessRouting, shouldSkipRef, } from "./improve-strategies.js";
|
|
15
|
+
/**
|
|
16
|
+
* The processes this report covers — only the ones {@link IMPROVE_PROCESS_ENGINE_CAPABILITIES}
|
|
17
|
+
* marks `"llm"`. `triage` ("runner" kind — its LLM cost, if any, is
|
|
18
|
+
* attributed to the separate `"triage.judgment"` pseudo-row, itself excluded
|
|
19
|
+
* below) and `proactiveMaintenance` (no engine at all) never make an
|
|
20
|
+
* attributable LLM call, so listing them as "zero calls" would always be a
|
|
21
|
+
* false positive, not a real finding.
|
|
22
|
+
*/
|
|
23
|
+
const LLM_BACKED_PROCESSES = new Set(Object.keys(IMPROVE_PROCESS_ENGINE_CAPABILITIES).filter((name) => IMPROVE_PROCESS_ENGINE_CAPABILITIES[name] === "llm"));
|
|
24
|
+
/** Ref-scoped processes `shouldSkipRef` understands — the only ones an eligible-ref count is meaningful for. */
|
|
25
|
+
const REF_SCOPED_PROCESSES = new Set(["reflect", "distill", "consolidate"]);
|
|
26
|
+
/** The autonomy lane (if any) gating each LLM-backed process, per `autonomy-gate.ts`'s `AUTONOMY_LANES`. */
|
|
27
|
+
const AUTONOMY_LANE_BY_PROCESS = {
|
|
28
|
+
memoryInference: "memoryInference",
|
|
29
|
+
};
|
|
30
|
+
/** `{reason -> count}` for the dominant-reason lookup in {@link deriveNoCallReason}. */
|
|
31
|
+
function dominantReason(counts) {
|
|
32
|
+
let best;
|
|
33
|
+
let bestCount = 0;
|
|
34
|
+
for (const [reason, count] of Object.entries(counts)) {
|
|
35
|
+
if (count > bestCount) {
|
|
36
|
+
best = reason;
|
|
37
|
+
bestCount = count;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return best;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The single decision point for why an LLM-backed process made zero LLM
|
|
44
|
+
* calls this run — every case `buildImproveUsageReport` needs to report,
|
|
45
|
+
* decided in one place instead of split between this function and its
|
|
46
|
+
* caller. Priority order: `"engine_unavailable"` (the row's engine or
|
|
47
|
+
* credential could not be resolved) beats everything else, including a
|
|
48
|
+
* process that also happens to be autonomy-gated; then, for a disabled row
|
|
49
|
+
* that IS resolvable, `"autonomy_gated"` when its lane was gated, else
|
|
50
|
+
* `undefined` (a `false`-in-config process never reached the routing table
|
|
51
|
+
* in the first place, so a disabled-and-not-gated row has no other
|
|
52
|
+
* explanation to report); for an enabled row, `"strategy_filtered_all_passes"`
|
|
53
|
+
* when every ref got filtered out, then the process's own dominant skip
|
|
54
|
+
* reason, else the `"no_signal"` fallback.
|
|
55
|
+
*
|
|
56
|
+
* Reuses the SAME reason strings already emitted elsewhere
|
|
57
|
+
* (`improve_skipped` events, reflect's `AkmReflectFailure.reason`, distill's
|
|
58
|
+
* `distillSkipped.byReason` keys) rather than inventing a translation layer —
|
|
59
|
+
* per the brief, never a fabricated category like `"limit_reached"`.
|
|
60
|
+
*/
|
|
61
|
+
export function deriveNoCallReason(args) {
|
|
62
|
+
const { row } = args;
|
|
63
|
+
if (row.unavailable)
|
|
64
|
+
return "engine_unavailable";
|
|
65
|
+
const lane = AUTONOMY_LANE_BY_PROCESS[row.process];
|
|
66
|
+
const laneGated = lane !== undefined && args.autonomyGated.some((gated) => gated.lane === lane);
|
|
67
|
+
if (!row.enabled)
|
|
68
|
+
return laneGated ? "autonomy_gated" : undefined;
|
|
69
|
+
if (args.eligibleRefs === 0 && args.strategyFilteredRefsCount > 0)
|
|
70
|
+
return "strategy_filtered_all_passes";
|
|
71
|
+
const dominant = args.skipReasonCounts ? dominantReason(args.skipReasonCounts) : undefined;
|
|
72
|
+
if (dominant)
|
|
73
|
+
return dominant;
|
|
74
|
+
return "no_signal";
|
|
75
|
+
}
|
|
76
|
+
/** Reflect's per-ref skip reason, read off `AkmReflectFailure.reason` for both cooldown and skipped actions. */
|
|
77
|
+
function countReflectSkipReasons(actions) {
|
|
78
|
+
const counts = {};
|
|
79
|
+
for (const action of actions) {
|
|
80
|
+
if (action.mode !== "reflect-cooldown" && action.mode !== "reflect-skipped")
|
|
81
|
+
continue;
|
|
82
|
+
const result = action.result;
|
|
83
|
+
const reason = typeof result?.reason === "string" && result.reason.trim() ? result.reason : "unknown";
|
|
84
|
+
counts[reason] = (counts[reason] ?? 0) + 1;
|
|
85
|
+
}
|
|
86
|
+
return counts;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Assemble this run's `usageReport` (#944): the process x engine x model
|
|
90
|
+
* cross-tab plus which enabled processes made zero calls and why. Pure — no
|
|
91
|
+
* I/O; the caller (`finalizeImproveResult`) supplies the cross-tab (already
|
|
92
|
+
* computed from this run's `llm_usage` events) and every other input from
|
|
93
|
+
* data it already has in scope.
|
|
94
|
+
*
|
|
95
|
+
* Returns `undefined` when both halves would be empty (e.g. a run whose
|
|
96
|
+
* active strategy enables no LLM-backed process), matching the envelope's
|
|
97
|
+
* existing convention of omitting empty optional sections.
|
|
98
|
+
*/
|
|
99
|
+
export function buildImproveUsageReport(args) {
|
|
100
|
+
// Only the LLM-backed processes (see LLM_BACKED_PROCESSES) — this also
|
|
101
|
+
// drops the "triage.judgment" pseudo-row (#947): judgment dispatch does
|
|
102
|
+
// not route through `withLlmStage`, so its calls (if any) are never
|
|
103
|
+
// attributable to a "triage.judgment" process in the cross-tab, and
|
|
104
|
+
// reporting it here would always read as a false "zero calls".
|
|
105
|
+
const routing = projectResolvedProcessRouting(args.resolvedPlan).filter((row) => row.process !== "triage.judgment" && LLM_BACKED_PROCESSES.has(row.process));
|
|
106
|
+
const calledProcesses = new Set(args.byProcessEngineModel.filter((row) => row.calls > 0).map((row) => row.process));
|
|
107
|
+
const reflectSkipCounts = countReflectSkipReasons(args.persistedActions);
|
|
108
|
+
const noCalls = [];
|
|
109
|
+
for (const row of routing) {
|
|
110
|
+
if (calledProcesses.has(row.process))
|
|
111
|
+
continue;
|
|
112
|
+
const eligibleRefs = REF_SCOPED_PROCESSES.has(row.process)
|
|
113
|
+
? args.loopRefs.filter((entry) => !shouldSkipRef(entry.ref, row.process, args.resolvedPlan.strategy.config).skip).length
|
|
114
|
+
: undefined;
|
|
115
|
+
const reason = deriveNoCallReason({
|
|
116
|
+
row,
|
|
117
|
+
autonomyGated: args.resolvedPlan.autonomyGated,
|
|
118
|
+
strategyFilteredRefsCount: args.strategyFilteredRefsCount,
|
|
119
|
+
eligibleRefs,
|
|
120
|
+
skipReasonCounts: row.process === "reflect"
|
|
121
|
+
? reflectSkipCounts
|
|
122
|
+
: row.process === "distill"
|
|
123
|
+
? args.distillSkippedAggregate?.byReason
|
|
124
|
+
: undefined,
|
|
125
|
+
});
|
|
126
|
+
if (reason === undefined)
|
|
127
|
+
continue;
|
|
128
|
+
noCalls.push({ process: row.process, ...(row.engine ? { engine: row.engine } : {}), reason });
|
|
129
|
+
}
|
|
130
|
+
if (args.byProcessEngineModel.length === 0 && noCalls.length === 0)
|
|
131
|
+
return undefined;
|
|
132
|
+
return { byProcessEngineModel: args.byProcessEngineModel, noCalls };
|
|
133
|
+
}
|
|
134
|
+
// ── Shared fixed-width text rendering ────────────────────────────────────────
|
|
135
|
+
function renderFixedWidthTable(headers, rows) {
|
|
136
|
+
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)));
|
|
137
|
+
const renderRow = (cells) => cells
|
|
138
|
+
.map((cell, index) => cell.padEnd(widths[index] ?? 0))
|
|
139
|
+
.join(" ")
|
|
140
|
+
.trimEnd();
|
|
141
|
+
return [renderRow(headers), ...rows.map(renderRow)];
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Render a `usageReport` as fixed-width plain text — the ONE formatter shared
|
|
145
|
+
* by the end-of-run `[improve] ...` stderr table (`improve-cli.ts`) and `akm
|
|
146
|
+
* improve report`'s `--format text` output
|
|
147
|
+
* (`src/output/text/improve-report.ts`), per the brief. `notes` surfaces
|
|
148
|
+
* degraded-precision caveats (e.g. a pre-0.9.15 run recomputed from raw
|
|
149
|
+
* events, per `improve-report.ts`).
|
|
150
|
+
*/
|
|
151
|
+
export function formatUsageReportTable(usageReport, notes) {
|
|
152
|
+
const lines = ["[improve] usage report (process x engine x model):"];
|
|
153
|
+
if (usageReport.byProcessEngineModel.length === 0) {
|
|
154
|
+
lines.push(" (no LLM calls recorded)");
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
const headers = ["process", "engine", "model", "calls", "failures", "promptTok", "complTok", "totalTok", "ms"];
|
|
158
|
+
const rows = usageReport.byProcessEngineModel.map((row) => [
|
|
159
|
+
row.process,
|
|
160
|
+
row.engine,
|
|
161
|
+
row.model,
|
|
162
|
+
String(row.calls),
|
|
163
|
+
String(row.failures),
|
|
164
|
+
String(row.promptTokens),
|
|
165
|
+
String(row.completionTokens),
|
|
166
|
+
String(row.totalTokens),
|
|
167
|
+
String(row.totalDurationMs),
|
|
168
|
+
]);
|
|
169
|
+
for (const line of renderFixedWidthTable(headers, rows))
|
|
170
|
+
lines.push(` ${line}`);
|
|
171
|
+
}
|
|
172
|
+
if (usageReport.noCalls.length > 0) {
|
|
173
|
+
lines.push("[improve] enabled processes with zero calls:");
|
|
174
|
+
const headers = ["process", "engine", "reason"];
|
|
175
|
+
const rows = usageReport.noCalls.map((row) => [row.process, row.engine ?? "-", row.reason]);
|
|
176
|
+
for (const line of renderFixedWidthTable(headers, rows))
|
|
177
|
+
lines.push(` ${line}`);
|
|
178
|
+
}
|
|
179
|
+
for (const note of notes ?? [])
|
|
180
|
+
lines.push(` note: ${note}`);
|
|
181
|
+
return lines.join("\n");
|
|
182
|
+
}
|