akm-cli 0.9.28-alpha.1 → 0.9.28-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/STABILITY.md +4 -0
- package/dist/assets/hints/cli-hints-full.md +3 -0
- package/dist/assets/hints/cli-hints-short.md +1 -0
- package/dist/assets/templates/html/metrics.html +977 -0
- package/dist/cli/shared.js +5 -4
- package/dist/cli.js +11 -1
- package/dist/commands/health/accept-rate.js +8 -4
- package/dist/commands/health/html-report.js +3 -8
- package/dist/commands/health/llm-usage.js +17 -6
- package/dist/commands/health/renderers.js +4 -4
- package/dist/commands/improve/improve-report.js +4 -2
- package/dist/commands/improve/improve.js +22 -10
- package/dist/commands/metrics/collect.js +439 -0
- package/dist/commands/metrics/html-report.js +82 -0
- package/dist/commands/metrics/md-report.js +44 -0
- package/dist/commands/metrics/metrics-cli.js +213 -0
- package/dist/commands/metrics/report-view.js +243 -0
- package/dist/commands/metrics/types.js +4 -0
- package/dist/commands/read/search.js +5 -0
- package/dist/indexer/indexer.js +64 -14
- package/dist/indexer/usage/usage-events.js +3 -1
- package/dist/integrations/session-logs/pre-filter.js +1 -0
- package/dist/llm/usage-persist.js +22 -11
- package/dist/llm/usage-telemetry.js +4 -0
- package/dist/output/html-render.js +15 -8
- package/dist/output/shapes/passthrough.js +1 -0
- package/dist/output/text/metrics.js +39 -0
- package/dist/output/text.js +2 -0
- package/dist/scripts/akm-migrate-node.js +281 -8
- package/dist/scripts/akm-migrate.js +281 -8
- package/dist/storage/repositories/index-utility-repository.js +24 -0
- package/dist/storage/repositories/metrics-repository.js +80 -0
- package/docs/reference/cli.md +63 -4
- package/docs/reference/data-and-telemetry.md +40 -7
- package/package.json +1 -1
package/dist/cli/shared.js
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import { defineCommand } from "citty";
|
|
11
11
|
import { stringify as yamlStringify } from "yaml";
|
|
12
12
|
import { renderHealthHtml } from "../commands/health/renderers.js";
|
|
13
|
+
import { renderMetricsHtml } from "../commands/metrics/html-report.js";
|
|
13
14
|
import { assertNever } from "../core/assert.js";
|
|
14
15
|
import { AkmError, UsageError } from "../core/errors.js";
|
|
15
16
|
import { getOutputMode } from "../output/context.js";
|
|
@@ -331,10 +332,10 @@ export function output(command, result) {
|
|
|
331
332
|
return;
|
|
332
333
|
}
|
|
333
334
|
case "html": {
|
|
334
|
-
// `akm health`
|
|
335
|
-
// called directly rather than through a
|
|
336
|
-
//
|
|
337
|
-
const rendered = command === "health" ? renderHealthHtml(shaped) : null;
|
|
335
|
+
// `akm health` and `akm metrics` are the only commands with a bespoke
|
|
336
|
+
// HTML report, so they are called directly rather than through a
|
|
337
|
+
// registry with two possible registrants (see `output/render-registry.ts`).
|
|
338
|
+
const rendered = command === "health" ? renderHealthHtml(shaped) : command === "metrics" ? renderMetricsHtml(shaped) : null;
|
|
338
339
|
deliverRendered(rendered ?? renderGenericHtml(command, shaped), mode.outputPath);
|
|
339
340
|
return;
|
|
340
341
|
}
|
package/dist/cli.js
CHANGED
|
@@ -70,8 +70,10 @@ import { secretCommand } from "./commands/env/secret-cli.js";
|
|
|
70
70
|
import { feedbackCommand } from "./commands/feedback-cli.js";
|
|
71
71
|
import { akmHealth } from "./commands/health.js";
|
|
72
72
|
import "./commands/health/renderers.js";
|
|
73
|
+
import "./commands/metrics/md-report.js";
|
|
73
74
|
import { parseWindowSpec } from "./commands/health/windows.js";
|
|
74
75
|
import { improveCommand } from "./commands/improve/improve-cli.js";
|
|
76
|
+
import { metricsCommand } from "./commands/metrics/metrics-cli.js";
|
|
75
77
|
import { migrateCommand } from "./commands/migrate-cli.js";
|
|
76
78
|
import { modelsCommand } from "./commands/models-cli.js";
|
|
77
79
|
import { logCommand } from "./commands/observability-cli.js";
|
|
@@ -94,6 +96,7 @@ import { DURATION_UNITS, parseDuration } from "./core/time.js";
|
|
|
94
96
|
import { plainize } from "./core/tty.js";
|
|
95
97
|
import { info, isQuiet, setQuiet, setVerbose, warn } from "./core/warn.js";
|
|
96
98
|
import { disposeDispatchResources } from "./integrations/agent/runner-dispatch.js";
|
|
99
|
+
import { installLlmUsagePersistenceIfAbsent } from "./llm/usage-persist.js";
|
|
97
100
|
import { EMBEDDED_HINTS, EMBEDDED_HINTS_FULL } from "./output/cli-hints.js";
|
|
98
101
|
import { getOutputMode, initOutputMode, parseDetailLevel } from "./output/context.js";
|
|
99
102
|
import { isFormatExemptCommand } from "./output/format-exempt.js";
|
|
@@ -448,6 +451,7 @@ const commands = {
|
|
|
448
451
|
setup: setupCommand,
|
|
449
452
|
index: indexCommand,
|
|
450
453
|
health: healthCommand,
|
|
454
|
+
metrics: metricsCommand,
|
|
451
455
|
info: infoCommand,
|
|
452
456
|
bundle: bundleCommand,
|
|
453
457
|
upgrade: upgradeCommand,
|
|
@@ -795,6 +799,7 @@ const HELP_SECTIONS = [
|
|
|
795
799
|
"index",
|
|
796
800
|
"lint",
|
|
797
801
|
"health",
|
|
802
|
+
"metrics",
|
|
798
803
|
"config",
|
|
799
804
|
"models",
|
|
800
805
|
"registry",
|
|
@@ -965,7 +970,7 @@ export function normalizeCittyCliError(error, rawArgs) {
|
|
|
965
970
|
* and returns, every direct call site here needs its own explicit `return;`
|
|
966
971
|
* to stop the rest of startup from running after a fatal early error.
|
|
967
972
|
*/
|
|
968
|
-
async function runCli() {
|
|
973
|
+
export async function runCli() {
|
|
969
974
|
try {
|
|
970
975
|
process.argv = consumeSchedulerContextArg(process.argv);
|
|
971
976
|
}
|
|
@@ -1056,6 +1061,10 @@ async function runCli() {
|
|
|
1056
1061
|
console.error(plainize("👋 First time with akm? Run `akm setup` to get started.\n Docs: https://github.com/itlackey/akm#readme\n"));
|
|
1057
1062
|
})();
|
|
1058
1063
|
const rawArgs = process.argv.slice(2);
|
|
1064
|
+
// Persist LLM usage for every command, not only `improve` and `proposal
|
|
1065
|
+
// drain`. A per-run sink installed inside a command layers over this one and
|
|
1066
|
+
// restores it on dispose.
|
|
1067
|
+
const disposeLlmUsageSink = installLlmUsagePersistenceIfAbsent();
|
|
1059
1068
|
try {
|
|
1060
1069
|
if (rawArgs.length === 0) {
|
|
1061
1070
|
process.stdout.write(`${await renderSectionedRootHelp()}\n`);
|
|
@@ -1119,6 +1128,7 @@ async function runCli() {
|
|
|
1119
1128
|
emitJsonError(error);
|
|
1120
1129
|
}
|
|
1121
1130
|
finally {
|
|
1131
|
+
disposeLlmUsageSink();
|
|
1122
1132
|
await disposeDispatchResources();
|
|
1123
1133
|
}
|
|
1124
1134
|
}
|
|
@@ -16,18 +16,22 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { resolveStashDir } from "../../core/common.js";
|
|
18
18
|
import { isProceduralRejection } from "../proposal/proposal-types.js";
|
|
19
|
-
import { listProposals } from "../proposal/repository.js";
|
|
19
|
+
import { listProposals, listProposalsReadOnly } from "../proposal/repository.js";
|
|
20
20
|
/**
|
|
21
21
|
* Compute accept-rate-per-source metrics from the proposal store. Defaults to
|
|
22
22
|
* the configured default stash when `stashDir` is omitted (same resolution
|
|
23
|
-
* `akm health` already uses for the rest of its report).
|
|
23
|
+
* `akm health` already uses for the rest of its report). A caller that passes
|
|
24
|
+
* `ctx` reads through {@link listProposalsReadOnly}, which never opens
|
|
25
|
+
* state.db for writing (`akm metrics`).
|
|
24
26
|
*/
|
|
25
|
-
export function computeAcceptRateBySource(stashDir) {
|
|
27
|
+
export function computeAcceptRateBySource(stashDir, ctx) {
|
|
26
28
|
const stash = stashDir ?? resolveStashDir();
|
|
27
29
|
const bySource = new Map();
|
|
28
30
|
const countProposals = (statuses, includeArchive) => {
|
|
29
31
|
for (const status of statuses) {
|
|
30
|
-
const proposals =
|
|
32
|
+
const proposals = ctx
|
|
33
|
+
? listProposalsReadOnly(stash, { status, includeArchive }, ctx)
|
|
34
|
+
: listProposals(stash, { status, includeArchive });
|
|
31
35
|
for (const p of proposals) {
|
|
32
36
|
// A stale-target auto-reject (STALE, R20) is procedural, not a
|
|
33
37
|
// judgement on the content — counting it would understate the
|
|
@@ -26,15 +26,10 @@
|
|
|
26
26
|
* pointed at the jsDelivr CDN, so viewing the report now requires network
|
|
27
27
|
* access. There is no env var or option to opt back into an offline report.
|
|
28
28
|
*/
|
|
29
|
-
import { escapeHtml } from "../../output/html-render.js";
|
|
29
|
+
import { escapeHtml, isoTimeTag } from "../../output/html-render.js";
|
|
30
30
|
import { pkgVersion } from "../../version.js";
|
|
31
31
|
import { buildHealthReportViewModel, compact, fmtMs, humanize, num, } from "./report-view-model.js";
|
|
32
32
|
const esc = escapeHtml;
|
|
33
|
-
/** Emit a <time> element that the browser's JS will reformat to the viewer's local timezone. */
|
|
34
|
-
function isoTimeTag(iso) {
|
|
35
|
-
const fallback = iso.slice(0, 16).replace("T", " ");
|
|
36
|
-
return `<time data-iso="${esc(iso)}">${esc(fallback)}</time>`;
|
|
37
|
-
}
|
|
38
33
|
function trendClass(direction) {
|
|
39
34
|
return direction === "up" ? "trend-up" : direction === "down" ? "trend-down" : "trend-flat";
|
|
40
35
|
}
|
|
@@ -79,8 +74,8 @@ const badgeByStatus = {
|
|
|
79
74
|
};
|
|
80
75
|
// ── ECharts delivery ─────────────────────────────────────────────────────────
|
|
81
76
|
const ECHARTS_CDN = "https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js";
|
|
82
|
-
/** Always CDN — see the module docstring's chunk-9 WI-9.4d note. */
|
|
83
|
-
function buildEchartsTag() {
|
|
77
|
+
/** Always CDN — see the module docstring's chunk-9 WI-9.4d note. Shared with the metrics dashboard. */
|
|
78
|
+
export function buildEchartsTag() {
|
|
84
79
|
return `<script src="${ECHARTS_CDN}"></script>`;
|
|
85
80
|
}
|
|
86
81
|
/**
|
|
@@ -68,7 +68,7 @@ export function readLlmUsageAggregate(stateDbPath, since, until) {
|
|
|
68
68
|
return summarizeLlmUsage(events);
|
|
69
69
|
}
|
|
70
70
|
/**
|
|
71
|
-
* Aggregate `llm_usage`
|
|
71
|
+
* Aggregate `llm_usage` records (#576) into a process x engine x model
|
|
72
72
|
* cross-tab (#944) — one row per distinct `(process, engine, model)` triple
|
|
73
73
|
* seen, each carrying the same call/failure/token/duration totals as
|
|
74
74
|
* {@link LlmUsageStageAggregate}. A call missing any one of the three
|
|
@@ -78,12 +78,9 @@ export function readLlmUsageAggregate(stateDbPath, since, until) {
|
|
|
78
78
|
* breakdowns — `akm health`'s existing consumers of those stay untouched.
|
|
79
79
|
* Row order is insertion order (first `(process, engine, model)` triple seen).
|
|
80
80
|
*/
|
|
81
|
-
export function
|
|
81
|
+
export function summarizeLlmUsageRecordsCrossTab(records) {
|
|
82
82
|
const rows = new Map();
|
|
83
|
-
for (const
|
|
84
|
-
const record = decodeLlmUsageRecord(event.metadata);
|
|
85
|
-
if (!record)
|
|
86
|
-
continue;
|
|
83
|
+
for (const record of records) {
|
|
87
84
|
const process = record.process ?? UNATTRIBUTED_STAGE;
|
|
88
85
|
const engine = record.engine ?? UNATTRIBUTED_STAGE;
|
|
89
86
|
const model = record.model ?? UNATTRIBUTED_STAGE;
|
|
@@ -104,3 +101,17 @@ export function summarizeLlmUsageCrossTab(events) {
|
|
|
104
101
|
}
|
|
105
102
|
return [...rows.values()];
|
|
106
103
|
}
|
|
104
|
+
/**
|
|
105
|
+
* The events form of {@link summarizeLlmUsageRecordsCrossTab}: decode each
|
|
106
|
+
* `llm_usage` event's metadata (dropping undecodable rows) and aggregate the
|
|
107
|
+
* records. Use it when only persisted events are available.
|
|
108
|
+
*/
|
|
109
|
+
export function summarizeLlmUsageCrossTab(events) {
|
|
110
|
+
const records = [];
|
|
111
|
+
for (const event of events) {
|
|
112
|
+
const record = decodeLlmUsageRecord(event.metadata);
|
|
113
|
+
if (record)
|
|
114
|
+
records.push(record);
|
|
115
|
+
}
|
|
116
|
+
return summarizeLlmUsageRecordsCrossTab(records);
|
|
117
|
+
}
|
|
@@ -15,10 +15,10 @@
|
|
|
15
15
|
*
|
|
16
16
|
* The Markdown renderer registers into the shared per-command registry (D7,
|
|
17
17
|
* `../../output/render-registry.ts`) alongside every other command's `--format
|
|
18
|
-
* md` renderer. The HTML renderer does not: `health`
|
|
19
|
-
*
|
|
20
|
-
* {@link renderHealthHtml} directly instead of going
|
|
21
|
-
*
|
|
18
|
+
* md` renderer. The HTML renderer does not: `health` and `metrics` are the only
|
|
19
|
+
* commands with a bespoke HTML report, so `cli/shared.ts` calls
|
|
20
|
+
* {@link renderHealthHtml} (and `renderMetricsHtml`) directly instead of going
|
|
21
|
+
* through a registry with two possible registrants.
|
|
22
22
|
*
|
|
23
23
|
* Returning `null` falls through to the generic renderer, so a result without
|
|
24
24
|
* the report dataset still renders — generically — instead of erroring or
|
|
@@ -25,8 +25,10 @@ import { summarizeLlmUsageCrossTab } from "../health/llm-usage.js";
|
|
|
25
25
|
const PRE_USAGE_REPORT_NOTE = "eligibility reasons unavailable for runs recorded before 0.9.15";
|
|
26
26
|
const TERMINATED_RUN_NOTE = "run terminated before completion; no usage report was recorded";
|
|
27
27
|
/**
|
|
28
|
-
* Recompute the cross-tab from
|
|
29
|
-
* row has no persisted one).
|
|
28
|
+
* Recompute the cross-tab from the `llm_usage` events in this run's wall-clock
|
|
29
|
+
* window (a pre-#944 row has no persisted one). Events carry no run id, so a
|
|
30
|
+
* recomputed cross-tab can include LLM calls that other akm processes made
|
|
31
|
+
* during the run's window. `eventsCtx` threads the state.db connection
|
|
30
32
|
* `runImproveReportQuery` already holds via `withStateDb`, so a `--since`
|
|
31
33
|
* window with several pre-0.9.15 or undecodable rows reads through that one
|
|
32
34
|
* open connection instead of each row opening and closing its own.
|
|
@@ -11,7 +11,7 @@ import path from "node:path";
|
|
|
11
11
|
import { parseRefInput } from "../../core/asset/resolve-ref.js";
|
|
12
12
|
import { bundlesToSourceEntries, loadConfig } from "../../core/config/config.js";
|
|
13
13
|
import { ConfigError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
14
|
-
import { appendEvent
|
|
14
|
+
import { appendEvent } from "../../core/events.js";
|
|
15
15
|
import { classifyImproveAction, foldDistillSkipped } from "../../core/improve-types.js";
|
|
16
16
|
import { resolveMutationTarget } from "../../core/mutation-target.js";
|
|
17
17
|
import { getDbPath, getStashLocksDir, getStateDbPathInDataDir } from "../../core/paths.js";
|
|
@@ -26,13 +26,13 @@ import { akmIndex } from "../../indexer/indexer.js";
|
|
|
26
26
|
import { collectPendingMemories } from "../../indexer/passes/memory-inference.js";
|
|
27
27
|
import { resolveEntryContentDir, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
28
28
|
import { collectEngineCredentialValues } from "../../integrations/agent/engine-resolution.js";
|
|
29
|
-
import { installLlmUsagePersistence
|
|
29
|
+
import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
|
|
30
30
|
import { withLlmStage } from "../../llm/usage-telemetry.js";
|
|
31
31
|
import { isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "../../sources/providers/git.js";
|
|
32
32
|
import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
|
|
33
33
|
import { getEntryCount } from "../../storage/repositories/index-entries-repository.js";
|
|
34
34
|
import { openSqliteReadSnapshot, SqliteReadSnapshotUnavailableError } from "../../storage/sqlite-read-snapshot.js";
|
|
35
|
-
import {
|
|
35
|
+
import { summarizeLlmUsageRecordsCrossTab } from "../health/llm-usage.js";
|
|
36
36
|
import { drainProposals } from "../proposal/drain.js";
|
|
37
37
|
import { describeGatedLanes, isAutonomyLaneAllowed } from "./autonomy-gate.js";
|
|
38
38
|
import { akmDistill } from "./distill.js";
|
|
@@ -118,6 +118,9 @@ export async function akmImprove(options = {}) {
|
|
|
118
118
|
// The usage sink resolves it per append.
|
|
119
119
|
let eventsCtx = { dbPath: resolvedStateDbPath };
|
|
120
120
|
let disposeLlmUsageSink = () => { };
|
|
121
|
+
// Every terminal LLM record this run's own sink sees, in call order. The
|
|
122
|
+
// usage report is built from these, not from a wall-clock read of state.db.
|
|
123
|
+
const usageRecords = [];
|
|
121
124
|
const releaseRunLock = () => {
|
|
122
125
|
const ownership = improveLockOwnership;
|
|
123
126
|
if (!ownership)
|
|
@@ -164,7 +167,8 @@ export async function akmImprove(options = {}) {
|
|
|
164
167
|
return buildLockSkippedResult(selectedStrategy.name, scope, options.runId);
|
|
165
168
|
}
|
|
166
169
|
improveLockOwnership = acquisition.ownership;
|
|
167
|
-
disposeLlmUsageSink = installLlmUsagePersistence(() => eventsCtx, () => {
|
|
170
|
+
disposeLlmUsageSink = installLlmUsagePersistence(() => eventsCtx, (record) => {
|
|
171
|
+
usageRecords.push(record);
|
|
168
172
|
firstEngineResponseSeen = true;
|
|
169
173
|
clearFirstResponseHeartbeat();
|
|
170
174
|
});
|
|
@@ -227,7 +231,15 @@ export async function akmImprove(options = {}) {
|
|
|
227
231
|
clearFirstResponseHeartbeat = () => clearTimeout(firstResponseTimer);
|
|
228
232
|
}
|
|
229
233
|
const seq = await runImproveStageSequence(setup, collected, preEnsureCleanupWarnings, eventsCtx);
|
|
230
|
-
const result = finalizeImproveResult({
|
|
234
|
+
const result = finalizeImproveResult({
|
|
235
|
+
run: setup,
|
|
236
|
+
seq,
|
|
237
|
+
collected,
|
|
238
|
+
triageDrain,
|
|
239
|
+
ensureIndexDurationMs,
|
|
240
|
+
eventsCtx,
|
|
241
|
+
usageRecords,
|
|
242
|
+
});
|
|
231
243
|
// The run's write provenance goes on the envelope before the sync, so
|
|
232
244
|
// `writtenPaths` is exactly the set the commit is scoped to.
|
|
233
245
|
const writtenPaths = describeRunWrittenPaths(setup, journal?.writtenPaths() ?? []);
|
|
@@ -843,7 +855,7 @@ async function runImproveStageSequence(run, collected, preEnsureCleanupWarnings,
|
|
|
843
855
|
}
|
|
844
856
|
/** Assemble the result envelope and emit `improve_completed`. */
|
|
845
857
|
function finalizeImproveResult(args) {
|
|
846
|
-
const { run, collected, triageDrain, ensureIndexDurationMs, eventsCtx } = args;
|
|
858
|
+
const { run, collected, triageDrain, ensureIndexDurationMs, eventsCtx, usageRecords } = args;
|
|
847
859
|
const { preparation, postLoop, reflectsWithErrorContext, finalActions } = args.seq;
|
|
848
860
|
const { options, startMs, resolvedPlan } = run;
|
|
849
861
|
const { memoryCleanupPlan, strategyFilteredRefs } = collected;
|
|
@@ -852,12 +864,12 @@ function finalizeImproveResult(args) {
|
|
|
852
864
|
const { memoryInferenceDurationMs } = postLoop;
|
|
853
865
|
// The per-ref distill-skipped rows fold into a bounded aggregate before persistence (C1).
|
|
854
866
|
const { actions: persistedActions, aggregate: distillSkippedAggregate } = foldDistillSkipped(finalActions);
|
|
855
|
-
// This run's LLM accounting (#944):
|
|
856
|
-
//
|
|
857
|
-
|
|
867
|
+
// This run's LLM accounting (#944): built from the records this run's own
|
|
868
|
+
// sink collected, so calls other akm processes persist to state.db while
|
|
869
|
+
// the run is in flight (llm_usage rows carry no run id) never count here.
|
|
858
870
|
const usageReport = buildImproveUsageReport({
|
|
859
871
|
resolvedPlan,
|
|
860
|
-
byProcessEngineModel:
|
|
872
|
+
byProcessEngineModel: summarizeLlmUsageRecordsCrossTab(usageRecords),
|
|
861
873
|
strategyFilteredRefsCount: strategyFilteredRefs.length,
|
|
862
874
|
loopRefs: preparation.loopRefs,
|
|
863
875
|
persistedActions,
|