akm-cli 0.9.28-alpha.1 → 0.9.28-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +43 -0
  2. package/STABILITY.md +4 -0
  3. package/dist/assets/hints/cli-hints-full.md +3 -0
  4. package/dist/assets/hints/cli-hints-short.md +1 -0
  5. package/dist/assets/templates/html/metrics.html +977 -0
  6. package/dist/cli/shared.js +5 -4
  7. package/dist/cli.js +11 -1
  8. package/dist/commands/health/accept-rate.js +8 -4
  9. package/dist/commands/health/html-report.js +3 -8
  10. package/dist/commands/health/llm-usage.js +17 -6
  11. package/dist/commands/health/renderers.js +4 -4
  12. package/dist/commands/improve/improve-report.js +4 -2
  13. package/dist/commands/improve/improve.js +22 -10
  14. package/dist/commands/metrics/collect.js +439 -0
  15. package/dist/commands/metrics/html-report.js +82 -0
  16. package/dist/commands/metrics/md-report.js +44 -0
  17. package/dist/commands/metrics/metrics-cli.js +213 -0
  18. package/dist/commands/metrics/report-view.js +243 -0
  19. package/dist/commands/metrics/types.js +4 -0
  20. package/dist/commands/read/search.js +5 -0
  21. package/dist/indexer/indexer.js +64 -14
  22. package/dist/indexer/usage/usage-events.js +3 -1
  23. package/dist/integrations/session-logs/pre-filter.js +1 -0
  24. package/dist/llm/usage-persist.js +22 -11
  25. package/dist/llm/usage-telemetry.js +4 -0
  26. package/dist/output/html-render.js +15 -8
  27. package/dist/output/shapes/passthrough.js +1 -0
  28. package/dist/output/text/metrics.js +39 -0
  29. package/dist/output/text.js +2 -0
  30. package/dist/scripts/akm-migrate-node.js +281 -8
  31. package/dist/scripts/akm-migrate.js +281 -8
  32. package/dist/storage/repositories/index-utility-repository.js +24 -0
  33. package/dist/storage/repositories/metrics-repository.js +80 -0
  34. package/docs/reference/cli.md +63 -4
  35. package/docs/reference/data-and-telemetry.md +40 -7
  36. package/package.json +1 -1
@@ -10,6 +10,7 @@
10
10
  import { defineCommand } from "citty";
11
11
  import { stringify as yamlStringify } from "yaml";
12
12
  import { renderHealthHtml } from "../commands/health/renderers.js";
13
+ import { renderMetricsHtml } from "../commands/metrics/html-report.js";
13
14
  import { assertNever } from "../core/assert.js";
14
15
  import { AkmError, UsageError } from "../core/errors.js";
15
16
  import { getOutputMode } from "../output/context.js";
@@ -331,10 +332,10 @@ export function output(command, result) {
331
332
  return;
332
333
  }
333
334
  case "html": {
334
- // `akm health` is the only command with a bespoke HTML report, so it is
335
- // called directly rather than through a registry with one possible
336
- // registrant (see `output/render-registry.ts`).
337
- const rendered = command === "health" ? renderHealthHtml(shaped) : null;
335
+ // `akm health` and `akm metrics` are the only commands with a bespoke
336
+ // HTML report, so they are called directly rather than through a
337
+ // registry with two possible registrants (see `output/render-registry.ts`).
338
+ const rendered = command === "health" ? renderHealthHtml(shaped) : command === "metrics" ? renderMetricsHtml(shaped) : null;
338
339
  deliverRendered(rendered ?? renderGenericHtml(command, shaped), mode.outputPath);
339
340
  return;
340
341
  }
package/dist/cli.js CHANGED
@@ -70,8 +70,10 @@ import { secretCommand } from "./commands/env/secret-cli.js";
70
70
  import { feedbackCommand } from "./commands/feedback-cli.js";
71
71
  import { akmHealth } from "./commands/health.js";
72
72
  import "./commands/health/renderers.js";
73
+ import "./commands/metrics/md-report.js";
73
74
  import { parseWindowSpec } from "./commands/health/windows.js";
74
75
  import { improveCommand } from "./commands/improve/improve-cli.js";
76
+ import { metricsCommand } from "./commands/metrics/metrics-cli.js";
75
77
  import { migrateCommand } from "./commands/migrate-cli.js";
76
78
  import { modelsCommand } from "./commands/models-cli.js";
77
79
  import { logCommand } from "./commands/observability-cli.js";
@@ -94,6 +96,7 @@ import { DURATION_UNITS, parseDuration } from "./core/time.js";
94
96
  import { plainize } from "./core/tty.js";
95
97
  import { info, isQuiet, setQuiet, setVerbose, warn } from "./core/warn.js";
96
98
  import { disposeDispatchResources } from "./integrations/agent/runner-dispatch.js";
99
+ import { installLlmUsagePersistenceIfAbsent } from "./llm/usage-persist.js";
97
100
  import { EMBEDDED_HINTS, EMBEDDED_HINTS_FULL } from "./output/cli-hints.js";
98
101
  import { getOutputMode, initOutputMode, parseDetailLevel } from "./output/context.js";
99
102
  import { isFormatExemptCommand } from "./output/format-exempt.js";
@@ -448,6 +451,7 @@ const commands = {
448
451
  setup: setupCommand,
449
452
  index: indexCommand,
450
453
  health: healthCommand,
454
+ metrics: metricsCommand,
451
455
  info: infoCommand,
452
456
  bundle: bundleCommand,
453
457
  upgrade: upgradeCommand,
@@ -795,6 +799,7 @@ const HELP_SECTIONS = [
795
799
  "index",
796
800
  "lint",
797
801
  "health",
802
+ "metrics",
798
803
  "config",
799
804
  "models",
800
805
  "registry",
@@ -965,7 +970,7 @@ export function normalizeCittyCliError(error, rawArgs) {
965
970
  * and returns, every direct call site here needs its own explicit `return;`
966
971
  * to stop the rest of startup from running after a fatal early error.
967
972
  */
968
- async function runCli() {
973
+ export async function runCli() {
969
974
  try {
970
975
  process.argv = consumeSchedulerContextArg(process.argv);
971
976
  }
@@ -1056,6 +1061,10 @@ async function runCli() {
1056
1061
  console.error(plainize("👋 First time with akm? Run `akm setup` to get started.\n Docs: https://github.com/itlackey/akm#readme\n"));
1057
1062
  })();
1058
1063
  const rawArgs = process.argv.slice(2);
1064
+ // Persist LLM usage for every command, not only `improve` and `proposal
1065
+ // drain`. A per-run sink installed inside a command layers over this one and
1066
+ // restores it on dispose.
1067
+ const disposeLlmUsageSink = installLlmUsagePersistenceIfAbsent();
1059
1068
  try {
1060
1069
  if (rawArgs.length === 0) {
1061
1070
  process.stdout.write(`${await renderSectionedRootHelp()}\n`);
@@ -1119,6 +1128,7 @@ async function runCli() {
1119
1128
  emitJsonError(error);
1120
1129
  }
1121
1130
  finally {
1131
+ disposeLlmUsageSink();
1122
1132
  await disposeDispatchResources();
1123
1133
  }
1124
1134
  }
@@ -16,18 +16,22 @@
16
16
  */
17
17
  import { resolveStashDir } from "../../core/common.js";
18
18
  import { isProceduralRejection } from "../proposal/proposal-types.js";
19
- import { listProposals } from "../proposal/repository.js";
19
+ import { listProposals, listProposalsReadOnly } from "../proposal/repository.js";
20
20
  /**
21
21
  * Compute accept-rate-per-source metrics from the proposal store. Defaults to
22
22
  * the configured default stash when `stashDir` is omitted (same resolution
23
- * `akm health` already uses for the rest of its report).
23
+ * `akm health` already uses for the rest of its report). A caller that passes
24
+ * `ctx` reads through {@link listProposalsReadOnly}, which never opens
25
+ * state.db for writing (`akm metrics`).
24
26
  */
25
- export function computeAcceptRateBySource(stashDir) {
27
+ export function computeAcceptRateBySource(stashDir, ctx) {
26
28
  const stash = stashDir ?? resolveStashDir();
27
29
  const bySource = new Map();
28
30
  const countProposals = (statuses, includeArchive) => {
29
31
  for (const status of statuses) {
30
- const proposals = listProposals(stash, { status, includeArchive });
32
+ const proposals = ctx
33
+ ? listProposalsReadOnly(stash, { status, includeArchive }, ctx)
34
+ : listProposals(stash, { status, includeArchive });
31
35
  for (const p of proposals) {
32
36
  // A stale-target auto-reject (STALE, R20) is procedural, not a
33
37
  // judgement on the content — counting it would understate the
@@ -26,15 +26,10 @@
26
26
  * pointed at the jsDelivr CDN, so viewing the report now requires network
27
27
  * access. There is no env var or option to opt back into an offline report.
28
28
  */
29
- import { escapeHtml } from "../../output/html-render.js";
29
+ import { escapeHtml, isoTimeTag } from "../../output/html-render.js";
30
30
  import { pkgVersion } from "../../version.js";
31
31
  import { buildHealthReportViewModel, compact, fmtMs, humanize, num, } from "./report-view-model.js";
32
32
  const esc = escapeHtml;
33
- /** Emit a <time> element that the browser's JS will reformat to the viewer's local timezone. */
34
- function isoTimeTag(iso) {
35
- const fallback = iso.slice(0, 16).replace("T", " ");
36
- return `<time data-iso="${esc(iso)}">${esc(fallback)}</time>`;
37
- }
38
33
  function trendClass(direction) {
39
34
  return direction === "up" ? "trend-up" : direction === "down" ? "trend-down" : "trend-flat";
40
35
  }
@@ -79,8 +74,8 @@ const badgeByStatus = {
79
74
  };
80
75
  // ── ECharts delivery ─────────────────────────────────────────────────────────
81
76
  const ECHARTS_CDN = "https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js";
82
- /** Always CDN — see the module docstring's chunk-9 WI-9.4d note. */
83
- function buildEchartsTag() {
77
+ /** Always CDN — see the module docstring's chunk-9 WI-9.4d note. Shared with the metrics dashboard. */
78
+ export function buildEchartsTag() {
84
79
  return `<script src="${ECHARTS_CDN}"></script>`;
85
80
  }
86
81
  /**
@@ -68,7 +68,7 @@ export function readLlmUsageAggregate(stateDbPath, since, until) {
68
68
  return summarizeLlmUsage(events);
69
69
  }
70
70
  /**
71
- * Aggregate `llm_usage` events (#576) into a process x engine x model
71
+ * Aggregate `llm_usage` records (#576) into a process x engine x model
72
72
  * cross-tab (#944) — one row per distinct `(process, engine, model)` triple
73
73
  * seen, each carrying the same call/failure/token/duration totals as
74
74
  * {@link LlmUsageStageAggregate}. A call missing any one of the three
@@ -78,12 +78,9 @@ export function readLlmUsageAggregate(stateDbPath, since, until) {
78
78
  * breakdowns — `akm health`'s existing consumers of those stay untouched.
79
79
  * Row order is insertion order (first `(process, engine, model)` triple seen).
80
80
  */
81
- export function summarizeLlmUsageCrossTab(events) {
81
+ export function summarizeLlmUsageRecordsCrossTab(records) {
82
82
  const rows = new Map();
83
- for (const event of events) {
84
- const record = decodeLlmUsageRecord(event.metadata);
85
- if (!record)
86
- continue;
83
+ for (const record of records) {
87
84
  const process = record.process ?? UNATTRIBUTED_STAGE;
88
85
  const engine = record.engine ?? UNATTRIBUTED_STAGE;
89
86
  const model = record.model ?? UNATTRIBUTED_STAGE;
@@ -104,3 +101,17 @@ export function summarizeLlmUsageCrossTab(events) {
104
101
  }
105
102
  return [...rows.values()];
106
103
  }
104
+ /**
105
+ * The events form of {@link summarizeLlmUsageRecordsCrossTab}: decode each
106
+ * `llm_usage` event's metadata (dropping undecodable rows) and aggregate the
107
+ * records. Use it when only persisted events are available.
108
+ */
109
+ export function summarizeLlmUsageCrossTab(events) {
110
+ const records = [];
111
+ for (const event of events) {
112
+ const record = decodeLlmUsageRecord(event.metadata);
113
+ if (record)
114
+ records.push(record);
115
+ }
116
+ return summarizeLlmUsageRecordsCrossTab(records);
117
+ }
@@ -15,10 +15,10 @@
15
15
  *
16
16
  * The Markdown renderer registers into the shared per-command registry (D7,
17
17
  * `../../output/render-registry.ts`) alongside every other command's `--format
18
- * md` renderer. The HTML renderer does not: `health` was, and remains, the
19
- * only command with a bespoke HTML report, so `cli/shared.ts` calls
20
- * {@link renderHealthHtml} directly instead of going through a registry with
21
- * exactly one possible registrant.
18
+ * md` renderer. The HTML renderer does not: `health` and `metrics` are the only
19
+ * commands with a bespoke HTML report, so `cli/shared.ts` calls
20
+ * {@link renderHealthHtml} (and `renderMetricsHtml`) directly instead of going
21
+ * through a registry with two possible registrants.
22
22
  *
23
23
  * Returning `null` falls through to the generic renderer, so a result without
24
24
  * the report dataset still renders — generically — instead of erroring or
@@ -25,8 +25,10 @@ import { summarizeLlmUsageCrossTab } from "../health/llm-usage.js";
25
25
  const PRE_USAGE_REPORT_NOTE = "eligibility reasons unavailable for runs recorded before 0.9.15";
26
26
  const TERMINATED_RUN_NOTE = "run terminated before completion; no usage report was recorded";
27
27
  /**
28
- * Recompute the cross-tab from this run's own `llm_usage` events (a pre-#944
29
- * row has no persisted one). `eventsCtx` threads the state.db connection
28
+ * Recompute the cross-tab from the `llm_usage` events in this run's wall-clock
29
+ * window (a pre-#944 row has no persisted one). Events carry no run id, so a
30
+ * recomputed cross-tab can include LLM calls that other akm processes made
31
+ * during the run's window. `eventsCtx` threads the state.db connection
30
32
  * `runImproveReportQuery` already holds via `withStateDb`, so a `--since`
31
33
  * window with several pre-0.9.15 or undecodable rows reads through that one
32
34
  * open connection instead of each row opening and closing its own.
@@ -11,7 +11,7 @@ import path from "node:path";
11
11
  import { parseRefInput } from "../../core/asset/resolve-ref.js";
12
12
  import { bundlesToSourceEntries, loadConfig } from "../../core/config/config.js";
13
13
  import { ConfigError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
14
- import { appendEvent, readEvents } from "../../core/events.js";
14
+ import { appendEvent } from "../../core/events.js";
15
15
  import { classifyImproveAction, foldDistillSkipped } from "../../core/improve-types.js";
16
16
  import { resolveMutationTarget } from "../../core/mutation-target.js";
17
17
  import { getDbPath, getStashLocksDir, getStateDbPathInDataDir } from "../../core/paths.js";
@@ -26,13 +26,13 @@ import { akmIndex } from "../../indexer/indexer.js";
26
26
  import { collectPendingMemories } from "../../indexer/passes/memory-inference.js";
27
27
  import { resolveEntryContentDir, resolveSourceEntries } from "../../indexer/search/search-source.js";
28
28
  import { collectEngineCredentialValues } from "../../integrations/agent/engine-resolution.js";
29
- import { installLlmUsagePersistence, LLM_USAGE_EVENT } from "../../llm/usage-persist.js";
29
+ import { installLlmUsagePersistence } from "../../llm/usage-persist.js";
30
30
  import { withLlmStage } from "../../llm/usage-telemetry.js";
31
31
  import { isGitBackedStash, listGitChangedPaths, resolveWritableOverride, saveGitStash, } from "../../sources/providers/git.js";
32
32
  import { closeDatabase, openExistingDatabase } from "../../storage/repositories/index-connection.js";
33
33
  import { getEntryCount } from "../../storage/repositories/index-entries-repository.js";
34
34
  import { openSqliteReadSnapshot, SqliteReadSnapshotUnavailableError } from "../../storage/sqlite-read-snapshot.js";
35
- import { summarizeLlmUsageCrossTab } from "../health/llm-usage.js";
35
+ import { summarizeLlmUsageRecordsCrossTab } from "../health/llm-usage.js";
36
36
  import { drainProposals } from "../proposal/drain.js";
37
37
  import { describeGatedLanes, isAutonomyLaneAllowed } from "./autonomy-gate.js";
38
38
  import { akmDistill } from "./distill.js";
@@ -118,6 +118,9 @@ export async function akmImprove(options = {}) {
118
118
  // The usage sink resolves it per append.
119
119
  let eventsCtx = { dbPath: resolvedStateDbPath };
120
120
  let disposeLlmUsageSink = () => { };
121
+ // Every terminal LLM record this run's own sink sees, in call order. The
122
+ // usage report is built from these, not from a wall-clock read of state.db.
123
+ const usageRecords = [];
121
124
  const releaseRunLock = () => {
122
125
  const ownership = improveLockOwnership;
123
126
  if (!ownership)
@@ -164,7 +167,8 @@ export async function akmImprove(options = {}) {
164
167
  return buildLockSkippedResult(selectedStrategy.name, scope, options.runId);
165
168
  }
166
169
  improveLockOwnership = acquisition.ownership;
167
- disposeLlmUsageSink = installLlmUsagePersistence(() => eventsCtx, () => {
170
+ disposeLlmUsageSink = installLlmUsagePersistence(() => eventsCtx, (record) => {
171
+ usageRecords.push(record);
168
172
  firstEngineResponseSeen = true;
169
173
  clearFirstResponseHeartbeat();
170
174
  });
@@ -227,7 +231,15 @@ export async function akmImprove(options = {}) {
227
231
  clearFirstResponseHeartbeat = () => clearTimeout(firstResponseTimer);
228
232
  }
229
233
  const seq = await runImproveStageSequence(setup, collected, preEnsureCleanupWarnings, eventsCtx);
230
- const result = finalizeImproveResult({ run: setup, seq, collected, triageDrain, ensureIndexDurationMs, eventsCtx });
234
+ const result = finalizeImproveResult({
235
+ run: setup,
236
+ seq,
237
+ collected,
238
+ triageDrain,
239
+ ensureIndexDurationMs,
240
+ eventsCtx,
241
+ usageRecords,
242
+ });
231
243
  // The run's write provenance goes on the envelope before the sync, so
232
244
  // `writtenPaths` is exactly the set the commit is scoped to.
233
245
  const writtenPaths = describeRunWrittenPaths(setup, journal?.writtenPaths() ?? []);
@@ -843,7 +855,7 @@ async function runImproveStageSequence(run, collected, preEnsureCleanupWarnings,
843
855
  }
844
856
  /** Assemble the result envelope and emit `improve_completed`. */
845
857
  function finalizeImproveResult(args) {
846
- const { run, collected, triageDrain, ensureIndexDurationMs, eventsCtx } = args;
858
+ const { run, collected, triageDrain, ensureIndexDurationMs, eventsCtx, usageRecords } = args;
847
859
  const { preparation, postLoop, reflectsWithErrorContext, finalActions } = args.seq;
848
860
  const { options, startMs, resolvedPlan } = run;
849
861
  const { memoryCleanupPlan, strategyFilteredRefs } = collected;
@@ -852,12 +864,12 @@ function finalizeImproveResult(args) {
852
864
  const { memoryInferenceDurationMs } = postLoop;
853
865
  // The per-ref distill-skipped rows fold into a bounded aggregate before persistence (C1).
854
866
  const { actions: persistedActions, aggregate: distillSkippedAggregate } = foldDistillSkipped(finalActions);
855
- // This run's LLM accounting (#944): llm_usage rows carry no run id, so the
856
- // read is bounded by the run's own wall clock.
857
- const usageEvents = readEvents({ since: new Date(startMs).toISOString(), type: LLM_USAGE_EVENT }, eventsCtx).events;
867
+ // This run's LLM accounting (#944): built from the records this run's own
868
+ // sink collected, so calls other akm processes persist to state.db while
869
+ // the run is in flight (llm_usage rows carry no run id) never count here.
858
870
  const usageReport = buildImproveUsageReport({
859
871
  resolvedPlan,
860
- byProcessEngineModel: summarizeLlmUsageCrossTab(usageEvents),
872
+ byProcessEngineModel: summarizeLlmUsageRecordsCrossTab(usageRecords),
861
873
  strategyFilteredRefsCount: strategyFilteredRefs.length,
862
874
  loopRefs: preparation.loopRefs,
863
875
  persistedActions,