akm-cli 0.9.27 → 0.9.28-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/STABILITY.md +4 -0
- package/dist/assets/hints/cli-hints-full.md +3 -0
- package/dist/assets/hints/cli-hints-short.md +1 -0
- package/dist/assets/prompts/consolidate-pair.md +1 -1
- package/dist/assets/templates/html/metrics.html +977 -0
- package/dist/cli/shared.js +5 -4
- package/dist/cli.js +11 -1
- package/dist/commands/health/accept-rate.js +8 -4
- package/dist/commands/health/html-report.js +3 -8
- package/dist/commands/health/llm-usage.js +17 -6
- package/dist/commands/health/renderers.js +4 -4
- package/dist/commands/improve/improve-report.js +4 -2
- package/dist/commands/improve/improve.js +22 -10
- package/dist/commands/improve/loop-stages.js +8 -1
- package/dist/commands/improve/preparation.js +16 -0
- package/dist/commands/improve/stage.js +1 -1
- package/dist/commands/metrics/collect.js +439 -0
- package/dist/commands/metrics/html-report.js +82 -0
- package/dist/commands/metrics/md-report.js +44 -0
- package/dist/commands/metrics/metrics-cli.js +213 -0
- package/dist/commands/metrics/report-view.js +243 -0
- package/dist/commands/metrics/types.js +4 -0
- package/dist/commands/read/search.js +5 -0
- package/dist/indexer/indexer.js +64 -14
- package/dist/indexer/usage/usage-events.js +3 -1
- package/dist/integrations/session-logs/pre-filter.js +1 -0
- package/dist/llm/usage-persist.js +22 -11
- package/dist/llm/usage-telemetry.js +4 -0
- package/dist/output/html-render.js +15 -8
- package/dist/output/shapes/passthrough.js +1 -0
- package/dist/output/text/metrics.js +39 -0
- package/dist/output/text.js +2 -0
- package/dist/scripts/akm-migrate-node.js +281 -8
- package/dist/scripts/akm-migrate.js +281 -8
- package/dist/storage/repositories/index-utility-repository.js +24 -0
- package/dist/storage/repositories/metrics-repository.js +80 -0
- package/docs/reference/cli.md +63 -4
- package/docs/reference/data-and-telemetry.md +41 -8
- package/package.json +1 -1
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* `akm metrics` — one read-only report over what akm has already recorded:
|
|
6
|
+
* asset usage, feedback, utility, LLM usage, task runs, proposals and
|
|
7
|
+
* workflow spend. This module reads the stores and hands the rows to the pure
|
|
8
|
+
* aggregation in `collect.ts`; the shape is `AkmMetricsResult` (`types.ts`).
|
|
9
|
+
*
|
|
10
|
+
* Nothing is written, and nothing is migrated: `state.db` is opened read-only,
|
|
11
|
+
* and one with migrations pending is skipped with a note. A missing `state.db`
|
|
12
|
+
* gives an empty report and a missing `index.db` an empty utility section, each
|
|
13
|
+
* with a note, never an error.
|
|
14
|
+
*/
|
|
15
|
+
import fs from "node:fs";
|
|
16
|
+
import { defineJsonCommand, output } from "../../cli/shared.js";
|
|
17
|
+
import { parseBundleRef } from "../../core/asset/asset-ref.js";
|
|
18
|
+
import { loadConfig } from "../../core/config/config.js";
|
|
19
|
+
import { UsageError } from "../../core/errors.js";
|
|
20
|
+
import { readEvents } from "../../core/events.js";
|
|
21
|
+
import { getDbPath } from "../../core/paths.js";
|
|
22
|
+
import { getStateDbPath, listPendingStateMigrations } from "../../core/state-db.js";
|
|
23
|
+
import { lookupBundleRefsReadonly } from "../../indexer/indexer.js";
|
|
24
|
+
import { USAGE_EVENT_RETENTION_DAYS } from "../../indexer/usage/usage-events.js";
|
|
25
|
+
import { LLM_USAGE_EVENT } from "../../llm/usage-persist.js";
|
|
26
|
+
import { getOutputMode } from "../../output/context.js";
|
|
27
|
+
import { openDatabase } from "../../storage/database.js";
|
|
28
|
+
import { closeDatabase, openReadonlyExistingDatabase } from "../../storage/repositories/index-connection.js";
|
|
29
|
+
import { listUtilityWithRefs } from "../../storage/repositories/index-utility-repository.js";
|
|
30
|
+
import { countProposalsByStatus, listIndexCompletedEvents, listLowestOutcomeAssets, listSelectEvents, listUsageEventRows, summarizeWorkflowRuns, } from "../../storage/repositories/metrics-repository.js";
|
|
31
|
+
import { queryTaskHistory } from "../../storage/repositories/task-history-repository.js";
|
|
32
|
+
import { applyReadonlyPragmas } from "../../storage/sqlite-pragmas.js";
|
|
33
|
+
import { parseHealthSince } from "../health.js";
|
|
34
|
+
import { computeAcceptRateBySource } from "../health/accept-rate.js";
|
|
35
|
+
import { emptyLlmUsageAggregate, summarizeLlmUsage } from "../health/llm-usage.js";
|
|
36
|
+
import { buildMetricsResult, DEFAULT_EVENT_RETENTION_DAYS, indexRunsFromEvents, llmRowsFromEvents, retentionNotes, } from "./collect.js";
|
|
37
|
+
const DEFAULT_SINCE = "30d";
|
|
38
|
+
/** Cap on every ranked list. */
|
|
39
|
+
const TOP = 20;
|
|
40
|
+
/** Usage rows from people; improve, task and audit rows are akm's own reads. */
|
|
41
|
+
const SOURCE = "user";
|
|
42
|
+
/**
|
|
43
|
+
* Whether the report carries the raw window rows: the HTML dashboard
|
|
44
|
+
* re-aggregates from them client-side, so `--format html` always includes
|
|
45
|
+
* them, and `--detail full` includes them for any other format.
|
|
46
|
+
*/
|
|
47
|
+
export function metricsIncludeRows(mode) {
|
|
48
|
+
return mode.format === "html" || mode.detail === "full";
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* `select` events keyed on the durable ref. The events stream stores the ref
|
|
52
|
+
* as typed (`knowledge/x` for the default bundle), while `usage_events` stores
|
|
53
|
+
* `bundle//conceptId`; resolving here keeps one asset from showing as two rows.
|
|
54
|
+
* A ref that does not resolve (registry results, removed assets) is kept as is.
|
|
55
|
+
*/
|
|
56
|
+
async function durableSelects(selects) {
|
|
57
|
+
const unqualified = [];
|
|
58
|
+
for (const ref of new Set(selects.map((select) => select.ref))) {
|
|
59
|
+
try {
|
|
60
|
+
const parsed = parseBundleRef(ref);
|
|
61
|
+
if (!parsed.bundle)
|
|
62
|
+
unqualified.push({ ref, parsed });
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
// Not a local bundle ref: keep the ref as recorded.
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const entries = await lookupBundleRefsReadonly(unqualified.map(({ parsed }) => parsed));
|
|
69
|
+
const resolved = new Map(unqualified.map(({ ref }, i) => [ref, entries[i]?.itemRef ?? ref]));
|
|
70
|
+
return selects.map((select) => ({ ts: select.ts, ref: resolved.get(select.ref) ?? select.ref }));
|
|
71
|
+
}
|
|
72
|
+
/** `index.db` utility rows, or a note saying why there are none. */
|
|
73
|
+
function readUtility() {
|
|
74
|
+
let db;
|
|
75
|
+
try {
|
|
76
|
+
db = openReadonlyExistingDatabase(getDbPath());
|
|
77
|
+
if (!db)
|
|
78
|
+
return { note: "index.db was not found, so the utility section is empty. Run 'akm index' to build it." };
|
|
79
|
+
return { utility: listUtilityWithRefs(db) };
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
83
|
+
return { note: `index.db could not be read (${message}), so the utility section is empty.` };
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
if (db)
|
|
87
|
+
closeDatabase(db);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
function readEventRetention() {
|
|
91
|
+
try {
|
|
92
|
+
const days = loadConfig().improve?.eventRetentionDays;
|
|
93
|
+
return { eventRetentionDays: typeof days === "number" ? days : DEFAULT_EVENT_RETENTION_DAYS };
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
97
|
+
return {
|
|
98
|
+
eventRetentionDays: DEFAULT_EVENT_RETENTION_DAYS,
|
|
99
|
+
note: `The config could not be loaded (${message}), so event retention assumes ${DEFAULT_EVENT_RETENTION_DAYS} days.`,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function readAcceptRate(stateDb, notes) {
|
|
104
|
+
try {
|
|
105
|
+
return computeAcceptRateBySource(undefined, { db: stateDb });
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
notes.push(`Proposal accept rate is unavailable (${error instanceof Error ? error.message : String(error)}).`);
|
|
109
|
+
return [];
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/** Read every store and build the report. */
|
|
113
|
+
export async function akmMetrics(options = {}) {
|
|
114
|
+
const nowMs = (options.now ?? Date.now)();
|
|
115
|
+
const sinceIso = parseHealthSince(options.since ?? DEFAULT_SINCE);
|
|
116
|
+
const untilIso = new Date(nowMs).toISOString();
|
|
117
|
+
if (Date.parse(sinceIso) >= Date.parse(untilIso)) {
|
|
118
|
+
throw new UsageError(`--since (${sinceIso}) must be earlier than now (${untilIso}).`, "INVALID_FLAG_VALUE");
|
|
119
|
+
}
|
|
120
|
+
const notes = [];
|
|
121
|
+
const config = readEventRetention();
|
|
122
|
+
if (config.note)
|
|
123
|
+
notes.push(config.note);
|
|
124
|
+
notes.push(...retentionNotes({
|
|
125
|
+
sinceIso,
|
|
126
|
+
nowMs,
|
|
127
|
+
usageRetentionDays: USAGE_EVENT_RETENTION_DAYS,
|
|
128
|
+
eventRetentionDays: config.eventRetentionDays,
|
|
129
|
+
}));
|
|
130
|
+
const index = readUtility();
|
|
131
|
+
if (index.note)
|
|
132
|
+
notes.push(index.note);
|
|
133
|
+
const filter = { sinceIso, untilIso, source: SOURCE };
|
|
134
|
+
notes.push("Selects come from the events stream, which records no source, so they are counted for every source.");
|
|
135
|
+
const input = {
|
|
136
|
+
window: { since: sinceIso, until: untilIso },
|
|
137
|
+
source: SOURCE,
|
|
138
|
+
top: TOP,
|
|
139
|
+
includeRows: options.includeRows === true,
|
|
140
|
+
usage: [],
|
|
141
|
+
selects: [],
|
|
142
|
+
utility: index.utility,
|
|
143
|
+
outcomes: [],
|
|
144
|
+
llm: emptyLlmUsageAggregate(),
|
|
145
|
+
llmRows: [],
|
|
146
|
+
indexRuns: [],
|
|
147
|
+
tasks: [],
|
|
148
|
+
proposals: { byStatus: {}, acceptRateBySource: [] },
|
|
149
|
+
workflows: { runs: 0, byStatus: {}, tokens: 0, byModel: {} },
|
|
150
|
+
notes,
|
|
151
|
+
};
|
|
152
|
+
const stateDbPath = getStateDbPath();
|
|
153
|
+
if (!fs.existsSync(stateDbPath)) {
|
|
154
|
+
notes.push("state.db was not found, so every section except utility is empty.");
|
|
155
|
+
return buildMetricsResult(input);
|
|
156
|
+
}
|
|
157
|
+
let pending;
|
|
158
|
+
try {
|
|
159
|
+
pending = listPendingStateMigrations(stateDbPath);
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
notes.push(`state.db could not be read (${error instanceof Error ? error.message : String(error)}), so every section except utility is empty.`);
|
|
163
|
+
return buildMetricsResult(input);
|
|
164
|
+
}
|
|
165
|
+
if (pending.length > 0) {
|
|
166
|
+
notes.push(`state.db has ${pending.length} pending migration${pending.length === 1 ? "" : "s"}; 'akm metrics' never migrates, so every section except utility is empty. Run 'akm migrate apply'.`);
|
|
167
|
+
return buildMetricsResult(input);
|
|
168
|
+
}
|
|
169
|
+
const stateDb = openDatabase(stateDbPath, { readonly: true, create: false });
|
|
170
|
+
let selects = [];
|
|
171
|
+
try {
|
|
172
|
+
applyReadonlyPragmas(stateDb);
|
|
173
|
+
input.usage = listUsageEventRows(stateDb, filter);
|
|
174
|
+
selects = listSelectEvents(stateDb, sinceIso, untilIso);
|
|
175
|
+
input.outcomes = listLowestOutcomeAssets(stateDb, TOP);
|
|
176
|
+
input.indexRuns = indexRunsFromEvents(listIndexCompletedEvents(stateDb, sinceIso, untilIso));
|
|
177
|
+
input.tasks = queryTaskHistory(stateDb, { since: sinceIso, until: untilIso });
|
|
178
|
+
input.proposals = {
|
|
179
|
+
byStatus: countProposalsByStatus(stateDb, sinceIso, untilIso),
|
|
180
|
+
acceptRateBySource: readAcceptRate(stateDb, notes),
|
|
181
|
+
};
|
|
182
|
+
input.workflows = summarizeWorkflowRuns(stateDb, sinceIso, untilIso);
|
|
183
|
+
const untilMs = Date.parse(untilIso);
|
|
184
|
+
const llmEvents = readEvents({ since: sinceIso, type: LLM_USAGE_EVENT }, { db: stateDb, readOnly: true }).events;
|
|
185
|
+
const inWindow = llmEvents.filter((event) => Date.parse(event.ts) < untilMs);
|
|
186
|
+
input.llm = summarizeLlmUsage(inWindow);
|
|
187
|
+
if (input.includeRows)
|
|
188
|
+
input.llmRows = llmRowsFromEvents(inWindow);
|
|
189
|
+
}
|
|
190
|
+
finally {
|
|
191
|
+
stateDb.close();
|
|
192
|
+
}
|
|
193
|
+
input.selects = await durableSelects(selects);
|
|
194
|
+
return buildMetricsResult(input);
|
|
195
|
+
}
|
|
196
|
+
export const metricsCommand = defineJsonCommand({
|
|
197
|
+
meta: {
|
|
198
|
+
name: "metrics",
|
|
199
|
+
description: "Experimental: report asset usage, feedback, utility, LLM usage, tasks, proposals and workflow spend from local records. Read-only. --format html writes a self-contained dashboard.",
|
|
200
|
+
},
|
|
201
|
+
args: {
|
|
202
|
+
since: {
|
|
203
|
+
type: "string",
|
|
204
|
+
description: `Window start: ISO, date, epoch ms, or 24h / 7d / 30d (default ${DEFAULT_SINCE})`,
|
|
205
|
+
},
|
|
206
|
+
},
|
|
207
|
+
async run({ args }) {
|
|
208
|
+
output("metrics", await akmMetrics({
|
|
209
|
+
since: args.since,
|
|
210
|
+
includeRows: metricsIncludeRows(getOutputMode()),
|
|
211
|
+
}));
|
|
212
|
+
},
|
|
213
|
+
});
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/** Lists are cut to this many rows at `--detail brief`; `normal`/`full` show what the top-20 cap kept. */
|
|
5
|
+
export const BRIEF_LIST_LIMIT = 5;
|
|
6
|
+
/** Narrow an unknown envelope to an `AkmMetricsResult`; renderers return `null` (generic fallback) otherwise. */
|
|
7
|
+
export function isMetricsResult(value) {
|
|
8
|
+
if (value === null || typeof value !== "object")
|
|
9
|
+
return false;
|
|
10
|
+
const r = value;
|
|
11
|
+
return r.schemaVersion === 1 && !!r.window && !!r.usage && !!r.feedback && !!r.llm;
|
|
12
|
+
}
|
|
13
|
+
const int = (n) => String(n);
|
|
14
|
+
const dash = (v) => (v === undefined || v === "" ? "-" : v);
|
|
15
|
+
/** A fraction as a percentage; `null` (zero denominator) is `n/a`, never `NaN`. */
|
|
16
|
+
const pct = (n) => (n === null ? "n/a" : `${(n * 100).toFixed(1)}%`);
|
|
17
|
+
const ms = (n) => (n === null ? "n/a" : `${Math.round(n)} ms`);
|
|
18
|
+
const fixed = (n, digits) => (n === null ? "n/a" : n.toFixed(digits));
|
|
19
|
+
function sourceFacts(counts) {
|
|
20
|
+
const entries = Object.entries(counts);
|
|
21
|
+
return entries.length === 0 ? "-" : entries.map(([k, v]) => `${k}=${v}`).join(" ");
|
|
22
|
+
}
|
|
23
|
+
function llmRows(group) {
|
|
24
|
+
return Object.entries(group).map(([name, a]) => [
|
|
25
|
+
name,
|
|
26
|
+
int(a.calls),
|
|
27
|
+
int(a.failures),
|
|
28
|
+
int(a.promptTokens),
|
|
29
|
+
int(a.completionTokens),
|
|
30
|
+
int(a.totalTokens),
|
|
31
|
+
ms(a.totalDurationMs),
|
|
32
|
+
]);
|
|
33
|
+
}
|
|
34
|
+
const LLM_HEADERS = ["name", "calls", "failed", "prompt", "completion", "total", "time"];
|
|
35
|
+
export function buildMetricsView(r, detail) {
|
|
36
|
+
const brief = detail === "brief";
|
|
37
|
+
const cut = (items) => (brief ? items.slice(0, BRIEF_LIST_LIMIT) : items);
|
|
38
|
+
const { usage, feedback, utility, llm } = r;
|
|
39
|
+
const searches = usage.totals.searches;
|
|
40
|
+
const sections = [
|
|
41
|
+
{
|
|
42
|
+
title: "Usage",
|
|
43
|
+
facts: [
|
|
44
|
+
["searches", int(searches)],
|
|
45
|
+
["shows", int(usage.totals.shows)],
|
|
46
|
+
["curates", int(usage.totals.curates)],
|
|
47
|
+
["selects", int(usage.totals.selects)],
|
|
48
|
+
["select rate", pct(usage.selectRate)],
|
|
49
|
+
[
|
|
50
|
+
"zero-result searches",
|
|
51
|
+
`${usage.totals.zeroResultSearches} (${pct(searches === 0 ? null : usage.totals.zeroResultSearches / searches)})`,
|
|
52
|
+
],
|
|
53
|
+
["search median", ms(usage.searchMedianMs)],
|
|
54
|
+
["distinct assets", int(usage.totals.distinctAssets)],
|
|
55
|
+
["distinct queries", int(usage.totals.distinctQueries)],
|
|
56
|
+
["by source", sourceFacts(usage.bySource)],
|
|
57
|
+
],
|
|
58
|
+
tables: [
|
|
59
|
+
{
|
|
60
|
+
title: "Top assets",
|
|
61
|
+
headers: ["ref", "shows", "search hits", "selects", "+", "-", "last used"],
|
|
62
|
+
rows: cut(usage.topAssets).map((a) => [
|
|
63
|
+
a.ref,
|
|
64
|
+
int(a.shows),
|
|
65
|
+
int(a.searchHits),
|
|
66
|
+
int(a.selects),
|
|
67
|
+
int(a.positive),
|
|
68
|
+
int(a.negative),
|
|
69
|
+
dash(a.lastUsedAt),
|
|
70
|
+
]),
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
title: "Top queries",
|
|
74
|
+
headers: ["query", "count", "avg results", "last"],
|
|
75
|
+
rows: cut(usage.topQueries).map((q) => [q.query, int(q.count), fixed(q.avgResults, 1), q.lastAt]),
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
title: "Zero-result queries",
|
|
79
|
+
headers: ["query", "count", "last"],
|
|
80
|
+
rows: cut(usage.zeroResultQueries).map((q) => [q.query, int(q.count), q.lastAt]),
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
title: "Daily",
|
|
84
|
+
headers: ["day", "search", "show", "curate", "feedback"],
|
|
85
|
+
rows: brief
|
|
86
|
+
? []
|
|
87
|
+
: usage.daily.map((d) => [d.day, int(d.search), int(d.show), int(d.curate), int(d.feedback)]),
|
|
88
|
+
hideWhenEmpty: true,
|
|
89
|
+
},
|
|
90
|
+
],
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
title: "Feedback",
|
|
94
|
+
facts: [
|
|
95
|
+
["positive", int(feedback.totals.positive)],
|
|
96
|
+
["negative", int(feedback.totals.negative)],
|
|
97
|
+
],
|
|
98
|
+
tables: [
|
|
99
|
+
{
|
|
100
|
+
title: "By asset",
|
|
101
|
+
headers: ["ref", "+", "-", "valence", "last"],
|
|
102
|
+
rows: cut(feedback.byAsset).map((a) => [
|
|
103
|
+
a.ref,
|
|
104
|
+
int(a.positive),
|
|
105
|
+
int(a.negative),
|
|
106
|
+
fixed(a.valence, 2),
|
|
107
|
+
a.lastAt,
|
|
108
|
+
]),
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
title: "By tag",
|
|
112
|
+
headers: ["tag", "+", "-"],
|
|
113
|
+
rows: cut(Object.entries(feedback.byTag)).map(([tag, c]) => [tag, int(c.positive), int(c.negative)]),
|
|
114
|
+
hideWhenEmpty: true,
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
title: "Recent negative",
|
|
118
|
+
headers: ["ref", "at", "reason", "tags"],
|
|
119
|
+
rows: cut(feedback.recentNegative).map((n) => [n.ref, n.at, dash(n.reason), dash(n.tags?.join(", "))]),
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
title: "Utility",
|
|
125
|
+
facts: [
|
|
126
|
+
["scored assets", int(utility.count)],
|
|
127
|
+
["never used", int(utility.neverUsed)],
|
|
128
|
+
],
|
|
129
|
+
tables: [
|
|
130
|
+
{
|
|
131
|
+
title: "Histogram",
|
|
132
|
+
headers: ["bucket", "count"],
|
|
133
|
+
rows: utility.histogram.map((h) => [h.bucket, int(h.count)]),
|
|
134
|
+
hideWhenEmpty: true,
|
|
135
|
+
},
|
|
136
|
+
{ title: "Lowest", headers: utilityHeaders(), rows: cut(utility.lowest).map(utilityRow) },
|
|
137
|
+
{ title: "Highest", headers: utilityHeaders(), rows: cut(utility.highest).map(utilityRow) },
|
|
138
|
+
{
|
|
139
|
+
title: "Lowest outcome",
|
|
140
|
+
headers: ["ref", "outcome", "retrievals", "negative", "accepted changes"],
|
|
141
|
+
rows: cut(r.outcomes.lowestOutcome).map((o) => [
|
|
142
|
+
o.ref,
|
|
143
|
+
fixed(o.outcomeScore, 2),
|
|
144
|
+
int(o.retrievalCount),
|
|
145
|
+
int(o.negativeFeedbackCount),
|
|
146
|
+
int(o.acceptedChangeCount),
|
|
147
|
+
]),
|
|
148
|
+
hideWhenEmpty: true,
|
|
149
|
+
},
|
|
150
|
+
],
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
title: "LLM",
|
|
154
|
+
facts: [
|
|
155
|
+
["calls", int(llm.calls)],
|
|
156
|
+
["failed", int(llm.failures)],
|
|
157
|
+
["prompt tokens", int(llm.promptTokens)],
|
|
158
|
+
["completion tokens", int(llm.completionTokens)],
|
|
159
|
+
["reasoning tokens", int(llm.reasoningTokens)],
|
|
160
|
+
["total tokens", int(llm.totalTokens)],
|
|
161
|
+
["time", ms(llm.totalDurationMs)],
|
|
162
|
+
],
|
|
163
|
+
tables: [
|
|
164
|
+
{ title: "By engine", headers: LLM_HEADERS, rows: cut(llmRows(llm.byEngine)), hideWhenEmpty: true },
|
|
165
|
+
{ title: "By process", headers: LLM_HEADERS, rows: cut(llmRows(llm.byProcess)), hideWhenEmpty: true },
|
|
166
|
+
{ title: "By stage", headers: LLM_HEADERS, rows: cut(llmRows(llm.byStage)), hideWhenEmpty: true },
|
|
167
|
+
],
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
title: "Index",
|
|
171
|
+
facts: [
|
|
172
|
+
["runs", int(r.index.runs)],
|
|
173
|
+
["median time", ms(r.index.medianMs)],
|
|
174
|
+
],
|
|
175
|
+
tables: [
|
|
176
|
+
{
|
|
177
|
+
title: "Recent runs",
|
|
178
|
+
headers: ["at", "mode", "time"],
|
|
179
|
+
rows: brief ? [] : r.index.recent.map((i) => [i.at, i.mode, ms(i.totalMs)]),
|
|
180
|
+
hideWhenEmpty: true,
|
|
181
|
+
},
|
|
182
|
+
],
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
title: "Tasks",
|
|
186
|
+
facts: [
|
|
187
|
+
["runs", int(r.tasks.runs)],
|
|
188
|
+
["failed", int(r.tasks.failed)],
|
|
189
|
+
["fail rate", pct(r.tasks.failRate)],
|
|
190
|
+
],
|
|
191
|
+
tables: [
|
|
192
|
+
{
|
|
193
|
+
title: "By task",
|
|
194
|
+
headers: ["task", "runs", "failed", "median"],
|
|
195
|
+
rows: cut(r.tasks.byTask).map((t) => [t.taskId, int(t.runs), int(t.failed), ms(t.medianMs)]),
|
|
196
|
+
hideWhenEmpty: true,
|
|
197
|
+
},
|
|
198
|
+
],
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
title: "Proposals",
|
|
202
|
+
facts: [["by status", sourceFacts(r.proposals.byStatus)]],
|
|
203
|
+
tables: [
|
|
204
|
+
{
|
|
205
|
+
title: "Accept rate by source",
|
|
206
|
+
headers: ["source", "total", "accepted", "rejected", "pending", "accept rate"],
|
|
207
|
+
rows: r.proposals.acceptRateBySource.map((p) => [
|
|
208
|
+
p.source,
|
|
209
|
+
int(p.total),
|
|
210
|
+
int(p.accepted),
|
|
211
|
+
int(p.rejected),
|
|
212
|
+
int(p.pending),
|
|
213
|
+
pct(p.acceptRate),
|
|
214
|
+
]),
|
|
215
|
+
hideWhenEmpty: true,
|
|
216
|
+
},
|
|
217
|
+
],
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
title: "Workflows",
|
|
221
|
+
facts: [
|
|
222
|
+
["runs", int(r.workflows.runs)],
|
|
223
|
+
["by status", sourceFacts(r.workflows.byStatus)],
|
|
224
|
+
["tokens", int(r.workflows.tokens)],
|
|
225
|
+
["tokens by model", sourceFacts(r.workflows.byModel)],
|
|
226
|
+
],
|
|
227
|
+
tables: [],
|
|
228
|
+
},
|
|
229
|
+
];
|
|
230
|
+
const f = r.filters;
|
|
231
|
+
return {
|
|
232
|
+
window: `${r.window.since} to ${r.window.until}`,
|
|
233
|
+
filters: `source=${f.source}`,
|
|
234
|
+
sections,
|
|
235
|
+
notes: r.notes,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
function utilityHeaders() {
|
|
239
|
+
return ["ref", "utility", "shows", "searches", "select rate", "last used"];
|
|
240
|
+
}
|
|
241
|
+
function utilityRow(u) {
|
|
242
|
+
return [u.ref, fixed(u.utility, 2), int(u.showCount), int(u.searchCount), pct(u.selectRate), dash(u.lastUsedAt)];
|
|
243
|
+
}
|
|
@@ -293,6 +293,11 @@ function logSearchEvent(rawQuery, response, mode = "keyword", eventSource = "use
|
|
|
293
293
|
registryHitCount,
|
|
294
294
|
resolvedCount: resolved.length,
|
|
295
295
|
mode,
|
|
296
|
+
// Whole-search latency plus the rank/embed phases when the local
|
|
297
|
+
// search reported them, so `akm metrics` can show timings later.
|
|
298
|
+
totalMs: response.timing?.totalMs,
|
|
299
|
+
rankMs: response.timing?.rankMs,
|
|
300
|
+
embedMs: response.timing?.embedMs,
|
|
296
301
|
}),
|
|
297
302
|
source: eventSource,
|
|
298
303
|
});
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -6,6 +6,7 @@ import path from "node:path";
|
|
|
6
6
|
import { detectAdapterId } from "../core/adapter/detect-adapter.js";
|
|
7
7
|
import { adapterForId } from "../core/adapter/registry.js";
|
|
8
8
|
import { compareCodePoints, isHttpUrl } from "../core/common.js";
|
|
9
|
+
import { appendEvent } from "../core/events.js";
|
|
9
10
|
import { classifyPathAccess, describeInaccessiblePath } from "../core/path-access.js";
|
|
10
11
|
import { getDbPath } from "../core/paths.js";
|
|
11
12
|
import { SCRIPT_EXTENSIONS } from "../core/recognition-util.js";
|
|
@@ -500,6 +501,32 @@ async function akmIndexReal(options) {
|
|
|
500
501
|
vacuumIndexDb(db, onProgress);
|
|
501
502
|
const totalEntries = getEntryCount(db);
|
|
502
503
|
const tFinalizeEnd = Date.now();
|
|
504
|
+
const timing = {
|
|
505
|
+
totalMs: Date.now() - t0,
|
|
506
|
+
walkMs: tWalkEnd - tWalkStart,
|
|
507
|
+
llmMs: tLlmEnd - tWalkEnd,
|
|
508
|
+
embedMs: tEmbedEnd - tLlmEnd,
|
|
509
|
+
ftsMs: tFtsEnd - tEmbedEnd,
|
|
510
|
+
finalizeMs: tFinalizeEnd - tFinalizeStart,
|
|
511
|
+
cleanMs: clean ? cleanEnd - cleanStart : 0,
|
|
512
|
+
preflightMs: t0 - requestedAt,
|
|
513
|
+
sourceCacheMs: sourceCacheEnd - sourceCacheStart,
|
|
514
|
+
endToEndMs: Date.now() - requestedAt,
|
|
515
|
+
};
|
|
516
|
+
// Persist the phase timings so `akm metrics` can report index latency
|
|
517
|
+
// after the command output is gone. Best-effort, like every appendEvent.
|
|
518
|
+
appendEvent({
|
|
519
|
+
eventType: "index_completed",
|
|
520
|
+
metadata: {
|
|
521
|
+
mode,
|
|
522
|
+
totalMs: timing.totalMs,
|
|
523
|
+
walkMs: timing.walkMs,
|
|
524
|
+
llmMs: timing.llmMs,
|
|
525
|
+
embedMs: timing.embedMs,
|
|
526
|
+
ftsMs: timing.ftsMs,
|
|
527
|
+
finalizeMs: timing.finalizeMs,
|
|
528
|
+
},
|
|
529
|
+
});
|
|
503
530
|
return {
|
|
504
531
|
stashDir,
|
|
505
532
|
totalEntries,
|
|
@@ -512,18 +539,7 @@ async function akmIndexReal(options) {
|
|
|
512
539
|
...(warnings.length > 0 ? { warnings } : {}),
|
|
513
540
|
...(Object.keys(persistedAdapters).length > 0 ? { configUpdated: { detectedAdapters: persistedAdapters } } : {}),
|
|
514
541
|
verification,
|
|
515
|
-
timing
|
|
516
|
-
totalMs: Date.now() - t0,
|
|
517
|
-
walkMs: tWalkEnd - tWalkStart,
|
|
518
|
-
llmMs: tLlmEnd - tWalkEnd,
|
|
519
|
-
embedMs: tEmbedEnd - tLlmEnd,
|
|
520
|
-
ftsMs: tFtsEnd - tEmbedEnd,
|
|
521
|
-
finalizeMs: tFinalizeEnd - tFinalizeStart,
|
|
522
|
-
cleanMs: clean ? cleanEnd - cleanStart : 0,
|
|
523
|
-
preflightMs: t0 - requestedAt,
|
|
524
|
-
sourceCacheMs: sourceCacheEnd - sourceCacheStart,
|
|
525
|
-
endToEndMs: Date.now() - requestedAt,
|
|
526
|
-
},
|
|
542
|
+
timing,
|
|
527
543
|
...(cleanResult !== undefined ? { clean: cleanResult } : {}),
|
|
528
544
|
};
|
|
529
545
|
}
|
|
@@ -1110,7 +1126,9 @@ async function resolveLookupSources() {
|
|
|
1110
1126
|
const { resolveSourceEntries } = await import("./search/search-source.js");
|
|
1111
1127
|
return resolveSourceEntries(undefined, loadConfig());
|
|
1112
1128
|
}
|
|
1113
|
-
async function lookupBundleRefWithResolutionUsing(ref, openLookupDatabase
|
|
1129
|
+
async function lookupBundleRefWithResolutionUsing(ref, openLookupDatabase,
|
|
1130
|
+
/** The opener hands out a handle the caller owns and closes. */
|
|
1131
|
+
borrowed = false) {
|
|
1114
1132
|
const sources = await resolveLookupSources();
|
|
1115
1133
|
if (sources.length === 0)
|
|
1116
1134
|
return { entry: null };
|
|
@@ -1171,7 +1189,7 @@ async function lookupBundleRefWithResolutionUsing(ref, openLookupDatabase) {
|
|
|
1171
1189
|
return resolved(null);
|
|
1172
1190
|
}
|
|
1173
1191
|
finally {
|
|
1174
|
-
if (db)
|
|
1192
|
+
if (db && !borrowed)
|
|
1175
1193
|
closeDatabase(db);
|
|
1176
1194
|
}
|
|
1177
1195
|
}
|
|
@@ -1200,6 +1218,38 @@ export async function lookupBundleRefReadonly(ref) {
|
|
|
1200
1218
|
throw resolution.indexError;
|
|
1201
1219
|
return resolution.entry;
|
|
1202
1220
|
}
|
|
1221
|
+
/**
|
|
1222
|
+
* {@link lookupBundleRefReadonly} for many refs through one snapshot of the
|
|
1223
|
+
* index, which is copied once rather than once per ref. A ref that does not
|
|
1224
|
+
* resolve, or an index that cannot be read, gives `null` for that ref.
|
|
1225
|
+
*/
|
|
1226
|
+
export async function lookupBundleRefsReadonly(refs) {
|
|
1227
|
+
if (refs.length === 0)
|
|
1228
|
+
return [];
|
|
1229
|
+
let db;
|
|
1230
|
+
try {
|
|
1231
|
+
db = openReadonlyExistingDatabase(getDbPath(), { isolatedSnapshot: true });
|
|
1232
|
+
}
|
|
1233
|
+
catch {
|
|
1234
|
+
// Unreadable index: every ref stays unresolved.
|
|
1235
|
+
}
|
|
1236
|
+
try {
|
|
1237
|
+
const entries = [];
|
|
1238
|
+
for (const ref of refs) {
|
|
1239
|
+
try {
|
|
1240
|
+
entries.push((await lookupBundleRefWithResolutionUsing(ref, () => db, true)).entry);
|
|
1241
|
+
}
|
|
1242
|
+
catch {
|
|
1243
|
+
entries.push(null);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
return entries;
|
|
1247
|
+
}
|
|
1248
|
+
finally {
|
|
1249
|
+
if (db)
|
|
1250
|
+
closeDatabase(db);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1203
1253
|
function readLookupEntry(db, id, fallbackConceptId, sourceRoot) {
|
|
1204
1254
|
const row = db
|
|
1205
1255
|
.prepare("SELECT file_path AS filePath, type, document_json AS documentJson, " +
|
|
@@ -134,7 +134,9 @@ export function purgeOldUsageEvents(db, retentionDays) {
|
|
|
134
134
|
return;
|
|
135
135
|
try {
|
|
136
136
|
const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
|
|
137
|
-
|
|
137
|
+
// `created_at` is `YYYY-MM-DD HH:MM:SS`, not ISO: normalize both sides so the
|
|
138
|
+
// space-vs-`T` ordering doesn't delete the whole cutoff date early.
|
|
139
|
+
db.prepare("DELETE FROM usage_events WHERE datetime(created_at) < datetime(?)").run(cutoff);
|
|
138
140
|
}
|
|
139
141
|
catch {
|
|
140
142
|
/* Table may not exist yet */
|