akm-cli 0.9.28-alpha.1 → 0.9.28-alpha.3

package/dist/commands/metrics/collect.js

@@ -0,0 +1,439 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { decodeLlmUsageRecord } from "../../llm/usage-telemetry.js";
+import { decodeTaskHistoryMetadata } from "../../storage/repositories/task-history-repository.js";
+import { computeWallTimeStats } from "../health/improve-metrics.js";
+import { computeValenceScore } from "../improve/feedback-valence.js";
+const DAY_MS = 86_400_000;
+/** Utility is a score in [0, 1], reported in ten equal buckets. */
+const UTILITY_BUCKETS = 10;
+/** Mean result counts are rounded to two decimals. */
+const MEAN_SCALE = 100;
+/** `improve.eventRetentionDays` default (`runRetentionPurgePass`); 0 disables the purge. */
+export const DEFAULT_EVENT_RETENTION_DAYS = 90;
+// ── Filters ─────────────────────────────────────────────────────────────────
+// ── Notes ───────────────────────────────────────────────────────────────────
+/**
+ * One note per store whose retention is shorter than the window, naming the
+ * store, how long it keeps rows and where the data effectively starts. A
+ * retention of 0 or less means the store is never purged.
+ */
+export function retentionNotes(args) {
+    const stores = [
+        { store: "usage_events (search, show, curate and feedback rows)", days: args.usageRetentionDays },
+        { store: "events (selects, LLM calls and index runs)", days: args.eventRetentionDays },
+    ];
+    const notes = [];
+    const sinceMs = Date.parse(args.sinceIso);
+    for (const { store, days } of stores) {
+        if (!Number.isFinite(days) || days <= 0)
+            continue;
+        const effectiveMs = args.nowMs - days * DAY_MS;
+        if (sinceMs >= effectiveMs)
+            continue;
+        notes.push(`${store} keeps ${days} days, so the window starts ${args.sinceIso} but its data starts no earlier than ${new Date(effectiveMs).toISOString()}.`);
+    }
+    return notes;
+}
+// ── Row mapping ─────────────────────────────────────────────────────────────
+/** `usage_events.created_at` (`YYYY-MM-DD HH:MM:SS`, UTC) as an ISO-8601 string. */
+export function usageCreatedAtToIso(createdAt) {
+    const text = createdAt.includes("T") ? createdAt : `${createdAt.replace(" ", "T")}Z`;
+    const ms = Date.parse(text);
+    return Number.isNaN(ms) ? createdAt : new Date(ms).toISOString();
+}
+function parseMetadata(text) {
+    if (!text)
+        return {};
+    try {
+        const parsed = JSON.parse(text);
+        return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
+            ? parsed
+            : {};
+    }
+    catch {
+        return {};
+    }
+}
+function finiteNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+/** Map one `usage_events` row to the shape reported (and carried to the HTML dashboard). */
+export function toUsageRow(row) {
+    const metadata = parseMetadata(row.metadata);
+    const resultCount = finiteNumber(metadata.resultCount);
+    const totalMs = finiteNumber(metadata.totalMs);
+    const reason = typeof metadata.reason === "string" && metadata.reason.trim() ? metadata.reason : undefined;
+    const tags = Array.isArray(metadata.tags)
+        ? metadata.tags.filter((tag) => typeof tag === "string")
+        : undefined;
+    return {
+        id: row.id,
+        at: usageCreatedAtToIso(row.created_at),
+        eventType: row.event_type,
+        ...(row.entry_ref ? { ref: row.entry_ref } : {}),
+        ...(row.query ? { query: row.query } : {}),
+        ...(row.signal === "positive" || row.signal === "negative" ? { signal: row.signal } : {}),
+        source: row.source,
+        ...(resultCount !== undefined ? { resultCount } : {}),
+        ...(totalMs !== undefined ? { totalMs } : {}),
+        ...(reason !== undefined ? { reason } : {}),
+        ...(tags && tags.length > 0 ? { tags } : {}),
+    };
+}
+/**
+ * `llm_usage` events summed per UTC day x stage x process x engine x model x
+ * outcome: the rows the dashboard re-aggregates. One row per call would be the
+ * largest thing in the page, and the page only ever filters them by date.
+ * Events that do not decode are skipped; rows come out in first-seen order.
+ */
+export function llmRowsFromEvents(events) {
+    const rows = new Map();
+    for (const event of events) {
+        const record = decodeLlmUsageRecord(event.metadata);
+        if (!record)
+            continue;
+        const day = event.ts.slice(0, 10);
+        const key = JSON.stringify([day, record.stage, record.process, record.engine, record.model, record.outcome]);
+        let row = rows.get(key);
+        if (!row) {
+            row = {
+                day,
+                ...(record.stage !== undefined ? { stage: record.stage } : {}),
+                ...(record.process !== undefined ? { process: record.process } : {}),
+                ...(record.engine !== undefined ? { engine: record.engine } : {}),
+                ...(record.model !== undefined ? { model: record.model } : {}),
+                outcome: record.outcome,
+                calls: 0,
+                durationMs: 0,
+                promptTokens: 0,
+                completionTokens: 0,
+                totalTokens: 0,
+                reasoningTokens: 0,
+            };
+            rows.set(key, row);
+        }
+        row.calls += 1;
+        row.durationMs += record.durationMs;
+        row.promptTokens += record.promptTokens ?? 0;
+        row.completionTokens += record.completionTokens ?? 0;
+        // A call without a total counts as prompt + completion, as the dashboard did per call.
+        row.totalTokens += record.totalTokens ?? (record.promptTokens ?? 0) + (record.completionTokens ?? 0);
+        row.reasoningTokens += record.reasoningTokens ?? 0;
+    }
+    return [...rows.values()];
+}
+/** `index_completed` events as index runs; rows without a numeric `totalMs` are skipped. */
+export function indexRunsFromEvents(events) {
+    const runs = [];
+    for (const event of events) {
+        const metadata = parseMetadata(event.metadata_json);
+        const totalMs = finiteNumber(metadata.totalMs);
+        if (totalMs === undefined)
+            continue;
+        runs.push({ at: event.ts, mode: metadata.mode === "full" ? "full" : "incremental", totalMs });
+    }
+    return runs;
+}
+// ── Small helpers ───────────────────────────────────────────────────────────
+function median(values) {
+    return values.length === 0 ? null : computeWallTimeStats(values).medianMs;
+}
+function mean(values) {
+    if (values.length === 0)
+        return null;
+    return Math.round((values.reduce((sum, value) => sum + value, 0) / values.length) * MEAN_SCALE) / MEAN_SCALE;
+}
+function byCountThenKey(key) {
+    return (a, b) => b.count - a.count || key(a).localeCompare(key(b));
+}
+// ── Sections ────────────────────────────────────────────────────────────────
+function buildUsage(input, rows) {
+    // A search or curate writes one summary row (no ref) plus one row per hit;
+    // the summary row is the unit counted.
+    const isUnit = (row) => row.ref === undefined;
+    const days = new Map();
+    const bump = (at, field) => {
+        const day = at.slice(0, 10);
+        const entry = days.get(day) ?? { day, search: 0, show: 0, curate: 0, feedback: 0 };
+        entry[field] += 1;
+        days.set(day, entry);
+    };
+    const assets = new Map();
+    const asset = (ref) => {
+        let entry = assets.get(ref);
+        if (!entry) {
+            entry = { ref, shows: 0, searchHits: 0, selects: 0, positive: 0, negative: 0 };
+            assets.set(ref, entry);
+        }
+        return entry;
+    };
+    const touch = (entry, at) => {
+        if (entry.lastUsedAt === undefined || at > entry.lastUsedAt)
+            entry.lastUsedAt = at;
+    };
+    const queries = new Map();
+    const zeroQueries = new Map();
+    const bySource = {};
+    const searchMs = [];
+    let searches = 0;
+    let searchesWithHits = 0;
+    let zeroResultSearches = 0;
+    let shows = 0;
+    let curates = 0;
+    for (const row of rows) {
+        bySource[row.source] = (bySource[row.source] ?? 0) + 1;
+        if (row.eventType === "search") {
+            if (row.ref !== undefined) {
+                const entry = asset(row.ref);
+                entry.searchHits += 1;
+                touch(entry, row.at);
+            }
+            if (!isUnit(row))
+                continue;
+            searches += 1;
+            bump(row.at, "search");
+            if ((row.resultCount ?? 0) > 0)
+                searchesWithHits += 1;
+            if (row.resultCount === 0)
+                zeroResultSearches += 1;
+            if (row.totalMs !== undefined)
+                searchMs.push(row.totalMs);
+            const query = row.query?.trim();
+            if (query) {
+                const entry = queries.get(query) ?? { count: 0, results: [], lastAt: row.at };
+                entry.count += 1;
+                if (row.resultCount !== undefined)
+                    entry.results.push(row.resultCount);
+                if (row.at > entry.lastAt)
+                    entry.lastAt = row.at;
+                queries.set(query, entry);
+                if (row.resultCount === 0) {
+                    const zero = zeroQueries.get(query) ?? { count: 0, lastAt: row.at };
+                    zero.count += 1;
+                    if (row.at > zero.lastAt)
+                        zero.lastAt = row.at;
+                    zeroQueries.set(query, zero);
+                }
+            }
+        }
+        else if (row.eventType === "curate") {
+            if (isUnit(row)) {
+                curates += 1;
+                bump(row.at, "curate");
+            }
+        }
+        else if (row.eventType === "show") {
+            shows += 1;
+            bump(row.at, "show");
+            if (row.ref !== undefined) {
+                const entry = asset(row.ref);
+                entry.shows += 1;
+                touch(entry, row.at);
+            }
+        }
+        else if (row.eventType === "feedback") {
+            bump(row.at, "feedback");
+            if (row.ref !== undefined && row.signal !== undefined) {
+                const entry = asset(row.ref);
+                entry[row.signal] += 1;
+                touch(entry, row.at);
+            }
+        }
+    }
+    for (const select of input.selects) {
+        const entry = asset(select.ref);
+        entry.selects += 1;
+        touch(entry, select.ts);
+    }
+    const toQueryCount = (query, entry) => ({
+        query,
+        count: entry.count,
+        avgResults: mean(entry.results),
+        lastAt: entry.lastAt,
+    });
+    return {
+        totals: {
+            searches,
+            shows,
+            curates,
+            selects: input.selects.length,
+            zeroResultSearches,
+            distinctAssets: assets.size,
+            distinctQueries: queries.size,
+        },
+        selectRate: searchesWithHits > 0 ? input.selects.length / searchesWithHits : null,
+        searchMedianMs: median(searchMs),
+        daily: [...days.values()].sort((a, b) => a.day.localeCompare(b.day)),
+        topAssets: [...assets.values()]
+            .sort((a, b) => b.shows - a.shows || b.searchHits - a.searchHits || b.selects - a.selects || a.ref.localeCompare(b.ref))
+            .slice(0, input.top),
+        topQueries: [...queries.entries()]
+            .map(([query, entry]) => toQueryCount(query, entry))
+            .sort(byCountThenKey((item) => item.query))
+            .slice(0, input.top),
+        zeroResultQueries: [...zeroQueries.entries()]
+            .map(([query, entry]) => ({ query, count: entry.count, avgResults: 0, lastAt: entry.lastAt }))
+            .sort(byCountThenKey((item) => item.query))
+            .slice(0, input.top),
+        bySource,
+    };
+}
+function buildFeedback(top, rows) {
+    const totals = { positive: 0, negative: 0 };
+    const byAsset = new Map();
+    const byTag = {};
+    const negatives = [];
+    for (const row of rows) {
+        if (row.eventType !== "feedback" || row.signal === undefined)
+            continue;
+        totals[row.signal] += 1;
+        if (row.ref !== undefined) {
+            const entry = byAsset.get(row.ref) ?? { positive: 0, negative: 0, lastAt: row.at };
+            entry[row.signal] += 1;
+            if (row.at > entry.lastAt)
+                entry.lastAt = row.at;
+            byAsset.set(row.ref, entry);
+        }
+        for (const tag of row.tags ?? []) {
+            byTag[tag] ??= { positive: 0, negative: 0 };
+            byTag[tag][row.signal] += 1;
+        }
+        if (row.signal === "negative" && row.ref !== undefined) {
+            negatives.push({
+                id: row.id,
+                ref: row.ref,
+                at: row.at,
+                ...(row.reason !== undefined ? { reason: row.reason } : {}),
+                ...(row.tags ? { tags: row.tags } : {}),
+            });
+        }
+    }
+    const assetRows = [...byAsset.entries()].map(([ref, entry]) => ({
+        ref,
+        positive: entry.positive,
+        negative: entry.negative,
+        valence: computeValenceScore(entry).valence,
+        lastAt: entry.lastAt,
+    }));
+    return {
+        totals,
+        byAsset: assetRows
+            .sort((a, b) => b.positive + b.negative - (a.positive + a.negative) || a.ref.localeCompare(b.ref))
+            .slice(0, top),
+        byTag: Object.fromEntries(Object.entries(byTag).sort(([a], [b]) => a.localeCompare(b))),
+        recentNegative: negatives
+            .sort((a, b) => b.at.localeCompare(a.at) || b.id - a.id)
+            .slice(0, top)
+            .map(({ id: _id, ...rest }) => rest),
+    };
+}
+function buildUtility(input) {
+    const histogram = Array.from({ length: UTILITY_BUCKETS }, (_, index) => ({
+        bucket: `${(index / UTILITY_BUCKETS).toFixed(1)}-${((index + 1) / UTILITY_BUCKETS).toFixed(1)}`,
+        count: 0,
+    }));
+    const scored = [];
+    let neverUsed = 0;
+    for (const entry of input.utility ?? []) {
+        if (!entry.score) {
+            neverUsed += 1;
+            continue;
+        }
+        const clamped = Math.min(1, Math.max(0, entry.score.utility));
+        const bucket = histogram[Math.min(UTILITY_BUCKETS - 1, Math.floor(clamped * UTILITY_BUCKETS))];
+        if (bucket)
+            bucket.count += 1;
+        scored.push({
+            ref: entry.ref,
+            utility: entry.score.utility,
+            showCount: entry.score.showCount,
+            searchCount: entry.score.searchCount,
+            selectRate: entry.score.selectRate,
+            ...(entry.score.lastUsedAt !== undefined ? { lastUsedAt: entry.score.lastUsedAt } : {}),
+        });
+    }
+    return {
+        count: scored.length,
+        histogram,
+        lowest: [...scored].sort((a, b) => a.utility - b.utility || a.ref.localeCompare(b.ref)).slice(0, input.top),
+        highest: [...scored].sort((a, b) => b.utility - a.utility || a.ref.localeCompare(b.ref)).slice(0, input.top),
+        neverUsed,
+    };
+}
+function taskDurationMs(row) {
+    try {
+        return decodeTaskHistoryMetadata(row.metadata_json).durationMs;
+    }
+    catch {
+        return undefined;
+    }
+}
+function buildTasks(top, rows) {
+    const byTask = new Map();
+    let failed = 0;
+    for (const row of rows) {
+        const entry = byTask.get(row.task_id) ?? { runs: 0, failed: 0, durations: [] };
+        entry.runs += 1;
+        if (row.status === "failed") {
+            entry.failed += 1;
+            failed += 1;
+        }
+        const durationMs = taskDurationMs(row);
+        if (durationMs !== undefined)
+            entry.durations.push(durationMs);
+        byTask.set(row.task_id, entry);
+    }
+    const summaries = [...byTask.entries()].map(([taskId, entry]) => ({
+        taskId,
+        runs: entry.runs,
+        failed: entry.failed,
+        medianMs: median(entry.durations),
+    }));
+    return {
+        runs: rows.length,
+        failed,
+        // Same definition as `akm health`'s task fail rate: failed runs over every run in the window.
+        failRate: rows.length > 0 ? failed / rows.length : null,
+        byTask: summaries.sort((a, b) => b.runs - a.runs || a.taskId.localeCompare(b.taskId)).slice(0, top),
+    };
+}
+function buildIndex(top, runs) {
+    return {
+        runs: runs.length,
+        medianMs: median(runs.map((run) => run.totalMs)),
+        // Input is oldest first; reversing before the stable sort puts later rows first within one timestamp.
+        recent: [...runs]
+            .reverse()
+            .sort((a, b) => b.at.localeCompare(a.at))
+            .slice(0, top),
+    };
+}
+// ── Entry point ─────────────────────────────────────────────────────────────
+export function buildMetricsResult(input) {
+    const rows = input.usage.map(toUsageRow);
+    const outcomes = input.outcomes.slice(0, input.top).map((row) => ({
+        ref: row.asset_ref,
+        outcomeScore: row.outcome_score,
+        retrievalCount: row.retrieval_count,
+        negativeFeedbackCount: row.negative_feedback_count,
+        acceptedChangeCount: row.accepted_change_count,
+    }));
+    return {
+        schemaVersion: 1,
+        window: input.window,
+        filters: { source: input.source },
+        usage: buildUsage(input, rows),
+        feedback: buildFeedback(input.top, rows),
+        utility: buildUtility(input),
+        outcomes: { lowestOutcome: outcomes },
+        llm: input.llm,
+        index: buildIndex(input.top, input.indexRuns),
+        tasks: buildTasks(input.top, input.tasks),
+        proposals: input.proposals,
+        workflows: input.workflows,
+        ...(input.includeRows ? { rows: { usage: rows, llm: input.llmRows } } : {}),
+        notes: input.notes,
+    };
+}

package/dist/commands/metrics/html-report.js

@@ -0,0 +1,82 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * `akm metrics --format html` — the self-contained dashboard.
+ *
+ * The page carries the window's data in a JSON island
+ * (`<script type="application/json" id="akm-data">`) and renders every panel
+ * client-side from it, so the viewer's filters re-aggregate without re-running
+ * the command. The template is `src/assets/templates/html/metrics.html`.
+ *
+ * Determinism: nothing here reads the clock. `%%GENERATED_AT%%` is
+ * `window.until`, so identical input gives byte-identical output.
+ */
+import { escapeHtml, isoTimeTag, renderHtml, resolveTemplatePath } from "../../output/html-render.js";
+import { pkgVersion } from "../../version.js";
+import { buildEchartsTag } from "../health/html-report.js";
+import { isMetricsResult } from "./report-view.js";
+const esc = escapeHtml;
+/** Most `rows.usage` entries the page carries; older rows are dropped with a note. */
+export const MAX_HTML_USAGE_ROWS = 50_000;
+/**
+ * The result as the page receives it: `rows` always present (empty when the
+ * caller left them out), usage rows cut to the most recent
+ * {@link MAX_HTML_USAGE_ROWS}, and the query text of per-hit rows (those with a
+ * ref) moved into one `rows.queries` table so a query that surfaced many assets
+ * is written once; the page puts it back on load. Never mutates the input.
+ */
+function toPageData(result) {
+    const notes = [...result.notes];
+    let usage = result.rows?.usage ?? [];
+    if (usage.length > MAX_HTML_USAGE_ROWS) {
+        // ISO timestamps sort lexically; id breaks ties so the cut is stable.
+        usage = [...usage].sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : a.id - b.id)).slice(-MAX_HTML_USAGE_ROWS);
+        notes.push(`The dashboard shows the most recent ${MAX_HTML_USAGE_ROWS} usage rows of this window; older rows were left out of the page (totals in the JSON output are complete).`);
+    }
+    const queries = [];
+    const queryIndex = new Map();
+    const pageUsage = usage.map((row) => {
+        if (row.ref === undefined || row.query === undefined)
+            return row;
+        const { query, ...rest } = row;
+        let q = queryIndex.get(query);
+        if (q === undefined) {
+            q = queries.length;
+            queries.push(query);
+            queryIndex.set(query, q);
+        }
+        return { ...rest, q };
+    });
+    return { ...result, notes, rows: { usage: pageUsage, llm: result.rows?.llm ?? [], queries } };
+}
+/**
+ * Serialize for an inline `<script type="application/json">`. Escaping `<` is
+ * enough: it stops `</script>` and `<!--` in a query or reason from ending the
+ * island, and `JSON.parse` turns `<` back into `<`.
+ */
+function islandJson(value) {
+    return JSON.stringify(value).replaceAll("<", "\\u003c");
+}
+export function buildMetricsHtmlReplacements(result) {
+    const { filters, window: win } = result;
+    return {
+        "%%ECHARTS_TAG%%": buildEchartsTag(),
+        "%%REPORT_TITLE%%": "akm metrics",
+        "%%WINDOW_HTML%%": `${isoTimeTag(win.since)} &rarr; ${isoTimeTag(win.until)}`,
+        "%%FILTERS_HTML%%": `source: ${esc(filters.source)}`,
+        "%%GENERATED_AT%%": esc(win.until),
+        "%%AKM_VERSION%%": esc(pkgVersion),
+        "%%DATA_JSON%%": islandJson(toPageData(result)),
+    };
+}
+/**
+ * `--format html` renderer for `akm metrics`, called directly from
+ * `cli/shared.ts`. Returns `null` for a payload that is not a metrics result so
+ * the generic renderer takes over.
+ */
+export function renderMetricsHtml(result) {
+    if (!isMetricsResult(result))
+        return null;
+    return renderHtml(resolveTemplatePath("metrics"), buildMetricsHtmlReplacements(result));
+}

package/dist/commands/metrics/md-report.js

@@ -0,0 +1,44 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { registerMdRenderer } from "../../output/render-registry.js";
+import { buildMetricsView, isMetricsResult } from "./report-view.js";
+/** Escape a cell so a query or reason containing `|` or a newline cannot break the table. */
+function cell(value) {
+    return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+}
+function renderTable(table) {
+    const row = (cells) => `| ${cells.map(cell).join(" | ")} |`;
+    return [
+        `### ${table.title}`,
+        "",
+        row(table.headers),
+        row(table.headers.map(() => "---")),
+        ...table.rows.map(row),
+        "",
+    ];
+}
+export function renderMetricsMd(result, detail) {
+    if (!isMetricsResult(result))
+        return null;
+    const view = buildMetricsView(result, detail);
+    const lines = ["# akm metrics", "", `- **Window:** ${view.window}`, `- **Filters:** ${view.filters}`, ""];
+    for (const section of view.sections) {
+        lines.push(`## ${section.title}`, "");
+        for (const [label, value] of section.facts)
+            lines.push(`- **${label}:** ${value}`);
+        if (section.facts.length > 0)
+            lines.push("");
+        for (const table of section.tables) {
+            if (table.rows.length > 0)
+                lines.push(...renderTable(table));
+            else if (!table.hideWhenEmpty)
+                lines.push(`### ${table.title}`, "", "_none_", "");
+        }
+    }
+    if (view.notes.length > 0) {
+        lines.push("## Notes", "", ...view.notes.map((n) => `- ${n}`), "");
+    }
+    return lines.join("\n").trimEnd();
+}
+registerMdRenderer("metrics", renderMetricsMd);