@klhapp/skillmux 1.7.1 → 1.8.0

package/src/stats.ts

@@ -1,5 +1,5 @@
 import type { Database } from "bun:sqlite";
-import type { AuditCandidate, AuditRow } from "./types";
+import type { AuditCandidate, AuditRow, FetchAuditRow } from "./types";
 
 export const SINCE_PATTERN = /^(\d+[hdwmy]|\d{4}-\d{2}-\d{2}([T ].+)?)$/;
 
@@ -20,6 +20,18 @@ export interface RetrievalTotals {
   lexical: number;
 }
 
+export type AcceptanceSignal =
+  | { available: false; uncorrelated_fetch_count: number }
+  | {
+      available: true;
+      resolves_with_candidates: number;
+      accepted_count: number;
+      acceptance_rate: number;
+      observed_mrr: number;
+      top1_acceptance_rate: number;
+      uncorrelated_fetch_count: number;
+    };
+
 export interface StatsResponse {
   since: string;
   until: string;
@@ -31,6 +43,8 @@ export interface StatsResponse {
   average_latency_ms: number;
   skills: SkillStat[];
   top_empty_shortlist_queries: EmptyShortlistQuery[];
+  acceptance: AcceptanceSignal;
+  top_unused_shortlist_queries: EmptyShortlistQuery[];
 }
 
 const RELATIVE_WINDOW = /^(\d+)([hdwmy])$/;
@@ -63,10 +77,16 @@ function compareCodeUnits(a: string, b: string): number {
   return 0;
 }
 
-export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsResponse {
+export function computeStats(
+  rows: AuditRow[],
+  since: Date,
+  until: Date,
+  fetchRows: FetchAuditRow[] = [],
+): StatsResponse {
   const retrieval_totals: RetrievalTotals = { exact: 0, reranked: 0, hybrid: 0, lexical: 0 };
   const skillCounts = new Map<string, number>();
   const emptyShortlistCounts = new Map<string, number>();
+  const resolvesWithCandidates = new Map<number, AuditRow>();
   let empty_shortlist_count = 0;
   let degraded_count = 0;
   let total_latency_ms = 0;
@@ -86,6 +106,7 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
       empty_shortlist_count++;
       emptyShortlistCounts.set(row.query, (emptyShortlistCounts.get(row.query) ?? 0) + 1);
     } else {
+      resolvesWithCandidates.set(row.id, row);
       const seenInRow = new Set<string>();
       for (const candidate of row.candidates) {
         if (!candidate.skill_id) continue;
@@ -96,6 +117,52 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
     }
   }
 
+  let uncorrelated_fetch_count = 0;
+  const firstFetchByResolve = new Map<number, FetchAuditRow>();
+  for (const fetch of fetchRows) {
+    if (fetch.resolve_audit_id === null) {
+      uncorrelated_fetch_count++;
+      continue;
+    }
+    if (!resolvesWithCandidates.has(fetch.resolve_audit_id)) continue;
+    const existing = firstFetchByResolve.get(fetch.resolve_audit_id);
+    if (!existing || fetch.ts < existing.ts) {
+      firstFetchByResolve.set(fetch.resolve_audit_id, fetch);
+    }
+  }
+
+  const acceptedResolveIds = new Set(firstFetchByResolve.keys());
+  const accepted_count = acceptedResolveIds.size;
+  const acceptance: AcceptanceSignal =
+    accepted_count > 0
+      ? (() => {
+          let reciprocalRankSum = 0;
+          let top1Count = 0;
+          for (const fetch of firstFetchByResolve.values()) {
+            const rank = fetch.rank_at_resolve;
+            if (rank !== null) {
+              reciprocalRankSum += 1 / rank;
+              if (rank === 1) top1Count++;
+            }
+          }
+          return {
+            available: true as const,
+            resolves_with_candidates: resolvesWithCandidates.size,
+            accepted_count,
+            acceptance_rate: accepted_count / resolvesWithCandidates.size,
+            observed_mrr: reciprocalRankSum / accepted_count,
+            top1_acceptance_rate: top1Count / accepted_count,
+            uncorrelated_fetch_count,
+          };
+        })()
+      : { available: false as const, uncorrelated_fetch_count };
+
+  const unusedShortlistCounts = new Map<string, number>();
+  for (const [id, row] of resolvesWithCandidates) {
+    if (acceptedResolveIds.has(id)) continue;
+    unusedShortlistCounts.set(row.query, (unusedShortlistCounts.get(row.query) ?? 0) + 1);
+  }
+
   const total_requests = rows.length;
   const empty_shortlist_rate = total_requests > 0 ? empty_shortlist_count / total_requests : 0;
   const average_latency_ms = total_requests > 0 ? total_latency_ms / total_requests : 0;
@@ -109,15 +176,8 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
       return compareCodeUnits(a.skill_id, b.skill_id);
     });
 
-  const top_empty_shortlist_queries: EmptyShortlistQuery[] = [...emptyShortlistCounts.entries()]
-    .map(([query, count]) => ({ query, count }))
-    .sort((a, b) => {
-      if (b.count !== a.count) {
-        return b.count - a.count;
-      }
-      return compareCodeUnits(a.query, b.query);
-    })
-    .slice(0, 20);
+  const top_empty_shortlist_queries = topQueryCounts(emptyShortlistCounts);
+  const top_unused_shortlist_queries = topQueryCounts(unusedShortlistCounts);
 
   return {
     since: since.toISOString(),
@@ -130,12 +190,27 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
     average_latency_ms,
     skills,
     top_empty_shortlist_queries,
+    acceptance,
+    top_unused_shortlist_queries,
   };
 }
 
+function topQueryCounts(counts: Map<string, number>): EmptyShortlistQuery[] {
+  return [...counts.entries()]
+    .map(([query, count]) => ({ query, count }))
+    .sort((a, b) => {
+      if (b.count !== a.count) {
+        return b.count - a.count;
+      }
+      return compareCodeUnits(a.query, b.query);
+    })
+    .slice(0, 20);
+}
+
 interface AuditTableRow {
   id: number;
   ts: string;
+  request_id: string | null;
   query: string;
   retrieval: AuditRow["retrieval"];
   degraded_from: string | null;
@@ -147,7 +222,7 @@ interface AuditTableRow {
 export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
   const rows = db
     .query(
-      "SELECT id, ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC",
+      "SELECT id, ts, request_id, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC",
     )
     .all(sinceIso) as AuditTableRow[];
 
@@ -178,6 +253,7 @@ export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
     const result: AuditRow = {
       id: row.id,
       ts: row.ts,
+      request_id: row.request_id,
       query: row.query,
       retrieval: row.retrieval,
       candidates,
@@ -193,10 +269,19 @@ export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
   });
 }
 
+export function queryFetchRows(db: Database, sinceIso: string): FetchAuditRow[] {
+  return db
+    .query(
+      "SELECT id, ts, skill_id, request_id, resolve_audit_id, rank_at_resolve FROM fetch WHERE ts >= ? ORDER BY ts ASC",
+    )
+    .all(sinceIso) as FetchAuditRow[];
+}
+
 export function getStats(db: Database, since: string, now: Date = new Date()): StatsResponse {
   const sinceDate = parseSince(since, now);
   const rows = queryAuditRows(db, sinceDate.toISOString());
-  return computeStats(rows, sinceDate, now);
+  const fetchRows = queryFetchRows(db, sinceDate.toISOString());
+  return computeStats(rows, sinceDate, now, fetchRows);
 }
 
 export function renderStatsText(stats: StatsResponse): string {
@@ -230,5 +315,26 @@ export function renderStatsText(stats: StatsResponse): string {
     }
   }
 
+  if (stats.acceptance.available) {
+    lines.push(
+      `acceptance: acceptance_rate=${stats.acceptance.acceptance_rate.toFixed(3)} ` +
+        `observed_mrr=${stats.acceptance.observed_mrr.toFixed(3)} ` +
+        `top1_acceptance_rate=${stats.acceptance.top1_acceptance_rate.toFixed(3)} ` +
+        `(accepted=${stats.acceptance.accepted_count}/${stats.acceptance.resolves_with_candidates}, ` +
+        `uncorrelated_fetch_count=${stats.acceptance.uncorrelated_fetch_count})`,
+    );
+  } else {
+    lines.push(`acceptance: unavailable (uncorrelated_fetch_count=${stats.acceptance.uncorrelated_fetch_count})`);
+  }
+
+  lines.push("top unused shortlist queries:");
+  if (stats.top_unused_shortlist_queries.length === 0) {
+    lines.push("  (none)");
+  } else {
+    for (const entry of stats.top_unused_shortlist_queries) {
+      lines.push(`  "${entry.query}" (${entry.count})`);
+    }
+  }
+
   return lines.join("\n");
 }

package/src/types.ts

@@ -107,6 +107,11 @@ export interface ConfigPolicy {
   environment_overrides?: boolean;
 }
 
+export interface AuditConfig {
+  /** Age in days beyond which audit rows are pruned. 0 disables pruning. */
+  retention_days: number;
+}
+
 export interface Config {
   config?: ConfigPolicy;
   vault_path: string;
@@ -116,6 +121,7 @@ export interface Config {
   output: OutputConfig;
   inference: InferenceConfig;
   server?: ServerConfig;
+  audit?: AuditConfig;
 }
 
 export interface RankedCandidate {
@@ -136,6 +142,7 @@ export type DegradationReason =
   | "reranker_protocol_error";
 
 export interface ResolveResult {
+  request_id: string;
   retrieval: RetrievalCapability;
   degraded_from?: "reranked" | "hybrid";
   degradation_reason?: DegradationReason;
@@ -151,6 +158,7 @@ export interface ResolveSkillInput {
 
 export interface FetchSkillInput {
   skill_id: string;
+  request_id?: string;
 }
 
 export interface FetchSkillResult {
@@ -169,6 +177,8 @@ export interface AuditCandidate {
 export interface AuditRow {
   id: number;
   ts: string;
+  /** Null for rows written before request_id existed (AC4). */
+  request_id: string | null;
   query: string;
   retrieval: RetrievalCapability;
   degraded_from?: "reranked" | "hybrid" | null;
@@ -177,6 +187,18 @@ export interface AuditRow {
   latency_ms: number;
 }
 
+export interface FetchAuditRow {
+  id: number;
+  ts: string;
+  skill_id: string;
+  /** Exactly as supplied by the caller, including an unknown value. Null when the caller sent none. */
+  request_id: string | null;
+  /** Null when the fetch is uncorrelated: no request_id, an unknown/malformed one, or a pruned resolve row. */
+  resolve_audit_id: number | null;
+  /** Rank of skill_id in the correlated resolve's shortlist. Null when uncorrelated or absent from the shortlist. */
+  rank_at_resolve: number | null;
+}
+
 export interface Clients {
   embed(texts: string[]): Promise<Float32Array[]>;
   rerank?: (query: string, docs: { skill_id: string; text: string }[]) => Promise<number[]>;