@klhapp/skillmux 1.7.1 → 1.8.0

package/src/db.ts

@@ -1,5 +1,5 @@
 import { Database } from "bun:sqlite";
-import { mkdirSync } from "node:fs";
+import { existsSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import type { AuditCandidate, AuditRow } from "./types";
 import type { VaultSkill } from "./vault";
@@ -12,6 +12,81 @@ export interface SkillRow {
   content_sha256: string;
 }
 
+export function openAudit(stateDir: string): Database {
+  mkdirSync(stateDir, { recursive: true });
+  const db = new Database(join(stateDir, "audit.sqlite3"), { create: true });
+  // auto_vacuum only takes on an empty database, so it must precede both the
+  // journal-mode switch and any CREATE TABLE. It is what lets a retention
+  // prune reclaim space without a full VACUUM.
+  db.run("PRAGMA auto_vacuum = INCREMENTAL");
+  db.run("PRAGMA journal_mode = WAL");
+  db.run("PRAGMA busy_timeout = 2000");
+  db.run(`CREATE TABLE IF NOT EXISTS audit (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    ts TEXT NOT NULL,
+    request_id TEXT,
+    query TEXT NOT NULL,
+    retrieval TEXT NOT NULL DEFAULT 'lexical',
+    degraded_from TEXT,
+    degradation_reason TEXT,
+    candidates TEXT NOT NULL,
+    latency_ms INTEGER NOT NULL
+  )`);
+  // CREATE TABLE IF NOT EXISTS no-ops on a table opened from before request_id
+  // existed (AC4), so add it explicitly when missing.
+  const auditColumns = new Set(
+    (db.query("PRAGMA table_info(audit)").all() as { name: string }[]).map((c) => c.name),
+  );
+  if (!auditColumns.has("request_id")) {
+    db.run("ALTER TABLE audit ADD COLUMN request_id TEXT");
+  }
+  db.run(`CREATE TABLE IF NOT EXISTS fetch (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    ts TEXT NOT NULL,
+    skill_id TEXT NOT NULL,
+    request_id TEXT,
+    resolve_audit_id INTEGER,
+    rank_at_resolve INTEGER
+  )`);
+  adoptAuditFromIndex(db, stateDir);
+  return db;
+}
+
+// Audit rows used to live in index.sqlite3. Move any that remain there into the
+// audit store, then drop the old table so the index carries no user queries.
+function adoptAuditFromIndex(db: Database, stateDir: string): void {
+  const indexPath = join(stateDir, "index.sqlite3");
+  if (!existsSync(indexPath)) return;
+
+  db.run("ATTACH DATABASE ? AS legacy", [indexPath]);
+  try {
+    const legacyAudit = db
+      .query("SELECT name FROM legacy.sqlite_master WHERE type = 'table' AND name = 'audit'")
+      .get();
+    if (!legacyAudit) return;
+
+    // Older audit tables predate the retrieval columns and carry outcome /
+    // degraded / selected_skill_id instead. Select what is actually there and
+    // let the canonical defaults stand in for the rest.
+    const legacyColumns = new Set(
+      (db.query("PRAGMA legacy.table_info(audit)").all() as { name: string }[]).map((c) => c.name),
+    );
+    const retrieval = legacyColumns.has("retrieval") ? "COALESCE(retrieval, 'lexical')" : "'lexical'";
+    const degradedFrom = legacyColumns.has("degraded_from") ? "degraded_from" : "NULL";
+    const degradationReason = legacyColumns.has("degradation_reason") ? "degradation_reason" : "NULL";
+
+    // SQLite commits atomically across attached databases, so the copy and the
+    // drop either both land or neither does.
+    db.transaction(() => {
+      db.run(`INSERT INTO audit (ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
+        SELECT ts, query, ${retrieval}, ${degradedFrom}, ${degradationReason}, candidates, latency_ms FROM legacy.audit`);
+      db.run("DROP TABLE legacy.audit");
+    })();
+  } finally {
+    db.run("DETACH DATABASE legacy");
+  }
+}
+
 export function openIndex(stateDir: string): Database {
   mkdirSync(stateDir, { recursive: true });
   const db = new Database(join(stateDir, "index.sqlite3"), { create: true });
@@ -39,48 +114,7 @@ export function openIndex(stateDir: string): Database {
   if (!vectorColumns.some((column) => column.name === "embedding_fingerprint")) {
     db.run("ALTER TABLE vectors ADD COLUMN embedding_fingerprint TEXT NOT NULL DEFAULT ''");
   }
-  db.run(`CREATE TABLE IF NOT EXISTS audit (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    ts TEXT NOT NULL,
-    query TEXT NOT NULL,
-    retrieval TEXT NOT NULL DEFAULT 'lexical',
-    degraded_from TEXT,
-    degradation_reason TEXT,
-    candidates TEXT NOT NULL,
-    latency_ms INTEGER NOT NULL
-  )`);
-  const auditColumns = db.query("PRAGMA table_info(audit)").all() as { name: string }[];
-  const columnNames = new Set(auditColumns.map((column) => column.name));
-  const hasLegacyColumns =
-    columnNames.has("outcome") ||
-    columnNames.has("selected_skill_id") ||
-    columnNames.has("degraded");
-  const missingCanonicalColumns =
-    !columnNames.has("retrieval") ||
-    !columnNames.has("degraded_from") ||
-    !columnNames.has("degradation_reason");
-
-  if (hasLegacyColumns || missingCanonicalColumns) {
-    db.transaction(() => {
-      db.run(`CREATE TABLE audit_new (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        ts TEXT NOT NULL,
-        query TEXT NOT NULL,
-        retrieval TEXT NOT NULL DEFAULT 'lexical',
-        degraded_from TEXT,
-        degradation_reason TEXT,
-        candidates TEXT NOT NULL,
-        latency_ms INTEGER NOT NULL
-      )`);
-      const retrievalExpr = columnNames.has("retrieval") ? "COALESCE(retrieval, 'lexical')" : "'lexical'";
-      const degradedFromExpr = columnNames.has("degraded_from") ? "degraded_from" : "NULL";
-      const degradationReasonExpr = columnNames.has("degradation_reason") ? "degradation_reason" : "NULL";
-      db.run(`INSERT INTO audit_new (id, ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
-        SELECT id, ts, query, ${retrievalExpr}, ${degradedFromExpr}, ${degradationReasonExpr}, candidates, latency_ms FROM audit`);
-      db.run("DROP TABLE audit");
-      db.run("ALTER TABLE audit_new RENAME TO audit");
-    })();
-  }
+  // Audit rows live in audit.sqlite3; openAudit adopts any left here.
   db.run(`CREATE TABLE IF NOT EXISTS index_meta (
     key TEXT PRIMARY KEY,
     value TEXT NOT NULL
@@ -281,6 +315,7 @@ export function vectorTopK(db: Database, query: Float32Array, k: number): SkillR
 
 export interface AuditInsert {
   ts: string;
+  request_id?: string | null;
   query: string;
   retrieval: AuditRow["retrieval"];
   degraded_from?: string | null;
@@ -291,10 +326,11 @@ export interface AuditInsert {
 
 export function insertAudit(db: Database, row: AuditInsert): void {
   db.run(
-    `INSERT INTO audit (ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
-     VALUES (?, ?, ?, ?, ?, ?, ?)`,
+    `INSERT INTO audit (ts, request_id, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
     [
       row.ts,
+      row.request_id ?? null,
       row.query,
       row.retrieval,
       row.degraded_from ?? null,
@@ -304,3 +340,74 @@ export function insertAudit(db: Database, row: AuditInsert): void {
     ],
   );
 }
+
+/**
+ * Correlation lookup for AC5/AC7: looks up the resolve that produced
+ * `requestId`, or null when it names no known resolve (including malformed
+ * input, which is never validated at the boundary per AC7).
+ */
+export function getAuditRowByRequestId(
+  db: Database,
+  requestId: string,
+): { id: number; candidates: AuditCandidate[] } | null {
+  const row = db
+    .query("SELECT id, candidates FROM audit WHERE request_id = ?")
+    .get(requestId) as { id: number; candidates: string } | null;
+  if (!row) return null;
+  return { id: row.id, candidates: JSON.parse(row.candidates) as AuditCandidate[] };
+}
+
+export interface FetchInsert {
+  ts: string;
+  skill_id: string;
+  request_id?: string | null;
+  resolve_audit_id?: number | null;
+  rank_at_resolve?: number | null;
+}
+
+export function insertFetch(db: Database, row: FetchInsert): void {
+  db.run(
+    `INSERT INTO fetch (ts, skill_id, request_id, resolve_audit_id, rank_at_resolve)
+     VALUES (?, ?, ?, ?, ?)`,
+    [
+      row.ts,
+      row.skill_id,
+      row.request_id ?? null,
+      row.resolve_audit_id ?? null,
+      row.rank_at_resolve ?? null,
+    ],
+  );
+}
+
+export interface PruneResult {
+  audit_deleted: number;
+  fetch_deleted: number;
+}
+
+/**
+ * Deletes resolve and fetch rows with ts before `cutoffIso`, each by its own
+ * timestamp; no FK ties them, so a fetch outliving its resolve row simply
+ * reads back uncorrelated (AC7's existing null path). Reclaims the freed
+ * pages with an incremental vacuum, which only touches audit.sqlite3 (AC16).
+ */
+export function pruneAuditBefore(db: Database, cutoffIso: string): PruneResult {
+  const auditResult = db.run("DELETE FROM audit WHERE ts < ?", [cutoffIso]);
+  const fetchResult = db.run("DELETE FROM fetch WHERE ts < ?", [cutoffIso]);
+  db.run("PRAGMA incremental_vacuum");
+
+  return { audit_deleted: auditResult.changes, fetch_deleted: fetchResult.changes };
+}
+
+/** AC12: retentionDays <= 0 disables pruning entirely. */
+export function pruneAudit(db: Database, retentionDays: number, now: Date = new Date()): PruneResult {
+  if (retentionDays <= 0) return { audit_deleted: 0, fetch_deleted: 0 };
+  const cutoff = new Date(now.getTime() - retentionDays * 86_400_000).toISOString();
+  return pruneAuditBefore(db, cutoff);
+}
+
+/** Dry-run counterpart of pruneAuditBefore: counts without deleting (AC15). */
+export function countPrunable(db: Database, cutoffIso: string): PruneResult {
+  const auditRow = db.query("SELECT count(*) AS n FROM audit WHERE ts < ?").get(cutoffIso) as { n: number };
+  const fetchRow = db.query("SELECT count(*) AS n FROM fetch WHERE ts < ?").get(cutoffIso) as { n: number };
+  return { audit_deleted: auditRow.n, fetch_deleted: fetchRow.n };
+}

package/src/eval.ts

@@ -1,3 +1,4 @@
+import type { Database } from "bun:sqlite";
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import {
@@ -58,6 +59,74 @@ export function parseEvalCases(raw: unknown): EvalCase[] {
   return result;
 }
 
+/** AC17: dedup key for promoted cases. Collapses whitespace and case differences that are not meaningfully distinct queries. */
+export function normalizeQuery(query: string): string {
+  return query.trim().replace(/\s+/g, " ").toLowerCase();
+}
+
+export interface ObservedFetch {
+  query: string;
+  skill_id: string;
+}
+
+/** AC17: groups fetched skill ids by normalized query into observed-split eval cases. */
+export function buildPromotedCases(fetches: ObservedFetch[]): EvalCase[] {
+  const byQuery = new Map<string, { query: string; skillIds: string[]; seen: Set<string> }>();
+  for (const fetch of fetches) {
+    const key = normalizeQuery(fetch.query);
+    if (!key) continue;
+    let entry = byQuery.get(key);
+    if (!entry) {
+      entry = { query: fetch.query, skillIds: [], seen: new Set() };
+      byQuery.set(key, entry);
+    }
+    if (!entry.seen.has(fetch.skill_id)) {
+      entry.seen.add(fetch.skill_id);
+      entry.skillIds.push(fetch.skill_id);
+    }
+  }
+  return [...byQuery.values()].map((entry) => ({
+    query: entry.query,
+    split: "observed",
+    relevant_skill_ids: entry.skillIds,
+  }));
+}
+
+/** AC18: never rewrites a case whose normalized query already exists in the target file. */
+export function excludeExistingCases(
+  cases: EvalCase[],
+  existing: EvalCase[],
+): { cases: EvalCase[]; skipped: number } {
+  const existingKeys = new Set(existing.map((c) => normalizeQuery(c.query)));
+  const kept: EvalCase[] = [];
+  let skipped = 0;
+  for (const c of cases) {
+    if (existingKeys.has(normalizeQuery(c.query))) {
+      skipped++;
+    } else {
+      kept.push(c);
+    }
+  }
+  return { cases: kept, skipped };
+}
+
+/**
+ * AC17: joins fetches to the resolve that produced them for promotion. Only
+ * correlated fetches (a known resolve_audit_id) carry a query to promote;
+ * uncorrelated fetches have no resolve to join against and are excluded.
+ */
+export function queryPromotableFetches(db: Database, sinceIso: string): ObservedFetch[] {
+  return db
+    .query(
+      `SELECT audit.query AS query, fetch.skill_id AS skill_id
+       FROM fetch
+       JOIN audit ON fetch.resolve_audit_id = audit.id
+       WHERE fetch.ts >= ?
+       ORDER BY fetch.ts ASC`,
+    )
+    .all(sinceIso) as ObservedFetch[];
+}
+
 export interface EvalMetrics {
   recall_at_5: number;
   recall_at_10: number;

package/src/router-core.ts

@@ -8,11 +8,15 @@ import {
   deleteSkill,
   findExactMatch,
   ftsSearch,
+  getAuditRowByRequestId,
   getIndexMeta,
   getSkillRow,
   ingestVault,
   insertAudit,
+  insertFetch,
+  openAudit,
   openIndex,
+  pruneAudit,
   replaceSkills,
   setIndexMeta,
   skillCount,
@@ -22,7 +26,7 @@ import {
   upsertVector,
   vectorTopK,
 } from "./db";
-import type { SkillRow } from "./db";
+import type { PruneResult, SkillRow } from "./db";
 import type {
   RankedCandidate,
   RetrievalCapability,
@@ -73,26 +77,45 @@ const defaultClients: Clients = {
 };
 
 let overrides: Overrides = {};
-let env: { config: Config; db: Database } | null = null;
+type Env = { config: Config; db: Database; auditDb: Database };
+
+let envPromise: Promise<Env> | null = null;
+let resolvedEnv: Env | null = null;
+let lastAuditPruneAt: number | null = null;
+
+const AUDIT_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
 
 /** Replace config/client overrides wholesale (tests, ops). Resets the cached index handle. */
 export function configure(opts: Overrides): void {
   overrides = opts;
-  env = null;
+  envPromise = null;
+  resolvedEnv = null;
+  lastAuditPruneAt = null;
 }
 
-async function getEnv(): Promise<{ config: Config; db: Database }> {
-  if (env) return env;
-  const config = overrides.config ?? (await loadConfig());
-  const db = openIndex(expandHome(config.state_dir));
-  if (skillCount(db) === 0) {
-    const vaultPath = expandHome(config.vault_path);
-    const localVaultPaths = config.local_vault_paths.map(expandHome);
-    ingestVault(db, await scanVaults(vaultPath, localVaultPaths));
-    setIndexMeta(db, "last_indexed_mtime", String(maxVaultMtime(vaultPath, localVaultPaths)));
-  }
-  env = { config, db };
-  return env;
+/**
+ * Memoizes the in-flight promise, not just the resolved value: startup fires
+ * initializeRuntime()'s getRuntime() and pruneAuditIfDue() back-to-back before
+ * either has awaited anything, so caching only the resolved env would let both
+ * open their own index/audit handles and race an ingestVault (AC14 regression).
+ */
+async function getEnv(): Promise<Env> {
+  if (envPromise) return envPromise;
+  envPromise = (async () => {
+    const config = overrides.config ?? (await loadConfig());
+    const stateDir = expandHome(config.state_dir);
+    const db = openIndex(stateDir);
+    const auditDb = openAudit(stateDir);
+    if (skillCount(db) === 0) {
+      const vaultPath = expandHome(config.vault_path);
+      const localVaultPaths = config.local_vault_paths.map(expandHome);
+      ingestVault(db, await scanVaults(vaultPath, localVaultPaths));
+      setIndexMeta(db, "last_indexed_mtime", String(maxVaultMtime(vaultPath, localVaultPaths)));
+    }
+    resolvedEnv = { config, db, auditDb };
+    return resolvedEnv;
+  })();
+  return envPromise;
 }
 
 function getClients(): Clients {
@@ -100,14 +123,21 @@ function getClients(): Clients {
 }
 
 /** Runtime accessor for the eval harness and CLI — not part of the MCP surface. */
-export async function getRuntime(): Promise<{ config: Config; db: Database; clients: Clients }> {
-  const { config, db } = await getEnv();
-  return { config, db, clients: getClients() };
+export async function getRuntime(): Promise<{
+  config: Config;
+  db: Database;
+  auditDb: Database;
+  clients: Clients;
+}> {
+  const { config, db, auditDb } = await getEnv();
+  return { config, db, auditDb, clients: getClients() };
 }
 
 export function closeRuntime(): void {
-  env?.db.close();
-  env = null;
+  resolvedEnv?.db.close();
+  resolvedEnv?.auditDb.close();
+  envPromise = null;
+  resolvedEnv = null;
 }
 
 /**
@@ -365,7 +395,7 @@ export async function startVaultWatcher(): Promise<() => void> {
 
 export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveResult> {
   const t0 = performance.now();
-  const { config, db } = await getEnv();
+  const { config, db, auditDb } = await getEnv();
   await syncVaultIfNeeded();
 
   if (input.top_k !== undefined) {
@@ -392,7 +422,10 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
       score: c.score,
     }));
 
+  const requestId = crypto.randomUUID();
+
   const result: ResolveResult = {
+    request_id: requestId,
     retrieval,
     ...(retrievalResult.degraded_from
       ? {
@@ -404,10 +437,11 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
   };
 
   insertAudit(
-    db,
+    auditDb,
     buildAuditRow({
       id: 0, // assigned by SQLite
       ts: new Date().toISOString(),
+      request_id: requestId,
       query: input.query,
       retrieval,
       degraded_from: retrievalResult.degraded_from ?? null,
@@ -592,11 +626,48 @@ export async function retrieveAndRerank(
   };
 }
 
+/**
+ * AC14: runs at most once per 24 hours per process. Callers must not await
+ * this on the startup or resolve path -- it is meant to be fired and left to
+ * resolve in the background so it never blocks readiness or a resolve.
+ */
+export async function pruneAuditIfDue(now: Date = new Date()): Promise<PruneResult | null> {
+  const { config, auditDb } = await getEnv();
+  const retentionDays = config.audit?.retention_days ?? 90;
+  if (retentionDays <= 0) return null;
+  if (lastAuditPruneAt !== null && now.getTime() - lastAuditPruneAt < AUDIT_PRUNE_INTERVAL_MS) {
+    return null;
+  }
+  lastAuditPruneAt = now.getTime();
+  return pruneAudit(auditDb, retentionDays, now);
+}
+
 export async function fetchSkill(input: FetchSkillInput): Promise<FetchSkillResult> {
-  const { config, db } = await getEnv();
+  const { config, db, auditDb } = await getEnv();
   await syncVaultIfNeeded();
   if (getSkillRow(db, input.skill_id) === null) {
     throw new Error(`SKILL_NOT_FOUND: no skill '${input.skill_id}' in the index`);
   }
-  return deliverSkill(db, config, input.skill_id);
+  const result = await deliverSkill(db, config, input.skill_id);
+
+  let resolveAuditId: number | null = null;
+  let rankAtResolve: number | null = null;
+  if (input.request_id) {
+    const resolveRow = getAuditRowByRequestId(auditDb, input.request_id);
+    if (resolveRow) {
+      resolveAuditId = resolveRow.id;
+      const index = resolveRow.candidates.findIndex((c) => c.skill_id === input.skill_id);
+      rankAtResolve = index === -1 ? null : index + 1;
+    }
+  }
+
+  insertFetch(auditDb, {
+    ts: new Date().toISOString(),
+    skill_id: input.skill_id,
+    request_id: input.request_id ?? null,
+    resolve_audit_id: resolveAuditId,
+    rank_at_resolve: rankAtResolve,
+  });
+
+  return result;
 }

package/src/server.ts

@@ -12,6 +12,7 @@ import {
   backfillEmbeddings,
   configure,
   fetchSkill,
+  pruneAuditIfDue,
   resolveSkill,
 } from "./router-core";
 import { closeRuntime, getRuntime, startVaultWatcher } from "./router-core";
@@ -113,12 +114,16 @@ export function createMcpServer(): McpServer {
     {
       description:
         "Fetch a skill's SKILL.md verbatim by skill_id, with sha256 and supporting-file paths. " +
-        "Independent of any prior resolve_skill outcome.",
-      inputSchema: { skill_id: z.string().regex(SKILL_ID_PATTERN) },
+        "Independent of any prior resolve_skill outcome. Pass the request_id from a prior " +
+        "resolve_skill call to link this fetch to it for quality measurement.",
+      inputSchema: {
+        skill_id: z.string().regex(SKILL_ID_PATTERN),
+        request_id: z.string().min(1).max(128).optional(),
+      },
     },
-    async ({ skill_id }) => {
+    async ({ skill_id, request_id }) => {
       try {
-        const result = await fetchSkill({ skill_id });
+        const result = await fetchSkill({ skill_id, request_id });
         const { body, ...meta } = result;
         return {
           content: [{ type: "text" as const, text: body }],
@@ -175,6 +180,14 @@ export async function startServer(opts?: {
     .then(() => metricsRegistry.setReadiness(readinessState.get()))
     .catch((err) => console.error("skillmux runtime init error:", err));
 
+  // AC14: fire-and-forget so this never delays readiness or blocks a resolve;
+  // not chained onto initPromise, which is awaited below for HTTP transport.
+  const runAuditPrune = () =>
+    pruneAuditIfDue().catch((err) => console.error("skillmux audit prune error:", err));
+  runAuditPrune();
+  const auditPruneInterval = setInterval(runAuditPrune, 24 * 60 * 60 * 1000);
+  auditPruneInterval.unref();
+
   const server = createMcpServer();
 
   const transportType = opts?.transport ?? "stdio";
@@ -345,13 +358,13 @@ export async function startServer(opts?: {
               { status: 400, headers: { "Content-Type": "application/json" } },
             );
           }
-          const { db } = await getRuntime();
+          const { auditDb } = await getRuntime();
           const headers = new Headers({ "Content-Type": "application/json" });
           if (allowOriginHeader)
             headers.set("Access-Control-Allow-Origin", allowOriginHeader);
           for (const [key, value] of Object.entries(rateLimitResult.headers))
             headers.set(key, value);
-          return new Response(JSON.stringify(getStats(db, since)), {
+          return new Response(JSON.stringify(getStats(auditDb, since)), {
             status: 200,
             headers,
           });
@@ -521,6 +534,7 @@ export async function startServer(opts?: {
       async stop() {
         if (stopped) return;
         stopped = true;
+        clearInterval(auditPruneInterval);
         readinessState.set({ ...readinessState.get(), status: "stopping" });
         metricsRegistry.setReadiness(readinessState.get());
         bunServer.stop(true);
@@ -540,6 +554,7 @@ export async function startServer(opts?: {
       async stop() {
         if (stopped) return;
         stopped = true;
+        clearInterval(auditPruneInterval);
         readinessState.set({ ...readinessState.get(), status: "stopping" });
         metricsRegistry.setReadiness(readinessState.get());
         configWatcher?.stop();