@klhapp/skillmux 1.7.1 → 1.8.0

package/docs/deployment.md

@@ -248,7 +248,7 @@ The HTTP server provides:
 
 The Docker health check calls `/health/ready`.
 
-`GET /stats` returns JSON aggregated from audit rows over a query window (using `?since=`, defaulting to 24 hours). The response contains `since`, `until`, `total_requests`, `empty_shortlist_count`, `empty_shortlist_rate`, `retrieval_totals` (counts for `exact`, `reranked`, `hybrid`, and `lexical`), `degraded_count`, `average_latency_ms`, per-skill candidate counts in `skills` (`skill_id`, `candidate_count`), and `top_empty_shortlist_queries` (`query`, `count`). See [Managing skills](skill-management.md#use-routing-data-to-tune-tiers) for using these statistics to tune delivery tiers.
+`GET /stats` returns JSON aggregated from audit rows over a query window (using `?since=`, defaulting to 24 hours). The response contains `since`, `until`, `total_requests`, `empty_shortlist_count`, `empty_shortlist_rate`, `retrieval_totals` (counts for `exact`, `reranked`, `hybrid`, and `lexical`), `degraded_count`, `average_latency_ms`, per-skill candidate counts in `skills` (`skill_id`, `candidate_count`), `top_empty_shortlist_queries` (`query`, `count`), an `acceptance` signal derived from fetches correlated to their resolve (`acceptance_rate`, `observed_mrr`, `top1_acceptance_rate`, `accepted_count`, `resolves_with_candidates`, and `uncorrelated_fetch_count`, or `available: false` with just the uncorrelated count when a window has no correlated fetches), and `top_unused_shortlist_queries` (`query`, `count`) for queries that returned candidates but received no correlated fetch. See [Managing skills](skill-management.md#use-routing-data-to-tune-tiers) for using these statistics to tune delivery tiers.
 
 Prometheus metrics cover request totals, resolve latency histograms, errors,
 rate-limit rejections, degraded retrieval totals by stage and reason, readiness
@@ -289,13 +289,26 @@ their mounted vault checkout and do not manage host agent directories.
 
 ## Persistent data and backups
 
-Persist `state_dir` to retain the index and audit log.
-Skill content remains in the server's vault checkout and should use its own
-backup or Git workflow.
-
-Treat the state database as sensitive because audit rows can contain raw user
+Persist `state_dir` to retain both state files it holds: `index.sqlite3`
+(retrieval index and vector matrix) and `audit.sqlite3` (resolve and fetch
+audit rows). Skill content remains in the server's vault checkout and should
+use its own backup or Git workflow.
+
+The two files are independent and can be backed up on different schedules.
+Copying or backing up `index.sqlite3` alone never carries raw user queries;
+only `audit.sqlite3` does. `audit.sqlite3` is created with WAL journaling and
+incremental auto-vacuum, so `skillmux audit prune` reclaims space without
+taking a lock on `index.sqlite3`. On first startup after an upgrade from a
+version that kept audit rows inside `index.sqlite3`, Skillmux migrates them
+into `audit.sqlite3` once, transactionally; the migration is idempotent, so
+it is safe to leave running unattended.
+
+Treat `audit.sqlite3` as sensitive because its rows can contain raw user
 queries. Stop the process or use SQLite-safe backup tooling before copying a
-live database.
+live database. `audit.retention_days` (default 90; `0` disables pruning) ages
+out old rows automatically at startup and at most once per 24 hours while the
+server runs; see [CLI reference](cli.md#observability-and-evaluation-skillmux-report-audit-eval)
+for the on-demand `skillmux audit prune` command.
 
 ## Native pins with shared retrieval
 

package/docs/getting-started.md

@@ -3,6 +3,17 @@
 Skillmux supports three setup paths. Pick the result you want before choosing
 an installation.
 
+```mermaid
+flowchart LR
+    Q{What do you need?}
+    Q -->|Managed links in client skill directories| A[Manage native skills]
+    Q -->|Local stdio MCP| B[Add local MCP retrieval]
+    Q -->|Streamable HTTP MCP for several clients| C[Run a shared MCP service]
+    A --> A1[Skillmux CLI]
+    B --> B1[Skillmux CLI]
+    C --> C1[Skillmux server, full image]
+```
+
 | Goal | Skill delivery | Recommended installation |
 | --- | --- | --- |
 | [Manage native skills](#manage-native-skills) | Managed links in client skill directories | Skillmux CLI |

package/docs/mcp-routing.md

@@ -79,6 +79,7 @@ Skillmux returns a ranked candidates response:
 
 ```json
 {
+  "request_id": "3fae2b8e-6c2d-4b1a-9d7a-2b6c5b6a9e10",
   "retrieval": "reranked",
   "candidates": [
     {
@@ -91,6 +92,7 @@ Skillmux returns a ranked candidates response:
 }
 ```
 
+- `request_id`: a unique id minted for this resolve. Pass it back to `fetch_skill` to correlate a fetch outcome with this resolve and the fetched skill's rank in this shortlist.
 - `retrieval`: the effective retrieval capability (`reranked`, `hybrid`, or `lexical`).
 - `candidates`: zero through effective `top_k` candidates ordered by descending score with contiguous 1-based ranks.
 - If reranking or embedding fails, degradation metadata (`degraded_from`, `degradation_reason`) is included.
@@ -101,13 +103,18 @@ Input:
 
 ```json
 {
-  "skill_id": "csv-formatter"
+  "skill_id": "csv-formatter",
+  "request_id": "3fae2b8e-6c2d-4b1a-9d7a-2b6c5b6a9e10"
 }
 ```
 
 The response contains the current `SKILL.md` body as text content.
 `structuredContent` contains the skill ID, title, content SHA-256, and
-supporting-file paths. Fetch does not depend on an earlier resolve call.
+supporting-file paths. `request_id` is optional. When it names a resolve that
+minted it, the recorded fetch outcome links to that resolve and its rank in
+the shortlist. An absent, unknown, or malformed `request_id` still succeeds
+and records an uncorrelated fetch — delivery never fails because telemetry
+could not correlate. Fetch does not depend on an earlier resolve call.
 
 The complete wire contract lives in [schema.json](schema.json).
 
@@ -117,11 +124,26 @@ Give the calling client these rules:
 
 1. Call `resolve_skill` when a task may benefit from a specialized workflow.
 2. Review the returned ranked candidates shortlist.
-3. If a relevant candidate exists, call `fetch_skill` with its `skill_id` to retrieve complete instructions.
+3. If a relevant candidate exists, call `fetch_skill` with its `skill_id` to retrieve complete instructions, passing back the resolve's `request_id` when the client retains it.
 4. If no candidate is relevant (or `candidates` is empty), continue under your normal workflow.
 
+Passing `request_id` is optional and never required for delivery, but it is
+what lets Skillmux measure whether a returned shortlist was actually used.
+
 ## Retrieval pipeline
 
+```mermaid
+flowchart LR
+    Q[Query] --> L[Lexical: FTS5 + BM25]
+    Q --> E[Embeddings: local or remote]
+    L --> F[Reciprocal-rank fusion]
+    E --> F
+    F --> RR[Optional reranker]
+    RR --> K[Ranked candidates: top_k]
+    F -. reranker unavailable .-> K
+    Q -. embedding unavailable: lexical only .-> K
+```
+
 Skillmux builds candidates in stages:
 
 1. SQLite FTS5 ranks lexical matches with BM25.
@@ -173,12 +195,24 @@ access.
 
 Each resolve request records:
 
-- timestamp and query;
+- timestamp, `request_id`, and query;
 - retrieval capability;
 - degradation metadata (`degraded_from`, `degradation_reason`) when degraded;
 - candidates with scores;
 - latency in milliseconds.
 
-Skillmux stores audit rows in the SQLite database under `state_dir`. Use
-`skillmux report` to summarize activity. Treat raw queries as private user
-data when backing up or sharing the database.
+Each fetch request records:
+
+- timestamp and `skill_id`;
+- the `request_id` supplied by the caller, or null when absent, unknown, or malformed;
+- the originating resolve's audit row id, or null when the fetch is uncorrelated;
+- `rank_at_resolve`: the fetched skill's rank in that resolve's shortlist, or null when the fetch is uncorrelated or the skill was absent from that shortlist.
+
+Skillmux stores audit rows in `audit.sqlite3` under `state_dir`, a file
+separate from the retrieval index. Use `skillmux report` to summarize
+activity, `skillmux audit prune` to reclaim space under
+`audit.retention_days` (default 90 days), and `skillmux eval promote` to turn
+correlated fetches into eval cases — see [CLI
+reference](cli.md#observability-and-evaluation-skillmux-report-audit-eval)
+for all three. Treat raw queries as private user data when backing up or
+sharing the database.

package/docs/schema.json

@@ -55,6 +55,18 @@
       "required": ["rank", "skill_id", "description", "score"],
       "additionalProperties": false
     },
+    "RequestId": {
+      "type": "string",
+      "description": "Per-resolve correlation id minted by resolve_skill. Lowercase UUIDv4.",
+      "format": "uuid",
+      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
+    },
+    "InboundRequestId": {
+      "type": "string",
+      "description": "request_id as supplied by the caller to fetch_skill. Deliberately not pattern-validated: an unknown or malformed value must still succeed and record an uncorrelated fetch (AC7); correlation validity is decided by the server, not the schema.",
+      "minLength": 1,
+      "maxLength": 128
+    },
     "ResolveSkillInput": {
       "type": "object",
       "properties": {
@@ -76,6 +88,7 @@
       "type": "object",
       "description": "Ranked candidates response from resolve_skill.",
       "properties": {
+        "request_id": { "$ref": "#/$defs/RequestId" },
         "retrieval": { "enum": ["exact", "reranked", "hybrid", "lexical"] },
         "degraded_from": { "enum": ["reranked", "hybrid"] },
         "degradation_reason": { "$ref": "#/$defs/DegradationReason" },
@@ -85,13 +98,14 @@
           "description": "Shortlist of candidates in descending score order with contiguous 1-based ranks."
         }
       },
-      "required": ["retrieval", "candidates"],
+      "required": ["request_id", "retrieval", "candidates"],
       "additionalProperties": false
     },
     "FetchSkillInput": {
       "type": "object",
       "properties": {
-        "skill_id": { "$ref": "#/$defs/SkillId" }
+        "skill_id": { "$ref": "#/$defs/SkillId" },
+        "request_id": { "$ref": "#/$defs/InboundRequestId" }
       },
       "required": ["skill_id"],
       "additionalProperties": false
@@ -146,7 +160,7 @@
         },
         "state_dir": {
           "type": "string",
-          "description": "All router writes are confined here (AC9): index SQLite (incl. audit log), vector matrix.",
+          "description": "All router writes are confined here (AC9): index SQLite (index.sqlite3), audit SQLite (audit.sqlite3), vector matrix.",
           "default": "~/.local/state/skillmux"
         },
         "recall": {
@@ -179,11 +193,24 @@
           "additionalProperties": false
         },
         "inference": { "$ref": "#/$defs/InferenceConfig" },
-        "server": { "$ref": "#/$defs/ServerConfig" }
+        "server": { "$ref": "#/$defs/ServerConfig" },
+        "audit": { "$ref": "#/$defs/AuditConfig" }
       },
       "required": ["vault_path", "state_dir", "recall", "output", "inference"],
       "additionalProperties": false
     },
+    "AuditConfig": {
+      "type": "object",
+      "properties": {
+        "retention_days": {
+          "type": "integer",
+          "minimum": 0,
+          "default": 90,
+          "description": "Age in days beyond which audit and fetch rows are pruned. 0 disables pruning and retains rows indefinitely."
+        }
+      },
+      "additionalProperties": false
+    },
     "InferenceConfig": {
       "oneOf": [
         {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@klhapp/skillmux",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Skill management and retrieval for AI agents: sync native skills across clients and route the long tail over MCP",
   "type": "module",
   "private": false,

package/src/audit.ts

@@ -5,6 +5,7 @@ export function buildAuditRow(row: AuditRow): AuditRow {
   const built: AuditRow = {
     id: row.id,
     ts: row.ts,
+    request_id: row.request_id,
     query: row.query,
     retrieval: row.retrieval,
     candidates: row.candidates.map((c) => ({ skill_id: c.skill_id, score: c.score })),

package/src/cli.ts

@@ -12,7 +12,7 @@ import {
   migrateLegacyPaths,
   resolveConfigPath,
 } from "./config";
-import { openIndex } from "./db";
+import { openAudit } from "./db";
 import { diagnose } from "./doctor";
 import { getEffectiveConfig } from "./config-service";
 import { evalVault } from "./eval";
@@ -106,7 +106,9 @@ import {
   suggestCorrection,
 } from "./output";
 import { generateCompletions, type ShellType } from "./completions";
+import { runAudit } from "./commands/audit";
 import { handleConfigCommand } from "./commands/config";
+import { runEvalPromote } from "./commands/eval";
 import { runCore } from "./commands/core";
 import { configuredTargetForSurface, runProject } from "./commands/project";
 import { confirmAction, confirmIfNeeded } from "./commands/shared";
@@ -124,6 +126,7 @@ const KNOWN_COMMANDS = [
   "target",
   "core",
   "report",
+  "audit",
   "scan",
   "install",
   "eval",
@@ -145,12 +148,15 @@ function isDockerHostManagementCommand(command: string, subCommand: string): boo
       "local-vault",
       "models",
       "context",
-      "eval",
     ].includes(command)
   ) {
     return true;
   }
 
+  // eval promote only touches the mounted state_dir, unlike bare `eval`
+  // (vault ranking evaluation), which needs local embeddings and the vault.
+  if (command === "eval" && subCommand !== "promote") return true;
+
   return command === "config" && ["init", "set"].includes(subCommand);
 }
 
@@ -322,6 +328,9 @@ async function main() {
       case "report":
         await runReport(rawArgv.slice(1), { isJson });
         break;
+      case "audit":
+        await runAudit(subCommand, commandArgs, { isJson, dryRun: isDryRun });
+        break;
       case "scan":
         await runScan(rawArgv.slice(1), { isJson });
         break;
@@ -329,7 +338,13 @@ async function main() {
         await runInstall(rawArgv.slice(1), { isJson });
         break;
       case "eval":
-        await runEval({ isJson });
+        if (subCommand === "promote") {
+          await runEvalPromote(commandArgs, { isJson, dryRun: isDryRun });
+        } else if (subCommand === "") {
+          await runEval({ isJson });
+        } else {
+          throw new Error(`usage: skillmux eval [promote --since <window> [--target <path>] [--dry-run] [--yes] [--json]]`);
+        }
         break;
       case "doctor":
         await runDoctor({ isJson });
@@ -359,7 +374,7 @@ async function main() {
         const suggestion = suggestCorrection(command, KNOWN_COMMANDS);
         const msg = suggestion
           ? `Unknown command "${command}". Did you mean "${suggestion}"?`
-          : `usage: skillmux <serve|index|sync|init|project|target|core pin/unpin|report|scan|install|eval|doctor|skill which|local-vault init|config show|models download>`;
+          : `usage: skillmux <serve|index|sync|init|project|target|core pin/unpin|report|audit prune|scan|install|eval|doctor|skill which|local-vault init|config show|models download>`;
         throw new Error(msg);
       }
     }
@@ -507,7 +522,7 @@ Default:
   serve --transport http
 
 Supported commands:
-  serve, index, doctor, report, scan, skill which
+  serve, index, doctor, report, audit prune, eval promote, scan, skill which
   config show|get|validate|diff|status
 
 Native skill management:
@@ -540,9 +555,14 @@ Init clients:
 Init targets:
   agent-skills, claude-code, codex, custom
 
+Operations:
+  skillmux report [--server <url> | --db <path>] --since <window> [--json]
+  skillmux audit prune [--older-than <window>] [--dry-run] [--yes] [--json]
+  skillmux eval promote --since <window> [--target <path>] [--dry-run] [--yes] [--json]
+
 Commands:
-  serve, index, sync, init, project, target, core, report, scan, install, eval, doctor, skill,
-  local-vault, config, models, context, completions`);
+  serve, index, sync, init, project, target, core, report, audit, scan, install, eval, doctor,
+  skill, local-vault, config, models, context, completions`);
 }
 
 // ---------------------------------------------------------------------------
@@ -1453,7 +1473,7 @@ async function runReport(
 
   const db = dbPath
     ? new Database(dbPath, { readonly: true })
-    : openIndex(expandHome((await loadConfig()).state_dir));
+    : openAudit(expandHome((await loadConfig()).state_dir));
   const stats = getStats(db, since);
   emitSuccess({ isJson: options.isJson }, stats, () =>
     console.log(renderStatsText(stats)),

package/src/commands/audit.ts

@@ -0,0 +1,82 @@
+import { expandHome, loadConfig } from "../config";
+import { countPrunable, openAudit, pruneAuditBefore } from "../db";
+import { parseSince } from "../stats";
+import { emitSuccess } from "../output";
+import { confirmIfNeeded } from "./shared";
+
+export async function runAudit(
+  subCommand: string,
+  args: string[],
+  options: { isJson: boolean; dryRun: boolean },
+): Promise<void> {
+  if (subCommand !== "prune") {
+    throw new Error("usage: skillmux audit prune [--older-than <window>] [--dry-run] [--yes] [--json]");
+  }
+
+  let olderThan: string | undefined;
+  let dryRun = options.dryRun;
+  let yes = false;
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "--older-than") olderThan = args[++i];
+    else if (arg === "--dry-run") dryRun = true;
+    else if (arg === "--yes") yes = true;
+    else if (arg === "--json") {
+      // handled globally by main()'s isJson flag; recognized here so it isn't rejected
+    } else if (arg?.startsWith("--")) {
+      throw new Error(`unknown audit prune option: ${arg}`);
+    }
+  }
+
+  const config = await loadConfig();
+  const stateDir = expandHome(config.state_dir);
+
+  let cutoff: Date;
+  if (olderThan) {
+    cutoff = parseSince(olderThan);
+  } else {
+    const retentionDays = config.audit?.retention_days ?? 90;
+    if (retentionDays <= 0) {
+      emitSuccess(
+        { isJson: options.isJson },
+        { audit_deleted: 0, fetch_deleted: 0, dry_run: dryRun, cutoff: null },
+        () => console.log("prune: audit.retention_days is 0 (pruning disabled); nothing to do"),
+      );
+      return;
+    }
+    cutoff = new Date(Date.now() - retentionDays * 86_400_000);
+  }
+  const cutoffIso = cutoff.toISOString();
+
+  const db = openAudit(stateDir);
+  try {
+    if (dryRun) {
+      const counts = countPrunable(db, cutoffIso);
+      emitSuccess(
+        { isJson: options.isJson },
+        { ...counts, dry_run: true, cutoff: cutoffIso },
+        () => console.log(`prune: audit=${counts.audit_deleted} fetch=${counts.fetch_deleted} (dry-run)`),
+      );
+      return;
+    }
+
+    if (
+      !(await confirmIfNeeded({
+        confirmed: yes,
+        isJson: options.isJson,
+        prompt: `prune audit rows older than ${cutoffIso}?`,
+        nonInteractiveError: "skillmux audit prune requires --yes when run non-interactively",
+      }))
+    )
+      return;
+
+    const counts = pruneAuditBefore(db, cutoffIso);
+    emitSuccess(
+      { isJson: options.isJson },
+      { ...counts, dry_run: false, cutoff: cutoffIso },
+      () => console.log(`prune: audit=${counts.audit_deleted} fetch=${counts.fetch_deleted}`),
+    );
+  } finally {
+    db.close();
+  }
+}

package/src/commands/eval.ts

@@ -0,0 +1,81 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { expandHome, loadConfig } from "../config";
+import { openAudit } from "../db";
+import { buildPromotedCases, excludeExistingCases, parseEvalCases, queryPromotableFetches } from "../eval";
+import { emitSuccess } from "../output";
+import { parseSince } from "../stats";
+import { confirmIfNeeded } from "./shared";
+
+export async function runEvalPromote(
+  args: string[],
+  options: { isJson: boolean; dryRun: boolean },
+): Promise<void> {
+  let since: string | undefined;
+  let target: string | undefined;
+  let dryRun = options.dryRun;
+  let yes = false;
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "--since") since = args[++i];
+    else if (arg === "--target") target = args[++i];
+    else if (arg === "--dry-run") dryRun = true;
+    else if (arg === "--yes") yes = true;
+    else if (arg === "--json") {
+      // handled globally by main()'s isJson flag; recognized here so it isn't rejected
+    } else if (arg?.startsWith("--")) {
+      throw new Error(`unknown eval promote option: ${arg}`);
+    }
+  }
+  if (!since) {
+    throw new Error("usage: skillmux eval promote --since <window> [--target <path>] [--dry-run] [--yes] [--json]");
+  }
+
+  const config = await loadConfig();
+  const stateDir = expandHome(config.state_dir);
+  const targetPath = target ?? join(stateDir, "eval-observed.json");
+  const sinceDate = parseSince(since);
+  const sinceIso = sinceDate.toISOString();
+
+  const db = openAudit(stateDir);
+  let candidates: ReturnType<typeof buildPromotedCases>;
+  try {
+    candidates = buildPromotedCases(queryPromotableFetches(db, sinceIso));
+  } finally {
+    db.close();
+  }
+
+  const existing = existsSync(targetPath) ? parseEvalCases(JSON.parse(readFileSync(targetPath, "utf-8"))) : [];
+  const { cases: newCases, skipped } = excludeExistingCases(candidates, existing);
+
+  console.error("warning: promoted eval cases contain raw user queries");
+
+  if (dryRun) {
+    emitSuccess(
+      { isJson: options.isJson },
+      { dry_run: true, since: sinceIso, target_path: targetPath, promoted: newCases.length, skipped_existing: skipped },
+      () => console.log(`promote: would write ${newCases.length} case(s) to ${targetPath} (skipped_existing=${skipped})`),
+    );
+    return;
+  }
+
+  if (
+    !(await confirmIfNeeded({
+      confirmed: yes,
+      isJson: options.isJson,
+      prompt: `promote ${newCases.length} eval case(s) to ${targetPath}?`,
+      nonInteractiveError: "skillmux eval promote requires --yes when run non-interactively",
+    }))
+  )
+    return;
+
+  if (newCases.length > 0) {
+    writeFileSync(targetPath, JSON.stringify([...existing, ...newCases], null, 2) + "\n");
+  }
+
+  emitSuccess(
+    { isJson: options.isJson },
+    { dry_run: false, since: sinceIso, target_path: targetPath, promoted: newCases.length, skipped_existing: skipped },
+    () => console.log(`promote: wrote ${newCases.length} case(s) to ${targetPath} (skipped_existing=${skipped})`),
+  );
+}

package/src/config.ts

@@ -88,6 +88,9 @@ const configSchema = z.object({
       token_env: z.string().min(1),
     }).strict().optional(),
   }).strict().optional(),
+  audit: z.object({
+    retention_days: z.number().int().min(0).default(90),
+  }).strict().default({ retention_days: 90 }),
 }).strict().refine((cfg) => {
   const hasReranker = cfg.inference.mode === "remote" && !!cfg.inference.reranker;
   if (hasReranker && cfg.output.max_top_k > cfg.recall.k_rerank) {
@@ -139,6 +142,9 @@ const DEFAULTS: Config = {
       requests_per_minute: 60,
     },
   },
+  audit: {
+    retention_days: 90,
+  },
 };
 
 export const DEFAULT_CONFIG_PATH = "~/.config/skillmux/config.toml";