@klhapp/skillmux 1.5.2 → 1.7.0

package/src/db.ts

@@ -43,24 +43,43 @@ export function openIndex(stateDir: string): Database {
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     ts TEXT NOT NULL,
     query TEXT NOT NULL,
-    outcome TEXT NOT NULL CHECK (outcome IN ('matched', 'ambiguous', 'no_match')),
-    degraded INTEGER NOT NULL,
     retrieval TEXT NOT NULL DEFAULT 'lexical',
     degraded_from TEXT,
     degradation_reason TEXT,
     candidates TEXT NOT NULL,
-    selected_skill_id TEXT,
     latency_ms INTEGER NOT NULL
   )`);
   const auditColumns = db.query("PRAGMA table_info(audit)").all() as { name: string }[];
-  if (!auditColumns.some((column) => column.name === "retrieval")) {
-    db.run("ALTER TABLE audit ADD COLUMN retrieval TEXT NOT NULL DEFAULT 'lexical'");
-  }
-  if (!auditColumns.some((column) => column.name === "degraded_from")) {
-    db.run("ALTER TABLE audit ADD COLUMN degraded_from TEXT");
-  }
-  if (!auditColumns.some((column) => column.name === "degradation_reason")) {
-    db.run("ALTER TABLE audit ADD COLUMN degradation_reason TEXT");
+  const columnNames = new Set(auditColumns.map((column) => column.name));
+  const hasLegacyColumns =
+    columnNames.has("outcome") ||
+    columnNames.has("selected_skill_id") ||
+    columnNames.has("degraded");
+  const missingCanonicalColumns =
+    !columnNames.has("retrieval") ||
+    !columnNames.has("degraded_from") ||
+    !columnNames.has("degradation_reason");
+
+  if (hasLegacyColumns || missingCanonicalColumns) {
+    db.transaction(() => {
+      db.run(`CREATE TABLE audit_new (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts TEXT NOT NULL,
+        query TEXT NOT NULL,
+        retrieval TEXT NOT NULL DEFAULT 'lexical',
+        degraded_from TEXT,
+        degradation_reason TEXT,
+        candidates TEXT NOT NULL,
+        latency_ms INTEGER NOT NULL
+      )`);
+      const retrievalExpr = columnNames.has("retrieval") ? "COALESCE(retrieval, 'lexical')" : "'lexical'";
+      const degradedFromExpr = columnNames.has("degraded_from") ? "degraded_from" : "NULL";
+      const degradationReasonExpr = columnNames.has("degradation_reason") ? "degradation_reason" : "NULL";
+      db.run(`INSERT INTO audit_new (id, ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
+        SELECT id, ts, query, ${retrievalExpr}, ${degradedFromExpr}, ${degradationReasonExpr}, candidates, latency_ms FROM audit`);
+      db.run("DROP TABLE audit");
+      db.run("ALTER TABLE audit_new RENAME TO audit");
+    })();
   }
   db.run(`CREATE TABLE IF NOT EXISTS index_meta (
     key TEXT PRIMARY KEY,
@@ -263,29 +282,24 @@ export function vectorTopK(db: Database, query: Float32Array, k: number): SkillR
 export interface AuditInsert {
   ts: string;
   query: string;
-  outcome: string;
   retrieval: AuditRow["retrieval"];
   degraded_from?: string | null;
   degradation_reason?: string | null;
   candidates: AuditCandidate[];
-  selected_skill_id: string | null;
   latency_ms: number;
 }
 
 export function insertAudit(db: Database, row: AuditInsert): void {
   db.run(
-    `INSERT INTO audit (ts, query, outcome, degraded, retrieval, degraded_from, degradation_reason, candidates, selected_skill_id, latency_ms)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    `INSERT INTO audit (ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
+     VALUES (?, ?, ?, ?, ?, ?, ?)`,
     [
       row.ts,
       row.query,
-      row.outcome,
-      row.retrieval === "lexical" || row.degradation_reason ? 1 : 0,
       row.retrieval,
       row.degraded_from ?? null,
       row.degradation_reason ?? null,
       JSON.stringify(row.candidates),
-      row.selected_skill_id,
       row.latency_ms,
     ],
   );

package/src/doctor.ts

@@ -1,9 +1,7 @@
 import { existsSync, mkdirSync } from "node:fs";
-import { computeCorpusFingerprint, getCalibrationRun, openCalibrateDb } from "./calibrate";
 import { createClients, RemoteInferenceError } from "./clients";
-import { embeddingDimension, embeddingFingerprint, expandHome, rerankerFingerprint } from "./config";
+import { embeddingDimension, expandHome } from "./config";
 import { describeDeployment, type DeploymentIdentity } from "./deployment";
-import { openIndex } from "./db";
 import { parseManifest, resolveManifestPath, validateManifest } from "./manifest";
 import { readSkillmuxMarker } from "./sync";
 import type { Config } from "./types";
@@ -32,83 +30,6 @@ export interface DoctorReport {
   checks: DoctorCheck[];
 }
 
-/**
- * Warn when live `inference.thresholds` didn't come from an applied
- * `skillmux calibrate` run — e.g. hand-copied from an example config.
- * Reranker score scales are not portable across models/adapters/corpora,
- * so uncalibrated thresholds routinely make automatic matching unreachable
- * without any visible error.
- */
-function checkCalibration(config: Config): DoctorCheck {
-  const inference = config.inference;
-  if (inference.mode !== "remote") throw new Error("checkCalibration requires remote inference mode");
-  const runId = inference.calibration?.run_id;
-  if (!runId) {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: "inference.thresholds are set but were never produced by `skillmux calibrate apply` — " +
-        "likely copied from an example config. Reranker scores are not portable across models, " +
-        "adapters, or corpora, so automatic matching may never trigger (or may trigger incorrectly). " +
-        "Run `skillmux calibrate`.",
-    };
-  }
-
-  const calibrateDb = openCalibrateDb(expandHome(config.state_dir));
-  let run;
-  try {
-    run = getCalibrationRun(calibrateDb, runId);
-  } finally {
-    calibrateDb.close();
-  }
-  if (!run) {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: `inference.calibration.run_id "${runId}" was not found in the local calibration ` +
-        "evidence store (state_dir may differ from where it was calibrated). Recalibrate.",
-    };
-  }
-  if (run.status !== "completed") {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: `inference.calibration.run_id "${runId}" has status "${run.status}" and should never ` +
-        "have been applied. Recalibrate.",
-    };
-  }
-
-  const indexDb = openIndex(expandHome(config.state_dir));
-  let currentCorpusFingerprint: string;
-  try {
-    currentCorpusFingerprint = computeCorpusFingerprint(indexDb);
-  } finally {
-    indexDb.close();
-  }
-
-  const stale = [
-    rerankerFingerprint(config) !== run.reranker_fingerprint ? "reranker" : null,
-    embeddingFingerprint(config) !== run.embedding_fingerprint ? "embedding" : null,
-    currentCorpusFingerprint !== run.corpus_fingerprint ? "vault contents" : null,
-    run.recall_settings && (
-      run.recall_settings.k_lexical !== config.recall.k_lexical ||
-      run.recall_settings.k_vector !== config.recall.k_vector ||
-      run.recall_settings.k_rerank !== (config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector))
-    ) ? "recall settings" : null,
-  ].filter((part): part is string => part !== null);
-
-  if (stale.length > 0) {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: `applied calibration run "${runId}" is stale — ${stale.join(", ")} changed since it was ` +
-        "calibrated. Recalibrate.",
-    };
-  }
-
-  return { name: "calibration", ok: true, detail: `thresholds from applied calibration run "${runId}"` };
-}
-
 export { describeDeployment };
 
 export async function diagnose(
@@ -246,10 +167,6 @@ export async function diagnose(
     }
   }
 
-  if (config.inference.mode === "remote" && config.inference.reranker && config.inference.thresholds) {
-    checks.push(checkCalibration(config));
-  }
-
   const inferenceReady = checks.some((check) => check.name === "embedding" && check.ok);
   const rerankerReady = checks.some((check) => check.name === "reranker" && check.ok);
   const coreReady = checks.some((check) => check.name === "vault" && check.ok)

package/src/eval.ts

@@ -1,31 +1,68 @@
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
-import { z } from "zod";
 import {
   backfillEmbeddings,
-  decideRetrievalResult,
   getRuntime,
   retrieveAndRerank,
 } from "./router-core";
 
 export interface EvalCase {
   query: string;
-  expected: string[];
+  split?: string;
+  relevant_skill_ids: string[];
 }
 
-const rawEvalItemSchema = z.object({
-  query: z.string().min(1),
-  expected: z.array(z.string().min(1)).optional(),
-  relevant_skill_ids: z.array(z.string()).optional(),
-  split: z.string().optional(),
-  expected_outcome: z.string().optional(),
-});
-
+export function parseEvalCases(raw: unknown): EvalCase[] {
+  if (!Array.isArray(raw)) throw new Error("Eval cases file must contain a JSON array");
+  const result: EvalCase[] = [];
+  for (let i = 0; i < raw.length; i++) {
+    const item = raw[i];
+    if (typeof item !== "object" || item === null) {
+      throw new Error(`Eval case at index ${i} must be an object`);
+    }
+    if ("expected" in item) {
+      throw new Error(
+        `Legacy 'expected' field at case ${i} is no longer supported in eval datasets; use 'relevant_skill_ids' instead.`,
+      );
+    }
+    if ("expected_outcome" in item) {
+      throw new Error(
+        `Legacy 'expected_outcome' field at case ${i} is no longer supported in eval datasets; remove it and provide 'relevant_skill_ids' instead.`,
+      );
+    }
+    if (typeof (item as any).query !== "string" || (item as any).query.trim().length === 0) {
+      throw new Error(`Eval case at index ${i} has invalid 'query': must be a non-empty string`);
+    }
+    if (
+      !Array.isArray((item as any).relevant_skill_ids) ||
+      (item as any).relevant_skill_ids.some((id: unknown) => typeof id !== "string")
+    ) {
+      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': must be an array of strings`);
+    }
+    const relevantSkillIds = (item as any).relevant_skill_ids as string[];
+    if (relevantSkillIds.some((id) => id.trim().length === 0)) {
+      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': IDs must be non-empty strings`);
+    }
+    if (new Set(relevantSkillIds).size !== relevantSkillIds.length) {
+      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': duplicate IDs are not allowed`);
+    }
+    if ("split" in item && (item as any).split !== undefined && typeof (item as any).split !== "string") {
+      throw new Error(`Eval case at index ${i} has invalid 'split': must be a string if present`);
+    }
+    result.push({
+      query: (item as any).query,
+      ...((item as any).split !== undefined ? { split: (item as any).split } : {}),
+      relevant_skill_ids: relevantSkillIds,
+    });
+  }
+  return result;
+}
 
 export interface EvalMetrics {
-  recall_at_3: number;
   recall_at_5: number;
+  recall_at_10: number;
   mrr: number;
+  ndcg_at_10: number;
 }
 
 export interface CandidateEvalDetail {
@@ -37,8 +74,7 @@ export interface CandidateEvalDetail {
 
 export interface EvalCaseResult {
   query: string;
-  expected: string[];
-  outcome: "matched" | "ambiguous" | "no_match";
+  relevant_skill_ids: string[];
   retrieval: string;
   degraded_from?: string | null;
   degradation_reason?: string | null;
@@ -53,48 +89,88 @@ export interface EvalCaseResult {
 
 export interface EvalReport {
   queries: number;
+  judged_queries: number;
+  unjudged_queries: number;
   lexical: EvalMetrics;
   hybrid: EvalMetrics;
   cases?: EvalCaseResult[];
 }
 
-function metrics(rankings: string[][], cases: EvalCase[]): EvalMetrics {
-  if (cases.length === 0) return { recall_at_3: 0, recall_at_5: 0, mrr: 0 };
-  let recall3 = 0;
-  let recall5 = 0;
-  let reciprocalRanks = 0;
-  rankings.forEach((ranking, index) => {
-    const expected = new Set(cases[index]!.expected);
-    if (ranking.slice(0, 3).some((id) => expected.has(id))) recall3++;
-    if (ranking.slice(0, 5).some((id) => expected.has(id))) recall5++;
-    const rank = ranking.findIndex((id) => expected.has(id));
-    if (rank >= 0) reciprocalRanks += 1 / (rank + 1);
-  });
+export function computeRankingMetrics(rankings: string[][], cases: EvalCase[]): EvalMetrics {
+  const judgedIndices: number[] = [];
+  for (let i = 0; i < cases.length; i++) {
+    if (cases[i]!.relevant_skill_ids && cases[i]!.relevant_skill_ids.length > 0) {
+      judgedIndices.push(i);
+    }
+  }
+
+  if (judgedIndices.length === 0) {
+    return {
+      recall_at_5: 0,
+      recall_at_10: 0,
+      mrr: 0,
+      ndcg_at_10: 0,
+    };
+  }
+
+  let totalRecall5 = 0;
+  let totalRecall10 = 0;
+  let totalMRR = 0;
+  let totalNDCG10 = 0;
+
+  for (const idx of judgedIndices) {
+    const c = cases[idx]!;
+    const ranking = rankings[idx] ?? [];
+    const relevantSet = new Set(c.relevant_skill_ids);
+    const numRelevant = c.relevant_skill_ids.length;
+
+    // Recall@5: fraction of relevant skill IDs present in the first 5 results
+    const top5 = ranking.slice(0, 5);
+    const count5 = top5.filter((id) => relevantSet.has(id)).length;
+    totalRecall5 += count5 / numRelevant;
+
+    // Recall@10: fraction of relevant skill IDs present in the first 10 results
+    const top10 = ranking.slice(0, 10);
+    const count10 = top10.filter((id) => relevantSet.has(id)).length;
+    totalRecall10 += count10 / numRelevant;
+
+    // MRR: reciprocal rank of the first relevant result (1-based), 0 if missing
+    const firstRelevantRank = ranking.findIndex((id) => relevantSet.has(id));
+    if (firstRelevantRank >= 0) {
+      totalMRR += 1 / (firstRelevantRank + 1);
+    }
+
+    // nDCG@10: binary relevance, DCG gain 1/log2(rank+1), IDCG over min(numRelevant, 10)
+    let dcg10 = 0;
+    for (let r = 0; r < Math.min(10, ranking.length); r++) {
+      if (relevantSet.has(ranking[r]!)) {
+        dcg10 += 1 / Math.log2(r + 2);
+      }
+    }
+    let idcg10 = 0;
+    const idealCount = Math.min(numRelevant, 10);
+    for (let r = 0; r < idealCount; r++) {
+      idcg10 += 1 / Math.log2(r + 2);
+    }
+    if (idcg10 > 0) {
+      totalNDCG10 += dcg10 / idcg10;
+    }
+  }
+
+  const judgedCount = judgedIndices.length;
   return {
-    recall_at_3: recall3 / cases.length,
-    recall_at_5: recall5 / cases.length,
-    mrr: reciprocalRanks / cases.length,
+    recall_at_5: totalRecall5 / judgedCount,
+    recall_at_10: totalRecall10 / judgedCount,
+    mrr: totalMRR / judgedCount,
+    ndcg_at_10: totalNDCG10 / judgedCount,
   };
 }
 
 export function loadEvalCases(path = join(import.meta.dir, "..", "eval", "queries.json")): EvalCase[] {
   const raw = JSON.parse(readFileSync(path, "utf8"));
-  if (!Array.isArray(raw)) throw new Error("Eval cases file must contain a JSON array");
-  const parsed = z.array(rawEvalItemSchema).parse(raw);
-  const result: EvalCase[] = [];
-  for (const item of parsed) {
-    const expected = item.expected ?? item.relevant_skill_ids;
-    if (expected && expected.length > 0) {
-      result.push({ query: item.query, expected });
-    }
-  }
-  if (parsed.length > 0 && result.length === 0) {
-    throw new Error("Eval cases file contained no cases with expected targets");
-  }
-  return result;
+  return parseEvalCases(raw);
 }
 
-
 export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
   const { config } = await getRuntime();
   await backfillEmbeddings();
@@ -107,27 +183,27 @@ export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
   for (const evalCase of cases) {
     const start = performance.now();
     const retrievalResult = await retrieveAndRerank({ query: evalCase.query });
-    const decision = decideRetrievalResult(config, retrievalResult);
-    const fusedRanking = retrievalResult.trace
-      .filter((candidate) => candidate.fused_rank !== null)
-      .sort((a, b) => a.fused_rank! - b.fused_rank!)
-      .map((candidate) => candidate.skill_id);
-
-    lexicalRankings.push(retrievalResult.trace
-      .filter((candidate) => candidate.lexical_rank !== null)
-      .sort((a, b) => a.lexical_rank! - b.lexical_rank!)
-      .map((candidate) => candidate.skill_id));
-    hybridRankings.push(fusedRanking.length > 0
-      ? fusedRanking
-      : retrievalResult.candidates.map((candidate) => candidate.skill_id));
+
+    // lexical metrics use lexical rank
+    lexicalRankings.push(
+      retrievalResult.trace
+        .filter((candidate) => candidate.lexical_rank !== null)
+        .sort((a, b) => a.lexical_rank! - b.lexical_rank!)
+        .map((candidate) => candidate.skill_id),
+    );
+
+    // retrieveAndRerank returns candidates in the exact order delivered to the
+    // caller: reranked when available, otherwise fused or lexical fallback.
+    hybridRankings.push(
+      retrievalResult.candidates.map((candidate) => candidate.skill_id),
+    );
 
     const latency_ms = Math.round(performance.now() - start);
     const candidateDetails: CandidateEvalDetail[] = retrievalResult.trace;
 
     caseResults.push({
       query: evalCase.query,
-      expected: evalCase.expected,
-      outcome: decision.outcome,
+      relevant_skill_ids: evalCase.relevant_skill_ids,
       retrieval: retrievalResult.retrieval,
       degraded_from: retrievalResult.degraded_from ?? null,
       degradation_reason: retrievalResult.degradation_reason ?? null,
@@ -141,10 +217,17 @@ export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
     });
   }
 
+  const judged_queries = cases.filter(
+    (c) => c.relevant_skill_ids && c.relevant_skill_ids.length > 0,
+  ).length;
+  const unjudged_queries = cases.length - judged_queries;
+
   return {
     queries: cases.length,
-    lexical: metrics(lexicalRankings, cases),
-    hybrid: metrics(hybridRankings, cases),
+    judged_queries,
+    unjudged_queries,
+    lexical: computeRankingMetrics(lexicalRankings, cases),
+    hybrid: computeRankingMetrics(hybridRankings, cases),
     cases: caseResults,
   };
 }

package/src/init.ts

@@ -357,15 +357,14 @@ export function applyInit(
   return manifest;
 }
 
-/** Verbatim from docs/sdd/skr-cli/think.md §3.3 — the shared instruction-stack paragraph. */
 export const DISCOVERY_PARAGRAPH =
   "Skills: only a curated core is loaded statically. Before improvising a " +
   "multi-step workflow, or when a task smells like a domain you have no loaded " +
   "skill for (career/resume, trading, SEO, i18n, design, one-off tooling), " +
-  "call `resolve_skill` with a one-line task description. `matched` → follow " +
-  "the returned SKILL.md. `ambiguous` → pick from the candidates and " +
-  "`fetch_skill`. `no_match` → proceed normally; don't force an unrelated " +
-  "skill.";
+  "call `resolve_skill` with a one-line task description. `resolve_skill` " +
+  "returns a ranked shortlist; inspect candidates, fetch one or more " +
+  "relevant skills with `fetch_skill`, or ignore all and proceed normally " +
+  "when none are useful.";
 
 export const MCP_REGISTRATION_SNIPPET = JSON.stringify(
   { mcpServers: { "skillmux": { command: "skillmux", args: ["serve"] } } },

package/src/metrics.ts

@@ -8,8 +8,7 @@ type MetricsDeploymentIdentity = Pick<
 
 export class MetricsRegistry {
   private requests = new Map<string, number>();
-  private outcomes = new Map<string, number>();
-  
+
   private buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0];
   private latencyBuckets = new Array(this.buckets.length).fill(0);
   private latencyInf = 0;
@@ -34,10 +33,6 @@ export class MetricsRegistry {
     this.requests.set(method, (this.requests.get(method) || 0) + 1);
   }
 
-  recordResolveOutcome(outcome: string) {
-    this.outcomes.set(outcome, (this.outcomes.get(outcome) || 0) + 1);
-  }
-
   recordDegradation(stage: "embedding" | "reranker", reason: string) {
     const key = `${stage}:${reason}`;
     this.degradations.set(key, (this.degradations.get(key) || 0) + 1);
@@ -72,13 +67,6 @@ export class MetricsRegistry {
       lines.push(`skill_router_requests_total{method="${method}"} ${count}`);
     }
 
-    // Resolve outcomes total
-    lines.push("# HELP skill_router_resolve_outcomes_total Total count of resolve_skill query outcomes.");
-    lines.push("# TYPE skill_router_resolve_outcomes_total counter");
-    for (const [outcome, count] of this.outcomes) {
-      lines.push(`skill_router_resolve_outcomes_total{outcome="${outcome}"} ${count}`);
-    }
-
     // Latency histogram
     lines.push("# HELP skill_router_resolve_latency_seconds Latency histogram of resolve_skill executions.");
     lines.push("# TYPE skill_router_resolve_latency_seconds histogram");