@klhapp/skillmux 1.6.0 → 1.7.1

package/src/config.ts

@@ -43,14 +43,11 @@ const configSchema = z.object({
     message: "recall.k_rerank cannot exceed k_lexical + k_vector",
   }),
   output: z.object({
-    ambiguous_candidate_limit: z.number().int().positive().default(5),
-  }).strict().optional(),
-  thresholds: z.object({
-    candidate_limit: z.number().int().positive().optional(),
-    match_score: z.number().optional(),
-    match_margin: z.number().nonnegative().optional(),
-    candidate_floor: z.number().optional(),
-  }).strict().optional(),
+    top_k: z.number().int().positive().default(10),
+    max_top_k: z.number().int().positive().default(50),
+  }).strict().refine((o) => o.top_k <= o.max_top_k, {
+    message: "output.top_k cannot exceed output.max_top_k",
+  }).default({ top_k: 10, max_top_k: 50 }),
   inference: z.discriminatedUnion("mode", [
     z.object({
       mode: z.literal("local"),
@@ -74,8 +71,6 @@ const configSchema = z.object({
         model: z.string().min(1),
         api_key_env: z.string().min(1).optional(),
       }).strict().optional(),
-      thresholds: remoteThresholdsSchema.optional(),
-      calibration: z.object({ run_id: z.string().min(1) }).strict().optional(),
     }).strict(),
   ]),
   server: z.object({
@@ -93,7 +88,15 @@ const configSchema = z.object({
       token_env: z.string().min(1),
     }).strict().optional(),
   }).strict().optional(),
-}).strict();
+}).strict().refine((cfg) => {
+  const hasReranker = cfg.inference.mode === "remote" && !!cfg.inference.reranker;
+  if (hasReranker && cfg.output.max_top_k > cfg.recall.k_rerank) {
+    return false;
+  }
+  return true;
+}, {
+  message: "output.max_top_k cannot exceed recall.k_rerank when reranking is enabled",
+});
 
 // Fallback values only; a config.toml (SKILLMUX_CONFIG or default path)
 // overrides them. The local bundle is the zero-config OSS path.
@@ -107,8 +110,7 @@ const DEFAULTS: Config = {
   local_vault_paths: [],
   state_dir: "~/.local/state/skillmux",
   recall: { k_lexical: 20, k_vector: 20, k_rerank: 10 },
-  thresholds: { candidate_limit: 5 },
-  output: { ambiguous_candidate_limit: 5 },
+  output: { top_k: 10, max_top_k: 50 },
   inference: {
     mode: "local",
     bundle: LOCAL_BUNDLE_ID,
@@ -232,6 +234,18 @@ export async function loadConfig(path?: string): Promise<Config> {
         "The old client appended /v1/embeddings.",
     );
   }
+  const removedLegacyOutputEnv = [
+    "SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT",
+    "AMBIGUOUS_CANDIDATE_LIMIT",
+    "SKILLMUX_CANDIDATE_LIMIT",
+    "CANDIDATE_LIMIT",
+  ].find((name) => process.env[name] !== undefined);
+  if (removedLegacyOutputEnv) {
+    throw new Error(
+      `${removedLegacyOutputEnv} is no longer supported. Use SKILLMUX_OUTPUT_TOP_K instead.`,
+    );
+  }
+
   const configPath = resolveConfigPath(path);
   const file = Bun.file(expandHome(configPath));
 
@@ -248,6 +262,26 @@ export async function loadConfig(path?: string): Promise<Config> {
     merged = baseConfig;
   } else {
     const parsed = Bun.TOML.parse(await file.text()) as Record<string, unknown>;
+    if ("thresholds" in parsed) {
+      throw new Error(
+        "The [thresholds] table is obsolete. Threshold calibration was removed; use [output] with top_k.",
+      );
+    }
+    if (isPlainObject(parsed.output) && "ambiguous_candidate_limit" in parsed.output) {
+      throw new Error(
+        "output.ambiguous_candidate_limit is obsolete. Use output.top_k instead.",
+      );
+    }
+    if (isPlainObject(parsed.inference) && "thresholds" in parsed.inference) {
+      throw new Error(
+        "inference.thresholds is obsolete. Threshold calibration was removed.",
+      );
+    }
+    if (isPlainObject(parsed.inference) && "calibration" in parsed.inference) {
+      throw new Error(
+        "inference.calibration is obsolete and should be deleted. Threshold calibration was removed; use skillmux eval for ranking evaluation.",
+      );
+    }
     if ("embedding" in parsed || "rerank" in parsed || "remote_timeout_ms" in parsed) {
       throw new Error(
         "Legacy inference config is not supported. Move [embedding], [rerank], and remote_timeout_ms under [inference] using config.remote.example.toml.",
@@ -279,30 +313,9 @@ export async function loadConfig(path?: string): Promise<Config> {
         throw new Error("Remote inference requires an inference.embedding section.");
       }
     }
-    if (parsed.thresholds && typeof parsed.thresholds === "object" && (parsed.thresholds as Record<string, unknown>).candidate_limit !== undefined) {
-      console.error("skillmux: thresholds.candidate_limit is deprecated, use output.ambiguous_candidate_limit instead");
-      if (!parsed.output || (parsed.output as Record<string, unknown>).ambiguous_candidate_limit === undefined) {
-        parsed.output = {
-          ...(typeof parsed.output === "object" && parsed.output !== null ? parsed.output : {}),
-          ambiguous_candidate_limit: (parsed.thresholds as Record<string, unknown>).candidate_limit,
-        };
-      }
-    }
 
     if (parsed.inference && typeof parsed.inference === "object" && "mode" in parsed.inference) {
       if (parsed.inference.mode === "remote") {
-        if ("thresholds" in parsed.inference) {
-          const rawRemoteThresholds = (parsed.inference as Record<string, unknown>).thresholds;
-          if (
-            typeof rawRemoteThresholds === "object" &&
-            rawRemoteThresholds !== null &&
-            !("match_score" in rawRemoteThresholds) &&
-            !("match_margin" in rawRemoteThresholds) &&
-            !("candidate_floor" in rawRemoteThresholds)
-          ) {
-            throw new Error("Invalid inference.thresholds in config.toml: must specify at least one threshold.");
-          }
-        }
         const withoutInference = { ...parsed };
         delete withoutInference.inference;
         merged = {
@@ -318,12 +331,7 @@ export async function loadConfig(path?: string): Promise<Config> {
   }
 
   if (!merged.output) {
-    merged.output = { ambiguous_candidate_limit: merged.thresholds?.candidate_limit ?? 5 };
-  }
-  if (!merged.thresholds) {
-    merged.thresholds = { candidate_limit: merged.output.ambiguous_candidate_limit };
-  } else if (merged.thresholds.candidate_limit === undefined) {
-    merged.thresholds.candidate_limit = merged.output.ambiguous_candidate_limit;
+    merged.output = { top_k: 10, max_top_k: 50 };
   }
 
   // Warn about deprecated generic environment variables regardless of override policy
@@ -333,8 +341,8 @@ export async function loadConfig(path?: string): Promise<Config> {
     RECALL_K_LEXICAL: "SKILLMUX_RECALL_K_LEXICAL",
     RECALL_K_VECTOR: "SKILLMUX_RECALL_K_VECTOR",
     RECALL_K_RERANK: "SKILLMUX_RECALL_K_RERANK",
-    AMBIGUOUS_CANDIDATE_LIMIT: "SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT",
-    CANDIDATE_LIMIT: "SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT",
+    OUTPUT_TOP_K: "SKILLMUX_OUTPUT_TOP_K",
+    OUTPUT_MAX_TOP_K: "SKILLMUX_OUTPUT_MAX_TOP_K",
     EMBED_MODEL: "SKILLMUX_EMBED_MODEL",
     EMBED_ENDPOINT: "SKILLMUX_EMBED_ENDPOINT",
     EMBED_DIMENSION: "SKILLMUX_EMBED_DIMENSION",
@@ -398,14 +406,17 @@ export async function loadConfig(path?: string): Promise<Config> {
       if (!Number.isInteger(k) || k < 1) throw new Error(`Invalid recall.k_rerank: ${kRerankStr}`);
       merged.recall.k_rerank = k;
     }
-    const ambiguousLimitStr =
-      getEnv("SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT", "AMBIGUOUS_CANDIDATE_LIMIT") ??
-      getEnv("SKILLMUX_CANDIDATE_LIMIT", "CANDIDATE_LIMIT");
-    if (ambiguousLimitStr) {
-      const lim = Number(ambiguousLimitStr);
-      if (!Number.isInteger(lim) || lim < 1) throw new Error(`Invalid output.ambiguous_candidate_limit: ${ambiguousLimitStr}`);
-      merged.output.ambiguous_candidate_limit = lim;
-      merged.thresholds.candidate_limit = lim;
+    const topKStr = getEnv("SKILLMUX_OUTPUT_TOP_K", "OUTPUT_TOP_K");
+    if (topKStr) {
+      const k = Number(topKStr);
+      if (!Number.isInteger(k) || k < 1) throw new Error(`Invalid output.top_k: ${topKStr}`);
+      merged.output.top_k = k;
+    }
+    const maxTopKStr = getEnv("SKILLMUX_OUTPUT_MAX_TOP_K", "OUTPUT_MAX_TOP_K");
+    if (maxTopKStr) {
+      const k = Number(maxTopKStr);
+      if (!Number.isInteger(k) || k < 1) throw new Error(`Invalid output.max_top_k: ${maxTopKStr}`);
+      merged.output.max_top_k = k;
     }
   }
 
@@ -441,16 +452,6 @@ export async function loadConfig(path?: string): Promise<Config> {
     if (merged.inference.reranker && (!merged.inference.reranker.endpoint || !merged.inference.reranker.model)) {
       throw new Error("Configured inference.reranker requires adapter, endpoint, and model.");
     }
-    if (merged.inference.reranker && !merged.inference.thresholds) {
-      const warningKey = "inference.reranker.without-thresholds";
-      if (!warnedEnv.has(warningKey)) {
-        warnedEnv.add(warningKey);
-        console.error(
-          "skillmux: configured reranker has no calibrated inference.thresholds; " +
-            "routing will remain ambiguous until you run `skillmux calibrate run`.",
-        );
-      }
-    }
     const embedEndpoint = getEnv("SKILLMUX_EMBED_ENDPOINT", "EMBED_ENDPOINT");
     const embedModel = getEnv("SKILLMUX_EMBED_MODEL", "EMBED_MODEL");
     const embedDimStr = getEnv("SKILLMUX_EMBED_DIMENSION", "EMBED_DIMENSION");

package/src/db.ts

@@ -43,24 +43,43 @@ export function openIndex(stateDir: string): Database {
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     ts TEXT NOT NULL,
     query TEXT NOT NULL,
-    outcome TEXT NOT NULL CHECK (outcome IN ('matched', 'ambiguous', 'no_match')),
-    degraded INTEGER NOT NULL,
     retrieval TEXT NOT NULL DEFAULT 'lexical',
     degraded_from TEXT,
     degradation_reason TEXT,
     candidates TEXT NOT NULL,
-    selected_skill_id TEXT,
     latency_ms INTEGER NOT NULL
   )`);
   const auditColumns = db.query("PRAGMA table_info(audit)").all() as { name: string }[];
-  if (!auditColumns.some((column) => column.name === "retrieval")) {
-    db.run("ALTER TABLE audit ADD COLUMN retrieval TEXT NOT NULL DEFAULT 'lexical'");
-  }
-  if (!auditColumns.some((column) => column.name === "degraded_from")) {
-    db.run("ALTER TABLE audit ADD COLUMN degraded_from TEXT");
-  }
-  if (!auditColumns.some((column) => column.name === "degradation_reason")) {
-    db.run("ALTER TABLE audit ADD COLUMN degradation_reason TEXT");
+  const columnNames = new Set(auditColumns.map((column) => column.name));
+  const hasLegacyColumns =
+    columnNames.has("outcome") ||
+    columnNames.has("selected_skill_id") ||
+    columnNames.has("degraded");
+  const missingCanonicalColumns =
+    !columnNames.has("retrieval") ||
+    !columnNames.has("degraded_from") ||
+    !columnNames.has("degradation_reason");
+
+  if (hasLegacyColumns || missingCanonicalColumns) {
+    db.transaction(() => {
+      db.run(`CREATE TABLE audit_new (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts TEXT NOT NULL,
+        query TEXT NOT NULL,
+        retrieval TEXT NOT NULL DEFAULT 'lexical',
+        degraded_from TEXT,
+        degradation_reason TEXT,
+        candidates TEXT NOT NULL,
+        latency_ms INTEGER NOT NULL
+      )`);
+      const retrievalExpr = columnNames.has("retrieval") ? "COALESCE(retrieval, 'lexical')" : "'lexical'";
+      const degradedFromExpr = columnNames.has("degraded_from") ? "degraded_from" : "NULL";
+      const degradationReasonExpr = columnNames.has("degradation_reason") ? "degradation_reason" : "NULL";
+      db.run(`INSERT INTO audit_new (id, ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
+        SELECT id, ts, query, ${retrievalExpr}, ${degradedFromExpr}, ${degradationReasonExpr}, candidates, latency_ms FROM audit`);
+      db.run("DROP TABLE audit");
+      db.run("ALTER TABLE audit_new RENAME TO audit");
+    })();
   }
   db.run(`CREATE TABLE IF NOT EXISTS index_meta (
     key TEXT PRIMARY KEY,
@@ -263,29 +282,24 @@ export function vectorTopK(db: Database, query: Float32Array, k: number): SkillR
 export interface AuditInsert {
   ts: string;
   query: string;
-  outcome: string;
   retrieval: AuditRow["retrieval"];
   degraded_from?: string | null;
   degradation_reason?: string | null;
   candidates: AuditCandidate[];
-  selected_skill_id: string | null;
   latency_ms: number;
 }
 
 export function insertAudit(db: Database, row: AuditInsert): void {
   db.run(
-    `INSERT INTO audit (ts, query, outcome, degraded, retrieval, degraded_from, degradation_reason, candidates, selected_skill_id, latency_ms)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    `INSERT INTO audit (ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms)
+     VALUES (?, ?, ?, ?, ?, ?, ?)`,
     [
       row.ts,
       row.query,
-      row.outcome,
-      row.retrieval === "lexical" || row.degradation_reason ? 1 : 0,
       row.retrieval,
       row.degraded_from ?? null,
       row.degradation_reason ?? null,
       JSON.stringify(row.candidates),
-      row.selected_skill_id,
       row.latency_ms,
     ],
   );

package/src/doctor.ts

@@ -1,9 +1,7 @@
 import { existsSync, mkdirSync } from "node:fs";
-import { computeCorpusFingerprint, getCalibrationRun, openCalibrateDb } from "./calibrate";
 import { createClients, RemoteInferenceError } from "./clients";
-import { embeddingDimension, embeddingFingerprint, expandHome, rerankerFingerprint } from "./config";
+import { embeddingDimension, expandHome } from "./config";
 import { describeDeployment, type DeploymentIdentity } from "./deployment";
-import { openIndex } from "./db";
 import { parseManifest, resolveManifestPath, validateManifest } from "./manifest";
 import { readSkillmuxMarker } from "./sync";
 import type { Config } from "./types";
@@ -32,83 +30,6 @@ export interface DoctorReport {
   checks: DoctorCheck[];
 }
 
-/**
- * Warn when live `inference.thresholds` didn't come from an applied
- * `skillmux calibrate` run — e.g. hand-copied from an example config.
- * Reranker score scales are not portable across models/adapters/corpora,
- * so uncalibrated thresholds routinely make automatic matching unreachable
- * without any visible error.
- */
-function checkCalibration(config: Config): DoctorCheck {
-  const inference = config.inference;
-  if (inference.mode !== "remote") throw new Error("checkCalibration requires remote inference mode");
-  const runId = inference.calibration?.run_id;
-  if (!runId) {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: "inference.thresholds are set but were never produced by `skillmux calibrate apply` — " +
-        "likely copied from an example config. Reranker scores are not portable across models, " +
-        "adapters, or corpora, so automatic matching may never trigger (or may trigger incorrectly). " +
-        "Run `skillmux calibrate`.",
-    };
-  }
-
-  const calibrateDb = openCalibrateDb(expandHome(config.state_dir));
-  let run;
-  try {
-    run = getCalibrationRun(calibrateDb, runId);
-  } finally {
-    calibrateDb.close();
-  }
-  if (!run) {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: `inference.calibration.run_id "${runId}" was not found in the local calibration ` +
-        "evidence store (state_dir may differ from where it was calibrated). Recalibrate.",
-    };
-  }
-  if (run.status !== "completed") {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: `inference.calibration.run_id "${runId}" has status "${run.status}" and should never ` +
-        "have been applied. Recalibrate.",
-    };
-  }
-
-  const indexDb = openIndex(expandHome(config.state_dir));
-  let currentCorpusFingerprint: string;
-  try {
-    currentCorpusFingerprint = computeCorpusFingerprint(indexDb);
-  } finally {
-    indexDb.close();
-  }
-
-  const stale = [
-    rerankerFingerprint(config) !== run.reranker_fingerprint ? "reranker" : null,
-    embeddingFingerprint(config) !== run.embedding_fingerprint ? "embedding" : null,
-    currentCorpusFingerprint !== run.corpus_fingerprint ? "vault contents" : null,
-    run.recall_settings && (
-      run.recall_settings.k_lexical !== config.recall.k_lexical ||
-      run.recall_settings.k_vector !== config.recall.k_vector ||
-      run.recall_settings.k_rerank !== (config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector))
-    ) ? "recall settings" : null,
-  ].filter((part): part is string => part !== null);
-
-  if (stale.length > 0) {
-    return {
-      name: "calibration",
-      ok: false,
-      detail: `applied calibration run "${runId}" is stale — ${stale.join(", ")} changed since it was ` +
-        "calibrated. Recalibrate.",
-    };
-  }
-
-  return { name: "calibration", ok: true, detail: `thresholds from applied calibration run "${runId}"` };
-}
-
 export { describeDeployment };
 
 export async function diagnose(
@@ -246,10 +167,6 @@ export async function diagnose(
     }
   }
 
-  if (config.inference.mode === "remote" && config.inference.reranker && config.inference.thresholds) {
-    checks.push(checkCalibration(config));
-  }
-
   const inferenceReady = checks.some((check) => check.name === "embedding" && check.ok);
   const rerankerReady = checks.some((check) => check.name === "reranker" && check.ok);
   const coreReady = checks.some((check) => check.name === "vault" && check.ok)

package/src/eval.ts

@@ -1,31 +1,68 @@
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
-import { z } from "zod";
 import {
   backfillEmbeddings,
-  decideRetrievalResult,
   getRuntime,
   retrieveAndRerank,
 } from "./router-core";
 
 export interface EvalCase {
   query: string;
-  expected: string[];
+  split?: string;
+  relevant_skill_ids: string[];
 }
 
-const rawEvalItemSchema = z.object({
-  query: z.string().min(1),
-  expected: z.array(z.string().min(1)).optional(),
-  relevant_skill_ids: z.array(z.string()).optional(),
-  split: z.string().optional(),
-  expected_outcome: z.string().optional(),
-});
-
+export function parseEvalCases(raw: unknown): EvalCase[] {
+  if (!Array.isArray(raw)) throw new Error("Eval cases file must contain a JSON array");
+  const result: EvalCase[] = [];
+  for (let i = 0; i < raw.length; i++) {
+    const item = raw[i];
+    if (typeof item !== "object" || item === null) {
+      throw new Error(`Eval case at index ${i} must be an object`);
+    }
+    if ("expected" in item) {
+      throw new Error(
+        `Legacy 'expected' field at case ${i} is no longer supported in eval datasets; use 'relevant_skill_ids' instead.`,
+      );
+    }
+    if ("expected_outcome" in item) {
+      throw new Error(
+        `Legacy 'expected_outcome' field at case ${i} is no longer supported in eval datasets; remove it and provide 'relevant_skill_ids' instead.`,
+      );
+    }
+    if (typeof (item as any).query !== "string" || (item as any).query.trim().length === 0) {
+      throw new Error(`Eval case at index ${i} has invalid 'query': must be a non-empty string`);
+    }
+    if (
+      !Array.isArray((item as any).relevant_skill_ids) ||
+      (item as any).relevant_skill_ids.some((id: unknown) => typeof id !== "string")
+    ) {
+      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': must be an array of strings`);
+    }
+    const relevantSkillIds = (item as any).relevant_skill_ids as string[];
+    if (relevantSkillIds.some((id) => id.trim().length === 0)) {
+      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': IDs must be non-empty strings`);
+    }
+    if (new Set(relevantSkillIds).size !== relevantSkillIds.length) {
+      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': duplicate IDs are not allowed`);
+    }
+    if ("split" in item && (item as any).split !== undefined && typeof (item as any).split !== "string") {
+      throw new Error(`Eval case at index ${i} has invalid 'split': must be a string if present`);
+    }
+    result.push({
+      query: (item as any).query,
+      ...((item as any).split !== undefined ? { split: (item as any).split } : {}),
+      relevant_skill_ids: relevantSkillIds,
+    });
+  }
+  return result;
+}
 
 export interface EvalMetrics {
-  recall_at_3: number;
   recall_at_5: number;
+  recall_at_10: number;
   mrr: number;
+  ndcg_at_10: number;
 }
 
 export interface CandidateEvalDetail {
@@ -37,8 +74,7 @@ export interface CandidateEvalDetail {
 
 export interface EvalCaseResult {
   query: string;
-  expected: string[];
-  outcome: "matched" | "ambiguous" | "no_match";
+  relevant_skill_ids: string[];
   retrieval: string;
   degraded_from?: string | null;
   degradation_reason?: string | null;
@@ -53,48 +89,88 @@ export interface EvalCaseResult {
 
 export interface EvalReport {
   queries: number;
+  judged_queries: number;
+  unjudged_queries: number;
   lexical: EvalMetrics;
   hybrid: EvalMetrics;
   cases?: EvalCaseResult[];
 }
 
-function metrics(rankings: string[][], cases: EvalCase[]): EvalMetrics {
-  if (cases.length === 0) return { recall_at_3: 0, recall_at_5: 0, mrr: 0 };
-  let recall3 = 0;
-  let recall5 = 0;
-  let reciprocalRanks = 0;
-  rankings.forEach((ranking, index) => {
-    const expected = new Set(cases[index]!.expected);
-    if (ranking.slice(0, 3).some((id) => expected.has(id))) recall3++;
-    if (ranking.slice(0, 5).some((id) => expected.has(id))) recall5++;
-    const rank = ranking.findIndex((id) => expected.has(id));
-    if (rank >= 0) reciprocalRanks += 1 / (rank + 1);
-  });
+export function computeRankingMetrics(rankings: string[][], cases: EvalCase[]): EvalMetrics {
+  const judgedIndices: number[] = [];
+  for (let i = 0; i < cases.length; i++) {
+    if (cases[i]!.relevant_skill_ids && cases[i]!.relevant_skill_ids.length > 0) {
+      judgedIndices.push(i);
+    }
+  }
+
+  if (judgedIndices.length === 0) {
+    return {
+      recall_at_5: 0,
+      recall_at_10: 0,
+      mrr: 0,
+      ndcg_at_10: 0,
+    };
+  }
+
+  let totalRecall5 = 0;
+  let totalRecall10 = 0;
+  let totalMRR = 0;
+  let totalNDCG10 = 0;
+
+  for (const idx of judgedIndices) {
+    const c = cases[idx]!;
+    const ranking = rankings[idx] ?? [];
+    const relevantSet = new Set(c.relevant_skill_ids);
+    const numRelevant = c.relevant_skill_ids.length;
+
+    // Recall@5: fraction of relevant skill IDs present in the first 5 results
+    const top5 = ranking.slice(0, 5);
+    const count5 = top5.filter((id) => relevantSet.has(id)).length;
+    totalRecall5 += count5 / numRelevant;
+
+    // Recall@10: fraction of relevant skill IDs present in the first 10 results
+    const top10 = ranking.slice(0, 10);
+    const count10 = top10.filter((id) => relevantSet.has(id)).length;
+    totalRecall10 += count10 / numRelevant;
+
+    // MRR: reciprocal rank of the first relevant result (1-based), 0 if missing
+    const firstRelevantRank = ranking.findIndex((id) => relevantSet.has(id));
+    if (firstRelevantRank >= 0) {
+      totalMRR += 1 / (firstRelevantRank + 1);
+    }
+
+    // nDCG@10: binary relevance, DCG gain 1/log2(rank+1), IDCG over min(numRelevant, 10)
+    let dcg10 = 0;
+    for (let r = 0; r < Math.min(10, ranking.length); r++) {
+      if (relevantSet.has(ranking[r]!)) {
+        dcg10 += 1 / Math.log2(r + 2);
+      }
+    }
+    let idcg10 = 0;
+    const idealCount = Math.min(numRelevant, 10);
+    for (let r = 0; r < idealCount; r++) {
+      idcg10 += 1 / Math.log2(r + 2);
+    }
+    if (idcg10 > 0) {
+      totalNDCG10 += dcg10 / idcg10;
+    }
+  }
+
+  const judgedCount = judgedIndices.length;
   return {
-    recall_at_3: recall3 / cases.length,
-    recall_at_5: recall5 / cases.length,
-    mrr: reciprocalRanks / cases.length,
+    recall_at_5: totalRecall5 / judgedCount,
+    recall_at_10: totalRecall10 / judgedCount,
+    mrr: totalMRR / judgedCount,
+    ndcg_at_10: totalNDCG10 / judgedCount,
   };
 }
 
 export function loadEvalCases(path = join(import.meta.dir, "..", "eval", "queries.json")): EvalCase[] {
   const raw = JSON.parse(readFileSync(path, "utf8"));
-  if (!Array.isArray(raw)) throw new Error("Eval cases file must contain a JSON array");
-  const parsed = z.array(rawEvalItemSchema).parse(raw);
-  const result: EvalCase[] = [];
-  for (const item of parsed) {
-    const expected = item.expected ?? item.relevant_skill_ids;
-    if (expected && expected.length > 0) {
-      result.push({ query: item.query, expected });
-    }
-  }
-  if (parsed.length > 0 && result.length === 0) {
-    throw new Error("Eval cases file contained no cases with expected targets");
-  }
-  return result;
+  return parseEvalCases(raw);
 }
 
-
 export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
   const { config } = await getRuntime();
   await backfillEmbeddings();
@@ -107,27 +183,27 @@ export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
   for (const evalCase of cases) {
     const start = performance.now();
     const retrievalResult = await retrieveAndRerank({ query: evalCase.query });
-    const decision = decideRetrievalResult(config, retrievalResult);
-    const fusedRanking = retrievalResult.trace
-      .filter((candidate) => candidate.fused_rank !== null)
-      .sort((a, b) => a.fused_rank! - b.fused_rank!)
-      .map((candidate) => candidate.skill_id);
-
-    lexicalRankings.push(retrievalResult.trace
-      .filter((candidate) => candidate.lexical_rank !== null)
-      .sort((a, b) => a.lexical_rank! - b.lexical_rank!)
-      .map((candidate) => candidate.skill_id));
-    hybridRankings.push(fusedRanking.length > 0
-      ? fusedRanking
-      : retrievalResult.candidates.map((candidate) => candidate.skill_id));
+
+    // lexical metrics use lexical rank
+    lexicalRankings.push(
+      retrievalResult.trace
+        .filter((candidate) => candidate.lexical_rank !== null)
+        .sort((a, b) => a.lexical_rank! - b.lexical_rank!)
+        .map((candidate) => candidate.skill_id),
+    );
+
+    // retrieveAndRerank returns candidates in the exact order delivered to the
+    // caller: reranked when available, otherwise fused or lexical fallback.
+    hybridRankings.push(
+      retrievalResult.candidates.map((candidate) => candidate.skill_id),
+    );
 
     const latency_ms = Math.round(performance.now() - start);
     const candidateDetails: CandidateEvalDetail[] = retrievalResult.trace;
 
     caseResults.push({
       query: evalCase.query,
-      expected: evalCase.expected,
-      outcome: decision.outcome,
+      relevant_skill_ids: evalCase.relevant_skill_ids,
       retrieval: retrievalResult.retrieval,
       degraded_from: retrievalResult.degraded_from ?? null,
       degradation_reason: retrievalResult.degradation_reason ?? null,
@@ -141,10 +217,17 @@ export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
     });
   }
 
+  const judged_queries = cases.filter(
+    (c) => c.relevant_skill_ids && c.relevant_skill_ids.length > 0,
+  ).length;
+  const unjudged_queries = cases.length - judged_queries;
+
   return {
     queries: cases.length,
-    lexical: metrics(lexicalRankings, cases),
-    hybrid: metrics(hybridRankings, cases),
+    judged_queries,
+    unjudged_queries,
+    lexical: computeRankingMetrics(lexicalRankings, cases),
+    hybrid: computeRankingMetrics(hybridRankings, cases),
     cases: caseResults,
   };
 }