@klhapp/skillmux 1.4.0 → 1.5.0

package/src/config.ts

@@ -26,16 +26,31 @@ const remoteThresholdsSchema = z.object({
 }).strict();
 
 const configSchema = z.object({
+  config: z.object({
+    environment_overrides: z.boolean().default(true),
+  }).strict().optional(),
   vault_path: z.string().min(1),
   local_vault_paths: z.array(z.string()),
   state_dir: z.string().min(1),
-  recall: z.object({ k_lexical: z.number().int().positive(), k_vector: z.number().int().positive() }).strict(),
+  recall: z.object({
+    k_lexical: z.number().int().positive(),
+    k_vector: z.number().int().positive(),
+    k_rerank: z.number().int().positive().optional(),
+  }).strict().transform((r) => ({
+    ...r,
+    k_rerank: r.k_rerank ?? Math.min(10, r.k_lexical + r.k_vector),
+  })).refine((r) => r.k_rerank <= r.k_lexical + r.k_vector, {
+    message: "recall.k_rerank cannot exceed k_lexical + k_vector",
+  }),
+  output: z.object({
+    ambiguous_candidate_limit: z.number().int().positive().default(5),
+  }).strict().optional(),
   thresholds: z.object({
-    candidate_limit: z.number().int().positive(),
+    candidate_limit: z.number().int().positive().optional(),
     match_score: z.number().optional(),
     match_margin: z.number().nonnegative().optional(),
     candidate_floor: z.number().optional(),
-  }).strict(),
+  }).strict().optional(),
   inference: z.discriminatedUnion("mode", [
     z.object({
       mode: z.literal("local"),
@@ -85,11 +100,15 @@ const configSchema = z.object({
 export const LOCAL_BUNDLE_ID = "gte-small-v1";
 
 const DEFAULTS: Config = {
+  config: {
+    environment_overrides: true,
+  },
   vault_path: "~/skills",
   local_vault_paths: [],
   state_dir: "~/.local/state/skillmux",
-  recall: { k_lexical: 20, k_vector: 20 },
+  recall: { k_lexical: 20, k_vector: 20, k_rerank: 10 },
   thresholds: { candidate_limit: 5 },
+  output: { ambiguous_candidate_limit: 5 },
   inference: {
     mode: "local",
     bundle: LOCAL_BUNDLE_ID,
@@ -259,25 +278,90 @@ export async function loadConfig(path?: string): Promise<Config> {
       if (!isPlainObject(parsed.inference.embedding)) {
         throw new Error("Remote inference requires an inference.embedding section.");
       }
-      const withoutInference = { ...parsed };
-      delete withoutInference.inference;
-      merged = {
-        ...deepMerge(baseConfig, withoutInference),
-        inference: configSchema.shape.inference.parse(parsed.inference),
-      };
+    }
+    if (parsed.thresholds && typeof parsed.thresholds === "object" && (parsed.thresholds as Record<string, unknown>).candidate_limit !== undefined) {
+      console.error("skillmux: thresholds.candidate_limit is deprecated, use output.ambiguous_candidate_limit instead");
+      if (!parsed.output || (parsed.output as Record<string, unknown>).ambiguous_candidate_limit === undefined) {
+        parsed.output = {
+          ...(typeof parsed.output === "object" && parsed.output !== null ? parsed.output : {}),
+          ambiguous_candidate_limit: (parsed.thresholds as Record<string, unknown>).candidate_limit,
+        };
+      }
+    }
+
+    if (parsed.inference && typeof parsed.inference === "object" && "mode" in parsed.inference) {
+      if (parsed.inference.mode === "remote") {
+        if ("thresholds" in parsed.inference) {
+          const rawRemoteThresholds = (parsed.inference as Record<string, unknown>).thresholds;
+          if (
+            typeof rawRemoteThresholds === "object" &&
+            rawRemoteThresholds !== null &&
+            !("match_score" in rawRemoteThresholds) &&
+            !("match_margin" in rawRemoteThresholds) &&
+            !("candidate_floor" in rawRemoteThresholds)
+          ) {
+            throw new Error("Invalid inference.thresholds in config.toml: must specify at least one threshold.");
+          }
+        }
+        const withoutInference = { ...parsed };
+        delete withoutInference.inference;
+        merged = {
+          ...deepMerge(baseConfig, withoutInference),
+          inference: configSchema.shape.inference.parse(parsed.inference),
+        };
+      } else {
+        merged = deepMerge(baseConfig, parsed);
+      }
     } else {
       merged = deepMerge(baseConfig, parsed);
     }
   }
 
-  // Environment variable overrides.
-  if (process.env.VAULT_PATH) {
-    merged.vault_path = process.env.VAULT_PATH;
+  if (!merged.output) {
+    merged.output = { ambiguous_candidate_limit: merged.thresholds?.candidate_limit ?? 5 };
   }
-  if (process.env.STATE_DIR) {
-    merged.state_dir = process.env.STATE_DIR;
+  if (!merged.thresholds) {
+    merged.thresholds = { candidate_limit: merged.output.ambiguous_candidate_limit };
+  } else if (merged.thresholds.candidate_limit === undefined) {
+    merged.thresholds.candidate_limit = merged.output.ambiguous_candidate_limit;
   }
+
+  // Warn about deprecated generic environment variables regardless of override policy
+  const GENERIC_ENV_MAPPINGS: Record<string, string> = {
+    VAULT_PATH: "SKILLMUX_VAULT_PATH",
+    STATE_DIR: "SKILLMUX_STATE_DIR",
+    RECALL_K_LEXICAL: "SKILLMUX_RECALL_K_LEXICAL",
+    RECALL_K_VECTOR: "SKILLMUX_RECALL_K_VECTOR",
+    RECALL_K_RERANK: "SKILLMUX_RECALL_K_RERANK",
+    AMBIGUOUS_CANDIDATE_LIMIT: "SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT",
+    CANDIDATE_LIMIT: "SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT",
+    EMBED_MODEL: "SKILLMUX_EMBED_MODEL",
+    EMBED_ENDPOINT: "SKILLMUX_EMBED_ENDPOINT",
+    EMBED_DIMENSION: "SKILLMUX_EMBED_DIMENSION",
+    EMBED_DEVICE: "SKILLMUX_EMBED_DEVICE",
+    EMBED_DTYPE: "SKILLMUX_EMBED_DTYPE",
+    RERANK_MODEL: "SKILLMUX_RERANK_MODEL",
+    RERANK_ENDPOINT: "SKILLMUX_RERANK_ENDPOINT",
+    RERANK_ADAPTER: "SKILLMUX_RERANK_ADAPTER",
+    HTTP_AUTH_ENABLED: "SKILLMUX_HTTP_AUTH_ENABLED",
+    HTTP_AUTH_TOKEN_ENV: "SKILLMUX_HTTP_AUTH_TOKEN_ENV",
+    HTTP_ALLOWED_ORIGINS: "SKILLMUX_HTTP_ALLOWED_ORIGINS",
+    HTTP_HOSTNAME: "SKILLMUX_HTTP_HOSTNAME",
+    HTTP_RATE_LIMIT_ENABLED: "SKILLMUX_HTTP_RATE_LIMIT_ENABLED",
+    HTTP_RATE_LIMIT_RPM: "SKILLMUX_HTTP_RATE_LIMIT_RPM",
+    HTTP_RATE_LIMIT_TRUST_PROXY: "SKILLMUX_HTTP_RATE_LIMIT_TRUST_PROXY",
+  };
+  for (const [generic, preferred] of Object.entries(GENERIC_ENV_MAPPINGS)) {
+    if (process.env[generic] !== undefined && !warnedEnv.has(generic)) {
+      warnedEnv.add(generic);
+      console.error(`skillmux: ${generic} is deprecated, use ${preferred} instead`);
+    }
+  }
+
+  const allowEnvOverrides = merged.config?.environment_overrides !== false;
+
   const getEnv = (newPrefixed: string, unprefixed: string) => {
+    if (!allowEnvOverrides) return undefined;
     const legacyPrefixed = newPrefixed.replace(/^SKILLMUX_/, "SKILL_ROUTER_");
     if (process.env[newPrefixed] !== undefined) return process.env[newPrefixed];
     if (process.env[legacyPrefixed] !== undefined) {
@@ -290,18 +374,57 @@ export async function loadConfig(path?: string): Promise<Config> {
     return process.env[unprefixed];
   };
 
+  // Environment variable overrides.
+  if (allowEnvOverrides) {
+    const vaultPath = getEnv("SKILLMUX_VAULT_PATH", "VAULT_PATH");
+    if (vaultPath) merged.vault_path = vaultPath;
+    const stateDir = getEnv("SKILLMUX_STATE_DIR", "STATE_DIR");
+    if (stateDir) merged.state_dir = stateDir;
+    const kLexicalStr = getEnv("SKILLMUX_RECALL_K_LEXICAL", "RECALL_K_LEXICAL");
+    if (kLexicalStr) {
+      const k = Number(kLexicalStr);
+      if (!Number.isInteger(k) || k < 1) throw new Error(`Invalid recall.k_lexical: ${kLexicalStr}`);
+      merged.recall.k_lexical = k;
+    }
+    const kVectorStr = getEnv("SKILLMUX_RECALL_K_VECTOR", "RECALL_K_VECTOR");
+    if (kVectorStr) {
+      const k = Number(kVectorStr);
+      if (!Number.isInteger(k) || k < 1) throw new Error(`Invalid recall.k_vector: ${kVectorStr}`);
+      merged.recall.k_vector = k;
+    }
+    const kRerankStr = getEnv("SKILLMUX_RECALL_K_RERANK", "RECALL_K_RERANK");
+    if (kRerankStr) {
+      const k = Number(kRerankStr);
+      if (!Number.isInteger(k) || k < 1) throw new Error(`Invalid recall.k_rerank: ${kRerankStr}`);
+      merged.recall.k_rerank = k;
+    }
+    const ambiguousLimitStr =
+      getEnv("SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT", "AMBIGUOUS_CANDIDATE_LIMIT") ??
+      getEnv("SKILLMUX_CANDIDATE_LIMIT", "CANDIDATE_LIMIT");
+    if (ambiguousLimitStr) {
+      const lim = Number(ambiguousLimitStr);
+      if (!Number.isInteger(lim) || lim < 1) throw new Error(`Invalid output.ambiguous_candidate_limit: ${ambiguousLimitStr}`);
+      merged.output.ambiguous_candidate_limit = lim;
+      merged.thresholds.candidate_limit = lim;
+    }
+  }
+
   if (merged.inference.mode === "local") {
-    let modelsDirEnv = process.env.SKILLMUX_MODELS_DIR;
-    if (modelsDirEnv === undefined && process.env.SKILL_ROUTER_MODELS_DIR !== undefined) {
-      if (!warnedEnv.has("SKILL_ROUTER_MODELS_DIR")) {
-        warnedEnv.add("SKILL_ROUTER_MODELS_DIR");
-        console.error("skillmux: SKILL_ROUTER_MODELS_DIR is deprecated, use SKILLMUX_MODELS_DIR instead");
+    if (allowEnvOverrides) {
+      let modelsDirEnv = process.env.SKILLMUX_MODELS_DIR;
+      if (modelsDirEnv === undefined && process.env.SKILL_ROUTER_MODELS_DIR !== undefined) {
+        if (!warnedEnv.has("SKILL_ROUTER_MODELS_DIR")) {
+          warnedEnv.add("SKILL_ROUTER_MODELS_DIR");
+          console.error("skillmux: SKILL_ROUTER_MODELS_DIR is deprecated, use SKILLMUX_MODELS_DIR instead");
+        }
+        modelsDirEnv = process.env.SKILL_ROUTER_MODELS_DIR;
       }
-      modelsDirEnv = process.env.SKILL_ROUTER_MODELS_DIR;
+      if (modelsDirEnv) merged.inference.models_dir = modelsDirEnv;
+      const embedDevice = getEnv("SKILLMUX_EMBED_DEVICE", "EMBED_DEVICE");
+      if (embedDevice) merged.inference.embedding.device = embedDevice as ONNXDevice;
+      const embedDtype = getEnv("SKILLMUX_EMBED_DTYPE", "EMBED_DTYPE");
+      if (embedDtype) merged.inference.embedding.dtype = embedDtype as ONNXDtype;
     }
-    if (modelsDirEnv) merged.inference.models_dir = modelsDirEnv;
-    if (process.env.EMBED_DEVICE) merged.inference.embedding.device = process.env.EMBED_DEVICE as ONNXDevice;
-    if (process.env.EMBED_DTYPE) merged.inference.embedding.dtype = process.env.EMBED_DTYPE as ONNXDtype;
   } else if (merged.inference.mode === "remote") {
     if (!merged.inference.embedding) {
       throw new Error("Remote inference requires an inference.embedding section.");
@@ -385,43 +508,49 @@ export async function loadConfig(path?: string): Promise<Config> {
 
   // HTTP server environment overrides
   if (merged.server) {
-    if (process.env.HTTP_AUTH_ENABLED) {
-      merged.server.auth_enabled = process.env.HTTP_AUTH_ENABLED === "true";
-    }
-    if (process.env.HTTP_AUTH_TOKEN_ENV) {
-      merged.server.auth_token_env = process.env.HTTP_AUTH_TOKEN_ENV;
-    }
-    if (process.env.HTTP_ALLOWED_ORIGINS) {
-      merged.server.allowed_origins = process.env.HTTP_ALLOWED_ORIGINS.split(",").map((o) => o.trim());
-    }
-    if (process.env.HTTP_HOSTNAME) {
-      merged.server.hostname = process.env.HTTP_HOSTNAME;
-    }
+    if (allowEnvOverrides) {
+      const authEnabledStr = getEnv("SKILLMUX_HTTP_AUTH_ENABLED", "HTTP_AUTH_ENABLED");
+      if (authEnabledStr !== undefined) {
+        merged.server.auth_enabled = authEnabledStr === "true";
+      }
+      const authTokenEnv = getEnv("SKILLMUX_HTTP_AUTH_TOKEN_ENV", "HTTP_AUTH_TOKEN_ENV");
+      if (authTokenEnv !== undefined) {
+        merged.server.auth_token_env = authTokenEnv;
+      }
+      const allowedOriginsStr = getEnv("SKILLMUX_HTTP_ALLOWED_ORIGINS", "HTTP_ALLOWED_ORIGINS");
+      if (allowedOriginsStr !== undefined) {
+        merged.server.allowed_origins = allowedOriginsStr.split(",").map((o) => o.trim());
+      }
+      const hostname = getEnv("SKILLMUX_HTTP_HOSTNAME", "HTTP_HOSTNAME");
+      if (hostname !== undefined) {
+        merged.server.hostname = hostname;
+      }
 
-    if (!merged.server.rate_limit) {
-      merged.server.rate_limit = { enabled: false, requests_per_minute: 60 };
-    }
+      if (!merged.server.rate_limit) {
+        merged.server.rate_limit = { enabled: false, requests_per_minute: 60 };
+      }
 
-    const rateLimitEnabledStr = getEnv("SKILLMUX_HTTP_RATE_LIMIT_ENABLED", "HTTP_RATE_LIMIT_ENABLED");
-    if (rateLimitEnabledStr) {
-      merged.server.rate_limit.enabled = rateLimitEnabledStr === "true";
-    }
+      const rateLimitEnabledStr = getEnv("SKILLMUX_HTTP_RATE_LIMIT_ENABLED", "HTTP_RATE_LIMIT_ENABLED");
+      if (rateLimitEnabledStr) {
+        merged.server.rate_limit.enabled = rateLimitEnabledStr === "true";
+      }
 
-    const rateLimitRPMStr = getEnv("SKILLMUX_HTTP_RATE_LIMIT_RPM", "HTTP_RATE_LIMIT_RPM");
-    if (rateLimitRPMStr) {
-      const rpm = Number(rateLimitRPMStr);
-      if (!Number.isInteger(rpm)) {
-        throw new Error(`Invalid rate limit RPM: ${rateLimitRPMStr}`);
+      const rateLimitRPMStr = getEnv("SKILLMUX_HTTP_RATE_LIMIT_RPM", "HTTP_RATE_LIMIT_RPM");
+      if (rateLimitRPMStr) {
+        const rpm = Number(rateLimitRPMStr);
+        if (!Number.isInteger(rpm)) {
+          throw new Error(`Invalid rate limit RPM: ${rateLimitRPMStr}`);
+        }
+        merged.server.rate_limit.requests_per_minute = rpm;
       }
-      merged.server.rate_limit.requests_per_minute = rpm;
-    }
 
-    const rateLimitTrustProxyStr = getEnv("SKILLMUX_HTTP_RATE_LIMIT_TRUST_PROXY", "HTTP_RATE_LIMIT_TRUST_PROXY");
-    if (rateLimitTrustProxyStr) {
-      merged.server.rate_limit.trust_proxy = rateLimitTrustProxyStr === "true";
+      const rateLimitTrustProxyStr = getEnv("SKILLMUX_HTTP_RATE_LIMIT_TRUST_PROXY", "HTTP_RATE_LIMIT_TRUST_PROXY");
+      if (rateLimitTrustProxyStr) {
+        merged.server.rate_limit.trust_proxy = rateLimitTrustProxyStr === "true";
+      }
     }
 
-    if (merged.server.rate_limit.enabled && merged.server.rate_limit.requests_per_minute === undefined) {
+    if (merged.server.rate_limit && merged.server.rate_limit.enabled && merged.server.rate_limit.requests_per_minute === undefined) {
       merged.server.rate_limit.requests_per_minute = 60;
     }
   }

package/src/db.ts

@@ -46,6 +46,8 @@ export function openIndex(stateDir: string): Database {
     outcome TEXT NOT NULL CHECK (outcome IN ('matched', 'ambiguous', 'no_match')),
     degraded INTEGER NOT NULL,
     retrieval TEXT NOT NULL DEFAULT 'lexical',
+    degraded_from TEXT,
+    degradation_reason TEXT,
     candidates TEXT NOT NULL,
     selected_skill_id TEXT,
     latency_ms INTEGER NOT NULL
@@ -54,6 +56,12 @@ export function openIndex(stateDir: string): Database {
   if (!auditColumns.some((column) => column.name === "retrieval")) {
     db.run("ALTER TABLE audit ADD COLUMN retrieval TEXT NOT NULL DEFAULT 'lexical'");
   }
+  if (!auditColumns.some((column) => column.name === "degraded_from")) {
+    db.run("ALTER TABLE audit ADD COLUMN degraded_from TEXT");
+  }
+  if (!auditColumns.some((column) => column.name === "degradation_reason")) {
+    db.run("ALTER TABLE audit ADD COLUMN degradation_reason TEXT");
+  }
   db.run(`CREATE TABLE IF NOT EXISTS index_meta (
     key TEXT PRIMARY KEY,
     value TEXT NOT NULL
@@ -257,6 +265,8 @@ export interface AuditInsert {
   query: string;
   outcome: string;
   retrieval: AuditRow["retrieval"];
+  degraded_from?: string | null;
+  degradation_reason?: string | null;
   candidates: AuditCandidate[];
   selected_skill_id: string | null;
   latency_ms: number;
@@ -264,14 +274,16 @@ export interface AuditInsert {
 
 export function insertAudit(db: Database, row: AuditInsert): void {
   db.run(
-    `INSERT INTO audit (ts, query, outcome, degraded, retrieval, candidates, selected_skill_id, latency_ms)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+    `INSERT INTO audit (ts, query, outcome, degraded, retrieval, degraded_from, degradation_reason, candidates, selected_skill_id, latency_ms)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
     [
       row.ts,
       row.query,
       row.outcome,
-      row.retrieval === "lexical" ? 1 : 0,
+      row.retrieval === "lexical" || row.degradation_reason ? 1 : 0,
       row.retrieval,
+      row.degraded_from ?? null,
+      row.degradation_reason ?? null,
       JSON.stringify(row.candidates),
       row.selected_skill_id,
       row.latency_ms,

package/src/doctor.ts

@@ -90,6 +90,11 @@ function checkCalibration(config: Config): DoctorCheck {
     rerankerFingerprint(config) !== run.reranker_fingerprint ? "reranker" : null,
     embeddingFingerprint(config) !== run.embedding_fingerprint ? "embedding" : null,
     currentCorpusFingerprint !== run.corpus_fingerprint ? "vault contents" : null,
+    run.recall_settings && (
+      run.recall_settings.k_lexical !== config.recall.k_lexical ||
+      run.recall_settings.k_vector !== config.recall.k_vector ||
+      run.recall_settings.k_rerank !== (config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector))
+    ) ? "recall settings" : null,
   ].filter((part): part is string => part !== null);
 
   if (stale.length > 0) {
@@ -109,8 +114,20 @@ export { describeDeployment };
 export async function diagnose(
   config: Config,
   environment: Record<string, string | undefined> = process.env,
+  sources: Record<string, "default" | "toml" | "environment" | "admin"> = {},
 ): Promise<DoctorReport> {
   const checks: DoctorCheck[] = [];
+  const envOverrides = config.config?.environment_overrides !== false;
+  checks.push({
+    name: "config_authority",
+    ok: true,
+    detail: envOverrides
+      ? "environment overrides enabled"
+      : "TOML authoritative (environment overrides disabled)",
+  });
+  for (const [key, source] of Object.entries(sources)) {
+    checks.push({ name: `config_source:${key}`, ok: true, detail: source });
+  }
   checks.push({ name: "vault", ok: existsSync(expandHome(config.vault_path)), detail: expandHome(config.vault_path) });
 
   for (const localPath of config.local_vault_paths) {

package/src/eval.ts

@@ -1,10 +1,12 @@
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import { z } from "zod";
-import type { SkillRow } from "./db";
-import { ftsSearch, vectorTopK } from "./db";
-import { reciprocalRankFusion } from "./rrf";
-import { backfillEmbeddings, getRuntime } from "./router-core";
+import {
+  backfillEmbeddings,
+  decideRetrievalResult,
+  getRuntime,
+  retrieveAndRerank,
+} from "./router-core";
 
 export interface EvalCase {
   query: string;
@@ -26,10 +28,34 @@ export interface EvalMetrics {
   mrr: number;
 }
 
+export interface CandidateEvalDetail {
+  skill_id: string;
+  lexical_rank: number | null;
+  fused_rank: number | null;
+  reranked_rank?: number | null;
+}
+
+export interface EvalCaseResult {
+  query: string;
+  expected: string[];
+  outcome: "matched" | "ambiguous" | "no_match";
+  retrieval: string;
+  degraded_from?: string | null;
+  degradation_reason?: string | null;
+  latency_ms: number;
+  recall_settings: {
+    k_lexical: number;
+    k_vector: number;
+    k_rerank: number;
+  };
+  candidates: CandidateEvalDetail[];
+}
+
 export interface EvalReport {
   queries: number;
   lexical: EvalMetrics;
   hybrid: EvalMetrics;
+  cases?: EvalCaseResult[];
 }
 
 function metrics(rankings: string[][], cases: EvalCase[]): EvalMetrics {
@@ -70,24 +96,55 @@ export function loadEvalCases(path = join(import.meta.dir, "..", "eval", "querie
 
 
 export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
-  const { config, db, clients } = await getRuntime();
-  if (config.inference.mode !== "local") throw new Error('Default evaluation requires inference.mode = "local".');
+  const { config } = await getRuntime();
   await backfillEmbeddings();
 
   const lexicalRankings: string[][] = [];
   const hybridRankings: string[][] = [];
+  const caseResults: EvalCaseResult[] = [];
+  const kRerank = config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector);
+
   for (const evalCase of cases) {
-    const lexical = ftsSearch(db, evalCase.query, config.recall.k_lexical);
-    const vector = (await clients.embed([evalCase.query]))[0];
-    if (!vector) throw new Error("Embedding client returned no query vector.");
-    const semantic = vectorTopK(db, vector, config.recall.k_vector);
-    lexicalRankings.push(lexical.map((row) => row.skill_id));
-    hybridRankings.push(reciprocalRankFusion<SkillRow>(lexical, semantic).map((row) => row.skill_id));
+    const start = performance.now();
+    const retrievalResult = await retrieveAndRerank({ query: evalCase.query });
+    const decision = decideRetrievalResult(config, retrievalResult);
+    const fusedRanking = retrievalResult.trace
+      .filter((candidate) => candidate.fused_rank !== null)
+      .sort((a, b) => a.fused_rank! - b.fused_rank!)
+      .map((candidate) => candidate.skill_id);
+
+    lexicalRankings.push(retrievalResult.trace
+      .filter((candidate) => candidate.lexical_rank !== null)
+      .sort((a, b) => a.lexical_rank! - b.lexical_rank!)
+      .map((candidate) => candidate.skill_id));
+    hybridRankings.push(fusedRanking.length > 0
+      ? fusedRanking
+      : retrievalResult.candidates.map((candidate) => candidate.skill_id));
+
+    const latency_ms = Math.round(performance.now() - start);
+    const candidateDetails: CandidateEvalDetail[] = retrievalResult.trace;
+
+    caseResults.push({
+      query: evalCase.query,
+      expected: evalCase.expected,
+      outcome: decision.outcome,
+      retrieval: retrievalResult.retrieval,
+      degraded_from: retrievalResult.degraded_from ?? null,
+      degradation_reason: retrievalResult.degradation_reason ?? null,
+      latency_ms,
+      recall_settings: {
+        k_lexical: config.recall.k_lexical,
+        k_vector: config.recall.k_vector,
+        k_rerank: kRerank,
+      },
+      candidates: candidateDetails,
+    });
   }
 
   return {
     queries: cases.length,
     lexical: metrics(lexicalRankings, cases),
     hybrid: metrics(hybridRankings, cases),
+    cases: caseResults,
   };
 }

package/src/metrics.ts

@@ -18,6 +18,7 @@ export class MetricsRegistry {
 
   private errors = 0;
   private rateLimitsExceeded = 0;
+  private degradations = new Map<string, number>();
   private readiness: ReadinessSnapshot | null = null;
   private deployment: MetricsDeploymentIdentity | null = null;
 
@@ -37,6 +38,11 @@ export class MetricsRegistry {
     this.outcomes.set(outcome, (this.outcomes.get(outcome) || 0) + 1);
   }
 
+  recordDegradation(stage: "embedding" | "reranker", reason: string) {
+    const key = `${stage}:${reason}`;
+    this.degradations.set(key, (this.degradations.get(key) || 0) + 1);
+  }
+
   recordResolveLatencySeconds(seconds: number) {
     this.latencySum += seconds;
     this.latencyCount++;
@@ -98,6 +104,14 @@ export class MetricsRegistry {
     lines.push("# TYPE skill_router_rate_limits_exceeded_total counter");
     lines.push(`skill_router_rate_limits_exceeded_total ${this.rateLimitsExceeded}`);
 
+    // Degraded retrieval total
+    lines.push("# HELP skill_router_degraded_retrieval_total Total count of degraded retrieval fallbacks labelled by stage and reason.");
+    lines.push("# TYPE skill_router_degraded_retrieval_total counter");
+    for (const [key, count] of this.degradations) {
+      const [stage, reason] = key.split(":");
+      lines.push(`skill_router_degraded_retrieval_total{stage="${stage}",reason="${reason}"} ${count}`);
+    }
+
     lines.push("# HELP skill_router_ready Whether the service is ready to route requests.");
     lines.push("# TYPE skill_router_ready gauge");
     lines.push(`skill_router_ready ${this.readiness?.status === "ready" ? 1 : 0}`);