@klhapp/skillmux 1.5.2 → 1.7.0

package/src/router-core.ts

@@ -23,7 +23,6 @@ import {
   vectorTopK,
 } from "./db";
 import type { SkillRow } from "./db";
-import { decideResolveOutcome, type Decision } from "./decision";
 import type {
   RankedCandidate,
   RetrievalCapability,
@@ -54,7 +53,6 @@ function maxVaultMtime(vaultPath: string, localVaultPaths: string[]): number {
 
 export { buildAuditRow } from "./audit";
 export { loadConfig } from "./config";
-export { decideResolveOutcome } from "./decision";
 export type * from "./types";
 
 const NO_MATCH_MESSAGE =
@@ -370,87 +368,40 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
   const { config, db } = await getEnv();
   await syncVaultIfNeeded();
 
-  // Short-circuit: exact match on skill_id, title, or alias (First Principles #1)
-  const exactMatch = findExactMatch(db, input.query);
-  if (exactMatch) {
-    const delivery = await deliverSkill(db, config, exactMatch.skill_id);
-    const result: ResolveResult = {
-      outcome: "matched",
-      retrieval: "exact",
-      skill_id: exactMatch.skill_id,
-      title: delivery.title,
-      content_sha256: delivery.content_sha256,
-      score: 1.0,
-      margin: 1.0,
-      body: delivery.body,
-      files: delivery.files,
-    };
-    insertAudit(
-      db,
-      buildAuditRow({
-        id: 0,
-        ts: new Date().toISOString(),
-        query: input.query,
-        outcome: "matched",
-        retrieval: "exact",
-        candidates: [{ skill_id: exactMatch.skill_id, score: 1.0 }],
-        selected_skill_id: exactMatch.skill_id,
-        latency_ms: Math.round(performance.now() - t0),
-      }),
-    );
-    return result;
+  if (input.top_k !== undefined) {
+    if (!Number.isInteger(input.top_k) || input.top_k < 1) {
+      throw new Error(`Invalid top_k: ${input.top_k} must be a positive integer`);
+    }
+    if (input.top_k > config.output.max_top_k) {
+      throw new Error(
+        `Invalid top_k: ${input.top_k} exceeds max_top_k of ${config.output.max_top_k}`,
+      );
+    }
   }
 
+  const effectiveTopK = input.top_k ?? config.output.top_k;
   const retrievalResult = await retrieveAndRerank(input);
   const { retrieval, candidates: rankedCandidates } = retrievalResult;
 
-  const decision = decideRetrievalResult(config, retrievalResult);
-
-  let result: ResolveResult;
-  if (decision.outcome === "matched") {
-    const delivery = await deliverSkill(db, config, decision.skill_id);
-    result = {
-      outcome: "matched",
-      retrieval: "reranked",
-      ...(retrievalResult.degraded_from
-        ? {
-            degraded_from: retrievalResult.degraded_from,
-            degradation_reason: retrievalResult.degradation_reason,
-          }
-        : {}),
-      skill_id: decision.skill_id,
-      title: delivery.title,
-      content_sha256: delivery.content_sha256,
-      score: decision.score,
-      margin: decision.margin,
-      body: delivery.body,
-      files: delivery.files,
-    };
-  } else if (decision.outcome === "ambiguous") {
-    result = {
-      outcome: "ambiguous",
-      retrieval,
-      ...(retrievalResult.degraded_from
-        ? {
-            degraded_from: retrievalResult.degraded_from,
-            degradation_reason: retrievalResult.degradation_reason,
-          }
-        : {}),
-      candidates: decision.candidates.map(({ score: _score, ...candidate }) => candidate),
-    };
-  } else {
-    result = {
-      outcome: "no_match",
-      retrieval,
-      ...(retrievalResult.degraded_from
-        ? {
-            degraded_from: retrievalResult.degraded_from,
-            degradation_reason: retrievalResult.degradation_reason,
-          }
-        : {}),
-      message: NO_MATCH_MESSAGE,
-    };
-  }
+  const candidates: RankedCandidate[] = rankedCandidates
+    .slice(0, effectiveTopK)
+    .map((c, index) => ({
+      rank: index + 1,
+      skill_id: c.skill_id,
+      description: c.description,
+      score: c.score,
+    }));
+
+  const result: ResolveResult = {
+    retrieval,
+    ...(retrievalResult.degraded_from
+      ? {
+          degraded_from: retrievalResult.degraded_from,
+          degradation_reason: retrievalResult.degradation_reason,
+        }
+      : {}),
+    candidates,
+  };
 
   insertAudit(
     db,
@@ -458,12 +409,10 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
       id: 0, // assigned by SQLite
       ts: new Date().toISOString(),
       query: input.query,
-      outcome: result.outcome,
       retrieval,
       degraded_from: retrievalResult.degraded_from ?? null,
       degradation_reason: retrievalResult.degradation_reason ?? null,
-      candidates: rankedCandidates.map((c) => ({ skill_id: c.skill_id, score: c.score })),
-      selected_skill_id: result.outcome === "matched" ? result.skill_id : null,
+      candidates: candidates.map((c) => ({ skill_id: c.skill_id, score: c.score })),
       latency_ms: Math.round(performance.now() - t0),
     }),
   );
@@ -471,11 +420,18 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
   return result;
 }
 
+export interface RawCandidate {
+  skill_id: string;
+  title: string;
+  description: string;
+  score: number | null;
+}
+
 export interface RetrievalResult {
   retrieval: Exclude<RetrievalCapability, "exact">;
   degraded_from?: "reranked" | "hybrid";
   degradation_reason?: DegradationReason;
-  candidates: RankedCandidate[];
+  candidates: RawCandidate[];
   trace: Array<{
     skill_id: string;
     lexical_rank: number | null;
@@ -484,18 +440,6 @@ export interface RetrievalResult {
   }>;
 }
 
-export function decideRetrievalResult(config: Config, result: RetrievalResult): Decision {
-  const candidateLimit = config.output?.ambiguous_candidate_limit ?? config.thresholds?.candidate_limit ?? 5;
-  const thresholds = result.retrieval === "reranked" && config.inference.mode === "remote"
-    ? { candidate_limit: candidateLimit, ...config.inference.thresholds }
-    : { ...config.thresholds, candidate_limit: candidateLimit };
-  return decideResolveOutcome({
-    reranked: result.retrieval === "reranked",
-    candidates: result.candidates,
-    thresholds,
-  });
-}
-
 export function classifyInferenceError(
   stage: "embedding" | "reranker",
   error: unknown,
@@ -530,64 +474,26 @@ export function classifyInferenceError(
   return stage === "embedding" ? "embedding_unavailable" : "reranker_unavailable";
 }
 
-export interface StageTiming {
-  embedding_ms: number;
-  lexical_ms: number;
-  vector_ms: number;
-  reranker_ms: number;
-}
-
-export interface RetrieveSnapshotOptions {
-  onTiming?: (timing: StageTiming) => void;
-}
-
 /**
- * Retrieve the full fused candidate set and rerank it once without applying
- * decision thresholds, synchronizing the vault before reading the index.
+ * Retrieve the full fused candidate set, optionally rerank it, and synchronize
+ * the vault before reading the index.
  */
 export async function retrieveAndRerank(
   input: ResolveSkillInput,
 ): Promise<RetrievalResult> {
   await syncVaultIfNeeded();
-  return retrieveAndRerankSnapshot(input);
-}
-
-/**
- * Retrieve candidates against an already-synchronized index snapshot without
- * triggering syncVaultIfNeeded(). Calibration uses this to avoid observing the
- * policy it is trying to replace while keeping one corpus for the whole run.
- */
-export async function retrieveAndRerankSnapshot(
-  input: ResolveSkillInput,
-  options?: RetrieveSnapshotOptions,
-): Promise<RetrievalResult> {
   const { config, db } = await getEnv();
   const clients = getClients();
-  const measureTiming = options?.onTiming !== undefined;
 
-  let embedding_ms = 0;
-  let lexical_ms = 0;
-  let vector_ms = 0;
-  let reranker_ms = 0;
-
-  const tEmbed0 = measureTiming ? performance.now() : 0;
   const embedPromise =
     !input.forceLexical && clients.embed
       ? clients.embed([input.query]).then(
-        (res) => {
-          if (measureTiming) embedding_ms = Math.max(0, performance.now() - tEmbed0);
-          return res;
-        },
-        (err) => {
-          if (measureTiming) embedding_ms = Math.max(0, performance.now() - tEmbed0);
-          return { error: err };
-        },
+        (res) => res,
+        (err) => ({ error: err }),
       )
       : null;
 
-  const tLex0 = measureTiming ? performance.now() : 0;
   const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
-  if (measureTiming) lexical_ms = Math.max(0, performance.now() - tLex0);
   const lexicalRanks = new Map(lexical.map((row, index) => [row.skill_id, index + 1]));
 
   let retrieval: RetrievalResult["retrieval"] = "lexical";
@@ -611,17 +517,14 @@ export async function retrieveAndRerankSnapshot(
         }),
       );
     } else {
-      const tVec0 = measureTiming ? performance.now() : 0;
       try {
         const queryVec = (embedRes as Float32Array[])[0];
         if (!queryVec) throw new Error("Embedding client returned no query vector.");
         const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
-        if (measureTiming) vector_ms = Math.max(0, performance.now() - tVec0);
         rows = reciprocalRankFusion(lexical, nearest);
         fusedRows = rows;
         retrieval = "hybrid";
       } catch (embedError) {
-        if (measureTiming) vector_ms = Math.max(0, performance.now() - tVec0);
         retrieval = "lexical";
         degraded_from = clients.rerank ? "reranked" : "hybrid";
         degradation_reason = classifyInferenceError("embedding", embedError);
@@ -641,17 +544,14 @@ export async function retrieveAndRerankSnapshot(
   if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
     const kRerank = config.recall.k_rerank ?? 10;
     const rerankCandidates = rows.slice(0, kRerank);
-    const tRerank0 = measureTiming ? performance.now() : 0;
     try {
       scores = await clients.rerank(
         input.query,
         rerankCandidates.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
       );
-      if (measureTiming) reranker_ms = Math.max(0, performance.now() - tRerank0);
       retrieval = "reranked";
       rows = rerankCandidates;
     } catch (rerankError) {
-      if (measureTiming) reranker_ms = Math.max(0, performance.now() - tRerank0);
       scores = null;
       degraded_from = "reranked";
       degradation_reason = classifyInferenceError("reranker", rerankError);
@@ -679,13 +579,6 @@ export async function retrieveAndRerankSnapshot(
     : new Map<string, number>();
   const traceRows = fusedRows ?? rows;
 
-  options?.onTiming?.({
-    embedding_ms,
-    lexical_ms,
-    vector_ms,
-    reranker_ms,
-  });
-
   return {
     retrieval,
     ...(degraded_from ? { degraded_from, degradation_reason } : {}),

package/src/server.ts

@@ -74,30 +74,29 @@ export function createMcpServer(): McpServer {
     "resolve_skill",
     {
       description:
-        "Route a natural-language task description to the most relevant skill in the vault. " +
-        "Returns outcome matched (skill delivered inline), ambiguous (shortlist — pick one, then call fetch_skill), " +
-        "or no_match (proceed under your normal workflow).",
-      inputSchema: { query: z.string().min(1) },
+        "Route a natural-language task description to candidate skills in the vault. " +
+        "Returns a ranked shortlist of candidates. Use fetch_skill to retrieve a candidate's complete instructions.",
+      inputSchema: {
+        query: z.string().min(1).describe("Natural-language task or prompt description to route."),
+        top_k: z
+          .number()
+          .int()
+          .min(1)
+          .optional()
+          .describe("Optional maximum number of ranked candidates to return (subject to server max_top_k limit)."),
+      },
     },
-    async ({ query }) => {
+    async ({ query, top_k }) => {
       const startTime = performance.now();
       try {
-        const result = await resolveSkill({ query });
+        const result = await resolveSkill({ query, top_k });
         const duration = (performance.now() - startTime) / 1000;
         metricsRegistry.recordResolveLatencySeconds(duration);
-        metricsRegistry.recordResolveOutcome(result.outcome);
         if (result.degradation_reason) {
           const stage = result.degradation_reason.startsWith("embedding_") ? "embedding" : "reranker";
           metricsRegistry.recordDegradation(stage, result.degradation_reason);
         }
 
-        if (result.outcome === "matched") {
-          const { body, ...meta } = result;
-          return {
-            content: [{ type: "text" as const, text: body }],
-            structuredContent: { ...meta },
-          };
-        }
         return {
           content: [{ type: "text" as const, text: JSON.stringify(result) }],
           structuredContent: { ...result },
@@ -394,7 +393,6 @@ export async function startServer(opts?: {
               JSON.stringify({
                 config_read: true,
                 config_write: !isExternallyManaged,
-                calibration: false,
                 persistence: isExternallyManaged
                   ? "externally_managed"
                   : "writable",
@@ -481,18 +479,6 @@ export async function startServer(opts?: {
             });
           }
 
-          if (url.pathname.startsWith("/admin/v1/calibrations")) {
-            return new Response(
-              JSON.stringify({
-                error: "not_implemented",
-                message:
-                  "Remote calibration is not implemented in this release. " +
-                  "Run `skillmux calibrate` against a local target.",
-              }),
-              { status: 501, headers },
-            );
-          }
-
           return new Response("Not Found", { status: 404, headers });
         }
 

package/src/stats.ts

@@ -5,22 +5,32 @@ export const SINCE_PATTERN = /^(\d+[hdwmy]|\d{4}-\d{2}-\d{2}([T ].+)?)$/;
 
 export interface SkillStat {
   skill_id: string;
-  matched_count: number;
   candidate_count: number;
 }
 
-export interface NoMatchQuery {
+export interface EmptyShortlistQuery {
   query: string;
   count: number;
 }
 
+export interface RetrievalTotals {
+  exact: number;
+  reranked: number;
+  hybrid: number;
+  lexical: number;
+}
+
 export interface StatsResponse {
   since: string;
   until: string;
-  outcome_totals: { matched: number; ambiguous: number; no_match: number };
-  ambiguous_rate: number;
+  total_requests: number;
+  empty_shortlist_count: number;
+  empty_shortlist_rate: number;
+  retrieval_totals: RetrievalTotals;
+  degraded_count: number;
+  average_latency_ms: number;
   skills: SkillStat[];
-  top_no_match_queries: NoMatchQuery[];
+  top_empty_shortlist_queries: EmptyShortlistQuery[];
 }
 
 const RELATIVE_WINDOW = /^(\d+)([hdwmy])$/;
@@ -47,58 +57,79 @@ export function parseSince(since: string, now: Date = new Date()): Date {
   return parsed;
 }
 
+function compareCodeUnits(a: string, b: string): number {
+  if (a < b) return -1;
+  if (a > b) return 1;
+  return 0;
+}
+
 export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsResponse {
-  const outcome_totals = { matched: 0, ambiguous: 0, no_match: 0 };
-  const skillStats = new Map<string, { matched_count: number; candidate_count: number }>();
-  const noMatchCounts = new Map<string, number>();
-
-  function statFor(skillId: string) {
-    let stat = skillStats.get(skillId);
-    if (!stat) {
-      stat = { matched_count: 0, candidate_count: 0 };
-      skillStats.set(skillId, stat);
-    }
-    return stat;
-  }
+  const retrieval_totals: RetrievalTotals = { exact: 0, reranked: 0, hybrid: 0, lexical: 0 };
+  const skillCounts = new Map<string, number>();
+  const emptyShortlistCounts = new Map<string, number>();
+  let empty_shortlist_count = 0;
+  let degraded_count = 0;
+  let total_latency_ms = 0;
 
   for (const row of rows) {
-    outcome_totals[row.outcome]++;
-
-    const seenInRow = new Set<string>();
-    for (const candidate of row.candidates) {
-      if (seenInRow.has(candidate.skill_id)) continue;
-      seenInRow.add(candidate.skill_id);
-      statFor(candidate.skill_id).candidate_count++;
+    if (row.retrieval in retrieval_totals) {
+      retrieval_totals[row.retrieval]++;
     }
 
-    if (row.outcome === "matched" && row.selected_skill_id) {
-      statFor(row.selected_skill_id).matched_count++;
+    if (row.degraded_from || row.degradation_reason) {
+      degraded_count++;
     }
 
-    if (row.outcome === "no_match") {
-      noMatchCounts.set(row.query, (noMatchCounts.get(row.query) ?? 0) + 1);
+    total_latency_ms += row.latency_ms;
+
+    if (row.candidates.length === 0) {
+      empty_shortlist_count++;
+      emptyShortlistCounts.set(row.query, (emptyShortlistCounts.get(row.query) ?? 0) + 1);
+    } else {
+      const seenInRow = new Set<string>();
+      for (const candidate of row.candidates) {
+        if (!candidate.skill_id) continue;
+        if (seenInRow.has(candidate.skill_id)) continue;
+        seenInRow.add(candidate.skill_id);
+        skillCounts.set(candidate.skill_id, (skillCounts.get(candidate.skill_id) ?? 0) + 1);
+      }
     }
   }
 
-  const total = outcome_totals.matched + outcome_totals.ambiguous + outcome_totals.no_match;
-  const ambiguous_rate = total > 0 ? outcome_totals.ambiguous / total : 0;
+  const total_requests = rows.length;
+  const empty_shortlist_rate = total_requests > 0 ? empty_shortlist_count / total_requests : 0;
+  const average_latency_ms = total_requests > 0 ? total_latency_ms / total_requests : 0;
 
-  const skills = [...skillStats.entries()]
-    .map(([skill_id, stat]) => ({ skill_id, ...stat }))
-    .sort((a, b) => b.matched_count - a.matched_count);
+  const skills: SkillStat[] = [...skillCounts.entries()]
+    .map(([skill_id, candidate_count]) => ({ skill_id, candidate_count }))
+    .sort((a, b) => {
+      if (b.candidate_count !== a.candidate_count) {
+        return b.candidate_count - a.candidate_count;
+      }
+      return compareCodeUnits(a.skill_id, b.skill_id);
+    });
 
-  const top_no_match_queries = [...noMatchCounts.entries()]
+  const top_empty_shortlist_queries: EmptyShortlistQuery[] = [...emptyShortlistCounts.entries()]
     .map(([query, count]) => ({ query, count }))
-    .sort((a, b) => b.count - a.count)
+    .sort((a, b) => {
+      if (b.count !== a.count) {
+        return b.count - a.count;
+      }
+      return compareCodeUnits(a.query, b.query);
+    })
     .slice(0, 20);
 
   return {
     since: since.toISOString(),
     until: until.toISOString(),
-    outcome_totals,
-    ambiguous_rate,
+    total_requests,
+    empty_shortlist_count,
+    empty_shortlist_rate,
+    retrieval_totals,
+    degraded_count,
+    average_latency_ms,
     skills,
-    top_no_match_queries,
+    top_empty_shortlist_queries,
   };
 }
 
@@ -106,27 +137,60 @@ interface AuditTableRow {
   id: number;
   ts: string;
   query: string;
-  outcome: AuditRow["outcome"];
   retrieval: AuditRow["retrieval"];
+  degraded_from: string | null;
+  degradation_reason: string | null;
   candidates: string;
-  selected_skill_id: string | null;
   latency_ms: number;
 }
 
 export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
   const rows = db
-    .query("SELECT id, ts, query, outcome, retrieval, candidates, selected_skill_id, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC")
+    .query(
+      "SELECT id, ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC",
+    )
     .all(sinceIso) as AuditTableRow[];
-  return rows.map((row) => ({
-    id: row.id,
-    ts: row.ts,
-    query: row.query,
-    outcome: row.outcome,
-    retrieval: row.retrieval,
-    candidates: JSON.parse(row.candidates) as AuditCandidate[],
-    selected_skill_id: row.selected_skill_id,
-    latency_ms: row.latency_ms,
-  }));
+
+  return rows.map((row) => {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(row.candidates);
+    } catch {
+      throw new Error(`Failed to parse candidates JSON for audit row ${row.id}`);
+    }
+    if (!Array.isArray(parsed)) {
+      throw new Error(`Invalid candidates JSON for audit row ${row.id}: expected array, got ${typeof parsed}`);
+    }
+    const candidates: AuditCandidate[] = parsed.map((c: any, index: number) => {
+      if (!c || typeof c !== "object" || Array.isArray(c) || typeof c.skill_id !== "string") {
+        throw new Error(`Invalid candidate at index ${index} for audit row ${row.id}: missing or invalid skill_id`);
+      }
+      const score = c.score;
+      if (score !== null && (typeof score !== "number" || !Number.isFinite(score))) {
+        throw new Error(`Invalid candidate at index ${index} for audit row ${row.id}: missing or invalid score`);
+      }
+      return {
+        skill_id: c.skill_id,
+        score,
+      };
+    });
+
+    const result: AuditRow = {
+      id: row.id,
+      ts: row.ts,
+      query: row.query,
+      retrieval: row.retrieval,
+      candidates,
+      latency_ms: row.latency_ms,
+    };
+    if (row.degraded_from !== null && row.degraded_from !== undefined) {
+      result.degraded_from = row.degraded_from as AuditRow["degraded_from"];
+    }
+    if (row.degradation_reason !== null && row.degradation_reason !== undefined) {
+      result.degradation_reason = row.degradation_reason as AuditRow["degradation_reason"];
+    }
+    return result;
+  });
 }
 
 export function getStats(db: Database, since: string, now: Date = new Date()): StatsResponse {
@@ -139,8 +203,13 @@ export function renderStatsText(stats: StatsResponse): string {
   const lines: string[] = [];
   lines.push(`window: ${stats.since} .. ${stats.until}`);
   lines.push(
-    `outcomes: matched=${stats.outcome_totals.matched} ambiguous=${stats.outcome_totals.ambiguous} ` +
-      `no_match=${stats.outcome_totals.no_match} (ambiguous_rate=${stats.ambiguous_rate.toFixed(3)})`,
+    `requests: total=${stats.total_requests} empty_shortlist=${stats.empty_shortlist_count} ` +
+      `(empty_shortlist_rate=${stats.empty_shortlist_rate.toFixed(3)}) ` +
+      `degraded=${stats.degraded_count} avg_latency_ms=${stats.average_latency_ms.toFixed(1)}`,
+  );
+  lines.push(
+    `retrieval: exact=${stats.retrieval_totals.exact} reranked=${stats.retrieval_totals.reranked} ` +
+      `hybrid=${stats.retrieval_totals.hybrid} lexical=${stats.retrieval_totals.lexical}`,
   );
 
   lines.push("skills:");
@@ -148,15 +217,15 @@ export function renderStatsText(stats: StatsResponse): string {
     lines.push("  (none)");
   } else {
     for (const skill of stats.skills) {
-      lines.push(`  ${skill.skill_id} matched=${skill.matched_count} candidate=${skill.candidate_count}`);
+      lines.push(`  ${skill.skill_id} candidate=${skill.candidate_count}`);
     }
   }
 
-  lines.push("top no_match queries:");
-  if (stats.top_no_match_queries.length === 0) {
+  lines.push("top empty shortlist queries:");
+  if (stats.top_empty_shortlist_queries.length === 0) {
     lines.push("  (none)");
   } else {
-    for (const entry of stats.top_no_match_queries) {
+    for (const entry of stats.top_empty_shortlist_queries) {
       lines.push(`  "${entry.query}" (${entry.count})`);
     }
   }