@klhapp/skillmux 1.0.0 → 1.1.0

package/src/config-watcher.ts

@@ -18,6 +18,13 @@ export const LIVE_RELOAD_KEYS = new Set([
   "recall.k_lexical",
   "recall.k_vector",
   "thresholds.candidate_limit",
+  "inference.embedding.endpoint",
+  "inference.embedding.api_key_env",
+  "inference.reranker.adapter",
+  "inference.reranker.endpoint",
+  "inference.reranker.model",
+  "inference.reranker.api_key_env",
+  "inference.timeout_ms",
 ]);
 
 // ---------------------------------------------------------------------------

package/src/config.ts

@@ -48,14 +48,14 @@ const configSchema = z.object({
       timeout_ms: z.number().int().min(100),
       embedding: z.object({
         provider: z.literal("openai"),
-        base_url: z.url(),
+        endpoint: z.url(),
         model: z.string().min(1),
         dimension: z.number().int().positive(),
         api_key_env: z.string().min(1).optional(),
       }).strict(),
       reranker: z.object({
-        provider: z.literal("infinity"),
-        base_url: z.url(),
+        adapter: z.enum(["jina-v1", "bifrost-v1"]),
+        endpoint: z.url(),
         model: z.string().min(1),
         api_key_env: z.string().min(1).optional(),
       }).strict().optional(),
@@ -133,6 +133,12 @@ export function embeddingFingerprint(config: Config): string {
   return `${implementation}:${inference.embedding.model}:${inference.embedding.dimension}`;
 }
 
+export function rerankerFingerprint(config: Config): string | undefined {
+  const inference = config.inference;
+  if (inference.mode !== "remote" || !inference.reranker) return undefined;
+  return `remote:${inference.reranker.adapter}:${inference.reranker.model}`;
+}
+
 export function expandHome(path: string): string {
   return path.startsWith("~") ? join(homedir(), path.slice(1)) : path;
 }
@@ -183,6 +189,30 @@ export function resolveConfigPath(path?: string): string {
 
 export async function loadConfig(path?: string): Promise<Config> {
   migrateLegacyPaths();
+  const removedRerankerEnv = [
+    "SKILLMUX_RERANK_BASE_URL",
+    "SKILL_ROUTER_RERANK_BASE_URL",
+    "RERANK_BASE_URL",
+  ].find((name) => process.env[name] !== undefined);
+  if (removedRerankerEnv) {
+    throw new Error(
+      `${removedRerankerEnv} is no longer supported. Configure ` +
+        "inference.reranker.endpoint with the complete request URL and " +
+        'inference.reranker.adapter (for example, "jina-v1"). The old client appended /rerank.',
+    );
+  }
+  const removedEmbeddingEnv = [
+    "SKILLMUX_EMBED_BASE_URL",
+    "SKILL_ROUTER_EMBED_BASE_URL",
+    "EMBED_BASE_URL",
+  ].find((name) => process.env[name] !== undefined);
+  if (removedEmbeddingEnv) {
+    throw new Error(
+      `${removedEmbeddingEnv} is no longer supported. Configure ` +
+        "inference.embedding.endpoint with the complete OpenAI-compatible embeddings request URL. " +
+        "The old client appended /v1/embeddings.",
+    );
+  }
   const configPath = resolveConfigPath(path);
   const file = Bun.file(expandHome(configPath));
 
@@ -204,6 +234,27 @@ export async function loadConfig(path?: string): Promise<Config> {
         "Legacy inference config is not supported. Move [embedding], [rerank], and remote_timeout_ms under [inference] using config.remote.example.toml.",
       );
     }
+    const rawReranker = isPlainObject(parsed.inference)
+      ? parsed.inference.reranker
+      : undefined;
+    const rawEmbedding = isPlainObject(parsed.inference)
+      ? parsed.inference.embedding
+      : undefined;
+    if (
+      isPlainObject(rawReranker) &&
+      ("provider" in rawReranker || "base_url" in rawReranker)
+    ) {
+      throw new Error(
+        "inference.reranker.provider and inference.reranker.base_url are no longer supported. " +
+          "Use adapter and the complete endpoint URL instead; the old client appended /rerank.",
+      );
+    }
+    if (isPlainObject(rawEmbedding) && "base_url" in rawEmbedding) {
+      throw new Error(
+        "inference.embedding.base_url is no longer supported. Use inference.embedding.endpoint " +
+          "with the complete OpenAI-compatible embeddings request URL; the old client appended /v1/embeddings.",
+      );
+    }
     if (isPlainObject(parsed.inference) && parsed.inference.mode === "remote") {
       if (!isPlainObject(parsed.inference.embedding)) {
         throw new Error("Remote inference requires an inference.embedding section.");
@@ -258,46 +309,76 @@ export async function loadConfig(path?: string): Promise<Config> {
     if (merged.inference.embedding?.provider !== "openai") {
       throw new Error('Remote inference.embedding.provider must be "openai".');
     }
-    if (merged.inference.reranker && merged.inference.reranker.provider !== "infinity") {
-      throw new Error('Remote inference.reranker.provider must be "infinity".');
-    }
     if (!Number.isInteger(merged.inference.timeout_ms) || merged.inference.timeout_ms < 100) {
       throw new Error("Remote inference.timeout_ms must be an integer of at least 100.");
     }
-    if (!merged.inference.embedding?.base_url || !merged.inference.embedding.model || !merged.inference.embedding.dimension) {
-      throw new Error("Remote inference requires inference.embedding base_url, model, and dimension.");
+    if (!merged.inference.embedding?.endpoint || !merged.inference.embedding.model || !merged.inference.embedding.dimension) {
+      throw new Error("Remote inference requires inference.embedding endpoint, model, and dimension.");
     }
-    if (merged.inference.reranker && (!merged.inference.reranker.base_url || !merged.inference.reranker.model)) {
-      throw new Error("Configured inference.reranker requires base_url and model.");
+    if (merged.inference.reranker && (!merged.inference.reranker.endpoint || !merged.inference.reranker.model)) {
+      throw new Error("Configured inference.reranker requires adapter, endpoint, and model.");
     }
     if (merged.inference.reranker && !merged.inference.thresholds) {
-      throw new Error("Configured inference.reranker requires calibrated inference.thresholds.");
-    }
-    for (const [name, value] of [
-      ["inference.embedding.base_url", merged.inference.embedding.base_url],
-      ...(merged.inference.reranker ? [["inference.reranker.base_url", merged.inference.reranker.base_url] as const] : []),
-    ] as const) {
-      try {
-        const url = new URL(value);
-        if (!['http:', 'https:'].includes(url.protocol)) throw new Error();
-      } catch {
-        throw new Error(`${name} must be an HTTP(S) URL.`);
+      const warningKey = "inference.reranker.without-thresholds";
+      if (!warnedEnv.has(warningKey)) {
+        warnedEnv.add(warningKey);
+        console.error(
+          "skillmux: configured reranker has no calibrated inference.thresholds; " +
+            "routing will remain ambiguous until you run `skillmux calibrate run`.",
+        );
       }
     }
-    const embedUrl = getEnv("SKILLMUX_EMBED_BASE_URL", "EMBED_BASE_URL");
+    const embedEndpoint = getEnv("SKILLMUX_EMBED_ENDPOINT", "EMBED_ENDPOINT");
     const embedModel = getEnv("SKILLMUX_EMBED_MODEL", "EMBED_MODEL");
     const embedDimStr = getEnv("SKILLMUX_EMBED_DIMENSION", "EMBED_DIMENSION");
-    const rerankUrl = getEnv("SKILLMUX_RERANK_BASE_URL", "RERANK_BASE_URL");
+    const rerankEndpoint = getEnv("SKILLMUX_RERANK_ENDPOINT", "RERANK_ENDPOINT");
+    const rerankAdapter = getEnv("SKILLMUX_RERANK_ADAPTER", "RERANK_ADAPTER");
     const rerankModel = getEnv("SKILLMUX_RERANK_MODEL", "RERANK_MODEL");
-    if (embedUrl) merged.inference.embedding.base_url = embedUrl;
+    if (embedEndpoint) merged.inference.embedding.endpoint = embedEndpoint;
     if (embedModel) merged.inference.embedding.model = embedModel;
-    if (rerankUrl && merged.inference.reranker) merged.inference.reranker.base_url = rerankUrl;
+    if (rerankEndpoint && merged.inference.reranker) merged.inference.reranker.endpoint = rerankEndpoint;
+    if (rerankAdapter && merged.inference.reranker) {
+      merged.inference.reranker.adapter = rerankAdapter as "jina-v1" | "bifrost-v1";
+    }
     if (rerankModel && merged.inference.reranker) merged.inference.reranker.model = rerankModel;
     if (embedDimStr) {
       const dimension = Number(embedDimStr);
       if (!Number.isInteger(dimension) || dimension < 1) throw new Error(`Invalid embedding dimension: ${embedDimStr}`);
       merged.inference.embedding.dimension = dimension;
     }
+    for (const [name, value, exactEndpoint] of [
+      ["inference.embedding.endpoint", merged.inference.embedding.endpoint, true],
+      ...(merged.inference.reranker
+        ? [["inference.reranker.endpoint", merged.inference.reranker.endpoint, true] as const]
+        : []),
+    ] as const) {
+      try {
+        const url = new URL(value);
+        if (!["http:", "https:"].includes(url.protocol)) throw new Error();
+        if (exactEndpoint && (url.username || url.password || url.hash)) throw new Error();
+      } catch {
+        throw new Error(
+          exactEndpoint
+            ? `${name} must be an absolute HTTP(S) URL without userinfo or a fragment.`
+            : `${name} must be an HTTP(S) URL.`,
+        );
+      }
+    }
+    for (const [name, apiKeyEnv] of [
+      ["inference.embedding.api_key_env", merged.inference.embedding.api_key_env],
+      ...(merged.inference.reranker
+        ? [["inference.reranker.api_key_env", merged.inference.reranker.api_key_env] as const]
+        : []),
+    ] as const) {
+      if (
+        apiKeyEnv !== undefined &&
+        (process.env[apiKeyEnv] === undefined || process.env[apiKeyEnv] === "")
+      ) {
+        throw new Error(
+          `${name} names environment variable "${apiKeyEnv}", but it is unset or empty.`,
+        );
+      }
+    }
   } else {
     throw new Error(`Invalid inference.mode: ${(merged.inference as { mode?: unknown }).mode}`);
   }

package/src/decision.ts

@@ -23,7 +23,10 @@ export function decideResolveOutcome({ reranked, candidates, thresholds }: Decis
     || thresholds.match_margin === undefined
     || thresholds.candidate_floor === undefined
   ) {
-    throw new Error("Reranked decisions require calibrated thresholds.");
+    return {
+      outcome: "ambiguous",
+      candidates: candidates.slice(0, thresholds.candidate_limit),
+    };
   }
   const { match_score, match_margin, candidate_floor } = thresholds;
 

package/src/doctor.ts

@@ -1,5 +1,5 @@
 import { existsSync, mkdirSync } from "node:fs";
-import { createClients } from "./clients";
+import { createClients, RemoteInferenceError } from "./clients";
 import { embeddingDimension, expandHome } from "./config";
 import { parseManifest, resolveManifestPath, validateManifest } from "./manifest";
 import { readSkillmuxMarker } from "./sync";
@@ -10,6 +10,7 @@ export interface DoctorCheck {
   name: string;
   ok: boolean;
   detail: string;
+  failure_kind?: "configuration" | "availability" | "protocol" | "unexpected";
 }
 
 export interface DoctorReport {
@@ -99,8 +100,13 @@ export async function diagnose(config: Config): Promise<DoctorReport> {
     }
   }
 
+  const inferenceFailure = (error: unknown): Pick<DoctorCheck, "detail" | "failure_kind"> =>
+    error instanceof RemoteInferenceError
+      ? { detail: error.message, failure_kind: error.kind }
+      : { detail: "unexpected inference failure", failure_kind: "unexpected" };
+
+  const clients = createClients(config);
   try {
-    const clients = createClients(config);
     const vectors = await clients.embed(["skill router diagnostic"]);
     const actualDimension = vectors[0]?.length ?? 0;
     checks.push({
@@ -108,14 +114,19 @@ export async function diagnose(config: Config): Promise<DoctorReport> {
       ok: actualDimension === embeddingDimension(config),
       detail: `dimension ${actualDimension}`,
     });
-    if (clients.rerank) {
+  } catch (error) {
+    checks.push({ name: "embedding", ok: false, ...inferenceFailure(error) });
+  }
+
+  if (clients.rerank) {
+    try {
       const scores = await clients.rerank("skill router diagnostic", [
         { skill_id: "doctor", text: "Routes a task to an appropriate skill." },
       ]);
       checks.push({ name: "reranker", ok: scores.length === 1 && Number.isFinite(scores[0]), detail: "one finite score" });
+    } catch (error) {
+      checks.push({ name: "reranker", ok: false, ...inferenceFailure(error) });
     }
-  } catch (error) {
-    checks.push({ name: "inference", ok: false, detail: String(error) });
   }
 
   const inferenceReady = checks.some((check) => check.name === "embedding" && check.ok);

package/src/eval.ts

@@ -78,7 +78,8 @@ export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
   const hybridRankings: string[][] = [];
   for (const evalCase of cases) {
     const lexical = ftsSearch(db, evalCase.query, config.recall.k_lexical);
-    const vector = (await clients.embed([evalCase.query]))[0]!;
+    const vector = (await clients.embed([evalCase.query]))[0];
+    if (!vector) throw new Error("Embedding client returned no query vector.");
     const semantic = vectorTopK(db, vector, config.recall.k_vector);
     lexicalRankings.push(lexical.map((row) => row.skill_id));
     hybridRankings.push(reciprocalRankFusion<SkillRow>(lexical, semantic).map((row) => row.skill_id));

package/src/router-core.ts

@@ -3,6 +3,7 @@ import { watch } from "node:fs";
 import { join } from "node:path";
 import { buildAuditRow } from "./audit";
 import { embeddingDimension, embeddingFingerprint, expandHome, loadConfig } from "./config";
+import { RemoteInferenceError } from "./clients";
 import {
   deleteSkill,
   findExactMatch,
@@ -269,12 +270,22 @@ export async function backfillEmbeddings(): Promise<number> {
     const chunk = pending.slice(i, i + BATCH_SIZE);
     try {
       const vectors = await clients.embed(chunk.map(rerankText));
-      chunk.forEach((row, j) => {
-        upsertVector(db, row.skill_id, row.content_sha256, fingerprint, vectors[j]!);
-      });
+      db.transaction(() => {
+        chunk.forEach((row, j) => {
+          const vector = vectors[j];
+          if (!vector) throw new Error("Embedding client returned an incomplete batch.");
+          upsertVector(db, row.skill_id, row.content_sha256, fingerprint, vector);
+        });
+      })();
       count += chunk.length;
     } catch (err) {
-      if (i === 0) throw err;
+      if (
+        i === 0 ||
+        (err instanceof RemoteInferenceError &&
+          (err.kind === "configuration" || err.kind === "protocol"))
+      ) {
+        throw err;
+      }
       break;
     }
   }
@@ -384,44 +395,7 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
     return result;
   }
 
-  const clients = getClients();
-
-  const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
-
-  let retrieval: RetrievalCapability = "lexical";
-  let rows = lexical;
-  if (!input.forceLexical) {
-    try {
-      const queryVec = (await clients.embed([input.query]))[0]!;
-      const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
-      rows = reciprocalRankFusion(lexical, nearest);
-      retrieval = "hybrid";
-    } catch {
-      retrieval = "lexical";
-    }
-  }
-
-  let scores: number[] | null = null;
-  if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
-    try {
-      scores = await clients.rerank(
-        input.query,
-        rows.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
-      );
-      retrieval = "reranked";
-    } catch {
-      scores = null;
-    }
-  }
-
-  const rankedCandidates: RankedCandidate[] = rows
-    .map((r, i) => ({
-      skill_id: r.skill_id,
-      title: r.title,
-      description: r.description,
-      score: scores?.[i] ?? null,
-    }))
-    .sort((a, b) => scores === null ? 0 : (b.score ?? -Infinity) - (a.score ?? -Infinity));
+  const { retrieval, candidates: rankedCandidates } = await retrieveAndRerank(input);
 
   const decisionThresholds = retrieval === "reranked" && config.inference.mode === "remote"
     ? { candidate_limit: config.thresholds.candidate_limit, ...config.inference.thresholds }
@@ -473,6 +447,63 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
   return result;
 }
 
+export interface RetrievalResult {
+  retrieval: Exclude<RetrievalCapability, "exact">;
+  candidates: RankedCandidate[];
+}
+
+/**
+ * Retrieve the full fused candidate set and rerank it once without applying
+ * decision thresholds. Calibration uses this to avoid observing the policy it
+ * is trying to replace.
+ */
+export async function retrieveAndRerank(
+  input: ResolveSkillInput,
+): Promise<RetrievalResult> {
+  const { config, db } = await getEnv();
+  await syncVaultIfNeeded();
+  const clients = getClients();
+  const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
+
+  let retrieval: RetrievalResult["retrieval"] = "lexical";
+  let rows = lexical;
+  if (!input.forceLexical) {
+    try {
+      const queryVec = (await clients.embed([input.query]))[0];
+      if (!queryVec) throw new Error("Embedding client returned no query vector.");
+      const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
+      rows = reciprocalRankFusion(lexical, nearest);
+      retrieval = "hybrid";
+    } catch {
+      retrieval = "lexical";
+    }
+  }
+
+  let scores: number[] | null = null;
+  if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
+    try {
+      scores = await clients.rerank(
+        input.query,
+        rows.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
+      );
+      retrieval = "reranked";
+    } catch {
+      scores = null;
+    }
+  }
+
+  const candidates = rows
+    .map((r, i) => ({
+      skill_id: r.skill_id,
+      title: r.title,
+      description: r.description,
+      score: scores?.[i] ?? null,
+    }))
+    .sort((a, b) => scores === null ? 0 : (b.score ?? -Infinity) - (a.score ?? -Infinity));
+
+  return { retrieval, candidates };
+}
+
 export async function fetchSkill(input: FetchSkillInput): Promise<FetchSkillResult> {
   const { config, db } = await getEnv();
   await syncVaultIfNeeded();

package/src/server.ts

@@ -4,7 +4,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { createClients } from "./clients";
-import { loadConfig, resolveConfigPath } from "./config";
+import { expandHome, loadConfig, rerankerFingerprint, resolveConfigPath } from "./config";
 import { ConfigWatcher, type ReloadStatus } from "./config-watcher";
 import { RuntimeSnapshotManager } from "./snapshot";
 import {
@@ -513,13 +513,16 @@ export async function startServer(opts?: {
                   JSON.stringify({ error: "Calibration run not found" }),
                   { status: 404, headers },
                 );
-              const { DEFAULT_CONFIG_PATH, expandHome } =
-                await import("./config");
+              const active = snapshots.acquire();
+              const currentRerankerFingerprint = rerankerFingerprint(
+                active.snapshot.config,
+              );
+              active.release();
               await applyCalibrationRun(
                 db,
                 runId,
-                expandHome(DEFAULT_CONFIG_PATH),
-                {},
+                expandHome(configPath),
+                { currentRerankerFingerprint },
               );
               return new Response(JSON.stringify({ ok: true, run_id: runId }), {
                 status: 200,

package/src/types.ts

@@ -54,15 +54,15 @@ export interface LocalInferenceConfig {
 
 export interface RemoteEmbeddingConfig {
   provider: "openai";
-  base_url: string;
+  endpoint: string;
   model: string;
   dimension: number;
   api_key_env?: string;
 }
 
 export interface RemoteRerankerConfig {
-  provider: "infinity";
-  base_url: string;
+  adapter: "jina-v1" | "bifrost-v1";
+  endpoint: string;
   model: string;
   api_key_env?: string;
 }