@klhapp/skillmux 1.0.1 → 1.2.0

package/src/cli.ts

@@ -399,12 +399,54 @@ async function handleCalibrateCommand(
 ) {
   if (sub === "run") {
     let datasetPath: string | undefined;
+    let minAutoMatchPrecision: number | undefined;
+    let minRetrievalRecallAtK: number | undefined;
+    let minDeliveredShortlistRecallAtK: number | undefined;
+    let minAutoMatchCount: number | undefined;
+    const readNumber = (flag: string, raw: string | undefined): number => {
+      if (raw === undefined) throw new Error(`${flag} requires a value`);
+      const value = Number(raw);
+      if (!Number.isFinite(value)) throw new Error(`${flag} must be a number`);
+      return value;
+    };
     for (let i = 0; i < args.length; i++) {
-      if (args[i] === "--dataset") datasetPath = args[++i];
+      const option = args[i];
+      if (option === "--dataset") {
+        datasetPath = args[++i];
+        if (!datasetPath) throw new Error("--dataset requires a path value");
+      } else if (option === "--min-auto-match-precision") {
+        minAutoMatchPrecision = readNumber(option, args[++i]);
+      } else if (option === "--min-retrieval-recall-at-k") {
+        minRetrievalRecallAtK = readNumber(option, args[++i]);
+      } else if (option === "--min-delivered-shortlist-recall-at-k") {
+        minDeliveredShortlistRecallAtK = readNumber(option, args[++i]);
+      } else if (option === "--min-auto-match-count") {
+        minAutoMatchCount = readNumber(option, args[++i]);
+        if (!Number.isInteger(minAutoMatchCount) || minAutoMatchCount < 1) {
+          throw new Error("--min-auto-match-count must be a positive integer");
+        }
+      } else {
+        throw new Error(`unknown calibrate run option: ${option}`);
+      }
     }
-    const res = await adapter.calibrateRun({ datasetPath });
+    for (const [flag, value] of [
+      ["--min-auto-match-precision", minAutoMatchPrecision],
+      ["--min-retrieval-recall-at-k", minRetrievalRecallAtK],
+      ["--min-delivered-shortlist-recall-at-k", minDeliveredShortlistRecallAtK],
+    ] as const) {
+      if (value !== undefined && (value < 0 || value > 1)) {
+        throw new Error(`${flag} must be between 0 and 1`);
+      }
+    }
+    const res = await adapter.calibrateRun({
+      datasetPath,
+      minAutoMatchPrecision,
+      minRetrievalRecallAtK,
+      minDeliveredShortlistRecallAtK,
+      minAutoMatchCount,
+    });
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       console.log(`Calibration run complete.`);
       if (res.result) console.log(JSON.stringify(res.result, null, 2));
     });
@@ -414,7 +456,7 @@ async function handleCalibrateCommand(
   if (sub === "list") {
     const res = await adapter.calibrateList();
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       renderTable(
         [
           { key: "run_id", header: "RUN_ID" },
@@ -432,7 +474,7 @@ async function handleCalibrateCommand(
     if (!runId) throw new Error("usage: skillmux calibrate show <run_id>");
     const res = await adapter.calibrateShow(runId);
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       console.log(JSON.stringify(res, null, 2));
     });
     return;
@@ -443,7 +485,7 @@ async function handleCalibrateCommand(
     if (!runId) throw new Error("usage: skillmux calibrate apply <run_id>");
     const res = await adapter.calibrateApply(runId);
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       console.log(`Applied calibration run "${runId}"`);
     });
     return;
@@ -459,6 +501,14 @@ async function handleCalibrateCommand(
   );
 }
 
+function renderCalibrationTarget(target: ResolvedTarget): void {
+  if (target.type === "local") {
+    console.log("Target: local");
+  } else {
+    console.log(`Target: remote (${target.name} -> ${target.server})`);
+  }
+}
+
 async function handleCompletionsCommand(shell: string) {
   if (shell !== "bash" && shell !== "zsh" && shell !== "fish") {
     throw new Error("usage: skillmux completions <bash|zsh|fish>");

package/src/clients.ts

@@ -1,13 +1,17 @@
-import type { Clients, Config } from "./types";
+import type { Clients, Config, RemoteRerankerConfig } from "./types";
 import { expandHome } from "./config";
 import type { pipeline as createPipeline } from "@huggingface/transformers";
 
-interface EmbeddingResponse {
-  data: { index: number; embedding: number[] }[];
-}
+export type RemoteErrorKind = "configuration" | "availability" | "protocol";
 
-interface RerankResponse {
-  results: { index: number; relevance_score: number }[];
+export class RemoteInferenceError extends Error {
+  constructor(
+    public readonly kind: RemoteErrorKind,
+    message: string,
+  ) {
+    super(message);
+    this.name = "RemoteInferenceError";
+  }
 }
 
 // Lazy-loaded model instances for in-process ONNX inference
@@ -15,6 +19,220 @@ type FeatureExtractor = Awaited<ReturnType<typeof createPipeline<"feature-extrac
 
 let localEmbedder: FeatureExtractor | null = null;
 
+function authorizationHeaders(
+  apiKeyEnv: string | undefined,
+  configKey: string,
+): Record<string, string> {
+  if (apiKeyEnv === undefined) return {};
+  const apiKey = process.env[apiKeyEnv];
+  if (apiKey === undefined || apiKey === "") {
+    throw new RemoteInferenceError(
+      "configuration",
+      `${configKey} names environment variable "${apiKeyEnv}", but it is unset or empty.`,
+    );
+  }
+  return { authorization: `Bearer ${apiKey}` };
+}
+
+function httpFailure(surface: string, status: number): RemoteInferenceError {
+  const kind: RemoteErrorKind =
+    status === 401 || status === 403
+      ? "configuration"
+      : status === 408 || status === 429 || status >= 500
+        ? "availability"
+        : "protocol";
+  return new RemoteInferenceError(kind, `${surface} returned HTTP ${status}`);
+}
+
+function finiteFloat32(
+  values: unknown[],
+  error: () => Error,
+): Float32Array {
+  const result = new Float32Array(values.length);
+  for (let index = 0; index < values.length; index++) {
+    const value = values[index];
+    if (typeof value !== "number" || !Number.isFinite(value)) throw error();
+    result[index] = value;
+    if (!Number.isFinite(result[index]!)) throw error();
+  }
+  return result;
+}
+
+function parseEmbeddingVectors(
+  value: unknown,
+  inputCount: number,
+  dimension: number,
+): Float32Array[] {
+  const response = value as { data?: unknown };
+  if (
+    typeof response !== "object" ||
+    response === null ||
+    !Array.isArray(response.data) ||
+    response.data.length !== inputCount
+  ) {
+    throw new RemoteInferenceError(
+      "protocol",
+      "embedding endpoint returned an incomplete data array",
+    );
+  }
+
+  const vectors = new Array<Float32Array>(inputCount);
+  const seen = new Set<number>();
+  for (const value of response.data) {
+    const entry = value as { index?: unknown; embedding?: unknown };
+    if (
+      typeof entry !== "object" ||
+      entry === null ||
+      !Number.isInteger(entry.index) ||
+      (entry.index as number) < 0 ||
+      (entry.index as number) >= inputCount ||
+      seen.has(entry.index as number) ||
+      !Array.isArray(entry.embedding) ||
+      entry.embedding.length !== dimension
+    ) {
+      throw new RemoteInferenceError(
+        "protocol",
+        "embedding endpoint returned invalid indexed vectors",
+      );
+    }
+    seen.add(entry.index as number);
+    vectors[entry.index as number] = finiteFloat32(
+      entry.embedding,
+      () => new RemoteInferenceError("protocol", "embedding endpoint returned invalid vector values"),
+    );
+  }
+  return vectors;
+}
+
+export function parseLocalEmbeddingVectors(
+  value: unknown,
+  inputCount: number,
+  dimension: number,
+): Float32Array[] {
+  if (!Array.isArray(value) || value.length !== inputCount) {
+    throw new Error("Embedding model returned an unexpected batch size.");
+  }
+  return value.map((row) => {
+    if (!Array.isArray(row) || row.length !== dimension) {
+      throw new Error("Embedding model returned unexpected vector dimensions.");
+    }
+    return finiteFloat32(
+      row,
+      () => new Error("Embedding model returned invalid vector values."),
+    );
+  });
+}
+
+function rerankerRequestBody(
+  reranker: RemoteRerankerConfig,
+  query: string,
+  docs: { skill_id: string; text: string }[],
+): Record<string, unknown> {
+  if (reranker.adapter === "jina-v1") {
+    return {
+      model: reranker.model,
+      query,
+      documents: docs.map((doc) => doc.text),
+    };
+  }
+  return {
+    model: reranker.model,
+    query,
+    documents: docs.map((doc) => ({
+      text: doc.text,
+      id: doc.skill_id,
+      meta: {},
+    })),
+    top_n: docs.length,
+    return_documents: false,
+  };
+}
+
+function parseRerankerScores(
+  adapter: RemoteRerankerConfig["adapter"],
+  value: unknown,
+  documentCount: number,
+): number[] {
+  const response = value as { results?: unknown };
+  if (
+    typeof response !== "object" ||
+    response === null ||
+    !Array.isArray(response.results) ||
+    response.results.length !== documentCount
+  ) {
+    throw new RemoteInferenceError(
+      "protocol",
+      `reranker adapter "${adapter}" returned an incomplete results array`,
+    );
+  }
+
+  const scores = new Array<number>(documentCount);
+  const seen = new Set<number>();
+  for (const value of response.results) {
+    const result = value as { index?: unknown; relevance_score?: unknown };
+    if (
+      typeof result !== "object" ||
+      result === null ||
+      !Number.isInteger(result.index) ||
+      (result.index as number) < 0 ||
+      (result.index as number) >= documentCount ||
+      seen.has(result.index as number) ||
+      typeof result.relevance_score !== "number" ||
+      !Number.isFinite(result.relevance_score)
+    ) {
+      throw new RemoteInferenceError(
+        "protocol",
+        `reranker adapter "${adapter}" returned invalid indexed scores`,
+      );
+    }
+    seen.add(result.index as number);
+    scores[result.index as number] = result.relevance_score;
+  }
+  return scores;
+}
+
+async function fetchRerankerScores(
+  reranker: RemoteRerankerConfig,
+  timeoutMs: number,
+  query: string,
+  docs: { skill_id: string; text: string }[],
+): Promise<number[]> {
+  if (docs.length === 0) return [];
+
+  let response: Response;
+  try {
+    response = await fetch(reranker.endpoint, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...authorizationHeaders(reranker.api_key_env, "inference.reranker.api_key_env"),
+      },
+      body: JSON.stringify(rerankerRequestBody(reranker, query, docs)),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+  } catch (error) {
+    if (error instanceof RemoteInferenceError) throw error;
+    throw new RemoteInferenceError(
+      "availability",
+      `reranker adapter "${reranker.adapter}" request failed`,
+    );
+  }
+  if (!response.ok) {
+    throw httpFailure(`reranker adapter "${reranker.adapter}"`, response.status);
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = await response.json();
+  } catch {
+    throw new RemoteInferenceError(
+      "protocol",
+      `reranker adapter "${reranker.adapter}" returned malformed JSON`,
+    );
+  }
+  return parseRerankerScores(reranker.adapter, parsed, docs.length);
+}
+
 function localInference(config: Config) {
   if (config.inference.mode !== "local") throw new Error("Local inference is not configured.");
   return config.inference;
@@ -56,37 +274,45 @@ export function createClients(config: Config): Clients {
         const pipe = await getLocalEmbedder(config);
         const output = await pipe(texts, { pooling: "mean", normalize: true });
         const dim = output.dims[1];
-        if (dim === undefined || output.dims.length !== 2 || output.dims[0] !== texts.length) {
+        if (
+          dim === undefined ||
+          output.dims.length !== 2 ||
+          output.dims[0] !== texts.length ||
+          dim !== config.inference.embedding.dimension
+        ) {
           throw new Error(`Embedding model returned unexpected dimensions: ${output.dims.join("x")}`);
         }
-        const result: Float32Array[] = [];
-        for (let i = 0; i < texts.length; i++) {
-          const row = output.slice(i, null).tolist();
-          if (!Array.isArray(row) || row.some((value) => typeof value !== "number")) {
-            throw new Error("Embedding model returned non-numeric values.");
-          }
-          result.push(Float32Array.from(row));
-        }
-        return result;
+        const rows = Array.from({ length: texts.length }, (_, index) =>
+          output.slice(index, null).tolist(),
+        );
+        return parseLocalEmbeddingVectors(rows, texts.length, dim);
       }
 
       const embedding = config.inference.embedding;
-      const apiKey = embedding.api_key_env ? process.env[embedding.api_key_env] : undefined;
-      const cleanBase = embedding.base_url.replace(/\/$/, "");
-      const embedPath = cleanBase.endsWith("/v1") ? "/embeddings" : "/v1/embeddings";
-      const response = await fetch(`${cleanBase}${embedPath}`, {
-        method: "POST",
-        headers: {
-          "content-type": "application/json",
-          ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}),
-        },
-        body: JSON.stringify({ model: embedding.model, input: texts }),
-        signal: AbortSignal.timeout(config.inference.timeout_ms),
-      });
-      if (!response.ok) throw new Error(`embeddings endpoint returned ${response.status}`);
-      const parsed = (await response.json()) as EmbeddingResponse;
-      const byIndex = [...parsed.data].sort((a, b) => a.index - b.index);
-      return byIndex.map((d) => Float32Array.from(d.embedding));
+      let response: Response;
+      try {
+        response = await fetch(embedding.endpoint, {
+          method: "POST",
+          headers: {
+            "content-type": "application/json",
+            ...authorizationHeaders(embedding.api_key_env, "inference.embedding.api_key_env"),
+          },
+          body: JSON.stringify({ model: embedding.model, input: texts }),
+          signal: AbortSignal.timeout(config.inference.timeout_ms),
+        });
+      } catch (error) {
+        if (error instanceof RemoteInferenceError) throw error;
+        throw new RemoteInferenceError("availability", "embedding endpoint request failed");
+      }
+      if (!response.ok) throw httpFailure("embedding endpoint", response.status);
+
+      let parsed: unknown;
+      try {
+        parsed = await response.json();
+      } catch {
+        throw new RemoteInferenceError("protocol", "embedding endpoint returned malformed JSON");
+      }
+      return parseEmbeddingVectors(parsed, texts.length, embedding.dimension);
     },
   };
   if (config.inference.mode === "remote" && config.inference.reranker) {
@@ -94,22 +320,12 @@ export function createClients(config: Config): Clients {
     clients.rerank = async (query, docs) => {
       const reranker = inference.reranker;
       if (!reranker) throw new Error("Reranker is not configured.");
-      const apiKey = reranker.api_key_env ? process.env[reranker.api_key_env] : undefined;
-      const response = await fetch(`${reranker.base_url.replace(/\/$/, "")}/rerank`, {
-        method: "POST",
-        headers: { "content-type": "application/json", ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}) },
-        body: JSON.stringify({
-          model: reranker.model,
-          query,
-          documents: docs.map((d) => d.text),
-        }),
-        signal: AbortSignal.timeout(inference.timeout_ms),
-      });
-      if (!response.ok) throw new Error(`rerank endpoint returned ${response.status}`);
-      const parsed = (await response.json()) as RerankResponse;
-      const scores = new Array<number>(docs.length).fill(0);
-      for (const result of parsed.results) scores[result.index] = result.relevance_score;
-      return scores;
+      return fetchRerankerScores(
+        reranker,
+        inference.timeout_ms,
+        query,
+        docs,
+      );
     };
   }
   return clients;

package/src/config-service.ts

@@ -44,6 +44,10 @@ export const RESTART_REQUIRED_KEYS = [
   "inference.bundle",
   "inference.models_dir",
   "state_dir",
+  "inference.embedding.model",
+  "inference.embedding.dimension",
+  "inference.embedding.device",
+  "inference.embedding.dtype",
 ];
 
 export const RELOADABLE_KEYS = [
@@ -54,12 +58,13 @@ export const RELOADABLE_KEYS = [
   "thresholds.match_score",
   "thresholds.match_margin",
   "thresholds.candidate_floor",
-  "inference.embedding.model",
-  "inference.embedding.dimension",
-  "inference.embedding.device",
-  "inference.embedding.dtype",
-  "inference.embedding.base_url",
+  "inference.embedding.endpoint",
   "inference.embedding.api_key_env",
+  "inference.reranker.adapter",
+  "inference.reranker.endpoint",
+  "inference.reranker.model",
+  "inference.reranker.api_key_env",
+  "inference.timeout_ms",
   "server.rate_limit.enabled",
   "server.rate_limit.requests_per_minute",
   "server.rate_limit.trust_proxy",
@@ -146,8 +151,12 @@ export async function getEffectiveConfig(configPath?: string): Promise<{
     "inference.embedding.dimension",
     "inference.embedding.device",
     "inference.embedding.dtype",
-    "inference.embedding.base_url",
+    "inference.embedding.endpoint",
     "inference.embedding.api_key_env",
+    "inference.reranker.adapter",
+    "inference.reranker.endpoint",
+    "inference.reranker.model",
+    "inference.reranker.api_key_env",
     "inference.timeout_ms",
     "server.auth_enabled",
     "server.auth_token_env",
@@ -178,8 +187,12 @@ export function isEnvMasked(key: string): boolean {
   if (key === "inference.models_dir" && (process.env.SKILLMUX_MODELS_DIR || process.env.SKILL_ROUTER_MODELS_DIR)) return true;
   if (key === "inference.embedding.device" && process.env.EMBED_DEVICE) return true;
   if (key === "inference.embedding.dtype" && process.env.EMBED_DTYPE) return true;
-  if (key === "inference.embedding.base_url" && (process.env.SKILLMUX_EMBED_BASE_URL || process.env.EMBED_BASE_URL)) return true;
+  if (key === "inference.embedding.endpoint" && (process.env.SKILLMUX_EMBED_ENDPOINT || process.env.EMBED_ENDPOINT)) return true;
   if (key === "inference.embedding.model" && (process.env.SKILLMUX_EMBED_MODEL || process.env.EMBED_MODEL)) return true;
+  if (key === "inference.embedding.dimension" && (process.env.SKILLMUX_EMBED_DIMENSION || process.env.EMBED_DIMENSION)) return true;
+  if (key === "inference.reranker.adapter" && (process.env.SKILLMUX_RERANK_ADAPTER || process.env.RERANK_ADAPTER)) return true;
+  if (key === "inference.reranker.endpoint" && (process.env.SKILLMUX_RERANK_ENDPOINT || process.env.RERANK_ENDPOINT)) return true;
+  if (key === "inference.reranker.model" && (process.env.SKILLMUX_RERANK_MODEL || process.env.RERANK_MODEL)) return true;
   if (key === "server.auth_enabled" && process.env.HTTP_AUTH_ENABLED) return true;
   if (key === "server.auth_token_env" && process.env.HTTP_AUTH_TOKEN_ENV) return true;
   if (key === "server.hostname" && process.env.HTTP_HOSTNAME) return true;
@@ -206,8 +219,12 @@ export function validateDottedKey(key: string): void {
     "inference.embedding.dimension",
     "inference.embedding.device",
     "inference.embedding.dtype",
-    "inference.embedding.base_url",
+    "inference.embedding.endpoint",
     "inference.embedding.api_key_env",
+    "inference.reranker.adapter",
+    "inference.reranker.endpoint",
+    "inference.reranker.model",
+    "inference.reranker.api_key_env",
     "inference.timeout_ms",
     "server.auth_enabled",
     "server.auth_token_env",

package/src/config-watcher.ts

@@ -18,6 +18,13 @@ export const LIVE_RELOAD_KEYS = new Set([
   "recall.k_lexical",
   "recall.k_vector",
   "thresholds.candidate_limit",
+  "inference.embedding.endpoint",
+  "inference.embedding.api_key_env",
+  "inference.reranker.adapter",
+  "inference.reranker.endpoint",
+  "inference.reranker.model",
+  "inference.reranker.api_key_env",
+  "inference.timeout_ms",
 ]);
 
 // ---------------------------------------------------------------------------