@klhapp/skillmux 1.6.0 → 1.7.1

package/src/cli.ts

@@ -4,7 +4,6 @@ import { Database } from "bun:sqlite";
 import { existsSync, lstatSync, mkdirSync, rmSync } from "node:fs";
 import { hostname } from "node:os";
 import { join } from "node:path";
-import { generateDataset } from "./dataset-generator";
 
 import { createClients } from "./clients";
 import {
@@ -116,7 +115,6 @@ import { runTarget } from "./commands/target";
 const KNOWN_COMMANDS = [
   "context",
   "config",
-  "calibrate",
   "completions",
   "serve",
   "index",
@@ -147,7 +145,6 @@ function isDockerHostManagementCommand(command: string, subCommand: string): boo
       "local-vault",
       "models",
       "context",
-      "calibrate",
       "eval",
     ].includes(command)
   ) {
@@ -210,7 +207,7 @@ async function main() {
     return;
   }
 
-  // Parse global flags for context/config/calibrate
+  // Parse global flags for context/config
   for (let i = 0; i < rawArgv.length; i++) {
     const arg = rawArgv[i];
     if (arg === "--json") isJson = true;
@@ -235,10 +232,10 @@ async function main() {
     return;
   }
 
-  // Only resolve target if command is target-aware or context/config/calibrate
+  // Only resolve target if command is target-aware or context/config
   const isLocalConfigInit = command === "config" && rawArgv[1] === "init";
   if (
-    (["context", "config", "calibrate"].includes(command) &&
+    (["context", "config"].includes(command) &&
       !isLocalConfigInit) ||
     flagContext ||
     flagServer
@@ -272,11 +269,9 @@ async function main() {
         });
         break;
       case "calibrate":
-        await handleCalibrateCommand(adapter, subCommand, rawArgv.slice(1), {
-          target: resolvedTarget,
-          isJson,
-        });
-        break;
+        throw new Error(
+          'skillmux calibrate was removed. Threshold calibration was removed; use "skillmux eval" for ranking evaluation.',
+        );
       case "completions":
         await handleCompletionsCommand(subCommand);
         break;
@@ -364,7 +359,7 @@ async function main() {
         const suggestion = suggestCorrection(command, KNOWN_COMMANDS);
         const msg = suggestion
           ? `Unknown command "${command}". Did you mean "${suggestion}"?`
-          : `usage: skillmux <serve|index|sync|init|project|target|core pin/unpin|report|scan|install|eval|doctor|skill which|local-vault init|config show|models download|calibrate generate-dataset>`;
+          : `usage: skillmux <serve|index|sync|init|project|target|core pin/unpin|report|scan|install|eval|doctor|skill which|local-vault init|config show|models download>`;
         throw new Error(msg);
       }
     }
@@ -463,188 +458,6 @@ async function handleContextCommand(
   throw new Error("usage: skillmux context <add|list|current|use|remove>");
 }
 
-async function handleCalibrateCommand(
-  adapter: TargetAdapter,
-  sub: string,
-  args: string[],
-  ctx: { target: ResolvedTarget; isJson: boolean },
-) {
-  if (sub === "run") {
-    let datasetPath: string | undefined;
-    let minAutoMatchPrecision: number | undefined;
-    let minRetrievalRecallAtK: number | undefined;
-    let minDeliveredShortlistRecallAtK: number | undefined;
-    let minAutoMatchCount: number | undefined;
-    let tuneAutoMatchPrecisionBuffer: number | undefined;
-    let tuneAutoMatchCountBuffer: number | undefined;
-    let tuneDeliveredShortlistRecallBuffer: number | undefined;
-    let concurrency: number | undefined;
-    let resumeRunId: string | undefined;
-    let timing = false;
-    const readNumber = (flag: string, raw: string | undefined): number => {
-      if (raw === undefined) throw new Error(`${flag} requires a value`);
-      const value = Number(raw);
-      if (!Number.isFinite(value)) throw new Error(`${flag} must be a number`);
-      return value;
-    };
-    for (let i = 1; i < args.length; i++) {
-      const option = args[i];
-      if (option === "--dataset") {
-        datasetPath = args[++i];
-        if (!datasetPath) throw new Error("--dataset requires a path value");
-      } else if (option === "--min-auto-match-precision") {
-        minAutoMatchPrecision = readNumber(option, args[++i]);
-      } else if (option === "--min-retrieval-recall-at-k") {
-        minRetrievalRecallAtK = readNumber(option, args[++i]);
-      } else if (option === "--min-delivered-shortlist-recall-at-k") {
-        minDeliveredShortlistRecallAtK = readNumber(option, args[++i]);
-      } else if (option === "--min-auto-match-count") {
-        minAutoMatchCount = readNumber(option, args[++i]);
-        if (!Number.isInteger(minAutoMatchCount) || minAutoMatchCount < 1) {
-          throw new Error("--min-auto-match-count must be a positive integer");
-        }
-      } else if (option === "--tune-auto-match-precision-buffer") {
-        tuneAutoMatchPrecisionBuffer = readNumber(option, args[++i]);
-      } else if (option === "--tune-auto-match-count-buffer") {
-        tuneAutoMatchCountBuffer = readNumber(option, args[++i]);
-        if (!Number.isInteger(tuneAutoMatchCountBuffer) || tuneAutoMatchCountBuffer < 0) {
-          throw new Error("--tune-auto-match-count-buffer must be a non-negative integer");
-        }
-      } else if (option === "--tune-delivered-shortlist-recall-buffer") {
-        tuneDeliveredShortlistRecallBuffer = readNumber(option, args[++i]);
-      } else if (option === "--concurrency") {
-        const raw = args[++i];
-        if (raw === undefined) throw new Error("--concurrency requires a value");
-        const val = Number(raw);
-        if (!Number.isInteger(val) || val < 1) {
-          throw new Error("--concurrency must be a positive integer");
-        }
-        concurrency = val;
-      } else if (option === "--resume") {
-        resumeRunId = args[++i];
-        if (!resumeRunId) throw new Error("--resume requires a run_id value");
-      } else if (option === "--timing") {
-        timing = true;
-      } else if (option === "--json") {
-        // Global flag accepted in the documented subcommand position
-      } else {
-        throw new Error(`unknown calibrate run option: ${option}`);
-      }
-    }
-    for (const [flag, value] of [
-      ["--min-auto-match-precision", minAutoMatchPrecision],
-      ["--min-retrieval-recall-at-k", minRetrievalRecallAtK],
-      ["--min-delivered-shortlist-recall-at-k", minDeliveredShortlistRecallAtK],
-      ["--tune-auto-match-precision-buffer", tuneAutoMatchPrecisionBuffer],
-      ["--tune-delivered-shortlist-recall-buffer", tuneDeliveredShortlistRecallBuffer],
-    ] as const) {
-      if (value !== undefined && (value < 0 || value > 1)) {
-        throw new Error(`${flag} must be between 0 and 1`);
-      }
-    }
-    const res = await adapter.calibrateRun({
-      datasetPath,
-      minAutoMatchPrecision,
-      minRetrievalRecallAtK,
-      minDeliveredShortlistRecallAtK,
-      minAutoMatchCount,
-      tuneAutoMatchPrecisionBuffer,
-      tuneAutoMatchCountBuffer,
-      tuneDeliveredShortlistRecallBuffer,
-      concurrency,
-      resumeRunId,
-      timing,
-      onTimingSummary: timing
-        ? (summary) => {
-            // Write timing report to stderr only — stdout remains valid JSON under --json.
-            // Cumulative fields are total worker time across concurrent queries and may
-            // exceed wall_ms. They do not sum to wall time.
-            process.stderr.write(
-              [
-                "--- calibrate run timing ---",
-                `cases_total:                ${summary.cases_total}`,
-                `cases_executed:             ${summary.cases_executed}  (retrieved in this invocation)`,
-                `cases_reused:               ${summary.cases_reused}  (loaded from prior interrupted run)`,
-                `wall_ms:                    ${summary.wall_ms.toFixed(1)}`,
-                `vault_sync_ms:              ${summary.vault_sync_ms.toFixed(1)}`,
-                "cumulative worker time (concurrent totals; may exceed wall_ms):",
-                `  cumulative_embedding_ms:  ${summary.cumulative_embedding_ms.toFixed(1)}`,
-                `  cumulative_lexical_ms:    ${summary.cumulative_lexical_ms.toFixed(1)}`,
-                `  cumulative_vector_ms:     ${summary.cumulative_vector_ms.toFixed(1)}`,
-                `  cumulative_reranker_ms:   ${summary.cumulative_reranker_ms.toFixed(1)}`,
-                `  cumulative_checkpoint_ms: ${summary.cumulative_checkpoint_ms.toFixed(1)}`,
-                `policy_evaluation_ms:       ${summary.policy_evaluation_ms.toFixed(1)}`,
-                "",
-              ].join("\n"),
-            );
-          }
-        : undefined,
-    });
-    emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderCalibrationTarget(ctx.target);
-      console.log(`Calibration run complete.`);
-      if (res.result) console.log(JSON.stringify(res.result, null, 2));
-    });
-    return;
-  }
-
-
-  if (sub === "list") {
-    const res = await adapter.calibrateList();
-    emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderCalibrationTarget(ctx.target);
-      renderTable(
-        [
-          { key: "run_id", header: "RUN_ID" },
-          { key: "created_at", header: "CREATED_AT" },
-          { key: "status", header: "STATUS" },
-        ],
-        res,
-      );
-    });
-    return;
-  }
-
-  if (sub === "show") {
-    const runId = args[1];
-    if (!runId) throw new Error("usage: skillmux calibrate show <run_id>");
-    const res = await adapter.calibrateShow(runId);
-    emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderCalibrationTarget(ctx.target);
-      console.log(JSON.stringify(res, null, 2));
-    });
-    return;
-  }
-
-  if (sub === "apply") {
-    const runId = args[1];
-    if (!runId) throw new Error("usage: skillmux calibrate apply <run_id>");
-    const res = await adapter.calibrateApply(runId);
-    emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderCalibrationTarget(ctx.target);
-      console.log(`Applied calibration run "${runId}"`);
-    });
-    return;
-  }
-
-  if (sub === "generate-dataset") {
-    await runCalibrateGenerateDataset(args.slice(1));
-    return;
-  }
-
-  throw new Error(
-    "usage: skillmux calibrate generate-dataset [--vault <path>] [--out <file>]",
-  );
-}
-
-function renderCalibrationTarget(target: ResolvedTarget): void {
-  if (target.type === "local") {
-    console.log("Target: local");
-  } else {
-    console.log(`Target: remote (${target.name} -> ${target.server})`);
-  }
-}
-
 async function handleCompletionsCommand(shell: string) {
   if (shell !== "bash" && shell !== "zsh" && shell !== "fish") {
     throw new Error("usage: skillmux completions <bash|zsh|fish>");
@@ -720,17 +533,6 @@ Setup:
   skillmux core <pin|unpin> <skill_id>... [--yes] [--dry-run] [--json]
   skillmux skill which <skill_id>
 
-Calibration:
-  skillmux calibrate run [--dataset <path>] [--concurrency <n>] [--resume <run_id>]
-                         [--min-auto-match-precision <0..1>] [--min-auto-match-count <n>]
-                         [--min-retrieval-recall-at-k <0..1>]
-                         [--min-delivered-shortlist-recall-at-k <0..1>]
-                         [--tune-auto-match-precision-buffer <0..1>]
-                         [--tune-auto-match-count-buffer <n>]
-                         [--tune-delivered-shortlist-recall-buffer <0..1>]
-                         [--timing] [--json]
-  skillmux calibrate <list|show|apply|generate-dataset>
-
 Init clients:
   claude-code, codex, gemini-cli, opencode, github-copilot, windsurf,
   antigravity, goose, hermes, skillmux-mcp
@@ -740,7 +542,7 @@ Init targets:
 
 Commands:
   serve, index, sync, init, project, target, core, report, scan, install, eval, doctor, skill,
-  local-vault, config, models, calibrate, context, completions`);
+  local-vault, config, models, context, completions`);
 }
 
 // ---------------------------------------------------------------------------
@@ -810,13 +612,17 @@ async function runEval(options: { isJson: boolean }): Promise<void> {
     throw new Error(`eval requires local embeddings: ${String(error)}`);
   });
   emitSuccess({ isJson: options.isJson }, report, () => {
-    console.log(`holdout queries: ${report.queries}`);
-    console.log(`lexical recall@3: ${report.lexical.recall_at_3.toFixed(3)}`);
-    console.log(`lexical recall@5: ${report.lexical.recall_at_5.toFixed(3)}`);
-    console.log(`lexical MRR:      ${report.lexical.mrr.toFixed(3)}`);
-    console.log(`hybrid recall@3:  ${report.hybrid.recall_at_3.toFixed(3)}`);
-    console.log(`hybrid recall@5:  ${report.hybrid.recall_at_5.toFixed(3)}`);
-    console.log(`hybrid MRR:       ${report.hybrid.mrr.toFixed(3)}`);
+    console.log(`holdout queries:   ${report.queries}`);
+    console.log(`judged queries:    ${report.judged_queries}`);
+    console.log(`unjudged queries:  ${report.unjudged_queries}`);
+    console.log(`lexical recall@5:  ${report.lexical.recall_at_5.toFixed(3)}`);
+    console.log(`lexical recall@10: ${report.lexical.recall_at_10.toFixed(3)}`);
+    console.log(`lexical MRR:       ${report.lexical.mrr.toFixed(3)}`);
+    console.log(`lexical nDCG@10:   ${report.lexical.ndcg_at_10.toFixed(3)}`);
+    console.log(`hybrid recall@5:   ${report.hybrid.recall_at_5.toFixed(3)}`);
+    console.log(`hybrid recall@10:  ${report.hybrid.recall_at_10.toFixed(3)}`);
+    console.log(`hybrid MRR:        ${report.hybrid.mrr.toFixed(3)}`);
+    console.log(`hybrid nDCG@10:    ${report.hybrid.ndcg_at_10.toFixed(3)}`);
   });
 }
 
@@ -1802,47 +1608,6 @@ async function runInstall(
   }
 }
 
-function parseCalibrateGenerateDatasetArgs(args: string[]): {
-  vault?: string;
-  out?: string;
-} {
-  let vault: string | undefined;
-  let out: string | undefined;
-  for (let i = 0; i < args.length; i++) {
-    const option = args[i];
-    if (option === "--vault") {
-      const value = args[++i];
-      if (!value) throw new Error("--vault requires a path value");
-      vault = value;
-    } else if (option === "--out") {
-      const value = args[++i];
-      if (!value) throw new Error("--out requires a file path value");
-      out = value;
-    } else {
-      throw new Error(`unknown calibrate option: ${option}`);
-    }
-  }
-  return { vault, out };
-}
-
-async function runCalibrateGenerateDataset(args: string[]): Promise<void> {
-  const { vault: vaultArg, out: outArg } =
-    parseCalibrateGenerateDatasetArgs(args);
-  const config = await loadConfig();
-  const vaultPath = expandHome(vaultArg ?? config.vault_path);
-  const outPath = expandHome(outArg ?? join(config.state_dir, "queries.json"));
-
-  const skills = await scanVault(vaultPath);
-  const dataset = generateDataset(skills);
-
-  const parentDir = join(outPath, "..");
-  mkdirSync(parentDir, { recursive: true });
-  await Bun.write(outPath, JSON.stringify(dataset, null, 2) + "\n");
-  console.log(
-    `generated synthetic dataset with ${dataset.length} cases at ${outPath}`,
-  );
-}
-
 if (import.meta.main) {
   await main();
 }

package/src/completions.ts

@@ -3,7 +3,6 @@ export type ShellType = "bash" | "zsh" | "fish";
 const TOP_LEVEL_COMMANDS: { name: string; description: string }[] = [
   { name: "context", description: "Manage connection contexts" },
   { name: "config", description: "Manage configuration" },
-  { name: "calibrate", description: "Manage policy calibration" },
   { name: "serve", description: "Start MCP server" },
   { name: "index", description: "Rebuild local search index" },
   { name: "sync", description: "Synchronize vault skills" },
@@ -45,9 +44,6 @@ _skillmux_completions() {
         config)
             COMPREPLY=( $(compgen -W "init show get validate diff set status" -- "$cur") )
             ;;
-        calibrate)
-            COMPREPLY=( $(compgen -W "run list show apply generate-dataset" -- "$cur") )
-            ;;
         completions)
             COMPREPLY=( $(compgen -W "bash zsh fish" -- "$cur") )
             ;;

package/src/config-service.ts

@@ -60,11 +60,8 @@ export const RELOADABLE_KEYS = [
   "recall.k_lexical",
   "recall.k_vector",
   "recall.k_rerank",
-  "output.ambiguous_candidate_limit",
-  "thresholds.candidate_limit",
-  "thresholds.match_score",
-  "thresholds.match_margin",
-  "thresholds.candidate_floor",
+  "output.top_k",
+  "output.max_top_k",
   "inference.embedding.endpoint",
   "inference.embedding.api_key_env",
   "inference.reranker.adapter",
@@ -123,7 +120,7 @@ export async function getEffectiveConfig(configPath?: string): Promise<{
   sources: ConfigSourceMap;
   rawToml: Record<string, unknown>;
 }> {
-  const path = configPath ?? DEFAULT_CONFIG_PATH;
+  const path = configPath ?? process.env.SKILLMUX_CONFIG ?? DEFAULT_CONFIG_PATH;
   const effective = await loadConfig(path);
   const rawToml: Record<string, unknown> = {};
 
@@ -150,11 +147,8 @@ export async function getEffectiveConfig(configPath?: string): Promise<{
     "recall.k_lexical",
     "recall.k_vector",
     "recall.k_rerank",
-    "output.ambiguous_candidate_limit",
-    "thresholds.candidate_limit",
-    "thresholds.match_score",
-    "thresholds.match_margin",
-    "thresholds.candidate_floor",
+    "output.top_k",
+    "output.max_top_k",
     "inference.mode",
     "inference.bundle",
     "inference.models_dir",
@@ -200,15 +194,8 @@ export function isEnvMasked(key: string, allowEnvOverrides: boolean = true): boo
   if (key === "recall.k_lexical" && (process.env.SKILLMUX_RECALL_K_LEXICAL || process.env.RECALL_K_LEXICAL)) return true;
   if (key === "recall.k_vector" && (process.env.SKILLMUX_RECALL_K_VECTOR || process.env.RECALL_K_VECTOR)) return true;
   if (key === "recall.k_rerank" && (process.env.SKILLMUX_RECALL_K_RERANK || process.env.RECALL_K_RERANK)) return true;
-  if (
-    (key === "output.ambiguous_candidate_limit" || key === "thresholds.candidate_limit") &&
-    (process.env.SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT ||
-      process.env.AMBIGUOUS_CANDIDATE_LIMIT ||
-      process.env.SKILLMUX_CANDIDATE_LIMIT ||
-      process.env.CANDIDATE_LIMIT)
-  ) {
-    return true;
-  }
+  if (key === "output.top_k" && (process.env.SKILLMUX_OUTPUT_TOP_K || process.env.OUTPUT_TOP_K)) return true;
+  if (key === "output.max_top_k" && (process.env.SKILLMUX_OUTPUT_MAX_TOP_K || process.env.OUTPUT_MAX_TOP_K)) return true;
   if (key === "inference.models_dir" && (process.env.SKILLMUX_MODELS_DIR || process.env.SKILL_ROUTER_MODELS_DIR)) return true;
   if (key === "inference.embedding.device" && (process.env.SKILLMUX_EMBED_DEVICE || process.env.EMBED_DEVICE)) return true;
   if (key === "inference.embedding.dtype" && (process.env.SKILLMUX_EMBED_DTYPE || process.env.EMBED_DTYPE)) return true;
@@ -229,6 +216,12 @@ export function isEnvMasked(key: string, allowEnvOverrides: boolean = true): boo
 }
 
 export function validateDottedKey(key: string): void {
+  if (key === "output.ambiguous_candidate_limit") {
+    throw new Error("output.ambiguous_candidate_limit is obsolete. Use output.top_k instead.");
+  }
+  if (key.startsWith("thresholds.") || key.startsWith("inference.thresholds.")) {
+    throw new Error("thresholds are obsolete. Threshold calibration was removed; use output.top_k.");
+  }
   const allowed = new Set([
     "config.environment_overrides",
     "vault_path",
@@ -236,11 +229,8 @@ export function validateDottedKey(key: string): void {
     "recall.k_lexical",
     "recall.k_vector",
     "recall.k_rerank",
-    "output.ambiguous_candidate_limit",
-    "thresholds.candidate_limit",
-    "thresholds.match_score",
-    "thresholds.match_margin",
-    "thresholds.candidate_floor",
+    "output.top_k",
+    "output.max_top_k",
     "inference.mode",
     "inference.bundle",
     "inference.models_dir",
@@ -279,11 +269,8 @@ export function parseDottedValue(key: string, valueStr: string): unknown {
     "recall.k_lexical",
     "recall.k_vector",
     "recall.k_rerank",
-    "output.ambiguous_candidate_limit",
-    "thresholds.candidate_limit",
-    "thresholds.match_score",
-    "thresholds.match_margin",
-    "thresholds.candidate_floor",
+    "output.top_k",
+    "output.max_top_k",
     "inference.embedding.dimension",
     "inference.timeout_ms",
     "server.rate_limit.requests_per_minute",
@@ -328,7 +315,7 @@ export async function setDottedKey(
     throw new Error(`Cannot set environment-masked configuration key "${key}"`);
   }
 
-  const path = opts?.configPath ?? DEFAULT_CONFIG_PATH;
+  const path = opts?.configPath ?? process.env.SKILLMUX_CONFIG ?? DEFAULT_CONFIG_PATH;
   const targetName = opts?.targetName ?? "local";
 
   const { effective: priorEffective, rawToml } = await getEffectiveConfig(path);
@@ -445,7 +432,7 @@ export async function getLocalConfigStatus(configPath?: string): Promise<ConfigS
 
   return {
     target: "local",
-    desired_source: configPath ?? DEFAULT_CONFIG_PATH,
+    desired_source: configPath ?? process.env.SKILLMUX_CONFIG ?? DEFAULT_CONFIG_PATH,
     desired_source_hash: hash,
     active_revision: hash,
     active_source_hash: hash,

package/src/config-watcher.ts

@@ -12,15 +12,11 @@ import type { Config } from "./types";
 
 export const LIVE_RELOAD_KEYS = new Set([
   "config.environment_overrides",
-  "inference.thresholds.match_score",
-  "inference.thresholds.match_margin",
-  "inference.thresholds.candidate_floor",
-  "inference.calibration.run_id",
   "recall.k_lexical",
   "recall.k_vector",
   "recall.k_rerank",
-  "output.ambiguous_candidate_limit",
-  "thresholds.candidate_limit",
+  "output.top_k",
+  "output.max_top_k",
   "inference.embedding.endpoint",
   "inference.embedding.api_key_env",
   "inference.reranker.adapter",