@klhapp/skillmux 1.5.1 → 1.6.0

package/src/cli.ts

@@ -475,6 +475,12 @@ async function handleCalibrateCommand(
     let minRetrievalRecallAtK: number | undefined;
     let minDeliveredShortlistRecallAtK: number | undefined;
     let minAutoMatchCount: number | undefined;
+    let tuneAutoMatchPrecisionBuffer: number | undefined;
+    let tuneAutoMatchCountBuffer: number | undefined;
+    let tuneDeliveredShortlistRecallBuffer: number | undefined;
+    let concurrency: number | undefined;
+    let resumeRunId: string | undefined;
+    let timing = false;
     const readNumber = (flag: string, raw: string | undefined): number => {
       if (raw === undefined) throw new Error(`${flag} requires a value`);
       const value = Number(raw);
@@ -497,6 +503,30 @@ async function handleCalibrateCommand(
         if (!Number.isInteger(minAutoMatchCount) || minAutoMatchCount < 1) {
           throw new Error("--min-auto-match-count must be a positive integer");
         }
+      } else if (option === "--tune-auto-match-precision-buffer") {
+        tuneAutoMatchPrecisionBuffer = readNumber(option, args[++i]);
+      } else if (option === "--tune-auto-match-count-buffer") {
+        tuneAutoMatchCountBuffer = readNumber(option, args[++i]);
+        if (!Number.isInteger(tuneAutoMatchCountBuffer) || tuneAutoMatchCountBuffer < 0) {
+          throw new Error("--tune-auto-match-count-buffer must be a non-negative integer");
+        }
+      } else if (option === "--tune-delivered-shortlist-recall-buffer") {
+        tuneDeliveredShortlistRecallBuffer = readNumber(option, args[++i]);
+      } else if (option === "--concurrency") {
+        const raw = args[++i];
+        if (raw === undefined) throw new Error("--concurrency requires a value");
+        const val = Number(raw);
+        if (!Number.isInteger(val) || val < 1) {
+          throw new Error("--concurrency must be a positive integer");
+        }
+        concurrency = val;
+      } else if (option === "--resume") {
+        resumeRunId = args[++i];
+        if (!resumeRunId) throw new Error("--resume requires a run_id value");
+      } else if (option === "--timing") {
+        timing = true;
+      } else if (option === "--json") {
+        // Global flag accepted in the documented subcommand position
       } else {
         throw new Error(`unknown calibrate run option: ${option}`);
       }
@@ -505,6 +535,8 @@ async function handleCalibrateCommand(
       ["--min-auto-match-precision", minAutoMatchPrecision],
       ["--min-retrieval-recall-at-k", minRetrievalRecallAtK],
       ["--min-delivered-shortlist-recall-at-k", minDeliveredShortlistRecallAtK],
+      ["--tune-auto-match-precision-buffer", tuneAutoMatchPrecisionBuffer],
+      ["--tune-delivered-shortlist-recall-buffer", tuneDeliveredShortlistRecallBuffer],
     ] as const) {
       if (value !== undefined && (value < 0 || value > 1)) {
         throw new Error(`${flag} must be between 0 and 1`);
@@ -516,6 +548,37 @@ async function handleCalibrateCommand(
       minRetrievalRecallAtK,
       minDeliveredShortlistRecallAtK,
       minAutoMatchCount,
+      tuneAutoMatchPrecisionBuffer,
+      tuneAutoMatchCountBuffer,
+      tuneDeliveredShortlistRecallBuffer,
+      concurrency,
+      resumeRunId,
+      timing,
+      onTimingSummary: timing
+        ? (summary) => {
+            // Write timing report to stderr only — stdout remains valid JSON under --json.
+            // Cumulative fields are total worker time across concurrent queries and may
+            // exceed wall_ms. They do not sum to wall time.
+            process.stderr.write(
+              [
+                "--- calibrate run timing ---",
+                `cases_total:                ${summary.cases_total}`,
+                `cases_executed:             ${summary.cases_executed}  (retrieved in this invocation)`,
+                `cases_reused:               ${summary.cases_reused}  (loaded from prior interrupted run)`,
+                `wall_ms:                    ${summary.wall_ms.toFixed(1)}`,
+                `vault_sync_ms:              ${summary.vault_sync_ms.toFixed(1)}`,
+                "cumulative worker time (concurrent totals; may exceed wall_ms):",
+                `  cumulative_embedding_ms:  ${summary.cumulative_embedding_ms.toFixed(1)}`,
+                `  cumulative_lexical_ms:    ${summary.cumulative_lexical_ms.toFixed(1)}`,
+                `  cumulative_vector_ms:     ${summary.cumulative_vector_ms.toFixed(1)}`,
+                `  cumulative_reranker_ms:   ${summary.cumulative_reranker_ms.toFixed(1)}`,
+                `  cumulative_checkpoint_ms: ${summary.cumulative_checkpoint_ms.toFixed(1)}`,
+                `policy_evaluation_ms:       ${summary.policy_evaluation_ms.toFixed(1)}`,
+                "",
+              ].join("\n"),
+            );
+          }
+        : undefined,
     });
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
       renderCalibrationTarget(ctx.target);
@@ -525,6 +588,7 @@ async function handleCalibrateCommand(
     return;
   }
 
+
   if (sub === "list") {
     const res = await adapter.calibrateList();
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
@@ -656,6 +720,17 @@ Setup:
   skillmux core <pin|unpin> <skill_id>... [--yes] [--dry-run] [--json]
   skillmux skill which <skill_id>
 
+Calibration:
+  skillmux calibrate run [--dataset <path>] [--concurrency <n>] [--resume <run_id>]
+                         [--min-auto-match-precision <0..1>] [--min-auto-match-count <n>]
+                         [--min-retrieval-recall-at-k <0..1>]
+                         [--min-delivered-shortlist-recall-at-k <0..1>]
+                         [--tune-auto-match-precision-buffer <0..1>]
+                         [--tune-auto-match-count-buffer <n>]
+                         [--tune-delivered-shortlist-recall-buffer <0..1>]
+                         [--timing] [--json]
+  skillmux calibrate <list|show|apply|generate-dataset>
+
 Init clients:
   claude-code, codex, gemini-cli, opencode, github-copilot, windsurf,
   antigravity, goose, hermes, skillmux-mcp

package/src/router-core.ts

@@ -530,18 +530,64 @@ export function classifyInferenceError(
   return stage === "embedding" ? "embedding_unavailable" : "reranker_unavailable";
 }
 
+export interface StageTiming {
+  embedding_ms: number;
+  lexical_ms: number;
+  vector_ms: number;
+  reranker_ms: number;
+}
+
+export interface RetrieveSnapshotOptions {
+  onTiming?: (timing: StageTiming) => void;
+}
+
 /**
  * Retrieve the full fused candidate set and rerank it once without applying
- * decision thresholds. Calibration uses this to avoid observing the policy it
- * is trying to replace.
+ * decision thresholds, synchronizing the vault before reading the index.
  */
 export async function retrieveAndRerank(
   input: ResolveSkillInput,
 ): Promise<RetrievalResult> {
-  const { config, db } = await getEnv();
   await syncVaultIfNeeded();
+  return retrieveAndRerankSnapshot(input);
+}
+
+/**
+ * Retrieve candidates against an already-synchronized index snapshot without
+ * triggering syncVaultIfNeeded(). Calibration uses this to avoid observing the
+ * policy it is trying to replace while keeping one corpus for the whole run.
+ */
+export async function retrieveAndRerankSnapshot(
+  input: ResolveSkillInput,
+  options?: RetrieveSnapshotOptions,
+): Promise<RetrievalResult> {
+  const { config, db } = await getEnv();
   const clients = getClients();
+  const measureTiming = options?.onTiming !== undefined;
+
+  let embedding_ms = 0;
+  let lexical_ms = 0;
+  let vector_ms = 0;
+  let reranker_ms = 0;
+
+  const tEmbed0 = measureTiming ? performance.now() : 0;
+  const embedPromise =
+    !input.forceLexical && clients.embed
+      ? clients.embed([input.query]).then(
+        (res) => {
+          if (measureTiming) embedding_ms = Math.max(0, performance.now() - tEmbed0);
+          return res;
+        },
+        (err) => {
+          if (measureTiming) embedding_ms = Math.max(0, performance.now() - tEmbed0);
+          return { error: err };
+        },
+      )
+      : null;
+
+  const tLex0 = measureTiming ? performance.now() : 0;
   const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
+  if (measureTiming) lexical_ms = Math.max(0, performance.now() - tLex0);
   const lexicalRanks = new Map(lexical.map((row, index) => [row.skill_id, index + 1]));
 
   let retrieval: RetrievalResult["retrieval"] = "lexical";
@@ -550,18 +596,12 @@ export async function retrieveAndRerank(
   let rows = lexical;
   let fusedRows: SkillRow[] | null = null;
 
-  if (!input.forceLexical) {
-    try {
-      const queryVec = (await clients.embed([input.query]))[0];
-      if (!queryVec) throw new Error("Embedding client returned no query vector.");
-      const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
-      rows = reciprocalRankFusion(lexical, nearest);
-      fusedRows = rows;
-      retrieval = "hybrid";
-    } catch (embedError) {
+  if (embedPromise) {
+    const embedRes = await embedPromise;
+    if (embedRes && typeof embedRes === "object" && "error" in embedRes) {
       retrieval = "lexical";
       degraded_from = clients.rerank ? "reranked" : "hybrid";
-      degradation_reason = classifyInferenceError("embedding", embedError);
+      degradation_reason = classifyInferenceError("embedding", embedRes.error);
       console.error(
         JSON.stringify({
           level: "warn",
@@ -570,6 +610,30 @@ export async function retrieveAndRerank(
           reason: degradation_reason,
         }),
       );
+    } else {
+      const tVec0 = measureTiming ? performance.now() : 0;
+      try {
+        const queryVec = (embedRes as Float32Array[])[0];
+        if (!queryVec) throw new Error("Embedding client returned no query vector.");
+        const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
+        if (measureTiming) vector_ms = Math.max(0, performance.now() - tVec0);
+        rows = reciprocalRankFusion(lexical, nearest);
+        fusedRows = rows;
+        retrieval = "hybrid";
+      } catch (embedError) {
+        if (measureTiming) vector_ms = Math.max(0, performance.now() - tVec0);
+        retrieval = "lexical";
+        degraded_from = clients.rerank ? "reranked" : "hybrid";
+        degradation_reason = classifyInferenceError("embedding", embedError);
+        console.error(
+          JSON.stringify({
+            level: "warn",
+            stage: "embedding",
+            degraded_from,
+            reason: degradation_reason,
+          }),
+        );
+      }
     }
   }
 
@@ -577,14 +641,17 @@ export async function retrieveAndRerank(
   if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
     const kRerank = config.recall.k_rerank ?? 10;
     const rerankCandidates = rows.slice(0, kRerank);
+    const tRerank0 = measureTiming ? performance.now() : 0;
     try {
       scores = await clients.rerank(
         input.query,
         rerankCandidates.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
       );
+      if (measureTiming) reranker_ms = Math.max(0, performance.now() - tRerank0);
       retrieval = "reranked";
       rows = rerankCandidates;
     } catch (rerankError) {
+      if (measureTiming) reranker_ms = Math.max(0, performance.now() - tRerank0);
       scores = null;
       degraded_from = "reranked";
       degradation_reason = classifyInferenceError("reranker", rerankError);
@@ -612,6 +679,13 @@ export async function retrieveAndRerank(
     : new Map<string, number>();
   const traceRows = fusedRows ?? rows;
 
+  options?.onTiming?.({
+    embedding_ms,
+    lexical_ms,
+    vector_ms,
+    reranker_ms,
+  });
+
   return {
     retrieval,
     ...(degraded_from ? { degraded_from, degradation_reason } : {}),